From 816cb57a4ca0d197b96e12d89c4e02a3b4b6d899 Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:37:23 -0800 Subject: [PATCH 1/6] [None][fix] FlashMLA: release the GIL in the pybind launch wrappers FlashMLA's pybind module binds its forward functions with plain m.def, so a call keeps the GIL while it blocks in a kernel launch. A CUDA host function that takes the GIL and sits ahead of those kernels on the stream then deadlocks the rank: the launch waits on the host function, and the host function waits on the GIL. The capturable guided decoder is such a host function (guided decoding beside one-model speculative decoding); its callbacks run on the stream and acquire the GIL in cpp/tensorrt_llm/nanobind/runtime/hostfunc.cpp. Under serving load the launch queue fills and an eager step that calls FlashMLA hangs. On main, FlashMLA serves the DSA sparse MLA backend (DeepSeek-V3.2 and the GLM models built on it), DeepSeek-V4, CSA2 and the GLM K-pool backend, so any of these with guided decoding and MTP or another one-model drafter can hit it. Found while bringing up Mistral Large 4 (xgrammar beside the DSpark drafter), where eager context steps deadlocked under load. A FetchContent patch adds pybind11::call_guard to the five bindings. Their arguments and results are tensors and scalars, converted before the GIL is released and after it is reacquired. FlashMLA is a FetchContent dependency (3rdparty/fetch_content.json), so the patch follows the patch_file convention of DeepGEMM and DeepEP rather than the vendored-source lock. Test: test_flash_mla_gil.py queues a GIL-taking host function behind GPU work and fills the launch queue with sparse_prefill_fwd calls in a subprocess; it deadlocks without the patch. Split out of #21693 (Mistral Large 4 merge-back); originally GitLab 876a93ccd7. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- 3rdparty/fetch_content.json | 3 +- 3rdparty/patches/flashmla_release_gil.patch | 17 +++++++ .../attention/sparse/test_flash_mla_gil.py | 51 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 3rdparty/patches/flashmla_release_gil.patch create mode 100644 tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py diff --git a/3rdparty/fetch_content.json b/3rdparty/fetch_content.json index 6ded1e29d280..412007acf9e0 100644 --- a/3rdparty/fetch_content.json +++ b/3rdparty/fetch_content.json @@ -62,7 +62,8 @@ "git_repository": "https://github.com/deepseek-ai/FlashMLA.git", "git_tag": "15f13e5030374295491c5ce31b02d7e63a7772c6", "git_submodules_recurse": true, - "source_subdir": "dont-add-this-project-with-add-subdirectory" + "source_subdir": "dont-add-this-project-with-add-subdirectory", + "patch_file": "patches/flashmla_release_gil.patch" }, { "name": "fmt", diff --git a/3rdparty/patches/flashmla_release_gil.patch b/3rdparty/patches/flashmla_release_gil.patch new file mode 100644 index 000000000000..8be7dbd6d2b9 --- /dev/null +++ b/3rdparty/patches/flashmla_release_gil.patch @@ -0,0 +1,17 @@ +--- a/csrc/api/api.cpp ++++ b/csrc/api/api.cpp +@@ -7,9 +7,9 @@ + + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.doc() = "FlashMLA"; +- m.def("sparse_decode_fwd", &sparse_attn_decode_interface); +- m.def("dense_decode_fwd", &dense_attn_decode_interface); +- m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface); +- m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun); +- m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun); ++ m.def("sparse_decode_fwd", &sparse_attn_decode_interface, pybind11::call_guard()); ++ m.def("dense_decode_fwd", &dense_attn_decode_interface, pybind11::call_guard()); ++ m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface, pybind11::call_guard()); ++ m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun, pybind11::call_guard()); ++ m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun, pybind11::call_guard()); + } diff --git a/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py b/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py new file mode 100644 index 000000000000..53d97b4f8104 --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""FlashMLA's launch wrappers release the GIL. + +A CUDA host function that takes the GIL (the capturable guided decoder's, for one) can be queued ahead of +FlashMLA's kernels on the same stream. Once the launch queue is full, a FlashMLA call that kept the GIL while +it blocks in a launch would wait on the host function, and the host function on the GIL. +""" + +import subprocess +import sys +import textwrap + +import pytest +import torch + +_CHILD = textwrap.dedent(""" + import time + + import torch + + import tensorrt_llm.flash_mla_cpp_tllm as flash_mla + from tensorrt_llm._torch.hostfunc import launch_hostfunc + + q = torch.randn(4, 64, 576, device="cuda", dtype=torch.bfloat16) + kv = torch.randn(256, 1, 576, device="cuda", dtype=torch.bfloat16) + indices = torch.randint(0, 256, (4, 1, 64), device="cuda", dtype=torch.int32) + flash_mla.sparse_prefill_fwd(q, kv, indices, 0.05, 512, None, None) + torch.cuda.synchronize() + + ran = [] + torch.cuda._sleep(int(3e9)) # the host function runs only after the device works through this + launch_hostfunc(lambda: ran.append(time.time())) + for _ in range(20000): # far more launches than the queue holds + flash_mla.sparse_prefill_fwd(q, kv, indices, 0.05, 512, None, None) + torch.cuda.synchronize() + print("host function ran", len(ran)) +""") + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="FlashMLA sparse prefill on SM100-family GPUs", +) +def test_sparse_prefill_does_not_hold_the_gil_while_launching(): + result = subprocess.run( + [sys.executable, "-c", _CHILD], capture_output=True, text=True, timeout=120 + ) + + assert result.returncode == 0, result.stderr[-2000:] + assert "host function ran 1" in result.stdout From b141beb7ec985427f0920784184ed6575a1bbdfd Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:32:02 -0800 Subject: [PATCH 2/6] [None][fix] Attention DP: give the padding dummy a cache in the paired draft KV pool With joint KV cache reuse the KV cache manager V2 scheduler admits every generation request in the target and the draft pool together (KVCacheV2Scheduler._try_allocate_generation). The attention-DP padding dummy was created with a target cache only, so its draft admission found no cache, the target growth was reverted, and the dummy was never scheduled. The idle rank then scheduled an empty batch, _can_queue vetoed the forward on every rank, and the live dummy kept the empty-batch padding from stepping in: the whole attention-DP group stopped making progress as soon as one rank went idle. The empty-batch padding path, which bypasses the scheduler, instead left the draft cache to the draft manager's prepare_resources mirror. That mirror raises when the draft pool is short, which takes down the executor loop; this follows from the code and was not observed in a run. Joint reuse is on for a one-model DFlash or DSpark drafter with its own draft KV pool, KV cache manager V2 and block reuse; attention DP keeps the separate draft pool for these standalone drafters. Found while bringing up Mistral Large 4 with its DSpark drafter: at DEP4 and DEP8 the first request never completed while the other ranks idled, and the same configuration ran to completion with this change. Every padding path now passes the paired draft manager to add_dummy_requests, so the dummy is created in both pools or in neither. A vetoed iteration already frees the draft pool's cache in _finalize_adp_dummy_allocation; a spec resource manager without a free slot now frees it as well. The non-transactional path also rolls back when the spec resource manager has no free slot, as the other two paths already did, instead of propagating NoFreeSlotsError with the KV allocated. add_dummy_requests reserves the dummy's whole forward in both pools, and the empty-batch dummy joins the batch after the scheduler, so it carries no scheduler allocation record. The draft manager's prepare_resources therefore takes a paired pool's existing dummy cache as already sized instead of growing it by another generation step: with a draft length of 8 that step would take the 24-token cache to 33 tokens, a second 32-token page, after the ranks have agreed to run the batch. An unpaired draft pool is not handed to add_dummy_requests; the mirror the scheduler admits for its dummy is empty and still grows in prepare_resources. Tests: test_adp_dummy_holds_paired_draft_kv builds real target and draft KVCacheManagerV2 pools with joint reuse and a KVCacheV2Scheduler, and checks for the plain, transactional and empty-batch padding paths that the dummy holds a cache in both pools, is scheduled, keeps its capacity through prepare_resources, and is released from both on a vetoed iteration. test_adp_dummy_grows_unpaired_draft_mirror runs the same paths over unpaired pools for two queued iterations and checks that the mirror grows by one generation step and takes the previous dummy's deferred update. test_draft_mirror_preserves_preallocated_dummy_capacity (CPU) pins which mirrors grow. The stub executors of test_py_executor.py, test_benchmark_disagg.py and test_connector_adp.py carry the new attribute (off). Split out of #21693 (Mistral Large 4 merge-back); originally GitLab 4932efdc24 ("Add native DSpark to StoneWillow Staircase specializations"). Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- .../kv_cache/kv_cache_manager_v2.py | 13 +- tensorrt_llm/_torch/pyexecutor/py_executor.py | 40 ++- .../disaggregation/test_benchmark_disagg.py | 1 + .../kv_cache/test_kvcm2_integration.py | 273 ++++++++++++++++++ .../_torch/pyexecutor/test_py_executor.py | 1 + tests/unittest/_torch/test_connector_adp.py | 1 + 6 files changed, 320 insertions(+), 9 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index f12801b255e1..edb9202addd4 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -6882,6 +6882,17 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests): self._after_shrink(req.py_request_id, kv_cache) for req in scheduled_batch.generation_requests: + # A paired pool's generation dummy has a fixed shape, and + # add_dummy_requests reserves its full forward capacity in both + # pools, even when it joins the batch after scheduler admission. + # An unpaired pool is not handed to add_dummy_requests: the + # mirror the scheduler admits for its dummy is empty and grows + # below like any other. + preallocated_dummy = ( + self.enable_joint_kv_cache_reuse + and req.is_dummy_request + and req.py_request_id in self.kv_cache_map + ) kv_cache = self._mirror_draft_kv_cache(req) if kv_cache is None: raise RuntimeError( @@ -6898,7 +6909,7 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests): ) # A paired scheduler already grew both pools atomically; the # recorded width says so, so do not charge this step twice. - if req.py_request_id in self._allocated_draft_lens: + if preallocated_dummy or req.py_request_id in self._allocated_draft_lens: continue new_cap = self._required_gen_capacity(req, kv_cache.capacity) if not kv_cache.resize(new_cap): diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index cd5703847df2..0ef8c9209f2d 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -5248,13 +5248,14 @@ def _finalize_adp_dummy_allocation(self, can_queue: bool) -> None: if spec_resource_manager is not None: spec_resource_manager.free_resources(dummy_request) self.kv_cache_manager.free_resources(dummy_request) - # The V2 scheduler also admits the draft pool's mirror while it - # schedules the dummy (KVCacheV2Scheduler._admit_unpaired_draft), so - # that pool holds an empty cache under the dummy ID too. Left behind, - # it outlives this dummy, and the deferred previous batch -- which - # still holds the earlier, finished dummy with the same ID -- resizes - # it to that dummy's history in update_resources ("History length - # cannot exceed capacity"). + # The draft pool holds a cache under the dummy ID too: a paired pool + # gets it with the dummy (add_dummy_requests), and the V2 scheduler + # admits an unpaired pool's mirror while it schedules the dummy + # (KVCacheV2Scheduler._admit_unpaired_draft). Left behind, it outlives + # this dummy, and the deferred previous batch -- which still holds the + # earlier, finished dummy with the same ID -- resizes it to that + # dummy's history in update_resources ("History length cannot exceed + # capacity"). draft_kv_cache_manager = getattr(self, "draft_kv_cache_manager", None) if (getattr(self, "_is_kv_manager_v2", False) and draft_kv_cache_manager is not None): @@ -9589,6 +9590,11 @@ def _pad_attention_dp_dummy_request(self): key="attention_dp_dummy_insufficient_kv_capacity") return + # A paired draft pool admits every scheduled request together with the + # target pool, so the dummy needs a cache in both. + draft_kv_cache_manager = (self.draft_kv_cache_manager + if self.enable_joint_kv_cache_reuse else None) + if (not self._enable_adp_dummy_fixes or self.kv_cache_transceiver is None): try: @@ -9597,6 +9603,7 @@ def _pad_attention_dp_dummy_request(self): token_nums=token_nums, is_gen=self._adp_dummy_is_gen, max_num_draft_tokens=self.max_total_draft_tokens, + draft_kv_cache_manager=draft_kv_cache_manager, ) except OutOfPagesError: dummy_requests = None @@ -9615,7 +9622,13 @@ def _pad_attention_dp_dummy_request(self): spec_resource_manager = self.resource_manager.get_resource_manager( ResourceManagerType.SPEC_RESOURCE_MANAGER) if spec_resource_manager is not None: - spec_resource_manager.add_dummy_requests(dummy_request_ids) + try: + spec_resource_manager.add_dummy_requests(dummy_request_ids) + except NoFreeSlotsError: + self.kv_cache_manager.free_resources(llm_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(llm_request) + return self.active_requests.append(llm_request) return @@ -9632,6 +9645,7 @@ def _pad_attention_dp_dummy_request(self): token_nums=token_nums, is_gen=self._adp_dummy_is_gen, max_num_draft_tokens=self.max_total_draft_tokens, + draft_kv_cache_manager=draft_kv_cache_manager, ) except OutOfPagesError: dummy_requests = None @@ -9648,6 +9662,8 @@ def _pad_attention_dp_dummy_request(self): spec_resource_manager.add_dummy_requests(dummy_request_ids) except NoFreeSlotsError: self.kv_cache_manager.free_resources(dummy_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(dummy_request) return assert dummy_request is not None @@ -9723,6 +9739,11 @@ def _pad_empty_attention_dp_batch( return dummy_request_ids = [ATTENTION_DP_DUMMY_REQUEST_ID] + # Allocate the paired draft cache with the dummy, where a shortage + # degrades to an empty batch, rather than leaving it to the draft + # manager's prepare_resources mirror, which raises on one. + draft_kv_cache_manager = (self.draft_kv_cache_manager + if self.enable_joint_kv_cache_reuse else None) try: # Degrade to an empty batch rather than propagate: the ranks have # yet to agree on can_queue, so a rank-local raise would strand the @@ -9733,6 +9754,7 @@ def _pad_empty_attention_dp_batch( is_gen=not context_dummy, max_num_draft_tokens=(0 if context_dummy else self.max_total_draft_tokens), + draft_kv_cache_manager=draft_kv_cache_manager, ) except (OutOfPagesError, NoFreeSlotsError): dummy_requests = None @@ -9751,6 +9773,8 @@ def _pad_empty_attention_dp_batch( spec_resource_manager.add_dummy_requests(dummy_request_ids) except NoFreeSlotsError: self.kv_cache_manager.free_resources(dummy_request) + if draft_kv_cache_manager is not None: + draft_kv_cache_manager.free_resources(dummy_request) return dummy_request.is_attention_dp_dummy = True diff --git a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py index b30177e29839..5dda01577ca5 100644 --- a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py +++ b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py @@ -755,6 +755,7 @@ def __init__( self.num_fetch_requests = num_fetch_requests self.benchmark_req_queues_size = benchmark_req_queues_size self.max_total_draft_tokens = 0 + self.enable_joint_kv_cache_reuse = False self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None self._enable_adp_dummy_fixes = True diff --git a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py index 9ddd9429d3b9..dc83dea4758c 100644 --- a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py +++ b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py @@ -17,6 +17,7 @@ import os import sys from collections.abc import Callable +from contextlib import ExitStack from dataclasses import dataclass, field, replace from types import SimpleNamespace from unittest.mock import ANY, Mock, call, patch @@ -57,6 +58,7 @@ from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm._torch.pyexecutor.scheduler.scheduler_v2 import KVCacheV2Scheduler from tensorrt_llm._torch.speculative.dflash import DFlashSpecMetadata, DFlashWorker from tensorrt_llm.bindings import DataType, SamplingConfig from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE @@ -66,6 +68,7 @@ from tensorrt_llm.conversation_params import ConversationParams from tensorrt_llm.llmapi.llm_args import ( BlockReuseConfig, + CapacitySchedulerPolicy, DecodingBaseConfig, DFlashDecodingConfig, DraftTargetDecodingConfig, @@ -3349,6 +3352,276 @@ def test_generation_dummy_uses_available_capacity(draft_len: int) -> None: manager.shutdown() +def _adp_dummy_executor( + cleanup: ExitStack, + spec: DFlashDecodingConfig, + *, + joint_reuse: bool, + transactional: bool, +) -> tuple[PyExecutor, dict[ResourceManagerType, KVCacheManagerV2]]: + """An attention-DP executor over real target and draft KV pools. + + ``joint_reuse`` pairs the two pools (block reuse on both); without it the + draft pool is an unpaired mirror of the target pool. + """ + managers = {} + for resource_type, is_draft in ( + (ResourceManagerType.KV_CACHE_MANAGER, False), + (ResourceManagerType.DRAFT_KV_CACHE_MANAGER, True), + ): + manager = KVCacheManagerV2( + KvCacheConfig(enable_block_reuse=joint_reuse, max_gpu_total_bytes=4 << 20), + CacheType.SELF, + num_layers=1, + num_kv_heads=2, + head_dim=64, + tokens_per_block=32, + max_seq_len=128, + max_batch_size=4, + max_num_tokens=128, + mapping=Mapping(), + spec_config=spec, + is_draft=is_draft, + joint_kv_cache_reuse=joint_reuse, + ) + cleanup.callback(manager.shutdown) + managers[resource_type] = manager + + executor = object.__new__(PyExecutor) + executor.resource_manager = ResourceManager(managers) + executor.kv_cache_manager = managers[ResourceManagerType.KV_CACHE_MANAGER] + executor.draft_kv_cache_manager = managers[ResourceManagerType.DRAFT_KV_CACHE_MANAGER] + executor.scheduler = KVCacheV2Scheduler( + max_batch_size=4, + max_num_tokens=128, + kv_cache_manager=executor.kv_cache_manager, + draft_kv_cache_manager=executor.draft_kv_cache_manager, + scheduler_policy=CapacitySchedulerPolicy.MAX_UTILIZATION, + ) + executor.enable_attention_dp = True + # Two attention-DP ranks: a single-rank group skips dummy padding. + executor.dist = SimpleNamespace(tp_size=2) + executor.kv_cache_transceiver = object() if transactional else None + executor.adp_router = SimpleNamespace(exclude_retiring_requests=False) + executor.active_requests = [] + executor.expected_num_active_requests = 1 + executor.max_num_active_requests = 4 + executor.max_total_draft_tokens = spec.max_draft_len + executor._adp_dummy_is_gen = True + executor._pending_adp_dummy_request = None + executor._enable_adp_dummy_fixes = True + executor._enable_scheduler_aware_adp_dummy = False + executor._enable_non_overlap_adp_forward_intent = False + executor._benchmark_fill_phase_active = False + executor._is_kv_manager_v2 = True + executor.enable_joint_kv_cache_reuse = joint_reuse + return executor, managers + + +@pytest.mark.parametrize("draft_len", [3, 8]) +@pytest.mark.parametrize("path", ["plain", "transactional", "empty_batch"]) +def test_adp_dummy_holds_paired_draft_kv(path: str, draft_len: int) -> None: + """An idle attention-DP rank's dummy needs a cache in both paired pools. + + Under joint KV cache reuse the scheduler admits a generation request in + the target and the draft pool together, so a dummy holding only a target + cache is never scheduled and the fleet cannot run a forward. Each padding + path must allocate the dummy in both pools, and a vetoed iteration must + release both so the singleton dummy ID can be allocated again. + """ + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + torch.cuda.init() + spec = DFlashDecodingConfig(max_draft_len=draft_len, speculative_model="draft") + with ExitStack() as cleanup: + executor, managers = _adp_dummy_executor( + cleanup, spec, joint_reuse=True, transactional=path == "transactional" + ) + + def assert_dummy_in_both_pools(dummy: LlmRequest) -> None: + for manager in managers.values(): + assert manager.kv_cache_map[dummy.py_request_id].is_active + + def assert_dummy_released(dummy: LlmRequest) -> None: + assert executor.active_requests == [] + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + assert dummy.py_request_id not in manager._allocated_draft_lens + assert dummy.py_request_id not in manager._inflight_draft_lens + + def prepare_without_growing(scheduled: ScheduledRequests, dummy: LlmRequest) -> None: + capacities = { + key: manager.kv_cache_map[dummy.py_request_id].capacity + for key, manager in managers.items() + } + executor.resource_manager.prepare_resources(scheduled) + for key, manager in managers.items(): + assert manager.kv_cache_map[dummy.py_request_id].capacity == capacities[key] + + if path == "empty_batch": + # The dummy joins the already scheduled batch directly; a vetoed + # iteration rolls it back, and the next one pads again. + for can_queue in (False, True): + scheduled = ScheduledRequests() + executor._pad_empty_attention_dp_batch(scheduled) + (dummy,) = scheduled.generation_requests + assert executor.active_requests == [dummy] + assert_dummy_in_both_pools(dummy) + executor._finalize_adp_dummy_allocation(can_queue) + if not can_queue: + assert_dummy_released(dummy) + else: + prepare_without_growing(scheduled, dummy) + executor.resource_manager.free_resources(dummy) + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + return + + # A rejected fleet vote must release both pools and permit the same + # singleton dummy ID to be allocated and admitted on the next pass. + for attempt in range(2 if path == "transactional" else 1): + executor._pad_attention_dp_dummy_request() + (dummy,) = executor.active_requests + scheduled = executor.scheduler.schedule_request(executor.active_requests, set()) + assert scheduled.generation_requests == [dummy] + assert scheduled.paused_requests == [] + assert_dummy_in_both_pools(dummy) + if path == "transactional" and attempt == 0: + executor._revert_gen_alloc(scheduled) + executor._finalize_adp_dummy_allocation(False) + assert_dummy_released(dummy) + else: + executor._finalize_adp_dummy_allocation(True) + prepare_without_growing(scheduled, dummy) + executor.resource_manager.free_resources(dummy) + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + + +@pytest.mark.parametrize("path", ["plain", "transactional", "empty_batch"]) +def test_adp_dummy_grows_unpaired_draft_mirror(path: str) -> None: + """An unpaired draft pool sizes the attention-DP dummy's mirror in prepare_resources. + + The dummy is created in the target pool only. The scheduler admits the + draft mirror while it schedules the dummy but leaves it empty, and the + empty-batch path leaves no mirror at all, so the draft manager has to grow + it by one generation step. An empty mirror cannot take the history of the + previous iteration's finished dummy, which shares its ID and which the + overlap loop applies once this iteration is prepared. + """ + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + torch.cuda.init() + spec = DFlashDecodingConfig(max_draft_len=8, speculative_model="draft") + with ExitStack() as cleanup: + executor, managers = _adp_dummy_executor( + cleanup, spec, joint_reuse=False, transactional=path == "transactional" + ) + draft = executor.draft_kv_cache_manager + assert executor.scheduler._joint_draft_manager is None + + previous_batch = None + for _ in range(2): + batch = ScheduledRequests() + if path == "empty_batch": + executor._pad_empty_attention_dp_batch(batch) + (dummy,) = batch.generation_requests + assert dummy.py_request_id not in draft.kv_cache_map + else: + executor._pad_attention_dp_dummy_request() + (dummy,) = executor.active_requests + scheduled = executor.scheduler.schedule_request(executor.active_requests, set()) + assert scheduled.generation_requests == [dummy] + batch.generation_requests.append(dummy) + # Admitted by the scheduler, not sized. + assert draft.kv_cache_map[dummy.py_request_id].capacity == 0 + executor._finalize_adp_dummy_allocation(True) + + executor.resource_manager.prepare_resources(batch) + draft_cache = draft.kv_cache_map[dummy.py_request_id] + assert draft_cache.capacity == draft._required_gen_capacity(dummy, 0) + + if previous_batch is not None: + # _process_previous_batch: the deferred batch still holds the + # finished dummy, whose ID now maps to this iteration's mirror. + executor.resource_manager.update_resources( + previous_batch, skip_kv_cache_manager=True + ) + # _terminate_attention_dp_dummy_requests. + dummy.state = LlmRequestState.GENERATION_COMPLETE + executor.resource_manager.free_resources(dummy) + executor.active_requests.remove(dummy) + previous_batch = batch + + for manager in managers.values(): + assert dummy.py_request_id not in manager.kv_cache_map + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "joint_reuse,is_dummy,mirror_capacity,grows", + [ + # Paired pool: add_dummy_requests sized the dummy's mirror with the dummy. + pytest.param(True, True, 24, False, id="paired-dummy-preallocated"), + # Unpaired pool: the scheduler admitted the dummy's mirror, still empty. + pytest.param(False, True, 0, True, id="unpaired-dummy-admitted-empty"), + # Unpaired pool, empty-batch padding: the dummy has no mirror yet. + pytest.param(False, True, None, True, id="unpaired-dummy-no-mirror"), + # Real requests are a control: they grow by one step in either pool. + pytest.param(True, False, 24, True, id="paired-request"), + pytest.param(False, False, 24, True, id="unpaired-request"), + pytest.param(False, False, None, True, id="unpaired-request-no-mirror"), + ], +) +def test_draft_mirror_preserves_preallocated_dummy_capacity( + joint_reuse: bool, is_dummy: bool, mirror_capacity: int | None, grows: bool +) -> None: + """A paired pool's preallocated dummy fits one page; charging another step would need two. + + Every other mirror, an unpaired pool's dummy included, is sized here. + """ + request = LlmRequest( + request_id=100, + max_new_tokens=1, + input_tokens=[1] * 9, + sampling_config=SamplingConfig(1), + is_streaming=False, + draft_tokens=[1] * 8, + ) + request.state = LlmRequestState.GENERATION_IN_PROGRESS + request.is_dummy_request = is_dummy + capacity = mirror_capacity or 0 + cache = Mock(capacity=capacity, is_active=True) + # Dummies have only one 32-token page available. Real requests are a + # control: unlike fixed-shape dummies they still need per-step growth. + cache.resize.side_effect = lambda capacity: capacity <= (32 if is_dummy else 64) + manager = object.__new__(KVCacheManagerV2) + manager.is_draft = True + manager.enable_joint_kv_cache_reuse = joint_reuse + manager._mirror_uses_draft_view = False + manager._kv_reserve_draft_tokens = 8 + manager._allocated_draft_lens = {} + manager.kv_cache_map = {} if mirror_capacity is None else {request.py_request_id: cache} + manager._mirror_draft_kv_cache = Mock(return_value=cache) + manager._resume_and_restore = Mock(return_value=True) + manager._ensure_generation_beam_width = Mock(return_value=True) + manager._mirror_resize_error = Mock(return_value=RuntimeError("draft pool exhausted")) + batch = ScheduledRequests() + batch.generation_requests.append(request) + + manager._prepare_draft_resources(batch) + + manager._resume_and_restore.assert_called_once_with(request.py_request_id, cache) + if grows: + # One generation step: the token itself and the eight draft slots. + cache.resize.assert_called_once_with(capacity + 9) + else: + cache.resize.assert_not_called() + # Recognizing a preallocated dummy must not fabricate scheduler rollback + # bookkeeping; the dummy is freed as a whole after the iteration. + assert manager._allocated_draft_lens == {} + + def test_generation_dummy_does_not_reuse_real_request_prefix() -> None: if not torch.cuda.is_available(): pytest.skip("requires CUDA") diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index cf8383290002..5e660220412a 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -3164,6 +3164,7 @@ def __init__( self.expected_num_active_requests = 1 self.max_num_active_requests = 8 self.max_total_draft_tokens = 0 + self.enable_joint_kv_cache_reuse = False self.max_num_tokens = max_num_tokens self._adp_dummy_is_gen = True self._pending_adp_dummy_request = None diff --git a/tests/unittest/_torch/test_connector_adp.py b/tests/unittest/_torch/test_connector_adp.py index c3cd2e1abcf2..f7d5fe59983c 100644 --- a/tests/unittest/_torch/test_connector_adp.py +++ b/tests/unittest/_torch/test_connector_adp.py @@ -116,6 +116,7 @@ def _configure_recovery_controls(executor: "PyExecutor") -> None: executor._profile_manager = MagicMock() executor.model_engine = SimpleNamespace(cuda_graph_runner=None) executor.draft_model_engine = None + executor.enable_joint_kv_cache_reuse = False executor.sampler = None executor.worker_thread = MagicMock() executor._shutdown_sleep_wakeup_listeners = MagicMock() From daead922f2c7d32a6bfb48e7b2d663570980632e Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:41:38 -0800 Subject: [PATCH 3/6] [None][fix] Decoder warmup: run the context warmup beside a capturable guided decoder A capturable guided decoder (guided decoding together with one-model speculative decoding: MTP, Eagle3 one-model, the external drafters such as DFlash and DSpark, or SA) turned the whole general warmup off, though it rules out only the warmup's generation requests: the autotuner already runs its context step with the decoder installed. Without the general warmup, the autotuner's max-token step was the engine's first forward. What a first forward creates lazily, including the attention-DP group's NCCL communicator (allocated through cuMem, outside the torch caching allocator, so it cannot reclaim the cache), then came in the middle of the autotuner's exploration with torch's cache full. Any model served with guided decoding and one of those drafters takes this path. It ran memory profiling out of memory while bringing up Mistral Large 4 (NVFP4 DEP4 at 1M context with the DSpark draft and xgrammar): rank 0 ran out of memory in the autotuner's max-token step, and the other ranks failed to allocate NCCL memory through cuMem. Without guided decoding, where the general warmup and its release run ahead of the autotuner, the same configuration profiles at a peak of 238.6 GiB per rank. The general warmup now runs its context requests in that case, followed by the same release as before, so the first forward and the release come ahead of the autotuner, as they do without guided decoding. The extra phase decision is agreed across ranks like can_run_general_warmup, which is already rank-uniform when it is taken. The general warmup skips a shape that runs a single rank out of memory and goes on to a smaller one. With the decoder installed, input preparation has already queued that shape's batch for the decoder's host functions, so the abandoned batch would stay at the head of the queue and every later forward, the first real guided request included, would be served the batch before its own. The recovery now waits for the decoder's stream, so that no host function of the failed forward is still pending, and drops the queued batches and both request snapshots (CapturableGuidedDecoder.reset_warmup_state). A failed reset stops the warmup, and a distributed forward still re-raises the out-of-memory error. Tests: test_guided_decoder_runs_the_context_warmup_before_the_autotuner installs a guided decoder and checks that the general warmup runs with context-only shapes (including the max-token one), followed by the MoE workspace release, the cache release and then the autotuner. test_general_warmup_oom_restores_guided_batch_queue fails a shape before and after its host function is submitted and checks that the next guided request is served its own batch. test_general_warmup_oom_policy and test_general_warmup_aborts_if_guided_state_cannot_be_reset cover when the reset runs and that its failure stops the warmup. Split out of #21693 (Mistral Large 4 merge-back); originally GitLab a2d3872e7b, rebased onto main's warmup after #21343. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- .../engine/runners/decoder/warmup.py | 23 ++- .../_torch/pyexecutor/guided_decoder.py | 14 ++ .../engine/runners/decoder/test_warmup.py | 134 +++++++++++++++++- 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py index e4763d201dd1..fd9bca875200 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py @@ -437,6 +437,17 @@ def _warmup_scheduled( generation_only = self.mapping.has_cp_helix() or is_decode_only( self.sparse_attention_config ) + # A capturable guided decoder rules out the general warmup's generation + # requests only: the autotuner below runs its context step with the + # decoder installed. Run the context requests anyway, so that the + # autotuner's max-token step is not the engine's first forward (with + # the buffers it allocates lazily) and runs after the release below, + # as it does without guided decoding. + can_run_context_warmup = can_run_general_warmup or self._agreement.agree_flag( + self._context_warmups_supported + and not generation_only + and not isinstance(kv_cache_manager, MambaHybridCacheManager) + ) log_mem_snapshot("warmup/before_warmup") # Compile the DSv4 indexer-Q CuTe DSL kernels before the first @@ -462,7 +473,7 @@ def _warmup_scheduled( resource_manager, can_run_general_warmup and not generation_only ) - if can_run_general_warmup: + if can_run_context_warmup: # Specialize torch.compile graphs across the key input shapes before CUDA graph capture. with self._warmup_timer.phase( "general", metrics=self._metrics, metric_name="general_warmup_seconds" @@ -472,6 +483,12 @@ def _warmup_scheduled( warmup_requests_configs = self._generation_warmup_shapes( warmup_requests_configs ) + elif not can_run_general_warmup: + warmup_requests_configs = [ + (num_tokens, num_gen_tokens) + for num_tokens, num_gen_tokens in warmup_requests_configs + if num_gen_tokens == 0 + ] warmup_requests_configs = self._agreement.agree_shapes(warmup_requests_configs) # Currently graph has not been captured, disable cuda graph for this warmup. with self._graphs.no_cuda_graph(): @@ -987,6 +1004,10 @@ def _general_warmup_impl( # Peers are inside the same forward's collectives and # cannot follow a rank-local skip. raise + if self.guided_decoder is not None: + # Input preparation enqueues the batch before model forward. + # An OOM may leave it unconsumed, or leave callbacks pending. + self.guided_decoder.reset_warmup_state() logger.warning( f"OOM during general warmup with {num_tokens} tokens, " f"{num_gen_tokens} generation tokens. Skipping." diff --git a/tensorrt_llm/_torch/pyexecutor/guided_decoder.py b/tensorrt_llm/_torch/pyexecutor/guided_decoder.py index 109d1b197cf7..a8f8fcfe4ae3 100644 --- a/tensorrt_llm/_torch/pyexecutor/guided_decoder.py +++ b/tensorrt_llm/_torch/pyexecutor/guided_decoder.py @@ -581,6 +581,20 @@ def _drain_host_functions(self) -> None: if not torch.cuda.is_current_stream_capturing(): self.bitmask_event.synchronize() + def reset_warmup_state(self) -> None: + """Discard abandoned dummy batches after an eager warmup failure. + + Only call during warmup, with no live requests or graph capture. Wait + on the stream itself: a failed forward may have enqueued callbacks + without reaching the bitmask-event record. Once they finish, no + callback can consume or overwrite the cleared request snapshots. + Warmup dummies have no grammar matchers to roll back. + """ + self.stream.synchronize() + self.queue = Queue() + self.requests = None + self.requests_hostfunc = None + @nvtx_range("GuidedDecoder.add_batch") def add_batch(self, scheduled_requests: ScheduledRequests, diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py index 87ee1978acec..685e974b0cf8 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py @@ -11,6 +11,7 @@ import unittest from collections.abc import Iterator from dataclasses import replace +from queue import Queue from types import ModuleType, SimpleNamespace from typing import TYPE_CHECKING from unittest import mock @@ -44,6 +45,7 @@ from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.graphs import DecoderCudaGraphs from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.metadata import DecoderMetadata from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.warmup import DecoderWarmup +from tensorrt_llm._torch.pyexecutor.guided_decoder import CapturableGuidedDecoder from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType from tensorrt_llm._torch.pyexecutor.runtime_hooks import DEFAULT_RUNTIME_HOOKS @@ -1009,6 +1011,28 @@ def test_empty_cache_count_under_default(self) -> None: f"Expected exactly 2 empty_cache calls; got order={calls}", ) + def test_guided_decoder_runs_the_context_warmup_before_the_autotuner(self) -> None: + """A capturable guided decoder rules out the general warmup's + generation requests, not its context requests: those still run, then + the release of the MoE workspaces and the cache, then the autotuner.""" + model_engine, resource_manager = self.enterContext(_warmup_and_resources()) + model_engine._inputs.guided_decoder = Mock() + tracker = _Tracker() + calls, _ = _run_warmup_tracked(model_engine, resource_manager, tracker=tracker) + (((_, configs), _),) = tracker.call_args["general_warmup"] + self.assertTrue(configs, "Expected context warmup requests") + self.assertTrue( + all(num_gen_tokens == 0 for _, num_gen_tokens in configs), + f"Expected context requests only; got {configs}", + ) + self.assertIn((model_engine._config.max_num_tokens, 0), configs) + general_idx = calls.index("general_warmup") + self.assertEqual( + calls[general_idx : general_idx + 5], + ["general_warmup", "moe_clear", "empty_cache", "autotuner", "empty_cache"], + f"Expected the context warmup and its release before the autotuner; got {calls}", + ) + def test_helix_cp_runs_only_the_generation_autotuner_warmup(self) -> None: """With Helix CP, can_run_general_warmup is False and the context warmups (general, Mamba hybrid) are skipped; the autotuner warmup still @@ -1539,12 +1563,17 @@ def test_native_mxfp8_respects_disabled_global_autotuner(self) -> None: @pytest.mark.cpu_only +@pytest.mark.parametrize("has_guided_decoder", [False, True]) @pytest.mark.parametrize( "world_size,dwdp_size,is_fatal", [(1, 0, False), (2, 0, True), (1, 4, True)], ) -def test_general_warmup_oom_policy(world_size: int, dwdp_size: int, is_fatal: bool) -> None: +def test_general_warmup_oom_policy( + world_size: int, dwdp_size: int, is_fatal: bool, has_guided_decoder: bool +) -> None: engine = _general_warmup_engine(world_size=world_size, dwdp_size=dwdp_size) + guided_decoder = Mock() if has_guided_decoder else None + engine._inputs.guided_decoder = guided_decoder error = torch.OutOfMemoryError("asymmetric OOM") engine._forward_warmup = mock.Mock(side_effect=error if is_fatal else [error, None]) @@ -1560,12 +1589,115 @@ def test_general_warmup_oom_policy(world_size: int, dwdp_size: int, is_fatal: bo # The remaining shape is never attempted: this rank's peers are # already stuck in the failed forward's collectives. engine._forward_warmup.assert_called_once() + if guided_decoder is not None: + guided_decoder.reset_warmup_state.assert_not_called() else: assert engine._forward_warmup.call_count == 2 # A retry after an OOM between dispatch() and combine() has to start # from a clean MoE all-to-all state. engine._reset_moe_alltoall_state.assert_called_once_with() empty_cache.assert_called_once_with() + if guided_decoder is not None: + guided_decoder.reset_warmup_state.assert_called_once_with() + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("callback_pending", [False, True]) +def test_general_warmup_oom_restores_guided_batch_queue(callback_pending: bool) -> None: + """A failed warmup must not shift the next request's grammar snapshot.""" + engine = _general_warmup_engine(world_size=1, dwdp_size=0) + decoder = object.__new__(CapturableGuidedDecoder) + decoder.queue = Queue() + decoder._num_added_batches = 0 + decoder.requests = None + decoder.requests_hostfunc = None + decoder.max_num_draft_tokens = 1 + engine._inputs.guided_decoder = decoder + pending_callbacks: list[Callable[[], None]] = [] + consumed: list[int] = [] + + def enqueue_callback(fn: Callable[..., None], *args: object, **kwargs: object) -> None: + pending_callbacks.append(lambda: fn(*args, **kwargs)) + + def synchronize() -> None: + while pending_callbacks: + pending_callbacks.pop(0)() + consumed.append(decoder.requests_hostfunc.requests[0].request_id) + + decoder.stream = Mock(synchronize=Mock(side_effect=synchronize)) + + def batch(request_id: int, *, guided: bool = False) -> SimpleNamespace: + request = SimpleNamespace( + py_request_id=request_id, + py_guided_decoding_params=object() if guided else None, + py_seq_slot=0, + py_batch_idx=None, + is_context_init_state=True, + is_last_context_chunk=True, + is_generation_in_progress_state=False, + llm_request_type=None, + py_decoding_iter=0, + get_last_tokens=lambda _: 1, + py_draft_tokens=[], + py_num_accepted_draft_tokens=0, + ) + return SimpleNamespace( + all_requests=lambda: [request], num_context_requests=1, num_generation_requests=0 + ) + + engine._dummy_requests = SimpleNamespace( + create_warmup_request=lambda _resources, tokens, _gen: batch(tokens), + release_batch_context=lambda requests, _resources: _released_batch(requests), + ) + + def forward(requests: SimpleNamespace, *_args: object, **_kwargs: object) -> None: + assert not pending_callbacks + assert decoder.queue.empty() + assert decoder.requests is None + assert decoder.requests_hostfunc is None + decoder.add_batch(requests) + if decoder.requests.requests[0].request_id == 128: + if callback_pending: + # OOM after submitting fetch_batch, before recording the + # completion event: recovery must drain the stream itself. + decoder.fetch_batch() + raise torch.OutOfMemoryError("warmup forward failed after enqueue") + decoder.fetch_batch() + decoder.stream.synchronize() + + engine._forward_warmup = forward + with ( + _no_cuda_side_effects(), + patch("tensorrt_llm._torch.hostfunc.launch_hostfunc", side_effect=enqueue_callback), + ): + engine._general_warmup_impl(object(), [(128, 0), (64, 0)]) + assert decoder.queue.empty() + assert consumed == ([128, 64] if callback_pending else [64]) + + decoder.add_batch(batch(1000, guided=True)) + decoder.fetch_batch() + decoder.stream.synchronize() + + assert consumed[-1] == 1000 + (guided_request,) = decoder.requests_hostfunc.valid_requests() + assert guided_request.require_matcher_init() + # The host functions hold the batch the host thread numbered last, so a + # build failure is reported under the real request's batch. + assert decoder.requests_hostfunc.batch_id == decoder.last_added_batch_id + assert decoder.queue.empty() + + +@pytest.mark.cpu_only +def test_general_warmup_aborts_if_guided_state_cannot_be_reset() -> None: + engine = _general_warmup_engine(world_size=1, dwdp_size=0) + engine._inputs.guided_decoder = Mock() + engine._inputs.guided_decoder.reset_warmup_state.side_effect = RuntimeError("drain failed") + engine._forward_warmup = Mock(side_effect=torch.OutOfMemoryError("warmup OOM")) + + with _no_cuda_side_effects(), pytest.raises(RuntimeError, match="drain failed"): + engine._general_warmup_impl(object(), [(128, 0), (64, 0)]) + + engine._forward_warmup.assert_called_once() @pytest.mark.cpu_only From 3f2deaf8988b9cc1000ec4ba6dbf56343ccf32aa Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:42:50 -0800 Subject: [PATCH 4/6] [None][fix] Attention DP: report each rank's own speculative-decoding stats Attention-DP iteration stats fan out one row per rank (#21131): the scheduling and KV fields of each row come from that rank's payload on the ADP rank-state allgather, but specdec_stats was in the list of nested objects copied from rank 0. Every rank's row therefore reported rank 0's draft and accepted token counts and acceptance length, so per-rank acceptance could not be told apart, an idle rank 0 showed zero acceptance for the whole group, and summing the rows multiplied rank 0's numbers by the number of ranks. Any attention-DP deployment with speculative decoding (for example DeepSeek-R1 DEP with MTP) and iteration stats enabled reports these rows, through LLM.get_stats() and the JSON iteration stats of trtllm-serve's /metrics; the Prometheus metrics (/prometheus/metrics) read spec-decoding stats from rank 0's row only and do not change. The acceptance-length gates of the accuracy tests sum every row (acceptance_length_from_iteration_stats), so under attention DP they measured rank 0's acceptance alone and could not see a drafting regression on the other ranks. Found while checking Mistral Large 4 DEP4 acceptance with its DSpark drafter. Each rank now snapshots its draft tokens, accepted tokens and requests with draft tokens (RankSpecDecodingStats) into its payload, and the fanout rebuilds the row's SpecDecodingStats from them, deriving the acceptance length from the rank's own counters. The draft timings stay rank 0's, like the row's other timings. A rank with no snapshot (an explicit zero placeholder) gets no spec-decoding stats rather than rank 0's. The snapshot frame gains a third presence flag, so its version moves from -1 to -2 and its header from four to five fields. Every rank of a job runs the same build, so all ranks write and read the same version; a payload with another version fails the header check instead of being misread. Tests: test_adp_router.py pins the new wire schema and round-trips the snapshot with and without KV snapshots and load fields, keeping zero counters distinct from an absent snapshot. test_iter_stats_populate.py fans out asymmetric work on two ranks (with rank 0 busy or idle, and the padding and CUDA-graph dummies excluded) and checks each row's counts and acceptance length, that the rows add up to the group's acceptance, and that a placeholder rank does not inherit rank 0's drafting. Split out of #21693 (Mistral Large 4 merge-back); originally GitLab 4932efdc24 ("Add native DSpark to StoneWillow Staircase specializations"), rebased onto the versioned snapshot frame of #21131. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- .../_torch/pyexecutor/adp_iter_stats.py | 59 ++++++++++++-- .../_torch/pyexecutor/scheduler/rank_state.py | 61 ++++++++++++--- .../_torch/pyexecutor/test_adp_router.py | 59 +++++++++++++- .../pyexecutor/test_iter_stats_populate.py | 76 ++++++++++++++++++- 4 files changed, 235 insertions(+), 20 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py index 45862bcc78df..129f96293d82 100644 --- a/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py @@ -12,11 +12,12 @@ IterationStats, KvCacheStats, RequestStats, + SpecDecodingStats, ) from tensorrt_llm.logger import logger from .scheduler.adp_router import RankIterStatsPayload, RankState -from .scheduler.rank_state import RankKvCacheStats +from .scheduler.rank_state import RankKvCacheStats, RankSpecDecodingStats _ITERATION_STATS_SCALAR_FIELDS = ( "timestamp", @@ -33,10 +34,7 @@ "pinned_mem_usage", ) -_ITERATION_STATS_OPTIONAL_FIELDS = ( - "static_batching_stats", - "specdec_stats", -) +_ITERATION_STATS_OPTIONAL_FIELDS = ("static_batching_stats",) def _snapshot_kv_cache_stats(stats: KvCacheStats | None) -> RankKvCacheStats | None: @@ -78,6 +76,51 @@ def _restore_kv_cache_stats(snapshot: RankKvCacheStats) -> KvCacheStats: return stats +def _snapshot_specdec_stats(stats: SpecDecodingStats | None) -> RankSpecDecodingStats | None: + """Capture a rank's speculative-decoding counters. + + Args: + stats: Local speculative-decoding stats, or None when they are unavailable. + + Returns: + Immutable counter snapshot, or None when unavailable. + """ + if stats is None: + return None + return RankSpecDecodingStats( + num_draft_tokens=stats.num_draft_tokens, + num_accepted_tokens=stats.num_accepted_tokens, + num_requests_with_draft_tokens=stats.num_requests_with_draft_tokens, + ) + + +def _restore_specdec_stats( + snapshot: RankSpecDecodingStats, rank0_stats: SpecDecodingStats | None +) -> SpecDecodingStats: + """Create a native speculative-decoding stats object for one rank's row. + + The counters and the acceptance length derived from them are the rank's + own. The draft timings keep rank 0's values, like the row's other timings. + + Args: + snapshot: Counters captured with the rank's iteration payload. + rank0_stats: Rank 0's speculative-decoding stats for the same iteration. + + Returns: + A fresh native stats object. + """ + stats = SpecDecodingStats() + for name, value in zip(snapshot._fields, snapshot): + setattr(stats, name, value) + num_requests = snapshot.num_requests_with_draft_tokens + stats.acceptance_length = ( + (snapshot.num_accepted_tokens + num_requests) / num_requests if num_requests else 0.0 + ) + stats.iter_latency_ms = rank0_stats.iter_latency_ms if rank0_stats is not None else 0.0 + stats.draft_overhead = rank0_stats.draft_overhead if rank0_stats is not None else 0.0 + return stats + + @dataclass class ADPIterStatsRecord: """Append-ready stats row produced by Attention-DP fanout.""" @@ -141,6 +184,7 @@ def make_payload(stats: IterationStats) -> RankIterStatsPayload: num_paused_kv_tokens=ifb.num_paused_kv_tokens, kv_cache_stats=_snapshot_kv_cache_stats(stats.kv_cache_stats), cross_kv_cache_stats=_snapshot_kv_cache_stats(stats.cross_kv_cache_stats), + specdec_stats=_snapshot_specdec_stats(stats.specdec_stats), ) def queue( @@ -266,6 +310,11 @@ def _make_rank_iter_stats( if nested_stats is not None: setattr(stats, attr, nested_stats) + if payload.specdec_stats is not None: + stats.specdec_stats = _restore_specdec_stats( + payload.specdec_stats, rank0_stats.specdec_stats + ) + ifb = InflightBatchingStats() ifb.num_context_requests = payload.num_context_requests ifb.num_ctx_tokens = payload.num_ctx_tokens diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py b/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py index b6c1e4ab21c6..d9c40995f8ed 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/rank_state.py @@ -59,6 +59,40 @@ def deserialize(cls, data: list[int | float]) -> RankKvCacheStats: ) +class RankSpecDecodingStats(NamedTuple): + """Immutable rank-local speculative-decoding counters for one iteration.""" + + num_draft_tokens: int + num_accepted_tokens: int + num_requests_with_draft_tokens: int + + @classmethod + def wire_size(cls) -> int: + """Return the number of scalar fields in a speculative-decoding snapshot. + + Returns: + Number of fields in the named tuple's wire schema. + """ + return len(cls._fields) + + @classmethod + def deserialize(cls, data: list[int | float]) -> RankSpecDecodingStats: + """Restore the counters from their scalar wire fields. + + Args: + data: Draft tokens, accepted draft tokens and requests with draft tokens. + + Returns: + The immutable speculative-decoding snapshot. + + Raises: + ValueError: The snapshot has an unexpected number of fields. + """ + if len(data) != cls.wire_size(): + raise ValueError(f"Speculative-decoding snapshot requires {cls.wire_size()} fields") + return cls(int(data[0]), int(data[1]), int(data[2])) + + @dataclass class RankIterStatsPayload: """Per-rank iteration payload piggybacked on the ADP allgather.""" @@ -66,8 +100,8 @@ class RankIterStatsPayload: # Legacy scalar widths are independent of the optional snapshot fields. _ITER_STATS_FIELD_COUNT: ClassVar[int] = 9 _KV_LOAD_FIELD_COUNT: ClassVar[int] = 3 - _SNAPSHOT_WIRE_VERSION: ClassVar[int] = -1 - _SNAPSHOT_HEADER_SIZE: ClassVar[int] = 4 + _SNAPSHOT_WIRE_VERSION: ClassVar[int] = -2 + _SNAPSHOT_HEADER_SIZE: ClassVar[int] = 5 has_iter_stats: int = 0 iter_stats_iter: int = -1 @@ -83,6 +117,7 @@ class RankIterStatsPayload: kv_load_timestamp_ns: int = 0 kv_cache_stats: RankKvCacheStats | None = None cross_kv_cache_stats: RankKvCacheStats | None = None + specdec_stats: RankSpecDecodingStats | None = None def serialize(self, include_kv_cache_load: bool = False) -> list[int | float]: """Serialize to a flat list for allgather transport.""" @@ -105,17 +140,17 @@ def serialize(self, include_kv_cache_load: bool = False) -> list[int | float]: self.kv_load_timestamp_ns, ] ) - if self.kv_cache_stats is not None or self.cross_kv_cache_stats is not None: + snapshots = (self.kv_cache_stats, self.cross_kv_cache_stats, self.specdec_stats) + if any(snapshot is not None for snapshot in snapshots): # The negative version cannot be confused with has_iter_stats (0/1). - # Presence flags frame both snapshots and the independent load fields. + # Presence flags frame the snapshots and the independent load fields. values = [ self._SNAPSHOT_WIRE_VERSION, int(include_kv_cache_load), - int(self.kv_cache_stats is not None), - int(self.cross_kv_cache_stats is not None), + *(int(snapshot is not None) for snapshot in snapshots), *values, ] - for snapshot in (self.kv_cache_stats, self.cross_kv_cache_stats): + for snapshot in snapshots: if snapshot is not None: values.extend(snapshot) return values @@ -138,12 +173,13 @@ def snapshot_payload_size(cls, data: list[int | float]) -> int: or data[0] != cls._SNAPSHOT_WIRE_VERSION or any(flag not in (0, 1) for flag in data[1 : cls._SNAPSHOT_HEADER_SIZE]) ): - raise ValueError("Invalid KV snapshot payload header") + raise ValueError("Invalid snapshot payload header") return ( cls._SNAPSHOT_HEADER_SIZE + cls._ITER_STATS_FIELD_COUNT + int(data[1]) * cls._KV_LOAD_FIELD_COUNT + int(data[2] + data[3]) * RankKvCacheStats.wire_size() + + int(data[4]) * RankSpecDecodingStats.wire_size() ) @classmethod @@ -153,7 +189,7 @@ def deserialize(cls, data: list[int | float]) -> RankIterStatsPayload: if values and values[0] < 0: expected_size = cls.snapshot_payload_size(values) if len(values) != expected_size: - raise ValueError(f"KV snapshot payload requires {expected_size} fields") + raise ValueError(f"Snapshot payload requires {expected_size} fields") scalar_end = ( cls._SNAPSHOT_HEADER_SIZE + cls._ITER_STATS_FIELD_COUNT @@ -170,6 +206,11 @@ def deserialize(cls, data: list[int | float]) -> RankIterStatsPayload: result.cross_kv_cache_stats = RankKvCacheStats.deserialize( values[cursor : cursor + RankKvCacheStats.wire_size()] ) + cursor += RankKvCacheStats.wire_size() + if values[4]: + result.specdec_stats = RankSpecDecodingStats.deserialize( + values[cursor : cursor + RankSpecDecodingStats.wire_size()] + ) return result scalar_field_count = cls._ITER_STATS_FIELD_COUNT + cls._KV_LOAD_FIELD_COUNT if len(values) > scalar_field_count: @@ -225,7 +266,7 @@ def deserialize(cls, data: list[int | float]) -> RankState: values[prefix_field_count:] ) if len(values) != prefix_field_count + stats_field_count + 4: - raise ValueError("KV snapshot rank state requires four trailing fields") + raise ValueError("Snapshot rank state requires four trailing fields") else: # Legacy messages have optional load telemetry and two or four tail # fields. Keep their framing independent of the Python dataclass size. diff --git a/tests/unittest/_torch/pyexecutor/test_adp_router.py b/tests/unittest/_torch/pyexecutor/test_adp_router.py index 068206650ff4..edbb49fccc7c 100644 --- a/tests/unittest/_torch/pyexecutor/test_adp_router.py +++ b/tests/unittest/_torch/pyexecutor/test_adp_router.py @@ -25,6 +25,7 @@ import inspect import random +from dataclasses import replace from types import SimpleNamespace from unittest.mock import MagicMock, Mock @@ -55,7 +56,10 @@ GenAdmissionConfig, GenAdmissionWaitingQueue, ) -from tensorrt_llm._torch.pyexecutor.scheduler.rank_state import RankKvCacheStats +from tensorrt_llm._torch.pyexecutor.scheduler.rank_state import ( + RankKvCacheStats, + RankSpecDecodingStats, +) from tensorrt_llm._torch.pyexecutor.scheduler.waiting_queue import PriorityWaitingQueue from tensorrt_llm.bindings.executor import RequestType from tensorrt_llm.conversation_params import ConversationParams @@ -311,8 +315,13 @@ def test_kv_snapshot_roundtrip_preserves_tail( assert RankIterStatsPayload.deserialize(payload.serialize(include_kv_cache_load)) == payload assert RankState.deserialize(state.serialize(include_kv_cache_load)) == state - def test_kv_snapshot_wire_schema_is_stable(self) -> None: + def test_snapshot_wire_schema_is_stable(self) -> None: """Pin field order and the complete versioned frame independently of roundtrips.""" + assert RankSpecDecodingStats._fields == ( + "num_draft_tokens", + "num_accepted_tokens", + "num_requests_with_draft_tokens", + ) assert RankKvCacheStats._fields == ( "max_num_blocks", "free_num_blocks", @@ -344,6 +353,7 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: kv_load_timestamp_ns=123, kv_cache_stats=RankKvCacheStats(100, 80, 20, 16, 40, 30, 7, 5, 0.375), cross_kv_cache_stats=RankKvCacheStats(60, 45, 15, 32, 25, 18, 4, 3, 0.25), + specdec_stats=RankSpecDecodingStats(15, 9, 5), ), num_context_requests=3, num_generation_requests=4, @@ -355,10 +365,11 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: 7, 321, 2, # Rank state prefix. - -1, + -2, 1, 1, - 1, # Version and load/primary/cross presence flags. + 1, + 1, # Version and load/primary/cross/speculative-decoding presence flags. 1, 9, 2, @@ -389,6 +400,9 @@ def test_kv_snapshot_wire_schema_is_stable(self) -> None: 4, 3, 0.25, # Cross KV snapshot. + 15, + 9, + 5, # Speculative-decoding snapshot. 3, 4, 6, @@ -418,6 +432,43 @@ def test_kv_snapshot_invalid_framing_is_rejected(self) -> None: with pytest.raises(ValueError, match="requires"): RankIterStatsPayload.deserialize(state.iter_stats.serialize()[:-1]) + @pytest.mark.parametrize("include_kv_cache_load", [False, True]) + @pytest.mark.parametrize("include_kv_cache_stats", [False, True]) + def test_specdec_snapshot_roundtrip_preserves_tail( + self, include_kv_cache_load: bool, include_kv_cache_stats: bool + ) -> None: + """Carry rank-local speculative-decoding counters with or without a KV snapshot.""" + payload = RankIterStatsPayload( + has_iter_stats=1, + iter_stats_iter=7, + num_context_requests=2, + num_gen_requests=5, + kv_used_blocks=4 if include_kv_cache_load else -1, + kv_total_blocks=10 if include_kv_cache_load else -1, + kv_load_timestamp_ns=123 if include_kv_cache_load else 0, + kv_cache_stats=( + RankKvCacheStats(10, 8, 2, 16, 3, 2, 1, 1, 0.5) if include_kv_cache_stats else None + ), + specdec_stats=RankSpecDecodingStats(15, 9, 5), + ) + state = RankState( + rank=2, + num_active_requests=7, + num_active_tokens=100, + num_retiring_requests=1, + iter_stats=payload, + num_context_requests=2, + num_generation_requests=5, + num_connector_transfers=3, + connector_poll_failed=True, + ) + + assert RankIterStatsPayload.deserialize(payload.serialize(include_kv_cache_load)) == payload + assert RankState.deserialize(state.serialize(include_kv_cache_load)) == state + # Zero counters are a measurement and stay distinct from an absent snapshot. + idle = replace(payload, specdec_stats=RankSpecDecodingStats(0, 0, 0)) + assert RankIterStatsPayload.deserialize(idle.serialize(include_kv_cache_load)) == idle + class TestDefaultADPRouter: # Router tests model strict placement, relaxed placement, and capacity diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py index 8afb1a0fed04..fef6e4aab1f6 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py @@ -940,6 +940,80 @@ def test_attention_dp_fanout_emits_rank_local_rows_with_rank0_queue(): assert buffer.next_payload() is None +@pytest.mark.parametrize("rank0_idle", [False, True]) +@pytest.mark.parametrize("include_kv_cache_load", [False, True]) +def test_attention_dp_specdec_fanout_uses_rank_local_counts( + rank0_idle: bool, include_kv_cache_load: bool +) -> None: + """Transport asymmetric real/dummy work without copying rank 0's AL.""" + real_requests = [ + [] if rank0_idle else [_StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2)], + [ + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=0), + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=1), + _StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2), + ], + ] + buffers = [] + rank_states = [] + for rank, requests in enumerate(real_requests): + dummies = [ + _StubRequest(py_draft_tokens=[1, 1, 1], is_attention_dp_dummy=True), + _StubRequest(py_draft_tokens=[1, 1, 1], is_cuda_graph_dummy=True), + ] + stats = _invoke_update_iter_stats( + _StubScheduledBatch(gen_reqs=requests + dummies), + [], + num_ctx_tokens=0, + specdec=True, + ) + stats.iter = 7 + buffer = _build_adp_stats_buffer(stats, is_rank0=rank == 0) + buffers.append(buffer) + state = RankState(rank=rank, iter_stats=buffer.next_payload()) + rank_states.append(RankState.deserialize(state.serialize(include_kv_cache_load))) + + records = buffers[0].finalize(rank_states, is_rank0=True) + assert len(records) == 2 + expected_counts = [(0, 0, 0) if rank0_idle else (3, 2, 1), (9, 3, 3)] + for record, (drafted, accepted, requests) in zip(records, expected_counts): + spec = record.stats.specdec_stats + assert spec is not None + assert spec.num_draft_tokens == drafted + assert spec.num_accepted_tokens == accepted + assert spec.num_requests_with_draft_tokens == requests + assert spec.acceptance_length == pytest.approx( + (accepted + requests) / requests if requests else 0.0 + ) + + accepted = sum(record.stats.specdec_stats.num_accepted_tokens for record in records) + requests = sum(record.stats.specdec_stats.num_requests_with_draft_tokens for record in records) + assert (accepted + requests) / requests == pytest.approx(2.0 if rank0_idle else 2.25) + + +def test_attention_dp_specdec_fanout_does_not_fill_synthetic_rank_from_rank0() -> None: + """An explicit zero payload must not inherit another rank's drafting.""" + stats = _invoke_update_iter_stats( + _StubScheduledBatch( + gen_reqs=[_StubRequest(num_draft_tokens=3, py_num_accepted_draft_tokens=2)] + ), + [], + num_ctx_tokens=0, + specdec=True, + ) + stats.iter = 7 + buffer = _build_adp_stats_buffer(stats) + records = buffer.finalize( + [ + RankState(rank=0, iter_stats=buffer.next_payload()), + RankState(rank=1, iter_stats=RankIterStatsPayload(has_iter_stats=1, iter_stats_iter=7)), + ], + is_rank0=True, + ) + assert records[0].stats.specdec_stats.num_requests_with_draft_tokens == 1 + assert records[1].stats.specdec_stats is None + + def test_attention_dp_fanout_waits_for_complete_matching_payloads(): """Verify ADP fanout waits until every rank reports the rank-0 iteration.""" rank0_stats = _make_adp_iteration_stats(iter_id=9, num_context_requests=1) @@ -1076,7 +1150,7 @@ def test_attention_dp_fanout_copy_lists_cover_iteration_stats_fields(): copied_fields = ( set(_ITERATION_STATS_SCALAR_FIELDS) | set(_ITERATION_STATS_OPTIONAL_FIELDS) - | {"inflight_batching_stats", "kv_cache_stats", "cross_kv_cache_stats"} + | {"inflight_batching_stats", "kv_cache_stats", "cross_kv_cache_stats", "specdec_stats"} ) assert copied_fields == actual_fields From d75541951e92515ba247c17c82e5d791afa14d70 Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:55:39 -0800 Subject: [PATCH 5/6] [None][chore] FlashMLA: explain the GIL deadlock beside the bindings, drop the subprocess test The patch now carries a comment above the bindings that explains the deadlock the GIL release prevents: what is seen, the three conditions that form the cycle, where TensorRT-LLM's host functions come from and why they can still be pending in an eager step, and why releasing the GIL is safe for these functions. The bindings themselves are unchanged. test_flash_mla_gil.py is removed. It reproduced the deadlock in a subprocess by filling the launch queue behind a host function, a setup that is awkward to run in CI. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- 3rdparty/patches/flashmla_release_gil.patch | 31 ++++++++++- .../attention/sparse/test_flash_mla_gil.py | 51 ------------------- 2 files changed, 30 insertions(+), 52 deletions(-) delete mode 100644 tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py diff --git a/3rdparty/patches/flashmla_release_gil.patch b/3rdparty/patches/flashmla_release_gil.patch index 8be7dbd6d2b9..7851ba9ea2d1 100644 --- a/3rdparty/patches/flashmla_release_gil.patch +++ b/3rdparty/patches/flashmla_release_gil.patch @@ -1,6 +1,6 @@ --- a/csrc/api/api.cpp +++ b/csrc/api/api.cpp -@@ -7,9 +7,9 @@ +@@ -7,9 +7,38 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.doc() = "FlashMLA"; @@ -9,6 +9,35 @@ - m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface); - m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun); - m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun); ++ // Every binding releases the GIL for the duration of its C++ call. ++ // ++ // A binding that kept the GIL could deadlock a process that also queues Python host functions on a ++ // CUDA stream, as TensorRT-LLM does. What is seen then: under load, the thread that runs the model ++ // never returns from a call into this module, and in a multi-rank deployment the other ranks stall ++ // in their next collective. ++ // ++ // Three facts make the cycle: ++ // 1. A kernel launch is normally asynchronous, but once the launch queue is full it blocks until ++ // the device has worked through part of the queue. ++ // 2. A host function queued with cudaLaunchHostFunc blocks the work queued behind it until it has ++ // returned, and one that calls back into Python must take the GIL before it can run. ++ // 3. pybind11 keeps the GIL for the whole call of a function bound with a plain m.def, also while ++ // a launch inside it blocks. ++ // So when a launch made from here blocks on a full queue that has a Python host function ahead of ++ // it, the launch waits for the host function and the host function waits for the GIL. ++ // ++ // In TensorRT-LLM the host functions come from tensorrt_llm/_torch/hostfunc.py; the capturable ++ // guided decoder (guided decoding beside one-model speculative decoding) queues them for every ++ // step. Those captured into a CUDA graph run when a replay reaches them, which is after the Python ++ // call that started the replay has returned, so they can still be pending when the next step runs ++ // eagerly and calls into this module. Whether the launch queue is full at that moment depends on ++ // the load, so the hang shows up under serving load and not on every request. ++ // ++ // Releasing the GIL is safe for the functions bound here: their arguments and results are tensors ++ // and scalars, which pybind11 converts before the guard is constructed and after it is destroyed; ++ // they never call into Python; and they keep no mutable state between calls, so calls from ++ // different Python threads need no serialization by the GIL. A binding that takes or returns ++ // Python objects, calls into Python, or shares mutable state must not get this guard. + m.def("sparse_decode_fwd", &sparse_attn_decode_interface, pybind11::call_guard()); + m.def("dense_decode_fwd", &dense_attn_decode_interface, pybind11::call_guard()); + m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface, pybind11::call_guard()); diff --git a/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py b/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py deleted file mode 100644 index 53d97b4f8104..000000000000 --- a/tests/unittest/_torch/attention/sparse/test_flash_mla_gil.py +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""FlashMLA's launch wrappers release the GIL. - -A CUDA host function that takes the GIL (the capturable guided decoder's, for one) can be queued ahead of -FlashMLA's kernels on the same stream. Once the launch queue is full, a FlashMLA call that kept the GIL while -it blocks in a launch would wait on the host function, and the host function on the GIL. -""" - -import subprocess -import sys -import textwrap - -import pytest -import torch - -_CHILD = textwrap.dedent(""" - import time - - import torch - - import tensorrt_llm.flash_mla_cpp_tllm as flash_mla - from tensorrt_llm._torch.hostfunc import launch_hostfunc - - q = torch.randn(4, 64, 576, device="cuda", dtype=torch.bfloat16) - kv = torch.randn(256, 1, 576, device="cuda", dtype=torch.bfloat16) - indices = torch.randint(0, 256, (4, 1, 64), device="cuda", dtype=torch.int32) - flash_mla.sparse_prefill_fwd(q, kv, indices, 0.05, 512, None, None) - torch.cuda.synchronize() - - ran = [] - torch.cuda._sleep(int(3e9)) # the host function runs only after the device works through this - launch_hostfunc(lambda: ran.append(time.time())) - for _ in range(20000): # far more launches than the queue holds - flash_mla.sparse_prefill_fwd(q, kv, indices, 0.05, 512, None, None) - torch.cuda.synchronize() - print("host function ran", len(ran)) -""") - - -@pytest.mark.skipif( - not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, - reason="FlashMLA sparse prefill on SM100-family GPUs", -) -def test_sparse_prefill_does_not_hold_the_gil_while_launching(): - result = subprocess.run( - [sys.executable, "-c", _CHILD], capture_output=True, text=True, timeout=120 - ) - - assert result.returncode == 0, result.stderr[-2000:] - assert "host function ran 1" in result.stdout From fa2a0ac248dd6244eaea8ccd965fd13f64095fb1 Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:52:05 -0800 Subject: [PATCH 6/6] [None][chore] FlashMLA: ask for a re-check of the GIL release conditions when the pin moves The conditions under which the bindings may release the GIL (tensor and scalar arguments and results, no calls into Python, no shared mutable state) are properties of the pinned FlashMLA commit only. The comment in the patch now says so and asks to check them again whenever the pin moves: the patch still applying does not show that they hold, because a binding added upstream gets no guard and a change inside a bound function leaves the patched file untouched. A _comment beside the pin in fetch_content.json points to that comment, as the ucxx entry does for its own check. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- 3rdparty/fetch_content.json | 3 ++- 3rdparty/patches/flashmla_release_gil.patch | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/3rdparty/fetch_content.json b/3rdparty/fetch_content.json index 412007acf9e0..4f7e1506b16d 100644 --- a/3rdparty/fetch_content.json +++ b/3rdparty/fetch_content.json @@ -63,7 +63,8 @@ "git_tag": "15f13e5030374295491c5ce31b02d7e63a7772c6", "git_submodules_recurse": true, "source_subdir": "dont-add-this-project-with-add-subdirectory", - "patch_file": "patches/flashmla_release_gil.patch" + "patch_file": "patches/flashmla_release_gil.patch", + "_comment": "When changing git_tag, check the conditions for releasing the GIL that the comment in patches/flashmla_release_gil.patch lists against the new sources; the patch still applying does not show that they hold." }, { "name": "fmt", diff --git a/3rdparty/patches/flashmla_release_gil.patch b/3rdparty/patches/flashmla_release_gil.patch index 7851ba9ea2d1..080a7bfc70f0 100644 --- a/3rdparty/patches/flashmla_release_gil.patch +++ b/3rdparty/patches/flashmla_release_gil.patch @@ -1,6 +1,6 @@ --- a/csrc/api/api.cpp +++ b/csrc/api/api.cpp -@@ -7,9 +7,38 @@ +@@ -7,9 +7,44 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.doc() = "FlashMLA"; @@ -38,6 +38,12 @@ + // they never call into Python; and they keep no mutable state between calls, so calls from + // different Python threads need no serialization by the GIL. A binding that takes or returns + // Python objects, calls into Python, or shares mutable state must not get this guard. ++ // ++ // These conditions were checked for the FlashMLA commit pinned in 3rdparty/fetch_content.json, and ++ // they are properties of that commit only. Whenever the pin moves, check them again for every ++ // binding of the new version and for the functions they bind. The patch still applying does not ++ // show that they hold: a binding added upstream gets no guard (and can deadlock as described above ++ // if it launches kernels), and a change inside a bound function leaves this file untouched. + m.def("sparse_decode_fwd", &sparse_attn_decode_interface, pybind11::call_guard()); + m.def("dense_decode_fwd", &dense_attn_decode_interface, pybind11::call_guard()); + m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface, pybind11::call_guard());