Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion 3rdparty/fetch_content.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@
"git_repository": "https://github.com/deepseek-ai/FlashMLA.git",
"git_tag": "15f13e5030374295491c5ce31b02d7e63a7772c6",
"git_submodules_recurse": true,
"source_subdir": "dont-add-this-project-with-add-subdirectory"
"source_subdir": "dont-add-this-project-with-add-subdirectory",
"patch_file": "patches/flashmla_release_gil.patch",
"_comment": "When changing git_tag, check the conditions for releasing the GIL that the comment in patches/flashmla_release_gil.patch lists against the new sources; the patch still applying does not show that they hold."
},
{
"name": "fmt",
Expand Down
52 changes: 52 additions & 0 deletions 3rdparty/patches/flashmla_release_gil.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
--- a/csrc/api/api.cpp
+++ b/csrc/api/api.cpp
@@ -7,9 +7,44 @@

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.doc() = "FlashMLA";
- m.def("sparse_decode_fwd", &sparse_attn_decode_interface);
- m.def("dense_decode_fwd", &dense_attn_decode_interface);
- m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface);
- m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun);
- m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun);
+ // Every binding releases the GIL for the duration of its C++ call.
+ //
+ // A binding that kept the GIL could deadlock a process that also queues Python host functions on a
+ // CUDA stream, as TensorRT-LLM does. What is seen then: under load, the thread that runs the model
+ // never returns from a call into this module, and in a multi-rank deployment the other ranks stall
+ // in their next collective.
+ //
+ // Three facts make the cycle:
+ // 1. A kernel launch is normally asynchronous, but once the launch queue is full it blocks until
+ // the device has worked through part of the queue.
+ // 2. A host function queued with cudaLaunchHostFunc blocks the work queued behind it until it has
+ // returned, and one that calls back into Python must take the GIL before it can run.
+ // 3. pybind11 keeps the GIL for the whole call of a function bound with a plain m.def, also while
+ // a launch inside it blocks.
+ // So when a launch made from here blocks on a full queue that has a Python host function ahead of
+ // it, the launch waits for the host function and the host function waits for the GIL.
+ //
+ // In TensorRT-LLM the host functions come from tensorrt_llm/_torch/hostfunc.py; the capturable
+ // guided decoder (guided decoding beside one-model speculative decoding) queues them for every
+ // step. Those captured into a CUDA graph run when a replay reaches them, which is after the Python
+ // call that started the replay has returned, so they can still be pending when the next step runs
+ // eagerly and calls into this module. Whether the launch queue is full at that moment depends on
+ // the load, so the hang shows up under serving load and not on every request.
+ //
+ // Releasing the GIL is safe for the functions bound here: their arguments and results are tensors
+ // and scalars, which pybind11 converts before the guard is constructed and after it is destroyed;
+ // they never call into Python; and they keep no mutable state between calls, so calls from
+ // different Python threads need no serialization by the GIL. A binding that takes or returns
+ // Python objects, calls into Python, or shares mutable state must not get this guard.
+ //
+ // These conditions were checked for the FlashMLA commit pinned in 3rdparty/fetch_content.json, and
+ // they are properties of that commit only. Whenever the pin moves, check them again for every
+ // binding of the new version and for the functions they bind. The patch still applying does not
+ // show that they hold: a binding added upstream gets no guard (and can deadlock as described above
+ // if it launches kernels), and a change inside a bound function leaves this file untouched.
+ m.def("sparse_decode_fwd", &sparse_attn_decode_interface, pybind11::call_guard<pybind11::gil_scoped_release>());
+ m.def("dense_decode_fwd", &dense_attn_decode_interface, pybind11::call_guard<pybind11::gil_scoped_release>());
+ m.def("sparse_prefill_fwd", &sparse_attn_prefill_interface, pybind11::call_guard<pybind11::gil_scoped_release>());
+ m.def("dense_prefill_fwd", &FMHACutlassSM100FwdRun, pybind11::call_guard<pybind11::gil_scoped_release>());
+ m.def("dense_prefill_bwd", &FMHACutlassSM100BwdRun, pybind11::call_guard<pybind11::gil_scoped_release>());
}
59 changes: 54 additions & 5 deletions tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
IterationStats,
KvCacheStats,
RequestStats,
SpecDecodingStats,
)
from tensorrt_llm.logger import logger

from .scheduler.adp_router import RankIterStatsPayload, RankState
from .scheduler.rank_state import RankKvCacheStats
from .scheduler.rank_state import RankKvCacheStats, RankSpecDecodingStats

_ITERATION_STATS_SCALAR_FIELDS = (
"timestamp",
Expand All @@ -33,10 +34,7 @@
"pinned_mem_usage",
)

_ITERATION_STATS_OPTIONAL_FIELDS = (
"static_batching_stats",
"specdec_stats",
)
_ITERATION_STATS_OPTIONAL_FIELDS = ("static_batching_stats",)


def _snapshot_kv_cache_stats(stats: KvCacheStats | None) -> RankKvCacheStats | None:
Expand Down Expand Up @@ -78,6 +76,51 @@ def _restore_kv_cache_stats(snapshot: RankKvCacheStats) -> KvCacheStats:
return stats


def _snapshot_specdec_stats(stats: SpecDecodingStats | None) -> RankSpecDecodingStats | None:
"""Capture a rank's speculative-decoding counters.

Args:
stats: Local speculative-decoding stats, or None when they are unavailable.

Returns:
Immutable counter snapshot, or None when unavailable.
"""
if stats is None:
return None
return RankSpecDecodingStats(
num_draft_tokens=stats.num_draft_tokens,
num_accepted_tokens=stats.num_accepted_tokens,
num_requests_with_draft_tokens=stats.num_requests_with_draft_tokens,
)


def _restore_specdec_stats(
snapshot: RankSpecDecodingStats, rank0_stats: SpecDecodingStats | None
) -> SpecDecodingStats:
"""Create a native speculative-decoding stats object for one rank's row.

The counters and the acceptance length derived from them are the rank's
own. The draft timings keep rank 0's values, like the row's other timings.

Args:
snapshot: Counters captured with the rank's iteration payload.
rank0_stats: Rank 0's speculative-decoding stats for the same iteration.

Returns:
A fresh native stats object.
"""
stats = SpecDecodingStats()
for name, value in zip(snapshot._fields, snapshot):
setattr(stats, name, value)
num_requests = snapshot.num_requests_with_draft_tokens
stats.acceptance_length = (
(snapshot.num_accepted_tokens + num_requests) / num_requests if num_requests else 0.0
)
stats.iter_latency_ms = rank0_stats.iter_latency_ms if rank0_stats is not None else 0.0
stats.draft_overhead = rank0_stats.draft_overhead if rank0_stats is not None else 0.0
return stats


@dataclass
class ADPIterStatsRecord:
"""Append-ready stats row produced by Attention-DP fanout."""
Expand Down Expand Up @@ -141,6 +184,7 @@ def make_payload(stats: IterationStats) -> RankIterStatsPayload:
num_paused_kv_tokens=ifb.num_paused_kv_tokens,
kv_cache_stats=_snapshot_kv_cache_stats(stats.kv_cache_stats),
cross_kv_cache_stats=_snapshot_kv_cache_stats(stats.cross_kv_cache_stats),
specdec_stats=_snapshot_specdec_stats(stats.specdec_stats),
)

def queue(
Expand Down Expand Up @@ -266,6 +310,11 @@ def _make_rank_iter_stats(
if nested_stats is not None:
setattr(stats, attr, nested_stats)

if payload.specdec_stats is not None:
stats.specdec_stats = _restore_specdec_stats(
payload.specdec_stats, rank0_stats.specdec_stats
)

ifb = InflightBatchingStats()
ifb.num_context_requests = payload.num_context_requests
ifb.num_ctx_tokens = payload.num_ctx_tokens
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,17 @@ def _warmup_scheduled(
generation_only = self.mapping.has_cp_helix() or is_decode_only(
self.sparse_attention_config
)
# A capturable guided decoder rules out the general warmup's generation
# requests only: the autotuner below runs its context step with the
# decoder installed. Run the context requests anyway, so that the
# autotuner's max-token step is not the engine's first forward (with
# the buffers it allocates lazily) and runs after the release below,
# as it does without guided decoding.
can_run_context_warmup = can_run_general_warmup or self._agreement.agree_flag(
self._context_warmups_supported
and not generation_only
and not isinstance(kv_cache_manager, MambaHybridCacheManager)
)

log_mem_snapshot("warmup/before_warmup")
# Compile the DSv4 indexer-Q CuTe DSL kernels before the first
Expand All @@ -462,7 +473,7 @@ def _warmup_scheduled(
resource_manager, can_run_general_warmup and not generation_only
)

if can_run_general_warmup:
if can_run_context_warmup:
Comment thread
yizhang-nv marked this conversation as resolved.
Resolved
# Specialize torch.compile graphs across the key input shapes before CUDA graph capture.
with self._warmup_timer.phase(
"general", metrics=self._metrics, metric_name="general_warmup_seconds"
Expand All @@ -472,6 +483,12 @@ def _warmup_scheduled(
warmup_requests_configs = self._generation_warmup_shapes(
warmup_requests_configs
)
elif not can_run_general_warmup:
warmup_requests_configs = [
(num_tokens, num_gen_tokens)
for num_tokens, num_gen_tokens in warmup_requests_configs
if num_gen_tokens == 0
]
warmup_requests_configs = self._agreement.agree_shapes(warmup_requests_configs)
# Currently graph has not been captured, disable cuda graph for this warmup.
with self._graphs.no_cuda_graph():
Expand Down Expand Up @@ -987,6 +1004,10 @@ def _general_warmup_impl(
# Peers are inside the same forward's collectives and
# cannot follow a rank-local skip.
raise
if self.guided_decoder is not None:
# Input preparation enqueues the batch before model forward.
# An OOM may leave it unconsumed, or leave callbacks pending.
self.guided_decoder.reset_warmup_state()
logger.warning(
f"OOM during general warmup with {num_tokens} tokens, "
f"{num_gen_tokens} generation tokens. Skipping."
Expand Down
14 changes: 14 additions & 0 deletions tensorrt_llm/_torch/pyexecutor/guided_decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,20 @@ def _drain_host_functions(self) -> None:
if not torch.cuda.is_current_stream_capturing():
self.bitmask_event.synchronize()

def reset_warmup_state(self) -> None:
"""Discard abandoned dummy batches after an eager warmup failure.

Only call during warmup, with no live requests or graph capture. Wait
on the stream itself: a failed forward may have enqueued callbacks
without reaching the bitmask-event record. Once they finish, no
callback can consume or overwrite the cleared request snapshots.
Warmup dummies have no grammar matchers to roll back.
"""
self.stream.synchronize()
self.queue = Queue()
self.requests = None
self.requests_hostfunc = None

@nvtx_range("GuidedDecoder.add_batch")
def add_batch(self,
scheduled_requests: ScheduledRequests,
Expand Down
13 changes: 12 additions & 1 deletion tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -6882,6 +6882,17 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests):
self._after_shrink(req.py_request_id, kv_cache)

for req in scheduled_batch.generation_requests:
# A paired pool's generation dummy has a fixed shape, and
# add_dummy_requests reserves its full forward capacity in both
# pools, even when it joins the batch after scheduler admission.
# An unpaired pool is not handed to add_dummy_requests: the
# mirror the scheduler admits for its dummy is empty and grows
# below like any other.
preallocated_dummy = (
self.enable_joint_kv_cache_reuse
and req.is_dummy_request
and req.py_request_id in self.kv_cache_map
)
kv_cache = self._mirror_draft_kv_cache(req)
if kv_cache is None:
raise RuntimeError(
Expand All @@ -6898,7 +6909,7 @@ def _prepare_draft_resources(self, scheduled_batch: ScheduledRequests):
)
# A paired scheduler already grew both pools atomically; the
# recorded width says so, so do not charge this step twice.
if req.py_request_id in self._allocated_draft_lens:
if preallocated_dummy or req.py_request_id in self._allocated_draft_lens:
Comment thread
yizhang-nv marked this conversation as resolved.
Resolved
continue
new_cap = self._required_gen_capacity(req, kv_cache.capacity)
if not kv_cache.resize(new_cap):
Expand Down
40 changes: 32 additions & 8 deletions tensorrt_llm/_torch/pyexecutor/py_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5248,13 +5248,14 @@ def _finalize_adp_dummy_allocation(self, can_queue: bool) -> None:
if spec_resource_manager is not None:
spec_resource_manager.free_resources(dummy_request)
self.kv_cache_manager.free_resources(dummy_request)
# The V2 scheduler also admits the draft pool's mirror while it
# schedules the dummy (KVCacheV2Scheduler._admit_unpaired_draft), so
# that pool holds an empty cache under the dummy ID too. Left behind,
# it outlives this dummy, and the deferred previous batch -- which
# still holds the earlier, finished dummy with the same ID -- resizes
# it to that dummy's history in update_resources ("History length
# cannot exceed capacity").
# The draft pool holds a cache under the dummy ID too: a paired pool
# gets it with the dummy (add_dummy_requests), and the V2 scheduler
# admits an unpaired pool's mirror while it schedules the dummy
# (KVCacheV2Scheduler._admit_unpaired_draft). Left behind, it outlives
# this dummy, and the deferred previous batch -- which still holds the
# earlier, finished dummy with the same ID -- resizes it to that
# dummy's history in update_resources ("History length cannot exceed
# capacity").
draft_kv_cache_manager = getattr(self, "draft_kv_cache_manager", None)
if (getattr(self, "_is_kv_manager_v2", False)
and draft_kv_cache_manager is not None):
Expand Down Expand Up @@ -9589,6 +9590,11 @@ def _pad_attention_dp_dummy_request(self):
key="attention_dp_dummy_insufficient_kv_capacity")
return

# A paired draft pool admits every scheduled request together with the
# target pool, so the dummy needs a cache in both.
draft_kv_cache_manager = (self.draft_kv_cache_manager
if self.enable_joint_kv_cache_reuse else None)

if (not self._enable_adp_dummy_fixes
or self.kv_cache_transceiver is None):
try:
Expand All @@ -9597,6 +9603,7 @@ def _pad_attention_dp_dummy_request(self):
token_nums=token_nums,
is_gen=self._adp_dummy_is_gen,
max_num_draft_tokens=self.max_total_draft_tokens,
draft_kv_cache_manager=draft_kv_cache_manager,
)
except OutOfPagesError:
dummy_requests = None
Expand All @@ -9615,7 +9622,13 @@ def _pad_attention_dp_dummy_request(self):
spec_resource_manager = self.resource_manager.get_resource_manager(
ResourceManagerType.SPEC_RESOURCE_MANAGER)
if spec_resource_manager is not None:
spec_resource_manager.add_dummy_requests(dummy_request_ids)
try:
spec_resource_manager.add_dummy_requests(dummy_request_ids)
except NoFreeSlotsError:
self.kv_cache_manager.free_resources(llm_request)
if draft_kv_cache_manager is not None:
draft_kv_cache_manager.free_resources(llm_request)
return
self.active_requests.append(llm_request)
return

Expand All @@ -9632,6 +9645,7 @@ def _pad_attention_dp_dummy_request(self):
token_nums=token_nums,
is_gen=self._adp_dummy_is_gen,
max_num_draft_tokens=self.max_total_draft_tokens,
draft_kv_cache_manager=draft_kv_cache_manager,
)
except OutOfPagesError:
dummy_requests = None
Expand All @@ -9648,6 +9662,8 @@ def _pad_attention_dp_dummy_request(self):
spec_resource_manager.add_dummy_requests(dummy_request_ids)
except NoFreeSlotsError:
self.kv_cache_manager.free_resources(dummy_request)
if draft_kv_cache_manager is not None:
draft_kv_cache_manager.free_resources(dummy_request)
return

assert dummy_request is not None
Expand Down Expand Up @@ -9723,6 +9739,11 @@ def _pad_empty_attention_dp_batch(
return

dummy_request_ids = [ATTENTION_DP_DUMMY_REQUEST_ID]
# Allocate the paired draft cache with the dummy, where a shortage
# degrades to an empty batch, rather than leaving it to the draft
# manager's prepare_resources mirror, which raises on one.
draft_kv_cache_manager = (self.draft_kv_cache_manager
if self.enable_joint_kv_cache_reuse else None)
try:
# Degrade to an empty batch rather than propagate: the ranks have
# yet to agree on can_queue, so a rank-local raise would strand the
Expand All @@ -9733,6 +9754,7 @@ def _pad_empty_attention_dp_batch(
is_gen=not context_dummy,
max_num_draft_tokens=(0 if context_dummy else
self.max_total_draft_tokens),
draft_kv_cache_manager=draft_kv_cache_manager,
)
except (OutOfPagesError, NoFreeSlotsError):
dummy_requests = None
Expand All @@ -9751,6 +9773,8 @@ def _pad_empty_attention_dp_batch(
spec_resource_manager.add_dummy_requests(dummy_request_ids)
except NoFreeSlotsError:
self.kv_cache_manager.free_resources(dummy_request)
if draft_kv_cache_manager is not None:
draft_kv_cache_manager.free_resources(dummy_request)
return

dummy_request.is_attention_dp_dummy = True
Expand Down
Loading
Loading