Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions tensorrt_llm/_torch/disaggregation/orchestration/coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ class DisaggTransferCoordinator:
# Since take_gen_receive_counters: most deferred receives pending at once, and the wait (s).
_max_deferred_gen_receives = 0
_gen_receive_ready_wait_s = 0.0
# The last poll skipped the status collectives, so a completed receive may not be reaped yet.
_gen_poll_skipped = False
_skipped_gen_polls = 0
# Most polls in a row a full batch skips: settles receives long before a transfer timeout and
# bounds how long cancelled or failed receives keep their KV pages.
_MAX_SKIPPED_GEN_POLLS = 127

def __init__(
self,
Expand Down Expand Up @@ -267,12 +273,30 @@ def prepare_context_schedulable(self, new_requests: List[LlmRequest]) -> None:
self._transceiver.prepare_context_requests(gen_first_ctx_requests)

@nvtx_range("poll_gen_transfers")
def poll_gen_transfers(self) -> None:
"""Poll receive-side transfers and their timeouts; rank-synchronized."""
def poll_gen_transfers(self, batch_full: bool = False) -> None:
Comment thread
ziyixiong-nv marked this conversation as resolved.
Resolved
"""Poll receive-side transfers and their timeouts; rank-synchronized.

``batch_full``, equal on every rank, skips the status collectives while
no completed receive could join the batch: at most
``_MAX_SKIPPED_GEN_POLLS`` polls in a row, and never under attention DP,
whose ranks hold different requests. Until the next unskipped poll,
which reaps receives first, completed, CTX-cancelled and failed
receives stay unreaped and keep their KV pages, timed-out receives are
not cancelled, and generation deadlines are not checked.
"""
if not uses_async_gen_transfer():
return
if self._defer_unready_gen_receives:
self._publish_ready_gen_receives()
skip = (
batch_full
and not self._enable_attention_dp
and self._skipped_gen_polls < self._MAX_SKIPPED_GEN_POLLS
)
self._skipped_gen_polls = self._skipped_gen_polls + 1 if skip else 0
self._gen_poll_skipped = skip
if skip:
return
# Gen-transfer status performs cross-rank consensus internally. Enter
# it symmetrically; ranks with no ready local future contribute an
# empty ready set.
Expand Down Expand Up @@ -417,6 +441,8 @@ def check_transfer_timeouts(self, only_with_context_sends: bool = False) -> None
):
req.py_kv_transfer_start_time = peer_ready_time
self._flag_transfer_timeout(req, "context", timeout_ms, current_time=now)
if self._gen_poll_skipped:
return
for req in self._registry.active_requests():
if req.is_disagg_generation_transmission_in_progress:
self._flag_transfer_timeout(req, "generation", timeout_ms)
Expand Down Expand Up @@ -1620,7 +1646,7 @@ def try_finish_gen_receive(self, request: LlmRequest) -> bool:
def prepare_context_schedulable(self, new_requests: List[LlmRequest]) -> None:
return None

def poll_gen_transfers(self) -> None:
def poll_gen_transfers(self, batch_full: bool = False) -> None:
return None

def poll_progress_when_idle(self) -> None:
Expand Down
6 changes: 5 additions & 1 deletion tensorrt_llm/_torch/models/modeling_deepseekv4.py
Original file line number Diff line number Diff line change
Expand Up @@ -3539,11 +3539,15 @@ def _finalize_hc_state(self, hc_state) -> torch.Tensor:

# The step-loop hooks validated on DeepSeek-V4 and V4.1.
_DEEPSEEK_V4_RUNTIME_HOOKS = RuntimeHooks(
stage_decode_inputs=True, prioritize_decode_graphs=True, prompts_to_owners=True
stage_decode_inputs=True,
prioritize_decode_graphs=True,
disagg_gen_receive_ahead=True,
prompts_to_owners=True,
)
_DEEPSEEK_V4_SPEC_RUNTIME_HOOKS = RuntimeHooks(
stage_decode_inputs=True,
prioritize_decode_graphs=True,
disagg_gen_receive_ahead=True,
loop_on_execution_stream=True,
prompts_to_owners=True,
skip_adp_count_gather=True,
Expand Down
34 changes: 19 additions & 15 deletions tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -6512,21 +6512,7 @@ def get_num_free_blocks(self) -> int:
"get_num_free_blocks is only used when the kv cache manager is empty"
)
if self.layer_specs is not None:
# Each lifecycle needs one slot per block. Coalesced layers share a
# lifecycle, while separate lifecycles can compete for the same pool.
slots_per_block: dict[int, int] = defaultdict(int)
for pool_id in self.impl.get_life_cycle_pool_group_indices(GPU_LEVEL):
slots_per_block[int(pool_id)] += 1

# Available slots already exclude the guard's permanent reservation.
stats = self._get_storage_statistics(GPU_LEVEL)
return min(
(
int(stats[pool_id].available) // slot_count
for pool_id, slot_count in slots_per_block.items()
),
default=0,
)
return self.num_free_gpu_blocks()

# Legacy managers estimate primary-pool capacity from page indices.
max_num_pages = max(
Expand All @@ -6542,6 +6528,24 @@ def get_num_free_blocks(self) -> int:
reserved_pages = sum(int(self.kv_cache_map[req_id].num_blocks) for req_id in reserved)
return max_num_pages // self.kv_factor - reserved_pages

def num_free_gpu_blocks(self) -> int:
"""How many one-block sequences the GPU pools can still take, from their available slots."""
# Each lifecycle needs one slot per block. Coalesced layers share a
# lifecycle, while separate lifecycles can compete for the same pool.
slots_per_block: dict[int, int] = defaultdict(int)
for pool_id in self.impl.get_life_cycle_pool_group_indices(GPU_LEVEL):
slots_per_block[int(pool_id)] += 1

# Available slots already exclude the guard's permanent reservation.
stats = self._get_storage_statistics(GPU_LEVEL)
return min(
(
int(stats[pool_id].available) // slot_count
for pool_id, slot_count in slots_per_block.items()
),
default=0,
)

def commit_scheduled_kv_cache_stats(self, scheduled_batch: ScheduledRequests) -> None:
if self.is_draft or (not self.enable_stats and not self._request_stats_enabled_ids):
return
Expand Down
Loading
Loading