From 26f5c109b4dfe9ba650ccbcbad7f5ceadca68b29 Mon Sep 17 00:00:00 2001 From: ziyixiong-nv <219238287+ziyixiong-nv@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:15:23 +0000 Subject: [PATCH] [TRTLLM-16622][perf] DeepSeek-V4: disagg GEN receives the next requests' KV while the batch decodes A disaggregated generation worker admits at most max_batch_size requests, counting those still receiving their KV, so at small batch sizes the next request's admission and KV transfer start only after a batch slot frees. With the new RuntimeHooks.disagg_gen_receive_ahead, set by DeepseekV4ForCausalLM (and so by DeepSeek-V4.1), generation-only requests may take receive-ahead slots beyond the batch: min(batch size, 4), and per rank min(batch size - 1, 4) under attention DP, leaving the padding dummy a seat. They receive their KV while the batch decodes and join it in arrival order as running requests free their slots. - On with KV cache manager V2, the asynchronous Python transceiver, one PP and CP rank and the overlap headroom: live requests, retiring ones included, stay within the per-request pools sized for two batches. Off with TLLM_DISAGG_GEN_RECEIVE_AHEAD=0, with GEN slot preemption, with TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 (received requests would take running decodes' slots) and in the disaggregated benchmark fill mode. - KVCacheV2Scheduler, once the executor enables receive-ahead (set_gen_init_receive_ahead), keeps allocating generation-init requests behind a full batch and schedules received requests after the others; the executor's per-rank cap still bounds the batch. One allocated with no seat left keeps its pages only while the GPU pools still hold a free block for every seat; after one does not fit, the next waits for more free blocks. - Under attention DP the per-rank cap rises by the slots while the queue is empty, and for a pass that pops only the lone generation-only requests at the queue head when they are more than the batch has room for; that pass counts up to that many retiring requests per rank. The pop returns these loads and the cap with the router's placement. - Without attention DP, while decodes holding their KV fill the batch and no receive ends at its first token, the coordinator skips the generation-transfer status poll, at most 127 times in a row. Deadlines of generation transfers, and the reaping of completed, CTX-cancelled and failed receives, wait for the next poll. Other models keep main's admission, scheduling and transfer polling. Measured with the original commits on the internal development branch (DeepSeek-V4.1-Flash on GB300, disaggregated: 1 context DEP2 + 4 generation TEP8 workers at batch 1, DSpark k=5, 30-minute window), against the same branch without them: output tokens/s per GPU +1.4%, TTFT p50/p90 -38%/-19%, tokens/s per user -0.1%. Those commits lacked the ordering and the page reserve, and do not exercise the attention-DP part. Numerics: unchanged per request (admission timing only). Test: test_adp_receive_ahead_admits_only_the_leading_generation_only_run (two attention-DP ranks through the real fetch path, router and queue). The post-merge DeepSeek-V4 disaggregated perf-sanity runs on GB300 exercise both paths (TEP8 batch-1 and DEP16 generation workers). Signed-off-by: ziyixiong-nv <219238287+ziyixiong-nv@users.noreply.github.com> --- .../orchestration/coordinator.py | 32 ++- .../_torch/models/modeling_deepseekv4.py | 6 +- .../kv_cache/kv_cache_manager_v2.py | 34 +-- tensorrt_llm/_torch/pyexecutor/py_executor.py | 229 +++++++++++++++--- .../_torch/pyexecutor/runtime_hooks.py | 4 + .../pyexecutor/scheduler/scheduler_v2.py | 96 +++++++- .../disaggregation/test_benchmark_disagg.py | 2 + .../test_disagg_loop_transcript.py | 2 +- .../test_overlap_admission_lifetime.py | 63 ++++- tests/unittest/_torch/test_connector_adp.py | 11 +- 10 files changed, 426 insertions(+), 53 deletions(-) diff --git a/tensorrt_llm/_torch/disaggregation/orchestration/coordinator.py b/tensorrt_llm/_torch/disaggregation/orchestration/coordinator.py index 04700b0725f8..6aa616a1a87b 100644 --- a/tensorrt_llm/_torch/disaggregation/orchestration/coordinator.py +++ b/tensorrt_llm/_torch/disaggregation/orchestration/coordinator.py @@ -97,6 +97,12 @@ class DisaggTransferCoordinator: # Since take_gen_receive_counters: most deferred receives pending at once, and the wait (s). _max_deferred_gen_receives = 0 _gen_receive_ready_wait_s = 0.0 + # The last poll skipped the status collectives, so a completed receive may not be reaped yet. + _gen_poll_skipped = False + _skipped_gen_polls = 0 + # Most polls in a row a full batch skips: settles receives long before a transfer timeout and + # bounds how long cancelled or failed receives keep their KV pages. + _MAX_SKIPPED_GEN_POLLS = 127 def __init__( self, @@ -267,12 +273,30 @@ def prepare_context_schedulable(self, new_requests: List[LlmRequest]) -> None: self._transceiver.prepare_context_requests(gen_first_ctx_requests) @nvtx_range("poll_gen_transfers") - def poll_gen_transfers(self) -> None: - """Poll receive-side transfers and their timeouts; rank-synchronized.""" + def poll_gen_transfers(self, batch_full: bool = False) -> None: + """Poll receive-side transfers and their timeouts; rank-synchronized. + + ``batch_full``, equal on every rank, skips the status collectives while + no completed receive could join the batch: at most + ``_MAX_SKIPPED_GEN_POLLS`` polls in a row, and never under attention DP, + whose ranks hold different requests. Until the next unskipped poll, + which reaps receives first, completed, CTX-cancelled and failed + receives stay unreaped and keep their KV pages, timed-out receives are + not cancelled, and generation deadlines are not checked. + """ if not uses_async_gen_transfer(): return if self._defer_unready_gen_receives: self._publish_ready_gen_receives() + skip = ( + batch_full + and not self._enable_attention_dp + and self._skipped_gen_polls < self._MAX_SKIPPED_GEN_POLLS + ) + self._skipped_gen_polls = self._skipped_gen_polls + 1 if skip else 0 + self._gen_poll_skipped = skip + if skip: + return # Gen-transfer status performs cross-rank consensus internally. Enter # it symmetrically; ranks with no ready local future contribute an # empty ready set. @@ -417,6 +441,8 @@ def check_transfer_timeouts(self, only_with_context_sends: bool = False) -> None ): req.py_kv_transfer_start_time = peer_ready_time self._flag_transfer_timeout(req, "context", timeout_ms, current_time=now) + if self._gen_poll_skipped: + return for req in self._registry.active_requests(): if req.is_disagg_generation_transmission_in_progress: self._flag_transfer_timeout(req, "generation", timeout_ms) @@ -1620,7 +1646,7 @@ def try_finish_gen_receive(self, request: LlmRequest) -> bool: def prepare_context_schedulable(self, new_requests: List[LlmRequest]) -> None: return None - def poll_gen_transfers(self) -> None: + def poll_gen_transfers(self, batch_full: bool = False) -> None: return None def poll_progress_when_idle(self) -> None: diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv4.py b/tensorrt_llm/_torch/models/modeling_deepseekv4.py index c9a6ea65f891..6211c17b6159 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv4.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv4.py @@ -3539,11 +3539,15 @@ def _finalize_hc_state(self, hc_state) -> torch.Tensor: # The step-loop hooks validated on DeepSeek-V4 and V4.1. _DEEPSEEK_V4_RUNTIME_HOOKS = RuntimeHooks( - stage_decode_inputs=True, prioritize_decode_graphs=True, prompts_to_owners=True + stage_decode_inputs=True, + prioritize_decode_graphs=True, + disagg_gen_receive_ahead=True, + prompts_to_owners=True, ) _DEEPSEEK_V4_SPEC_RUNTIME_HOOKS = RuntimeHooks( stage_decode_inputs=True, prioritize_decode_graphs=True, + disagg_gen_receive_ahead=True, loop_on_execution_stream=True, prompts_to_owners=True, skip_adp_count_gather=True, diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index 8091b2d8be0c..9d8a7cd50ab5 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -6512,21 +6512,7 @@ def get_num_free_blocks(self) -> int: "get_num_free_blocks is only used when the kv cache manager is empty" ) if self.layer_specs is not None: - # Each lifecycle needs one slot per block. Coalesced layers share a - # lifecycle, while separate lifecycles can compete for the same pool. - slots_per_block: dict[int, int] = defaultdict(int) - for pool_id in self.impl.get_life_cycle_pool_group_indices(GPU_LEVEL): - slots_per_block[int(pool_id)] += 1 - - # Available slots already exclude the guard's permanent reservation. - stats = self._get_storage_statistics(GPU_LEVEL) - return min( - ( - int(stats[pool_id].available) // slot_count - for pool_id, slot_count in slots_per_block.items() - ), - default=0, - ) + return self.num_free_gpu_blocks() # Legacy managers estimate primary-pool capacity from page indices. max_num_pages = max( @@ -6542,6 +6528,24 @@ def get_num_free_blocks(self) -> int: reserved_pages = sum(int(self.kv_cache_map[req_id].num_blocks) for req_id in reserved) return max_num_pages // self.kv_factor - reserved_pages + def num_free_gpu_blocks(self) -> int: + """How many one-block sequences the GPU pools can still take, from their available slots.""" + # Each lifecycle needs one slot per block. Coalesced layers share a + # lifecycle, while separate lifecycles can compete for the same pool. + slots_per_block: dict[int, int] = defaultdict(int) + for pool_id in self.impl.get_life_cycle_pool_group_indices(GPU_LEVEL): + slots_per_block[int(pool_id)] += 1 + + # Available slots already exclude the guard's permanent reservation. + stats = self._get_storage_statistics(GPU_LEVEL) + return min( + ( + int(stats[pool_id].available) // slot_count + for pool_id, slot_count in slots_per_block.items() + ), + default=0, + ) + def commit_scheduled_kv_cache_stats(self, scheduled_batch: ScheduledRequests) -> None: if self.is_draft or (not self.enable_stats and not self._request_stats_enabled_ids): return diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 9a1ee2f90ff7..c52f76fccae3 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -17,8 +17,8 @@ from enum import IntEnum from queue import Queue from typing import (TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List, - Literal, NoReturn, Optional, Sequence, Set, Tuple, Union, - cast) + Literal, NamedTuple, NoReturn, Optional, Sequence, Set, + Tuple, Union, cast) import torch from strenum import StrEnum @@ -398,6 +398,74 @@ def _fill_stall_timeout_sec() -> float: # max_input_len truncation. _UNBOUNDED_PAUSE_MAX_INPUT_LEN = 0x7fffffff +# Receive-ahead slots per generation worker (at most its batch size): more than +# the receives that start while one KV transfer is in flight. +_RECEIVE_AHEAD_SLOTS = 4 + + +class _AdpRouting(NamedTuple): + """What ``ADPRouter.route_requests`` takes for one admission pass.""" + rank_states: List[RankState] + rank_cap: int + # Request id -> rank the router planned; empty when none are planned. + placement: Dict[int, int] + + +def _is_lone_generation_only(item: RequestQueueItem) -> bool: + """Whether a queued request is generation-only and has no child requests.""" + return (item.request is not None and not item.child_req_ids + and item.request.request_type + == RequestType.REQUEST_TYPE_GENERATION_ONLY) + + +class _GenerationOnlyRun: + """The lone generation-only requests at the head of a waiting queue. + + Offers the part of ``WaitingQueue`` that ``ADPRouter.pop_requests`` uses; + peeking or popping past those requests raises. + """ + + def __init__(self, queue: WaitingQueue): + self._queue = queue + + def __bool__(self) -> bool: + return bool(self._queue) and _is_lone_generation_only( + self._queue.peek_request()) + + def __len__(self) -> int: + return len(self.peek_requests(len(self._queue))) + + def __iter__(self) -> Iterator[RequestQueueItem]: + return iter(self.peek_requests(len(self._queue))) + + def peek_request(self) -> RequestQueueItem: + if not self: + raise IndexError("no generation-only request heads the queue") + return self._queue.peek_request() + + def peek_requests(self, k: int) -> List[RequestQueueItem]: + run = [] + for item in self._queue.peek_requests(k): + if not _is_lone_generation_only(item): + break + run.append(item) + return run + + def pop_request(self) -> RequestQueueItem: + if not self: + raise IndexError("no generation-only request heads the queue") + return self._queue.pop_request() + + def prepend_requests(self, requests: Iterable[RequestQueueItem]) -> None: + self._queue.prepend_requests(requests) + + def admission_groups(self, requests: List[RequestQueueItem]) -> List[int]: + return self._queue.admission_groups(requests) + + def mark_admitted(self, requests: Iterable[RequestQueueItem]) -> None: + self._queue.mark_admitted(requests) + + # Period of the native MPI progress thread that pipeline-parallel ranks run; # a value <= 0 disables the thread, and a value above the maximum is malformed. PP_MPI_PROGRESS_INTERVAL_US_ENV_VAR_NAME = "TLLM_PP_MPI_PROGRESS_INTERVAL_US" @@ -775,6 +843,8 @@ class PyExecutor: _loop_iteration = 0 _staged_at = 0 _fetch_may_block = False + # Receive-ahead admission (set by _initialize_execution); 0 is off. + _receive_ahead_slots = 0 # Holds context final windows across passes when that empties decoder passes (set by __init__). ctx_decoder_batcher: Optional[CtxDecoderBatcher] = None @@ -1710,6 +1780,31 @@ def on_detected(): self._disagg_transfer_admission_controller) self.is_benchmark_disagg = (self.benchmark_req_queues_size > 0 and self.kv_cache_transceiver is not None) + # Receive-ahead (RuntimeHooks.disagg_gen_receive_ahead; off with + # TLLM_DISAGG_GEN_RECEIVE_AHEAD=0): live requests, retiring ones + # included, stay within the per-request pools sized for two batches by + # the overlap headroom, and under attention DP one seat stays free for + # the padding dummy. Off with the slot policies it would defeat: GEN + # slot preemption frees slots for at-risk waiters, and the first-token + # priority would let every received request take a running decode's + # slot. + self._receive_ahead_slots = 0 + if (model_hooks.disagg_gen_receive_ahead + and os.environ.get("TLLM_DISAGG_GEN_RECEIVE_AHEAD", "1") != "0" + and isinstance(self.scheduler, KVCacheV2Scheduler) + and not self.scheduler.prioritizes_first_token_gen + and not self.is_benchmark_disagg and self.dist.cp_size == 1 + and self._gen_preemption is None + and self.model_engine.enable_overlap_headroom + and transfer_window_bypass_eligible(self.kv_cache_transceiver, + self.dist, + self._is_kv_manager_v2)): + self._receive_ahead_slots = min( + self.max_num_active_requests - int(self.enable_attention_dp), + _RECEIVE_AHEAD_SLOTS) + if self._receive_ahead_slots: + self.scheduler.set_gen_init_receive_ahead( + self.max_num_active_requests) # True while the benchmark disagg fill phase is in progress (waiting # for all benchmark requests to complete KV transfer before the first # forward pass). Cleared by _check_benchmark_disagg_gate when the @@ -6040,7 +6135,8 @@ def _prepare_and_schedule_batch( new_requests = self._progress_context_cancellations(new_requests) new_requests = self._cancel_waiting_disagg_requests(new_requests) self.disagg.prepare_context_schedulable(new_requests) - self.disagg.poll_gen_transfers() + self.disagg.poll_gen_transfers( + batch_full=self._gen_transfer_batch_full()) self._activate_completed_remote_tails() self.disagg.check_transfer_timeouts() # After the polls so a transfer that completed this iteration releases @@ -9792,14 +9888,13 @@ def _pop_from_waiting_queue( waiting_queue: WaitingQueue, total_num_active_requests: int, all_rank_states: Optional[List[RankState]] = None - ) -> Tuple[List[RequestQueueItem], Dict[int, int]]: + ) -> Tuple[List[RequestQueueItem], Optional[_AdpRouting]]: """Pop requests from waiting_queue based on available capacity. - Returns the requests and, with attention DP, the ranks the router - planned for them (request id -> rank; empty when none are planned). + Returns the requests and, with attention DP, how to route them. """ admission_capacity = self._get_request_admission_capacity() - + receive_ahead_slots = getattr(self, "_receive_ahead_slots", 0) max_new_requests = admission_capacity - total_num_active_requests # Benchmark disagg fill-phase admission throttle (slow-start ramp). @@ -9826,11 +9921,41 @@ def _pop_from_waiting_queue( kv_balance = not (self.is_warmup or (self.is_benchmark_disagg and self._benchmark_fill_phase_active)) - return self.adp_router.pop_requests(waiting_queue, - all_rank_states, - max_new_requests, - self.max_num_active_requests, - kv_balance=kv_balance) + queue = waiting_queue + rank_states = all_rank_states + rank_cap = self.max_num_active_requests + if receive_ahead_slots: + # Receive-ahead raises each rank's cap by its slots: with an + # empty queue to bound the router's expected per-rank load, else + # for a pass that pops only the lone generation-only requests at + # the queue head, when they are more than the batch has room + # for. Retiring requests count against the slots, so live + # requests fit the pools for two batches, and no rank's cap is + # lower than without receive-ahead. + room = max(max_new_requests, 0) + run = _GenerationOnlyRun(waiting_queue) + if not waiting_queue: + rank_cap += receive_ahead_slots + elif len(run.peek_requests(room + 1)) > room: + queue = run + rank_cap += receive_ahead_slots + rank_states = [ + dataclasses.replace( + state, + num_active_requests=state.num_active_requests + + min(state.num_retiring_requests, + receive_ahead_slots)) + for state in all_rank_states + ] + max_new_requests = self.dist.tp_size * rank_cap - sum( + state.num_active_requests for state in rank_states) + new_requests, placement = self.adp_router.pop_requests( + queue, + rank_states, + max_new_requests, + rank_cap, + kv_balance=kv_balance) + return new_requests, _AdpRouting(rank_states, rank_cap, placement) mm_admission = getattr(self, "_mm_encoder_item_scheduling_enabled", False) new_requests = get_from_waiting_queue( @@ -9840,12 +9965,17 @@ def _pop_from_waiting_queue( max_num_active_requests=self.max_num_active_requests, mark_admitted=not mm_admission) if not mm_admission: - return new_requests, {} + if receive_ahead_slots: + num_live = len(self.active_requests) + sum( + 1 + len(item.child_req_ids or ()) for item in new_requests) + new_requests += self._pop_receive_ahead_requests( + waiting_queue, num_live) + return new_requests, None # Deferred requests go back with their arrival order, so only the rest are admitted. admitted = (self._apply_mm_encoder_admission( waiting_queue, new_requests) if new_requests else []) waiting_queue.mark_admitted(admitted) - return admitted, {} + return admitted, None def _publish_adp_iter_stats(self, all_rank_states: List[RankState], idle_wait_follows: bool) -> None: @@ -9882,6 +10012,45 @@ def _publish_adp_iter_stats(self, all_rank_states: List[RankState], # from it (see _plan_idle_heartbeat_wait). return + def _pop_receive_ahead_requests(self, waiting_queue: WaitingQueue, + num_live: int) -> List[RequestQueueItem]: + """Pop queued generation-only requests into the receive-ahead slots. + + ``num_live`` counts retiring requests, which still hold their slots. + """ + room = (self.max_num_active_requests + self._receive_ahead_slots - + num_live) + popped = [] + while (room > 0 and waiting_queue + and _is_lone_generation_only(waiting_queue.peek_request())): + popped.append(waiting_queue.pop_request()) + room -= 1 + if popped: + waiting_queue.mark_admitted(popped) + return popped + + def _gen_transfer_batch_full(self) -> bool: + """Whether no completed receive could join the batch this iteration. + + True while decodes holding their KV fill the batch (a decode suspended + for pages leaves a seat) and no receive ends at its first token, which + needs no seat. Without attention DP every rank holds the same requests + and decides alike. + """ + if not self._receive_ahead_slots or self.enable_attention_dp: + return False + generating = LlmRequestState.GENERATION_IN_PROGRESS.value + resident = self.kv_cache_manager.is_request_active + if sum(1 for req in self.active_requests + if req.state_value == generating + and resident(req.py_request_id)) < self.max_num_active_requests: + return False + # A generation-first receive learns its first token only on arrival. + return not any(req.is_disagg_generation_transmission_in_progress and + (req.py_max_new_tokens <= 1 + or self._first_gen_token_finish_reason(req) is not None) + for req in self.active_requests) + @nvtx_range("_fetch_new_requests") def _fetch_new_requests( self, waiting_queue: WaitingQueue, @@ -10041,7 +10210,7 @@ def _fetch_new_requests( preemption.maybe_log() # 3. Pop requests from waiting queue - new_requests, placement = self._pop_from_waiting_queue( + new_requests, routing = self._pop_from_waiting_queue( waiting_queue, total_num_active_requests, all_rank_states) # 4. Update performance metrics (before DP scheduling to clear all start_times) @@ -10059,8 +10228,8 @@ def _fetch_new_requests( all_ranks_new_requests, self.expected_num_active_requests = \ self.adp_router.route_requests( - all_rank_states, new_requests, - self.max_num_active_requests, placement=placement) + routing.rank_states, new_requests, + routing.rank_cap, placement=routing.placement) self.request_broadcaster.deliver_prompts(all_ranks_new_requests) new_requests_cur_rank = all_ranks_new_requests[self.dist.tp_rank] @@ -11304,13 +11473,18 @@ def _pad_attention_dp_dummy_request(self): # test below, and a rank holding surplus requests needs no dummy, # so tolerate the surplus. Asserting here took down the executor # event loop on every affected rank at once, leaving the survivors - # to HangDetector-abort. - logger.warning( - f"routable active_requests " - f"({num_routable_active_requests}) exceeds " - f"expected_num_active_requests " - f"({expected_num_active_requests}); tolerating (a busy rank " - f"needs no attention-DP dummy).") + # to HangDetector-abort. With receive-ahead, a surplus of up to its + # slots is expected: the router's cap includes them only while the + # queue head is generation-only. + receive_ahead_slots = getattr(self, "_receive_ahead_slots", 0) + if not (receive_ahead_slots and num_routable_active_requests + <= self.max_num_active_requests + receive_ahead_slots): + logger.warning( + f"routable active_requests " + f"({num_routable_active_requests}) exceeds " + f"expected_num_active_requests " + f"({expected_num_active_requests}); tolerating (a busy " + f"rank needs no attention-DP dummy).") expected_num_active_requests = num_routable_active_requests num_active_request = self._count_schedulable_active_requests() @@ -11478,9 +11652,10 @@ def _pad_empty_attention_dp_batch( # recovery while peers still need this owner to participate in forward. if self.expected_num_active_requests <= 0: return - # This path can still hold active requests, so respect the same cap - # that `_pad_attention_dp_dummy_request` asserts against. - if len(self.active_requests) >= self.max_num_active_requests: + # This path can still hold active requests, so a rank already at its + # request cap (raised by the receive-ahead slots) gets no dummy. + if (len(self.active_requests) >= self.max_num_active_requests + + getattr(self, "_receive_ahead_slots", 0)): return # The fill gate suppresses forwards on purpose; dummies added then are # never terminated (termination follows a forward pass). diff --git a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py index 734079d8ad37..f1a3e4f37dfb 100644 --- a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py +++ b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py @@ -253,6 +253,10 @@ class RuntimeHooks: # Under multi-rank MPI attention DP, rank 0 sends each admitted request's prompt only to the rank # that owns it instead of broadcasting every prompt to every rank (RequestBroadcaster). prompts_to_owners: bool = False + # A disaggregated generation worker admits up to min(batch size, 4) generation-only requests beyond its batch + # (per rank under attention DP): they receive their KV while the batch decodes and, once received, join it as + # running requests free their slots. + disagg_gen_receive_ahead: bool = False # After autotuning, keep the fastest profiled tactic per shape on every # tensor-parallel rank, so the ranks capture the same kernels. merge_tuned_tactics_across_ranks: bool = False diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py index 0d1ac19f788c..95e309300713 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py @@ -200,6 +200,11 @@ class KVCacheV2Scheduler(RequestScheduler): _num_gen_init_transfer_budget_holds = 0 _num_ctx_decoder_hold_passes = 0 _num_ctx_decoder_held_requests = 0 + # Scheduling for the executor's receive-ahead admission (set_gen_init_receive_ahead). + _gen_init_receive_ahead = False + _gen_init_seats = 0 + # (request id, free GPU blocks before its receive-ahead allocation) for the last one that did not fit. + _receive_ahead_floor: Optional[tuple[int, int]] = None def __init__( self, @@ -452,6 +457,19 @@ def set_disagg_transfer_admission_controller(self, controller) -> None: """ self._disagg_transfer_controller = controller + @property + def prioritizes_first_token_gen(self) -> bool: + """Whether first-token generation-only requests go ahead of running decodes + (TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN).""" + return self._prioritize_first_token_gen + + def set_gen_init_receive_ahead(self, seats: int) -> None: + """Schedule for the executor's receive-ahead admission, whose per-rank request cap is ``seats``: + generation-init requests behind a full batch still allocate and start receiving, and received + ones join behind running decodes.""" + self._gen_init_receive_ahead = True + self._gen_init_seats = seats + def _pregrown_context_tokens(self, req: LlmRequest) -> int: """KV capacity a context request holds past its position, left by a skipped forward. @@ -722,9 +740,12 @@ def _schedule_loop( scheduled_beam_width = 0 has_chunking = False + # With receive-ahead a rank holds more requests than the executor's cap, which still bounds the batch. budget = BudgetTracker( self.max_num_tokens, - self.max_num_requests, + min(self.max_num_requests, self._gen_init_seats) + if self._gen_init_receive_ahead + else self.max_num_requests, self.peft_cache_manager, ) @@ -760,6 +781,34 @@ def _schedule_loop( ) ) + # With receive-ahead a rank holds more requests than its batch, so a received request + # (FIFO among them) takes only a slot that no running decode holds. A generation-init + # request past the batch seats receives ahead only while the GPU pools keep a free block + # for the next step of every request in them. + receive_ahead_seats = None + receive_ahead_closed = False + if self._gen_init_receive_ahead: + trans_complete = LlmRequestState.DISAGG_GENERATION_TRANS_COMPLETE.value + requests_list.sort(key=lambda req: req.state_value == trans_complete) + seated = ( + LlmRequestState.GENERATION_IN_PROGRESS.value, + LlmRequestState.DISAGG_GENERATION_TRANS_IN_PROGRESS.value, + trans_complete, + self._context_init_state_value, + self._encoder_init_state_value, + ) + # Counted over every active request: a remote-tail phase pass leaves some out. + receive_ahead_seats = self._gen_init_seats - sum( + 1 for req in active_requests if req.state_value in seated + ) + # A floor lasts only while the request that set it still waits to allocate. + if self._receive_ahead_floor is not None and not any( + req.py_request_id == self._receive_ahead_floor[0] + and req.state_value == self._disagg_gen_init_state_value + for req in active_requests + ): + self._receive_ahead_floor = None + # Resume deadlines take precedence over the first-token preference. if self._gen_resumed_ids: requests_list.sort(key=lambda req: req.py_request_id not in self._gen_resumed_ids) @@ -782,6 +831,8 @@ def _schedule_loop( pending_ctx: RequestList = [] # Index of each pending context request in requests_list. pending_ctx_index: dict[int, int] = {} + # Index of the last disagg gen-init request, found once the batch is full. + last_gen_init_it = None # Pre-claim PEFT pages for terminal requests and, with GEN preemption, # every live decode, including requests omitted from this batch. @@ -854,12 +905,35 @@ def _schedule_loop( req_it += 1 continue + receives_ahead = receive_ahead_seats is not None and receive_ahead_seats <= 0 + if receives_ahead and not receive_ahead_closed: + free_blocks = self.kv_cache_manager.num_free_gpu_blocks() + # After one did not fit, allocate again only once more blocks are free. + floor = self._receive_ahead_floor + receive_ahead_closed = floor is not None and free_blocks <= floor[1] + if receives_ahead and receive_ahead_closed: + req_it += 1 + continue action, tokens = self._try_schedule_disagg_gen_init(req, budget) if action is ScheduleAction.STOP: break if action is ScheduleAction.SKIP: + if receives_ahead: + self._receive_ahead_floor = (req.py_request_id, free_blocks) + receive_ahead_closed = True req_it += 1 continue + if receives_ahead: + if self.kv_cache_manager.num_free_gpu_blocks() < self._gen_init_seats: + # No receive is posted yet, so its pages can go back. + self.kv_cache_manager.revert_allocate_context(req) + self._receive_ahead_floor = (req.py_request_id, free_blocks) + receive_ahead_closed = True + req_it += 1 + continue + self._receive_ahead_floor = None + if receive_ahead_seats is not None: + receive_ahead_seats -= 1 disagg_candidates.append(req) if disagg_gate is not None: disagg_gate.observe_candidate(req) @@ -874,9 +948,25 @@ def _schedule_loop( continue # Budget check for non-disagg requests (disagg bypasses this - # because it doesn't participate in the forward pass). + # because it doesn't participate in the forward pass). With + # receive-ahead, gen-init requests behind a full batch still + # allocate and start receiving. if budget.requests_full: - break + if not self._gen_init_receive_ahead: + break + if last_gen_init_it is None: + last_gen_init_it = max( + ( + it + for it in range(req_it + 1, req_it_end) + if requests_list[it].state_value == self._disagg_gen_init_state_value + ), + default=-1, + ) + if req_it > last_gen_init_it: + break + req_it += 1 + continue if not ( req_state_value >= self._no_schedule_until_state_value diff --git a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py index f4fc63f53891..7ed1bbf3fd12 100644 --- a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py +++ b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py @@ -964,6 +964,7 @@ def _make_executor(tp_size: int = 4, fill_phase_active: bool = True): ex.dist = Mock() ex.dist.tp_size = tp_size ex.adp_router = Mock() + ex.adp_router.pop_requests.return_value = ([], {}) return ex @staticmethod @@ -1745,6 +1746,7 @@ def test_full_lifecycle(self): # pulls only tp_size requests per iteration, rather than the full # tp_size * max_batch_size queue. ex.adp_router = Mock() + ex.adp_router.pop_requests.return_value = ([], {}) ex._pop_from_waiting_queue( waiting_queue=Mock(), total_num_active_requests=0, diff --git a/tests/unittest/_torch/disaggregation/test_disagg_loop_transcript.py b/tests/unittest/_torch/disaggregation/test_disagg_loop_transcript.py index a9fb89dfebda..fe924077456f 100644 --- a/tests/unittest/_torch/disaggregation/test_disagg_loop_transcript.py +++ b/tests/unittest/_torch/disaggregation/test_disagg_loop_transcript.py @@ -227,7 +227,7 @@ def _pp_executor(monkeypatch, calls: list, *, rank: int) -> PyExecutor: ("handle_errors_synced",), ("start_deferred_gen_receives",), ("prepare_context_schedulable", []), - ("poll_gen_transfers",), + ("poll_gen_transfers", "batch_full=False"), ("check_transfer_timeouts",), ("admit", []), ("align_gen_admission_across_cp", []), diff --git a/tests/unittest/_torch/pyexecutor/test_overlap_admission_lifetime.py b/tests/unittest/_torch/pyexecutor/test_overlap_admission_lifetime.py index 3d54a18099cb..2cc56bca4d44 100644 --- a/tests/unittest/_torch/pyexecutor/test_overlap_admission_lifetime.py +++ b/tests/unittest/_torch/pyexecutor/test_overlap_admission_lifetime.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from types import MethodType, SimpleNamespace +from unittest.mock import patch import numpy as np import pytest @@ -17,7 +18,8 @@ from tensorrt_llm._torch.pyexecutor.scheduler import FCFSWaitingQueue, ScheduledRequests from tensorrt_llm._torch.pyexecutor.scheduler.adp_router import DefaultADPRouter from tensorrt_llm._torch.pyexecutor.seq_slot_manager import SeqSlotManager -from tensorrt_llm.bindings.executor import Request +from tensorrt_llm.bindings.executor import Request, RequestType +from tensorrt_llm.executor.request import DEFAULT_REQUEST_PRIORITY from tensorrt_llm.mapping import Mapping pytestmark = pytest.mark.cpu_only @@ -175,3 +177,62 @@ def test_adp_replacements_acquire_slots_before_retiring_requests_release_them( for request in requests: manager.free_resources(request) assert len(manager.slot_manager.free_slots) == capacity + + +def _generation_only_item(request_id: int, child_req_ids=None) -> RequestQueueItem: + request = SimpleNamespace( + request_type=RequestType.REQUEST_TYPE_GENERATION_ONLY, + num_input_tokens=2, + py_scheduling_params=None, + priority=DEFAULT_REQUEST_PRIORITY, + ) + return RequestQueueItem(request_id, request, child_req_ids=child_req_ids) + + +@pytest.mark.parametrize("blocker", ["context", "child-bearing"]) +def test_adp_receive_ahead_admits_only_the_leading_generation_only_run(blocker: str) -> None: + # Under attention DP the executor gives batch 3 two slots, leaving the padding dummy a seat. + rank_count, batch_size, slots = 2, 3, 2 + mapping = Mapping(world_size=rank_count, tp_size=rank_count, enable_attention_dp=True) + headroom = should_enable_overlap_headroom(mapping, False, kv_cache_manager_is_v2=True) + capacity = compute_max_num_sequences( + mapping, batch_size, False, enable_overlap_headroom=headroom + ) + executors = [ + _admission_executor(rank, rank_count, batch_size, headroom) for rank in range(rank_count) + ] + queues = [FCFSWaitingQueue() for _ in executors] + _enqueue_cohort(queues, 1, rank_count * batch_size) + resident = _fetch_all_ranks(executors, queues, [[] for _ in executors]) + for requests in resident: + for request in requests: + request.state = LlmRequestState.GENERATION_IN_PROGRESS + # Rank 0 decodes a full batch; rank 1 decodes one request beside a retiring one. + resident[1][0].state = LlmRequestState.GENERATION_TO_COMPLETE + + for queue in queues: + queue.add_requests( + [ + _generation_only_item(101), + _generation_only_item(102), + RequestQueueItem(104, Request(input_token_ids=[1, 2], max_tokens=1)) + if blocker == "context" + else _generation_only_item(104, child_req_ids=[1104]), + _generation_only_item(105), + ] + ) + for executor in executors: + executor._receive_ahead_slots = slots + with patch( + "tensorrt_llm._torch.pyexecutor.py_executor.merge_requests", + side_effect=lambda requests, **_: requests, + ): + incoming = _fetch_all_ranks(executors, queues, resident) + + # The batches have room for one request: the rest of the leading run goes + # beyond them, and the blocker and the request behind it wait, though the + # raised room would fit the blocker too. + assert sorted(item.id for items in incoming for item in items) == [101, 102] + assert all([item.id for item in queue] == [104, 105] for queue in queues) + for active, items in zip(resident, incoming, strict=True): + assert len(active) + len(items) <= min(batch_size + slots, capacity) diff --git a/tests/unittest/_torch/test_connector_adp.py b/tests/unittest/_torch/test_connector_adp.py index 0a2e5804e6bc..a679124d4484 100644 --- a/tests/unittest/_torch/test_connector_adp.py +++ b/tests/unittest/_torch/test_connector_adp.py @@ -85,6 +85,13 @@ def _manager(rank: int = 1) -> KvCacheConnectorManager: return KvCacheConnectorManager(worker, scheduler, enable_attention_dp=True) +def _admit_nothing(executor: "PyExecutor", _waiting_queue, _total_num_active, all_rank_states): + """``_pop_from_waiting_queue`` that admits nothing and routes under the executor's cap.""" + from tensorrt_llm._torch.pyexecutor.py_executor import _AdpRouting + + return [], _AdpRouting(all_rank_states, executor.max_num_active_requests, {}) + + def _configure_recovery_controls(executor: "PyExecutor") -> None: from tensorrt_llm._torch.pyexecutor.executor_request_queue import ExecutorRequestQueue @@ -1255,7 +1262,7 @@ def test_idle_fleet_polls_saves_without_new_traffic(outcome: str) -> None: executor.waiting_queue.__len__.return_value = 0 executor.request_broadcaster = MagicMock() executor.request_broadcaster.broadcast.return_value = [] - executor._pop_from_waiting_queue = MagicMock(return_value=([], None)) + executor._pop_from_waiting_queue = MagicMock(side_effect=partial(_admit_nothing, executor)) executor.executor_request_queue.get_from_request_queue = MagicMock(return_value=[]) executor._enable_non_overlap_adp_forward_intent = True executor.disable_overlap_scheduler = False @@ -1405,7 +1412,7 @@ def test_control_drain_keeps_owner_collectives_aligned(loading_rank: int) -> Non executor.waiting_queue = MagicMock() executor.waiting_queue.__len__.return_value = 0 executor.enable_iter_perf_stats = False - executor._pop_from_waiting_queue = MagicMock(return_value=([], None)) + executor._pop_from_waiting_queue = MagicMock(side_effect=partial(_admit_nothing, executor)) executor._enable_non_overlap_adp_forward_intent = True executor.disable_overlap_scheduler = False executor.previous_batch = None