Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions tensorrt_llm/_torch/pyexecutor/llm_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -1004,6 +1004,7 @@ def __init__(
encoder_input_tokens, "tolist") else list(encoder_input_tokens))
self.encoder_tokens = encoder_tokens
self.py_return_encoder_output = return_encoder_output
self.py_request_terminated = False
self.py_client_id = client_id
self.py_request_id = self.request_id
self.py_llm_request_type = self.llm_request_type
Expand Down Expand Up @@ -1208,6 +1209,8 @@ def _initialize_execution_state(self,
seq_slot: Optional[int],
orig_prompt_len: int,
clear_draft_tokens: bool = False) -> None:
self.py_request_resources_released = False
self.py_request_pp_termination_agreed = False
self.py_prompt_len = self.prompt_len
self.py_orig_prompt_len = orig_prompt_len
self.py_max_new_tokens = self.max_new_tokens
Expand Down
122 changes: 92 additions & 30 deletions tensorrt_llm/_torch/pyexecutor/py_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2091,7 +2091,8 @@ def _release_transfer(self, request: LlmRequest) -> None:
self.disagg.release_transfer(request)
return
if self.async_transfer_manager.end_transfer(request):
if not self.force_terminate_ctx_for_partial_reuse:
if not (self.force_terminate_ctx_for_partial_reuse and getattr(
request, "py_request_terminated", False) is True):
self._terminate_request(request)

def _flush_pending_transfer_responses(self, merge: bool = False):
Expand All @@ -2118,8 +2119,17 @@ def _flush_pending_transfer_responses(self, merge: bool = False):
# Even when this rank has no responses we must participate in the
# collective when ADP is enabled so that the other rank's gather
# can complete.
self._enqueue_responses(responses)
for request in requests_to_terminate:
self._publish_responses_and_terminate(responses,
requests_to_terminate)
else:
self._terminate_requests(requests_to_terminate)

def _publish_responses_and_terminate(
self, responses: list[tuple[int, LlmResponse]],
requests: Iterable[LlmRequest]) -> None:
"""Publish at the caller's collective boundary before removing routing."""
self._enqueue_responses(responses)
for request in requests:
self._terminate_request(request)

def _flush_iter_stats_synced(self):
Expand Down Expand Up @@ -5634,9 +5644,7 @@ def _snapshot_remote_tail_phase_work(
generation_ids = set()
for request in active_requests:
if (request.is_dummy or request.request_id in self.inflight_req_ids
or
not self.scheduler.is_request_in_schedulable_state(request)
):
or not self._is_request_schedulable(request)):
continue
if request.state in (LlmRequestState.CONTEXT_INIT,
LlmRequestState.ENCODER_INIT):
Expand Down Expand Up @@ -11155,7 +11163,7 @@ def _count_schedulable_active_requests(self) -> int:
or req.is_disagg_generation_transmission_in_progress))

return sum(1 for req in self.active_requests
if self.scheduler.is_request_in_schedulable_state(req))
if self._is_request_schedulable(req))

def _get_non_overlap_adp_forward_intent(
self) -> tuple[int, _ADPForwardIntent]:
Expand All @@ -11168,9 +11176,8 @@ def _get_non_overlap_adp_forward_intent(
local_schedulable_count = 0
local_intent = _ADPForwardIntent.NONE
for request in self.active_requests:
if (request.is_attention_dp_dummy or
not self.scheduler.is_request_in_schedulable_state(request)
):
if (request.is_attention_dp_dummy
or not self._is_request_schedulable(request)):
continue

local_schedulable_count += 1
Expand Down Expand Up @@ -12732,9 +12739,8 @@ def _handle_errors(self,
self._pending_transfer_responses = []
pending_terminations = self._pending_response_terminations
self._pending_response_terminations = []
self._enqueue_responses(pending + list(error_responses.items()))
for request in pending_terminations:
self._terminate_request(request)
self._publish_responses_and_terminate(
pending + list(error_responses.items()), pending_terminations)

if not defer_termination:
for request in failed_requests:
Expand All @@ -12748,36 +12754,78 @@ def _handle_errors(self,
# tears down every rank together via is_shutdown.
raise self._fatal_error

def _is_request_schedulable(self, request: LlmRequest) -> bool:
"""State readiness before capacity admission, excluding parked requests."""
return (not self._is_preemption_pending(request)
and request.py_request_id not in getattr(
self, "_pending_recompute_pause_ids", ())
and self.scheduler.is_request_in_schedulable_state(request))

def _is_final_response_ready(self, request: LlmRequest) -> bool:
"""Native completion permits a final response, even during a context send."""
return request.is_finished and not self._is_preemption_pending(request)

def _has_pending_response(self, request: LlmRequest) -> bool:
return any(
req_id == request.py_request_id
for req_id, _ in getattr(self, "_pending_transfer_responses", ()))

def _can_release_request(self, request: LlmRequest) -> bool:
"""Whether execution owners permit teardown; PP consensus is separate.

Connector loads register their deferred termination before this check.
Legacy connector-only saves pin detached blocks and allow early teardown;
V2 saves and transceiver sends retain the request's allocation itself.
"""
if self._is_preemption_pending(request) or self._has_pending_response(
request):
return False
connector = getattr(self, "kv_connector_manager", None)
if connector is not None and connector.has_pending_load(request):
return False
if (getattr(self, "_is_kv_manager_v2", False)
or getattr(self, "kv_cache_transceiver", None) is not None):
transfers = getattr(self, "async_transfer_manager", None)
if (transfers is not None and request.py_request_id
in transfers.requests_in_transfer()):
return False
return True

def _terminate_request(self, request: LlmRequest) -> None:
if getattr(request, "py_request_terminated", False) is True:
return
if self.disagg.defer_gen_termination(request) is True:
return
connector = getattr(self, "kv_connector_manager", None)
if connector is not None:
connector.release_unstarted_prefix_loads(request)
if connector.defer_load_termination(request):
return
if (getattr(self, "_is_kv_manager_v2", False)
or getattr(self, "kv_cache_transceiver", None) is not None):
transfers = getattr(self, "async_transfer_manager", None)
if (transfers is not None and request.py_request_id
in transfers.requests_in_transfer()):
# A transfer still holds a claim on this request's blocks.
# end_transfer pops the claim and the release path re-enters
# _terminate_request once the last owner lets go; freeing here
# would race the fabric and then double-free on that re-entry.
return
if self._has_pending_response(request):
# A second transfer owner can finish before the first owner's
# staged response is flushed. Keep routing until that publication.
if not any(req is request
for req in self._pending_response_terminations):
self._pending_response_terminations.append(request)
return
if not self._can_release_request(request):
return
# Dummy requests don't participate in disagg KV cache transfers,
# so they must bypass the PP termination handler to avoid stale
# sequences in the KV cache manager (the handler delays removal,
# but the dummy ID is reused every iteration).
if (self._disagg_pp_termination_handler is not None
and not request.is_dummy_request):
and not request.is_dummy_request
and getattr(request, "py_request_pp_termination_agreed",
False) is not True):
self._disagg_pp_termination_handler.terminate(request)
else:
self._do_terminate_request(request)

def _free_request_resources(self, request: LlmRequest) -> None:
"""Release execution resources without removing response routing."""
if getattr(request, "py_request_resources_released", False) is True:
return
connector = getattr(self, "kv_connector_manager", None)
if (self.enable_attention_dp and connector is not None
and connector.has_pending_transfers(request.request_id)):
Expand Down Expand Up @@ -12806,8 +12854,14 @@ def _free_request_resources(self, request: LlmRequest) -> None:
"release_request_state", None)
if release is not None:
release(request.py_request_id)
request.py_request_resources_released = True

def _do_terminate_request(self, request: LlmRequest) -> None:
if getattr(request, "py_request_terminated", False) is True:
return
if not self._can_release_request(request):
self._terminate_request(request)
return
self._free_request_resources(request)
# Cancellation and request-scoped failures can terminate before the
# normal post-prefill release point, including with a partial buffer.
Expand All @@ -12817,10 +12871,14 @@ def _do_terminate_request(self, request: LlmRequest) -> None:
request.py_encoder_output_ready_event = None
if self.gather_all_responses or self.dist.rank == 0:
self._response_channel.release(request.py_request_id)
request.py_request_terminated = True

def _on_disagg_pp_termination(self, request: LlmRequest) -> None:
"""Finish a PP-synchronized termination and any deferred recompute."""
request_id = request.py_request_id
# Consensus remains valid while local response or DMA owners drain;
# peers may already have released and must not vote a second time.
request.py_request_pp_termination_agreed = True
if request_id not in self._pending_recompute_pause_ids:
self._do_terminate_request(request)
return
Expand Down Expand Up @@ -13636,8 +13694,9 @@ def _handle_responses(self, emit_first_iter: bool = True):
if request.return_perf_metrics and request.py_decoding_iter >= 1:
request.update_perf_metrics(self.iter_counter)

final_response_ready = self._is_final_response_ready(request)
request_done = False
if is_finished:
if final_response_ready:
# Guard the engine lookup -- minimal executors (unit tests)
# may have no engine.
route_capture = getattr(getattr(self, "model_engine", None),
Expand All @@ -13655,7 +13714,8 @@ def _handle_responses(self, emit_first_iter: bool = True):
# every earlier step only costs a native call. Requests that
# return perf metrics keep the call: it finalizes their ctx GPU
# totals.
if (should_emit and not is_finished and not request.streaming
if (should_emit and not final_response_ready
and not request.streaming
and not request.return_perf_metrics):
should_emit = False
if should_emit:
Expand All @@ -13664,7 +13724,7 @@ def _handle_responses(self, emit_first_iter: bool = True):
if response:
if first_gen_emit:
request.py_gen_first_emitted = True
request_done = request.is_finished
request_done = final_response_ready
new_responses.append((req_id, response))

if request_done:
Expand All @@ -13691,9 +13751,8 @@ def _handle_responses(self, emit_first_iter: bool = True):
self.active_requests.clear()
self.active_requests.extend(new_active_requests)
# Request should be terminated after enqueueing response to ensure we can enqueue response successfully.
self._enqueue_responses(new_responses)
for request in requests_to_terminate:
self._terminate_request(request)
self._publish_responses_and_terminate(new_responses,
requests_to_terminate)
# Under multi-rank ADP the coordinator defers the error response to
# its synced drain; an in-place tp_allgather here would desync
# (reached from per-rank-divergent gates).
Expand Down Expand Up @@ -13727,6 +13786,9 @@ def _terminate_requests(
def _pause_requests(self, requests_to_pause: Iterable[LlmRequest]) -> None:
for req in requests_to_pause:
req.pause(self.max_input_len)
req.py_request_resources_released = False
req.py_request_terminated = False
req.py_request_pp_termination_agreed = False

def _pause_recompute_request(self, req: LlmRequest) -> None:
req.reset_for_recompute(_UNBOUNDED_PAUSE_MAX_INPUT_LEN)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ def _make_v2_kv_cache_manager() -> Mock:
manager = Mock()
manager.enable_block_reuse = True
manager.disk_prefetch_num_reqs = 0
manager.is_preemption_pending.return_value = False
return manager


Expand Down Expand Up @@ -814,6 +815,8 @@ def __init__(

_pad_attention_dp_dummy_request = PyExecutor._pad_attention_dp_dummy_request
_count_schedulable_active_requests = PyExecutor._count_schedulable_active_requests
_is_request_schedulable = PyExecutor._is_request_schedulable
_is_preemption_pending = PyExecutor._is_preemption_pending
_get_non_overlap_adp_forward_intent = PyExecutor._get_non_overlap_adp_forward_intent
_has_adp_dummy_kv_capacity = PyExecutor._has_adp_dummy_kv_capacity
_should_skip_dummy_for_benchmark_disagg = PyExecutor._should_skip_dummy_for_benchmark_disagg
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,11 @@ def _executor_with_real_error_path(monkeypatch) -> tuple:
executor.force_terminate_ctx_for_partial_reuse = False
transceiver = FakeKvCacheTransceiver()
executor.kv_cache_transceiver = transceiver
kv_cache_manager = Mock(spec=["store_blocks_for_reuse", "unpin_blocks_by_id"])
kv_cache_manager = Mock(
spec=["store_blocks_for_reuse", "unpin_blocks_by_id", "is_preemption_pending"]
)
kv_cache_manager.is_preemption_pending.return_value = False
executor.kv_cache_manager = kv_cache_manager
transfers = AsyncTransferManager(
SimpleNamespace(resource_managers={ResourceManagerType.KV_CACHE_MANAGER: kv_cache_manager})
)
Expand Down
Loading
Loading