Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 32 additions & 8 deletions tensorrt_llm/_torch/disaggregation/kv_cache_transceiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
MambaHybridCacheManagerV2, MixedMambaHybridCacheManager)
from ..pyexecutor.llm_request import LlmRequest
from ..pyexecutor.resource_manager import KVCacheManager
from .orchestration.cancellation import ReceiveCancellation

CacheTransceiverCpp = tensorrt_llm.bindings.internal.batch_manager.CacheTransceiver
AttentionTypeCpp = tensorrt_llm.bindings.internal.batch_manager.AttentionType
Expand Down Expand Up @@ -440,23 +441,46 @@ def cancel_request(self, req: LlmRequest) -> bool:
def cancel_gen_logically(
self,
req: LlmRequest,
ready: Optional["torch.cuda.Event"] = None) -> bool:
"""Accept logical GEN cancellation without authorizing memory release.
ready: Optional["torch.cuda.Event"] = None,
*,
prior_failure: bool = False) -> ReceiveCancellation:
"""Close GEN publication and return its logical cancellation disposition.

Args:
req: Model-idle generation request to cancel.
ready: Unpublished destination's outstanding admission fence, if any.
prior_failure: An executor failure that preceded cancellation.

Returns:
Whether cancellation was accepted under a qualified retirement policy.
The caller must still use cancel_request to establish safe retirement.
Under that policy, the transceiver retains a passed ready fence until
cancel_request returns True, even if an earlier failure makes this
method return False. Unsupported implementations return False without
taking ownership of ready.
NOT_HANDLED leaves ready with the caller. PENDING, CANCELLED and
FAILED retain it under the runtime's physical-retirement policy.
Repeated calls preserve a committed result, including prior failure.
The caller must use cancel_request separately for release permission.
The receive owner alone writes py_kv_receive_cancel_outcome, once
after successful owner closure, as a duplicate-call tombstone. That
field is not independent access-end evidence. Only the executor
applies this disposition to LlmRequest.state.
"""
return ReceiveCancellation.NOT_HANDLED

def supports_group_receive_cancellation(self) -> bool:
"""Whether this runtime binds owned access, immutable IDs and shared progress.

This rank-uniform capability permits cancellation agreement in the
existing receive-progress collective. It does not qualify a new profile.
"""
return False

def take_receive_cancellation_intents(self) -> dict[int, bool]:
"""Take peer intent observed by the existing receive-progress exchange.

Returns:
Immutable transfer IDs mapped to prior failure, not release evidence.
The coordinator must close local publication and supply a real local
cancellation vote, including any unpublished readiness fence it owns.
"""
return {}

def supports_inflight_request_cancellation(self) -> bool:
"""Return True when in-flight transfers can be cancelled safely."""
return False
Expand Down
38 changes: 38 additions & 0 deletions tensorrt_llm/_torch/disaggregation/lifecycle/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,31 @@ supported configuration.
| Native containment | `RetirementWatchdog` latches a controller-wide fatal state, closes admission, and notifies containment once. | Preserve native's executor-fatal callback and keep sessions, manager loans, pools, and agents reachable. |
| Logical outcome | Remains with the integration, outside this package. | Commit failure/cancellation once; bind a metadata-only timeout callback. Late physical completion must not replace that outcome. |

`orchestration/cancellation.py` provides receive cancellation agreement without
owning resources or calling a backend. Each binding supplies one prior-failure
and access-ended vote per required participant. The first complete vote commits
`CANCELLED` or `FAILED`; later votes can permit retirement but cannot rewrite the
outcome. `PENDING` means agreement is not complete, and `NOT_HANDLED` means this
binding did not accept the request. None of these values alone authorizes reuse.

The coordinator exposes this semantic capability to the executor. Native still
collects the evidence, closes real owners and supplies its existing collective;
the executor alone applies the cancellation disposition to `LlmRequest.state`.
After successful closure, the receive owner writes the request's optional
cancellation outcome once for duplicate calls. It is not a second release vote.
Native qualification remains restricted to the existing BF16 context-first gate.
Another binding must demonstrate immutable participant identity, closed admission,
strong resource roots, access-end evidence and symmetric progress before enabling
the same capability. This helper does not qualify another layout or runtime.

Peer cancellation intent uses the existing receive-progress exchange. The
coordinator matches its immutable transfer ID to the local request and hands off
any unpublished CUDA fence; a missing local record is not a safe vote. Pending
handoffs keep publication closed and retry without resetting the retirement clock.
Qualified group retirement must reach executor failure cleanup on the same poll
across ranks; rank-local retry pacing must not delay an already-retired outcome.
Requests outside this binding retain the legacy rank-aligned cancellation path.

`timeout_s` is the mandatory transfer safety budget from first exposure, not the
overall inference-request deadline. The optional `grace_s` override controls the
non-resettable drain window; omitting it or passing `None` uses `timeout_s`, preserving
Expand All @@ -38,6 +63,19 @@ A backend error, elapsed time, logical failure, or empty report count is not
quiescence. The existing native fatal policy is unchanged: retirement at or after
fatal expiry remains prohibited even if DONE arrives later.

The watchdog reports two fatal kinds. `deadline` means an existing owner still
has unproven access at its fixed grace deadline. `ownership_handoff` means an
already-admitted accessor cannot enter bounded retirement tracking. For example,
an unpublished receive can still have a local CUDA writer when shutdown closes
watchdog admission. Retaining its request and fence prevents reuse, but the
unregistered record has no retirement clock; an unsafe TP vote alone cannot
bound that wait, especially if watchdog progress has stopped. The binding must
keep those roots and call `fail_closed()` rather than release them or wait
indefinitely. This does not make ordinary shutdown fatal: owners registered
before shutdown continue their existing drain. A separate local-wait alternative
would need its own bounded progress, root retention, and access-end contract.
Both fatal kinds preserve the first decision and notify containment only once.

Future bindings may use qualified lease isolation or binding/domain quarantine
instead of executor termination. The scope depends on the proven access boundary,
not host versus GPU memory alone. For example, host staging can retain affected
Expand Down
39 changes: 38 additions & 1 deletion tensorrt_llm/_torch/disaggregation/lifecycle/retirement.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@

@dataclass(frozen=True)
class QuiescenceFatalEvent:
"""Immutable evidence that an exposed owner missed its retirement deadline."""
"""Immutable evidence that safe physical retirement cannot continue."""

request_id: int
direction: Literal["send", "receive"]
reason: str
started_at: float
deadline: float
expired_at: float
failure_kind: Literal["deadline", "ownership_handoff"] = "deadline"


class RetirementDeadline:
Expand Down Expand Up @@ -322,6 +323,42 @@ def progress(self) -> None:
# fatal decision or destroy the owners that still retain memory.
logger.error(f"KV retirement containment callback failed; owners retained: {error}")

def fail_closed(
Comment thread
chienchunhung marked this conversation as resolved.
Resolved
self, request_id: int, direction: Literal["send", "receive"], reason: str
) -> None:
"""Contain an admitted access that cannot join bounded ownership tracking.

Args:
request_id: Exact request whose roots the caller must retain.
direction: Source or destination access that cannot be tracked.
reason: Description of the failed ownership handoff.
"""
with self.lock:
if self.fatal is None:
now = self.clock()
self.fatal = QuiescenceFatalEvent(
request_id,
direction,
reason,
now,
now,
now,
failure_kind="ownership_handoff",
)
self.wake.set()
# Reuse the existing once-only callback outside the lock, including
# when the background progress thread has already stopped.
try:
self.progress()
finally:
with self.lock:
if self._stopped.is_set() or self._thread is None or not self._thread.is_alive():
# Fatal admission stays closed. Reuse the persistent progress
# thread to root the callback graph if containment returns.
self._stopped.clear()
self._thread = None
self.start()

def request_shutdown(self) -> None:
"""Start drain on every tracked owner before any resource teardown."""
with self.lock:
Expand Down
23 changes: 23 additions & 0 deletions tensorrt_llm/_torch/disaggregation/native/messenger.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from abc import ABC, abstractmethod
from threading import Event, Lock, Thread
from typing import Callable, Optional
Expand Down Expand Up @@ -125,6 +128,26 @@ def send(self, messages: list[bytes], recipient: Optional[bytes] = None) -> None
else:
self._socket.send_multipart(messages)

def try_send(self, messages: list[bytes]) -> bool:
"""Queue one DEALER message without waiting for transport capacity.

Args:
messages: Complete multipart message to queue.

Returns:
False on backpressure; the caller retains responsibility for retry.

Raises:
ValueError: The messenger is not a DEALER socket.
"""
if self._mode != "DEALER":
raise ValueError("try_send requires a DEALER socket")
try:
self._socket.send_multipart(messages, flags=zmq.DONTWAIT)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Nice to have] (R1: PR process) This PR overlaps open PRs that the description does not name; please state the merge order:

except zmq.Again:
return False
return True

def receive(self, timeout_ms: Optional[int] = None) -> list[bytes]:
if timeout_ms is not None and not self._socket.poll(timeout_ms, zmq.POLLIN):
raise TimeoutError("Timed out waiting for a peer response")
Expand Down
56 changes: 49 additions & 7 deletions tensorrt_llm/_torch/disaggregation/native/transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -3657,8 +3657,16 @@ def _get_sender_info(self, params: DisaggregatedParams) -> RankInfo:
else:
return self._sender_ep_instance_map[info_endpoint]

def send_cancel_to_senders(self, unique_rid: int, sender_endpoints: set[str]) -> set[str]:
"""Notify senders and return the endpoints whose cancel message was sent."""
def send_cancel_to_senders(
self, unique_rid: int, sender_endpoints: set[str], *, nonblocking: bool = False
) -> set[str]:
"""Notify senders and return only successfully queued endpoints.

Args:
unique_rid: Immutable receive identity.
sender_endpoints: Writers that still need cancellation notification.
nonblocking: Leave backpressured endpoints unsent for later progress.
"""
message = [MessageType.CANCEL_SESSION, str(unique_rid).encode("ascii")]
session = self._get_session(unique_rid)
if session is not None and session._enforce_physical_ownership:
Expand All @@ -3669,7 +3677,12 @@ def send_cancel_to_senders(self, unique_rid: int, sender_endpoints: set[str]) ->
sent = set()
for endpoint in sender_endpoints:
try:
self._get_or_connect_dealer(endpoint).send(message)
dealer = self._get_or_connect_dealer(endpoint)
if nonblocking:
if not dealer.try_send(message):
continue
else:
dealer.send(message)
sent.add(endpoint)
except Exception as e:
logger.warning(f"send_cancel_to_senders: failed for rid={unique_rid}: {e}")
Expand Down Expand Up @@ -4507,17 +4520,36 @@ def capture_cancel_targets(self) -> set[str]:
with self.lock:
return set(self._sender_endpoints)

def notify_cancel(self, sender_endpoints: Optional[set[str]] = None) -> None:
with self._publication_lock:
def notify_cancel(
self, sender_endpoints: Optional[set[str]] = None, *, nonblocking: bool = False
) -> None:
"""Notify writers after publication, retaining unsent endpoints for retry.

Args:
sender_endpoints: Explicit targets or the session's published writers.
nonblocking: Do not wait for publication serialization or socket capacity.
"""
if not self._publication_lock.acquire(blocking=not nonblocking):
return
try:
if sender_endpoints is None:
sender_endpoints = self.capture_cancel_targets()
if self._enforce_physical_ownership:
with self.lock:
self._cancel_notification_started = True
sender_endpoints = sender_endpoints - self._cancelled_sender_endpoints
sent = self._receiver.send_cancel_to_senders(self.disagg_request_id, sender_endpoints)
if nonblocking:
sent = self._receiver.send_cancel_to_senders(
self.disagg_request_id, sender_endpoints, nonblocking=True
)
else:
sent = self._receiver.send_cancel_to_senders(
self.disagg_request_id, sender_endpoints
)
if self._enforce_physical_ownership and sent:
self._cancelled_sender_endpoints.update(sent)
finally:
self._publication_lock.release()

def process_session_quiesced(self, peer_rank: int) -> None:
"""Settle missing pieces only after every published candidate fences writes."""
Expand Down Expand Up @@ -4931,7 +4963,7 @@ def _on_quiescence_fatal(self, event: QuiescenceFatalEvent) -> None:
"""Forward fatal evidence while rooting all worker-owned resources.

Args:
event: Immutable deadline decision; never authorizes resource release.
event: Immutable physical-retirement failure; never authorizes resource release.
"""
callback = self._config.quiescence_fatal_callback
assert callback is not None
Expand Down Expand Up @@ -4986,6 +5018,16 @@ def prepare_rank_info(self, request: LlmRequest) -> None:
"""Authenticate and cache the context peer without publishing receive buffers."""
self._receiver._get_sender_info(request.py_disaggregated_params)

def fail_receive_retirement(self, request_id: int, reason: str) -> None:
"""Contain a retained receive that cannot enroll in physical ownership.

Args:
request_id: Exact destination request retained by the transceiver.
reason: Description of the failed ownership handoff.
"""
assert self._retirement_watchdog is not None
self._retirement_watchdog.fail_closed(request_id, "receive", reason)

def create_rx_session(self, request: LlmRequest) -> RxSession:
params = request.py_disaggregated_params
assert params is not None
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Request-level cancellation agreement, independent of backend evidence collection."""

from dataclasses import dataclass
from enum import Enum
from typing import Sequence


class ReceiveCancellation(Enum):
"""Logical cancellation disposition; no member authorizes memory reuse."""

NOT_HANDLED = "not_handled"
PENDING = "pending"
CANCELLED = "cancelled"
FAILED = "failed"


@dataclass(frozen=True)
class ReceiveCancellationVote:
"""Facts supplied by one qualified owner after closing further admission."""

prior_failure: bool
access_ended: bool


def agree_receive_cancellation(
committed: ReceiveCancellation,
votes: Sequence[ReceiveCancellationVote | None],
) -> tuple[ReceiveCancellation, bool]:
"""Commit one logical outcome and separately agree physical retirement.

Args:
committed: PENDING until the first complete vote; terminal afterwards.
votes: One entry per required participant; None means no vote yet.

Returns:
The immutable logical outcome and whether every participant supplied
access-end evidence. The binding must still close its real owners.

Raises:
ValueError: An unsupported request was included in agreement.
"""
if committed is ReceiveCancellation.NOT_HANDLED:
raise ValueError("Unsupported cancellation cannot enter group agreement")
if not votes or any(vote is None for vote in votes):
return committed, False
present = [vote for vote in votes if vote is not None]
if committed is ReceiveCancellation.PENDING:
committed = (
ReceiveCancellation.FAILED
if any(vote.prior_failure for vote in present)
else ReceiveCancellation.CANCELLED
)
return committed, all(vote.access_ended for vote in present)
Loading
Loading