diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index d354939c4179..be65c9f57e11 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -1004,6 +1004,7 @@ def __init__( encoder_input_tokens, "tolist") else list(encoder_input_tokens)) self.encoder_tokens = encoder_tokens self.py_return_encoder_output = return_encoder_output + self.py_request_terminated = False self.py_client_id = client_id self.py_request_id = self.request_id self.py_llm_request_type = self.llm_request_type @@ -1208,6 +1209,8 @@ def _initialize_execution_state(self, seq_slot: Optional[int], orig_prompt_len: int, clear_draft_tokens: bool = False) -> None: + self.py_request_resources_released = False + self.py_request_pp_termination_agreed = False self.py_prompt_len = self.prompt_len self.py_orig_prompt_len = orig_prompt_len self.py_max_new_tokens = self.max_new_tokens diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 9a1ee2f90ff7..118fae33a03b 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -2091,7 +2091,8 @@ def _release_transfer(self, request: LlmRequest) -> None: self.disagg.release_transfer(request) return if self.async_transfer_manager.end_transfer(request): - if not self.force_terminate_ctx_for_partial_reuse: + if not (self.force_terminate_ctx_for_partial_reuse and getattr( + request, "py_request_terminated", False) is True): self._terminate_request(request) def _flush_pending_transfer_responses(self, merge: bool = False): @@ -2118,8 +2119,17 @@ def _flush_pending_transfer_responses(self, merge: bool = False): # Even when this rank has no responses we must participate in the # collective when ADP is enabled so that the other rank's gather # can complete. - self._enqueue_responses(responses) - for request in requests_to_terminate: + self._publish_responses_and_terminate(responses, + requests_to_terminate) + else: + self._terminate_requests(requests_to_terminate) + + def _publish_responses_and_terminate( + self, responses: list[tuple[int, LlmResponse]], + requests: Iterable[LlmRequest]) -> None: + """Publish at the caller's collective boundary before removing routing.""" + self._enqueue_responses(responses) + for request in requests: self._terminate_request(request) def _flush_iter_stats_synced(self): @@ -5634,9 +5644,7 @@ def _snapshot_remote_tail_phase_work( generation_ids = set() for request in active_requests: if (request.is_dummy or request.request_id in self.inflight_req_ids - or - not self.scheduler.is_request_in_schedulable_state(request) - ): + or not self._is_request_schedulable(request)): continue if request.state in (LlmRequestState.CONTEXT_INIT, LlmRequestState.ENCODER_INIT): @@ -11155,7 +11163,7 @@ def _count_schedulable_active_requests(self) -> int: or req.is_disagg_generation_transmission_in_progress)) return sum(1 for req in self.active_requests - if self.scheduler.is_request_in_schedulable_state(req)) + if self._is_request_schedulable(req)) def _get_non_overlap_adp_forward_intent( self) -> tuple[int, _ADPForwardIntent]: @@ -11168,9 +11176,8 @@ def _get_non_overlap_adp_forward_intent( local_schedulable_count = 0 local_intent = _ADPForwardIntent.NONE for request in self.active_requests: - if (request.is_attention_dp_dummy or - not self.scheduler.is_request_in_schedulable_state(request) - ): + if (request.is_attention_dp_dummy + or not self._is_request_schedulable(request)): continue local_schedulable_count += 1 @@ -12732,9 +12739,8 @@ def _handle_errors(self, self._pending_transfer_responses = [] pending_terminations = self._pending_response_terminations self._pending_response_terminations = [] - self._enqueue_responses(pending + list(error_responses.items())) - for request in pending_terminations: - self._terminate_request(request) + self._publish_responses_and_terminate( + pending + list(error_responses.items()), pending_terminations) if not defer_termination: for request in failed_requests: @@ -12748,7 +12754,46 @@ def _handle_errors(self, # tears down every rank together via is_shutdown. raise self._fatal_error + def _is_request_schedulable(self, request: LlmRequest) -> bool: + """State readiness before capacity admission, excluding parked requests.""" + return (not self._is_preemption_pending(request) + and request.py_request_id not in getattr( + self, "_pending_recompute_pause_ids", ()) + and self.scheduler.is_request_in_schedulable_state(request)) + + def _is_final_response_ready(self, request: LlmRequest) -> bool: + """Native completion permits a final response, even during a context send.""" + return request.is_finished and not self._is_preemption_pending(request) + + def _has_pending_response(self, request: LlmRequest) -> bool: + return any( + req_id == request.py_request_id + for req_id, _ in getattr(self, "_pending_transfer_responses", ())) + + def _can_release_request(self, request: LlmRequest) -> bool: + """Whether execution owners permit teardown; PP consensus is separate. + + Connector loads register their deferred termination before this check. + Legacy connector-only saves pin detached blocks and allow early teardown; + V2 saves and transceiver sends retain the request's allocation itself. + """ + if self._is_preemption_pending(request) or self._has_pending_response( + request): + return False + connector = getattr(self, "kv_connector_manager", None) + if connector is not None and connector.has_pending_load(request): + return False + if (getattr(self, "_is_kv_manager_v2", False) + or getattr(self, "kv_cache_transceiver", None) is not None): + transfers = getattr(self, "async_transfer_manager", None) + if (transfers is not None and request.py_request_id + in transfers.requests_in_transfer()): + return False + return True + def _terminate_request(self, request: LlmRequest) -> None: + if getattr(request, "py_request_terminated", False) is True: + return if self.disagg.defer_gen_termination(request) is True: return connector = getattr(self, "kv_connector_manager", None) @@ -12756,28 +12801,31 @@ def _terminate_request(self, request: LlmRequest) -> None: connector.release_unstarted_prefix_loads(request) if connector.defer_load_termination(request): return - if (getattr(self, "_is_kv_manager_v2", False) - or getattr(self, "kv_cache_transceiver", None) is not None): - transfers = getattr(self, "async_transfer_manager", None) - if (transfers is not None and request.py_request_id - in transfers.requests_in_transfer()): - # A transfer still holds a claim on this request's blocks. - # end_transfer pops the claim and the release path re-enters - # _terminate_request once the last owner lets go; freeing here - # would race the fabric and then double-free on that re-entry. - return + if self._has_pending_response(request): + # A second transfer owner can finish before the first owner's + # staged response is flushed. Keep routing until that publication. + if not any(req is request + for req in self._pending_response_terminations): + self._pending_response_terminations.append(request) + return + if not self._can_release_request(request): + return # Dummy requests don't participate in disagg KV cache transfers, # so they must bypass the PP termination handler to avoid stale # sequences in the KV cache manager (the handler delays removal, # but the dummy ID is reused every iteration). if (self._disagg_pp_termination_handler is not None - and not request.is_dummy_request): + and not request.is_dummy_request + and getattr(request, "py_request_pp_termination_agreed", + False) is not True): self._disagg_pp_termination_handler.terminate(request) else: self._do_terminate_request(request) def _free_request_resources(self, request: LlmRequest) -> None: """Release execution resources without removing response routing.""" + if getattr(request, "py_request_resources_released", False) is True: + return connector = getattr(self, "kv_connector_manager", None) if (self.enable_attention_dp and connector is not None and connector.has_pending_transfers(request.request_id)): @@ -12806,8 +12854,14 @@ def _free_request_resources(self, request: LlmRequest) -> None: "release_request_state", None) if release is not None: release(request.py_request_id) + request.py_request_resources_released = True def _do_terminate_request(self, request: LlmRequest) -> None: + if getattr(request, "py_request_terminated", False) is True: + return + if not self._can_release_request(request): + self._terminate_request(request) + return self._free_request_resources(request) # Cancellation and request-scoped failures can terminate before the # normal post-prefill release point, including with a partial buffer. @@ -12817,10 +12871,14 @@ def _do_terminate_request(self, request: LlmRequest) -> None: request.py_encoder_output_ready_event = None if self.gather_all_responses or self.dist.rank == 0: self._response_channel.release(request.py_request_id) + request.py_request_terminated = True def _on_disagg_pp_termination(self, request: LlmRequest) -> None: """Finish a PP-synchronized termination and any deferred recompute.""" request_id = request.py_request_id + # Consensus remains valid while local response or DMA owners drain; + # peers may already have released and must not vote a second time. + request.py_request_pp_termination_agreed = True if request_id not in self._pending_recompute_pause_ids: self._do_terminate_request(request) return @@ -13636,8 +13694,9 @@ def _handle_responses(self, emit_first_iter: bool = True): if request.return_perf_metrics and request.py_decoding_iter >= 1: request.update_perf_metrics(self.iter_counter) + final_response_ready = self._is_final_response_ready(request) request_done = False - if is_finished: + if final_response_ready: # Guard the engine lookup -- minimal executors (unit tests) # may have no engine. route_capture = getattr(getattr(self, "model_engine", None), @@ -13655,7 +13714,8 @@ def _handle_responses(self, emit_first_iter: bool = True): # every earlier step only costs a native call. Requests that # return perf metrics keep the call: it finalizes their ctx GPU # totals. - if (should_emit and not is_finished and not request.streaming + if (should_emit and not final_response_ready + and not request.streaming and not request.return_perf_metrics): should_emit = False if should_emit: @@ -13664,7 +13724,7 @@ def _handle_responses(self, emit_first_iter: bool = True): if response: if first_gen_emit: request.py_gen_first_emitted = True - request_done = request.is_finished + request_done = final_response_ready new_responses.append((req_id, response)) if request_done: @@ -13691,9 +13751,8 @@ def _handle_responses(self, emit_first_iter: bool = True): self.active_requests.clear() self.active_requests.extend(new_active_requests) # Request should be terminated after enqueueing response to ensure we can enqueue response successfully. - self._enqueue_responses(new_responses) - for request in requests_to_terminate: - self._terminate_request(request) + self._publish_responses_and_terminate(new_responses, + requests_to_terminate) # Under multi-rank ADP the coordinator defers the error response to # its synced drain; an in-place tp_allgather here would desync # (reached from per-rank-divergent gates). @@ -13727,6 +13786,9 @@ def _terminate_requests( def _pause_requests(self, requests_to_pause: Iterable[LlmRequest]) -> None: for req in requests_to_pause: req.pause(self.max_input_len) + req.py_request_resources_released = False + req.py_request_terminated = False + req.py_request_pp_termination_agreed = False def _pause_recompute_request(self, req: LlmRequest) -> None: req.reset_for_recompute(_UNBOUNDED_PAUSE_MAX_INPUT_LEN) diff --git a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py index f4fc63f53891..f6b305a610ef 100644 --- a/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py +++ b/tests/unittest/_torch/disaggregation/test_benchmark_disagg.py @@ -97,6 +97,7 @@ def _make_v2_kv_cache_manager() -> Mock: manager = Mock() manager.enable_block_reuse = True manager.disk_prefetch_num_reqs = 0 + manager.is_preemption_pending.return_value = False return manager @@ -814,6 +815,8 @@ def __init__( _pad_attention_dp_dummy_request = PyExecutor._pad_attention_dp_dummy_request _count_schedulable_active_requests = PyExecutor._count_schedulable_active_requests + _is_request_schedulable = PyExecutor._is_request_schedulable + _is_preemption_pending = PyExecutor._is_preemption_pending _get_non_overlap_adp_forward_intent = PyExecutor._get_non_overlap_adp_forward_intent _has_adp_dummy_kv_capacity = PyExecutor._has_adp_dummy_kv_capacity _should_skip_dummy_for_benchmark_disagg = PyExecutor._should_skip_dummy_for_benchmark_disagg diff --git a/tests/unittest/_torch/disaggregation/test_disagg_coordinator_errors.py b/tests/unittest/_torch/disaggregation/test_disagg_coordinator_errors.py index 33e0bb41995c..5279d78105c3 100644 --- a/tests/unittest/_torch/disaggregation/test_disagg_coordinator_errors.py +++ b/tests/unittest/_torch/disaggregation/test_disagg_coordinator_errors.py @@ -402,7 +402,11 @@ def _executor_with_real_error_path(monkeypatch) -> tuple: executor.force_terminate_ctx_for_partial_reuse = False transceiver = FakeKvCacheTransceiver() executor.kv_cache_transceiver = transceiver - kv_cache_manager = Mock(spec=["store_blocks_for_reuse", "unpin_blocks_by_id"]) + kv_cache_manager = Mock( + spec=["store_blocks_for_reuse", "unpin_blocks_by_id", "is_preemption_pending"] + ) + kv_cache_manager.is_preemption_pending.return_value = False + executor.kv_cache_manager = kv_cache_manager transfers = AsyncTransferManager( SimpleNamespace(resource_managers={ResourceManagerType.KV_CACHE_MANAGER: kv_cache_manager}) ) diff --git a/tests/unittest/_torch/disaggregation/test_request_lifecycle.py b/tests/unittest/_torch/disaggregation/test_request_lifecycle.py new file mode 100644 index 000000000000..aaf76d19b69d --- /dev/null +++ b/tests/unittest/_torch/disaggregation/test_request_lifecycle.py @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Response publication and allocation lifetime across independent transfer owners.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from coordinator_harness import CoordinatorHarness, TransferRequest + +from tensorrt_llm._torch.disaggregation.orchestration.coordinator import NoopDisaggCoordinator +from tensorrt_llm._torch.disaggregation.orchestration.pp_termination import ( + DisaggPPTerminationHandler, +) +from tensorrt_llm._torch.pyexecutor.disagg_adapter import ( + PyExecutorEffects, + PyExecutorRequestRegistry, +) +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, executor_request_to_llm_request +from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor +from tensorrt_llm._torch.pyexecutor.request_delivery import ResponseChannel +from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm.bindings import LlmRequestState +from tensorrt_llm.bindings import executor as trtllm +from tensorrt_llm.bindings.internal.batch_manager import ReqIdsSet + +pytestmark = pytest.mark.cpu_only + + +def _executor() -> PyExecutor: + executor = object.__new__(PyExecutor) + executor.enable_attention_dp = False + executor._is_kv_manager_v2 = True + executor.kv_cache_manager = Mock() + executor.kv_cache_manager.is_preemption_pending.return_value = False + executor.kv_cache_transceiver = None + executor.kv_connector_manager = None + executor.async_transfer_manager = None + executor._disagg_coordinator = NoopDisaggCoordinator() + executor._disagg_pp_termination_handler = None + executor._pending_recompute_pause_ids = set() + executor._pending_transfer_responses = [] + executor._pending_response_terminations = [] + executor._prefetched_request_ids = set() + executor.inflight_req_ids = ReqIdsSet() + executor.sampler = Mock() + executor.resource_manager = Mock() + executor.model_engine = None + executor.draft_model_engine = None + executor._release_multimodal_resources = Mock() + executor.gather_all_responses = False + executor.dist = SimpleNamespace(rank=0) + executor._response_channel = ResponseChannel() + executor.active_requests = [] + executor.canceled_req_ids = [] + executor._enqueue_responses = Mock() + executor.scheduler = Mock() + executor.scheduler.is_request_in_schedulable_state.return_value = True + return executor + + +def _request() -> LlmRequest: + return executor_request_to_llm_request( + 1, trtllm.Request(input_token_ids=[1, 2], max_tokens=8), [], False + ) + + +@pytest.mark.parametrize("attention_dp", [False, True]) +@pytest.mark.parametrize("first_owner", ["connector", "transceiver"]) +def test_both_transfer_owners_finish_before_publication( + first_owner: str, attention_dp: bool +) -> None: + """The second completion must not remove routing for the first staged response.""" + executor = _executor() + executor.enable_attention_dp = attention_dp + harness = CoordinatorHarness() + executor.active_requests = harness.active + executor.async_transfer_manager = harness.transfers + executor.kv_cache_transceiver = harness.transceiver + executor._disagg_coordinator = harness.coordinator + harness.coordinator._effects = PyExecutorEffects(executor) + harness.coordinator._registry = PyExecutorRequestRegistry(executor) + response = SimpleNamespace(result=SimpleNamespace()) + request = TransferRequest(1, is_dummy_request=False) + request.create_response = Mock(side_effect=[response, None]) + executor.active_requests.append(request) + executor._response_channel.result_wait_queues[1] = object() + harness.send(request) + harness.transfers.start_transfer(request) # independent connector claim + delivered = [] + + def publish(responses) -> None: + assert 1 in executor._response_channel.result_wait_queues + executor.resource_manager.free_resources.assert_not_called() + delivered.extend(responses) + + executor._enqueue_responses.side_effect = publish + + def finish(owner) -> None: + if owner == "transceiver": + harness.transceiver.finish_send(request) + harness.coordinator.reap_context_sends(0) + else: + executor._release_transfer(request) + + finish(first_owner) + assert harness.in_transfer(request) + executor.resource_manager.free_resources.assert_not_called() + finish("transceiver" if first_owner == "connector" else "connector") + assert not harness.in_transfer(request) + assert 1 in executor._response_channel.result_wait_queues + executor.resource_manager.free_resources.assert_not_called() + + executor._flush_pending_transfer_responses() + executor._terminate_request(request) # repeated terminal callback + + assert delivered == [(1, response)] + assert 1 not in executor._response_channel.result_wait_queues + executor.sampler.release_request.assert_called_once_with(request) + executor.resource_manager.free_resources.assert_called_once_with(request) + executor._release_multimodal_resources.assert_called_once_with(request) + harness.kv_cache_manager.unpin_blocks_by_id.assert_called_once_with(1) + + +@pytest.mark.parametrize("pp_consensus", [False, True]) +def test_terminal_release_is_idempotent(pp_consensus: bool) -> None: + executor = _executor() + request = _request() + executor._response_channel.result_wait_queues[request.py_request_id] = object() + if pp_consensus: + executor._disagg_pp_termination_handler = Mock() + executor._terminate_request(request) + executor._terminate_request(request) + if pp_consensus: + executor.resource_manager.free_resources.assert_not_called() + assert request.py_request_id in executor._response_channel.result_wait_queues + executor._on_disagg_pp_termination(request) + executor._on_disagg_pp_termination(request) + executor._terminate_request(request) + executor.resource_manager.free_resources.assert_called_once_with(request) + executor.sampler.release_request.assert_called_once_with(request) + executor._release_multimodal_resources.assert_called_once_with(request) + assert request.py_request_terminated + assert not executor._response_channel.result_wait_queues + + +@pytest.mark.parametrize("cancel_before_reset", [False, True]) +def test_recompute_frees_each_allocation_once(cancel_before_reset: bool) -> None: + executor = _executor() + request = _request() + executor.active_requests.append(request) + executor._response_channel.result_wait_queues[request.py_request_id] = object() + batch = ScheduledRequests() + batch.recompute_paused_requests = [request] + executor._terminate_recompute_paused_requests(batch) + if cancel_before_reset: + request.state = LlmRequestState.GENERATION_COMPLETE + executor._pause_recompute_paused_requests(batch) + assert not request.py_request_terminated + assert request.py_request_id in executor._response_channel.result_wait_queues + executor._terminate_request(request) + expected = 1 if cancel_before_reset else 2 + assert executor.resource_manager.free_resources.call_count == expected + assert executor.sampler.release_request.call_count == expected + executor._terminate_request(request) + assert executor.resource_manager.free_resources.call_count == expected + + +def test_legacy_pause_starts_a_fresh_allocation_lifetime() -> None: + executor = _executor() + executor.max_input_len = 128 + request = _request() + executor._terminate_request(request) + executor._pause_requests([request]) + assert not request.py_request_terminated + assert not request.py_request_resources_released + executor._terminate_request(request) + assert executor.resource_manager.free_resources.call_count == 2 + + +@pytest.mark.parametrize("parked", [False, True]) +@pytest.mark.parametrize("in_transfer", [False, True]) +@pytest.mark.parametrize("loading", [False, True]) +def test_final_response_and_safe_release_have_independent_readiness( + parked: bool, in_transfer: bool, loading: bool +) -> None: + executor = _executor() + request = _request() + request.state = LlmRequestState.DISAGG_CONTEXT_TRANS_IN_PROGRESS + executor.kv_cache_manager.is_preemption_pending.return_value = parked + executor.async_transfer_manager = Mock() + executor.async_transfer_manager.requests_in_transfer.return_value = ( + {request.py_request_id: request} if in_transfer else {} + ) + executor.kv_connector_manager = Mock() + executor.kv_connector_manager.has_pending_load.return_value = loading + assert executor._is_final_response_ready(request) is (not parked) + assert executor._can_release_request(request) is (not (parked or in_transfer or loading)) + + +@pytest.mark.parametrize("parked", [False, True]) +@pytest.mark.parametrize("recomputing", [False, True]) +def test_parked_and_recomputing_requests_are_not_schedulable( + parked: bool, recomputing: bool +) -> None: + executor = _executor() + request = _request() + executor.kv_cache_manager.is_preemption_pending.return_value = parked + if recomputing: + executor._pending_recompute_pause_ids.add(request.py_request_id) + assert executor._is_request_schedulable(request) is (not (parked or recomputing)) + + +@pytest.mark.parametrize("owner", ["response", "load", "save"]) +def test_pp_consensus_does_not_override_a_remaining_owner(owner: str) -> None: + executor = _executor() + request = _request() + executor._disagg_pp_termination_handler = Mock() + executor._response_channel.result_wait_queues[request.py_request_id] = object() + response = SimpleNamespace(result=SimpleNamespace()) + if owner == "response": + executor._pending_transfer_responses = [(request.py_request_id, response)] + elif owner == "load": + executor.kv_connector_manager = Mock() + executor.kv_connector_manager.has_pending_load.return_value = True + executor.kv_connector_manager.defer_load_termination.return_value = True + else: + executor.async_transfer_manager = Mock() + executor.async_transfer_manager.requests_in_transfer.return_value = { + request.py_request_id: request + } + + executor._on_disagg_pp_termination(request) + executor.resource_manager.free_resources.assert_not_called() + assert request.py_request_id in executor._response_channel.result_wait_queues + if owner == "response": + executor._flush_pending_transfer_responses() + executor._enqueue_responses.assert_called_once_with([(request.py_request_id, response)]) + else: + if owner == "load": + executor.kv_connector_manager.has_pending_load.return_value = False + executor.kv_connector_manager.defer_load_termination.return_value = False + else: + executor.async_transfer_manager.requests_in_transfer.return_value = {} + executor._terminate_request(request) + executor._disagg_pp_termination_handler.terminate.assert_not_called() + executor.resource_manager.free_resources.assert_called_once_with(request) + assert request.py_request_pp_termination_agreed + + +@pytest.mark.parametrize("early_release", [False, True]) +def test_last_owner_retries_teardown_deferred_by_the_early_path(early_release: bool) -> None: + executor = _executor() + harness = CoordinatorHarness() + executor._is_kv_manager_v2 = not early_release + executor.force_terminate_ctx_for_partial_reuse = early_release + harness.transfers.should_store_blocks = early_release + executor.async_transfer_manager = harness.transfers + executor.kv_cache_transceiver = harness.transceiver + executor._disagg_coordinator = harness.coordinator + harness.coordinator._effects = PyExecutorEffects(executor) + request = TransferRequest(1, is_dummy_request=False) + harness.send(request) + executor._terminate_request(request) + executor.resource_manager.free_resources.assert_not_called() + harness.transceiver.finish_send(request) + harness.coordinator.reap_context_sends(0) + executor.resource_manager.free_resources.assert_called_once_with(request) + + +def test_pp_ring_releases_a_delayed_owner_without_another_consensus() -> None: + mailboxes = [[], []] + sends = [] + + def dist(rank: int) -> SimpleNamespace: + def send(payload: dict, dest: int, tag: object) -> Mock: + sends.append((rank, dest, tag)) + mailboxes[dest].append(deepcopy(payload)) + return Mock() + + return SimpleNamespace( + rank=rank, + pp_rank=rank, + pp_size=2, + is_first_pp_rank=rank == 0, + is_last_pp_rank=rank == 1, + prev_pp_rank=1 - rank, + next_pp_rank=1 - rank, + isend_object=send, + recv_object=lambda src, tag: mailboxes[rank].pop(0), + ) + + executors = [_executor(), _executor()] + requests = [_request(), _request()] + for rank, (executor, request) in enumerate(zip(executors, requests)): + executor.dist = dist(rank) + executor._disagg_pp_termination_handler = DisaggPPTerminationHandler( + executor.dist, executor._on_disagg_pp_termination + ) + executor._response_channel.result_wait_queues[request.py_request_id] = object() + executor._terminate_request(request) + executor._disagg_pp_termination_handler.terminate_pending_requests() + + # Both ranks voted, but rank 0 must still publish its response before freeing. + response = SimpleNamespace(result=SimpleNamespace()) + executors[0]._pending_transfer_responses = [(requests[0].py_request_id, response)] + for executor in executors: + executor._disagg_pp_termination_handler.terminate_pending_requests() + executors[0].resource_manager.free_resources.assert_not_called() + executors[1].resource_manager.free_resources.assert_called_once_with(requests[1]) + executors[0]._flush_pending_transfer_responses() + + assert len(sends) == 4 + for executor, request in zip(executors, requests): + executor.resource_manager.free_resources.assert_called_once_with(request) + assert executor._disagg_pp_termination_handler._pending_termination == {} + assert request.py_request_pp_termination_agreed diff --git a/tests/unittest/_torch/pyexecutor/test_kv_connector_executor_lifetime.py b/tests/unittest/_torch/pyexecutor/test_kv_connector_executor_lifetime.py index aa402e2e61de..d5692fbdd23c 100644 --- a/tests/unittest/_torch/pyexecutor/test_kv_connector_executor_lifetime.py +++ b/tests/unittest/_torch/pyexecutor/test_kv_connector_executor_lifetime.py @@ -45,6 +45,7 @@ def _executor() -> PyExecutor: executor.kv_connector_manager.defer_load_termination.return_value = False executor.kv_connector_manager.has_outstanding_save.return_value = False executor.kv_connector_manager.has_pending_loads.return_value = False + executor.kv_connector_manager.has_pending_load.return_value = False executor.kv_connector_manager.get_finished.return_value = [] executor.kv_connector_manager.take_finished_load_terminations.return_value = [] executor.kv_cache_manager = Mock() diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index eb0e73244181..5ec8916ff406 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -2273,6 +2273,7 @@ def test_pp1_retries_deferred_termination_on_transfer_complete(self): final claim is released, the departed request must be terminated even when the PP=1 early-termination optimization was requested.""" req = Mock() + req.py_request_terminated = True coordinator, _, effects = _make_transfer_coordinator( active_requests=[], # early termination deferred until transfer release ) @@ -3271,6 +3272,8 @@ def _add_dummy(**kwargs): def _run_pad(stub): for helper in ( "_count_schedulable_active_requests", + "_is_request_schedulable", + "_is_preemption_pending", "_get_non_overlap_adp_forward_intent", "_has_adp_dummy_kv_capacity", "_should_skip_dummy_for_benchmark_disagg", @@ -4181,6 +4184,8 @@ def test_pad_dummy_no_op_when_attention_dp_disabled(): def _run_pad_empty(stub, scheduled_batch): for helper in ( "_count_schedulable_active_requests", + "_is_request_schedulable", + "_is_preemption_pending", "_has_adp_dummy_kv_capacity", "_should_skip_dummy_for_benchmark_disagg", ):