diff --git a/tensorrt_llm/executor/rpc_worker_mixin.py b/tensorrt_llm/executor/rpc_worker_mixin.py index a01ad3ee2441..874d9b7ea3d9 100644 --- a/tensorrt_llm/executor/rpc_worker_mixin.py +++ b/tensorrt_llm/executor/rpc_worker_mixin.py @@ -24,6 +24,7 @@ from ..logger import logger from .request import GenerationRequest from .rpc import RPCServer +from .utils import ErrorResponse class RpcWorkerMixin: @@ -104,8 +105,6 @@ def init_postproc_workers(self) -> None: # which deadlocks inside a Ray actor (verified empirically). self._postproc_pool = ProcessPoolExecutor(max_workers=num) - from .utils import ErrorResponse - def _make_on_postproc_worker_done(shard: int): def _on_postproc_worker_done(fut) -> None: # ProcessPoolExecutor stores task exceptions on the Future and @@ -232,8 +231,12 @@ def fetch_responses(self, timeout: Optional[float] = None) -> list: actual_timeout = ( timeout if timeout is not None else getattr(self, "_fetch_timeout", 0.1) ) + # Snapshot before waiting so the poll drains all responses flushed before shutdown. + loop_error = self.engine._event_loop_error if self.engine.is_shutdown else None responses = super().await_responses(timeout=actual_timeout) responses = self._await_response_helper.process_and_handle_responses(responses) + if loop_error is not None: + self._fail_pending_requests(loop_error) logger_debug(f"[worker] Fetched {len(responses)} responses", color="green") qsize = self._response_queue.qsize() @@ -245,6 +248,17 @@ def fetch_responses(self, timeout: Optional[float] = None) -> list: all_responses.extend(self._response_queue.get()) return all_responses + def _fail_pending_requests(self, error: BaseException) -> None: + client_ids = list(self._results) + if not client_ids: + return + error_msg = f"Event loop terminated with error: {error}" + errors = [ErrorResponse(client_id, error_msg, -1) for client_id in client_ids] + logger.error(f"Failing {len(errors)} pending request(s) after event-loop crash: {error}") + for client_id in client_ids: + self._pop_result(client_id) + self._response_queue.put(errors) + async def fetch_responses_async(self, timeout: Optional[float] = None) -> list: """Async version of fetch_responses using asyncio.to_thread.""" # Use asyncio.to_thread to avoid blocking the event loop diff --git a/tests/unittest/executor/test_rpc_worker.py b/tests/unittest/executor/test_rpc_worker.py index 8e8c400b9262..0eb3128ab2c5 100644 --- a/tests/unittest/executor/test_rpc_worker.py +++ b/tests/unittest/executor/test_rpc_worker.py @@ -3,8 +3,11 @@ import asyncio import time +import pytest + from tensorrt_llm.executor.request import GenerationRequest from tensorrt_llm.executor.rpc_worker import RpcWorker +from tensorrt_llm.executor.utils import ErrorResponse from tensorrt_llm.llmapi.llm_args import KvCacheConfig, TorchLlmArgs from tensorrt_llm.sampling_params import SamplingParams @@ -82,3 +85,45 @@ def test_fetch_kv_cache_events_async(self): # Fetch kv cache events events = asyncio.run(self.worker.fetch_kv_cache_events_async()) assert isinstance(events, list) + + def test_fetch_responses_after_event_loop_crash( + self, monkeypatch: pytest.MonkeyPatch) -> None: + engine = self.worker.engine + start_iteration = engine._start_response_staging_iteration + crash_message = "injected RPC executor event-loop crash" + sampling_params = SamplingParams(max_tokens=10, ignore_eos=True) + pending = GenerationRequest(prompt_token_ids=[3, 4, 5], + sampling_params=sampling_params).set_id(1) + + def crash_with_pending_request() -> None: + if pending.id in self.worker._client_id_to_request_id: + raise RuntimeError(crash_message) + start_iteration() + + monkeypatch.setattr(engine, "_start_response_staging_iteration", + crash_with_pending_request) + self.worker.submit(pending) + + deadline = time.monotonic() + 10 + responses = [] + while not responses and time.monotonic() < deadline: + responses = self.worker.fetch_responses(timeout=0.1) + if not responses: + time.sleep(0.01) + assert responses, "Timed out waiting for the pending request's crash response" + assert responses == [ + ErrorResponse(pending.id, + f"Event loop terminated with error: {crash_message}", + -1) + ] + + late = GenerationRequest(prompt_token_ids=[3, 4, 5], + sampling_params=sampling_params).set_id(2) + self.worker.submit(late) + + assert self.worker.fetch_responses(timeout=0.1) == [ + ErrorResponse(late.id, + f"Event loop terminated with error: {crash_message}", + -1) + ] + assert not engine._event_loop_error_delivered.is_set() diff --git a/tests/unittest/executor/test_rpc_worker_mixin.py b/tests/unittest/executor/test_rpc_worker_mixin.py index f5cd3f263aef..eb2f1138ab08 100644 --- a/tests/unittest/executor/test_rpc_worker_mixin.py +++ b/tests/unittest/executor/test_rpc_worker_mixin.py @@ -13,11 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections.abc import Callable from queue import Queue +from threading import Event +from types import SimpleNamespace import pytest -from tensorrt_llm.executor.base_worker import AwaitResponseHelper +from tensorrt_llm.executor.base_worker import AwaitResponseHelper, BaseWorker from tensorrt_llm.executor.rpc_worker_mixin import RpcWorkerMixin from tensorrt_llm.executor.utils import ErrorResponse @@ -27,12 +30,24 @@ class _WorkerBaseStub: def await_responses(self, timeout): self.await_responses_timeout = timeout - return self.engine_responses + responses = self.engine_responses + self.engine_responses = [] + if self.await_responses_hook is not None: + self.await_responses_hook() + return responses class _RpcWorkerStub(RpcWorkerMixin, _WorkerBaseStub): + _pop_result = BaseWorker._pop_result + def __init__(self, engine_responses): self.engine_responses = engine_responses + self.await_responses_hook: Callable[[], None] | None = None + self.engine = SimpleNamespace( + is_shutdown=False, _event_loop_error=None, _event_loop_error_delivered=Event() + ) + self._results: dict[int, object] = {} + self._client_id_to_request_id: dict[int, int] = {} self.rank = 0 self._fetch_timeout = 0.1 self._response_queue = Queue() @@ -41,6 +56,10 @@ def __init__(self, engine_responses): self._await_response_helper.responses_handler = self._responses_handler self.handler_responses = None + def _register_pending_request(self, client_id: int, request_id: int) -> None: + self._results[client_id] = object() + self._client_id_to_request_id[client_id] = request_id + def _responses_handler(self, responses): self.handler_responses = responses if responses: @@ -77,3 +96,60 @@ def test_fetch_responses_dispatches_engine_responses_and_submission_errors( assert responses[-1] is submission_error assert engine_responses == original_responses assert worker._await_response_helper.temp_error_responses.empty() + + +def test_fetch_responses_fails_pending_requests_once_after_loop_crash() -> None: + worker = _RpcWorkerStub([]) + worker.engine._event_loop_error = RuntimeError("loop crashed") + worker.engine.is_shutdown = True + worker._register_pending_request(1, 11) + worker._register_pending_request(2, 12) + + assert worker.fetch_responses() == [ + ErrorResponse(1, "Event loop terminated with error: loop crashed", -1), + ErrorResponse(2, "Event loop terminated with error: loop crashed", -1), + ] + assert not worker._results + assert not worker._client_id_to_request_id + assert not worker.engine._event_loop_error_delivered.is_set() + assert worker.fetch_responses() == [] + + +def test_fetch_responses_fails_request_registered_after_loop_crash() -> None: + worker = _RpcWorkerStub([]) + worker.engine._event_loop_error = RuntimeError("loop crashed") + worker.engine.is_shutdown = True + assert worker.fetch_responses() == [] + + worker._register_pending_request(3, 13) + + assert worker.fetch_responses() == [ + ErrorResponse(3, "Event loop terminated with error: loop crashed", -1), + ] + + +def test_fetch_responses_dispatches_flushed_final_response_before_crash_errors() -> None: + worker = _RpcWorkerStub([]) + worker.engine._event_loop_error = RuntimeError("loop crashed") + worker._register_pending_request(1, 11) + worker._register_pending_request(2, 12) + final_response = SimpleNamespace(client_id=1) + + def finish_cleanup() -> None: + worker.engine_responses = [final_response] + worker.engine.is_shutdown = True + worker.await_responses_hook = None + + def dispatch_final_responses(responses: list[SimpleNamespace]) -> None: + worker._responses_handler(responses) + for response in responses: + worker._pop_result(response.client_id) + + worker.await_responses_hook = finish_cleanup + worker._await_response_helper.responses_handler = dispatch_final_responses + + assert worker.fetch_responses() == [] + assert worker.fetch_responses() == [ + final_response, + ErrorResponse(2, "Event loop terminated with error: loop crashed", -1), + ]