Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions tensorrt_llm/executor/rpc_worker_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from ..logger import logger
from .request import GenerationRequest
from .rpc import RPCServer
from .utils import ErrorResponse


class RpcWorkerMixin:
Expand Down Expand Up @@ -104,8 +105,6 @@ def init_postproc_workers(self) -> None:
# which deadlocks inside a Ray actor (verified empirically).
self._postproc_pool = ProcessPoolExecutor(max_workers=num)

from .utils import ErrorResponse

def _make_on_postproc_worker_done(shard: int):
def _on_postproc_worker_done(fut) -> None:
# ProcessPoolExecutor stores task exceptions on the Future and
Expand Down Expand Up @@ -232,8 +231,12 @@ def fetch_responses(self, timeout: Optional[float] = None) -> list:
actual_timeout = (
timeout if timeout is not None else getattr(self, "_fetch_timeout", 0.1)
)
# Snapshot before waiting so the poll drains all responses flushed before shutdown.
loop_error = self.engine._event_loop_error if self.engine.is_shutdown else None
responses = super().await_responses(timeout=actual_timeout)
responses = self._await_response_helper.process_and_handle_responses(responses)
if loop_error is not None:
self._fail_pending_requests(loop_error)
logger_debug(f"[worker] Fetched {len(responses)} responses", color="green")

qsize = self._response_queue.qsize()
Expand All @@ -245,6 +248,17 @@ def fetch_responses(self, timeout: Optional[float] = None) -> list:
all_responses.extend(self._response_queue.get())
return all_responses

def _fail_pending_requests(self, error: BaseException) -> None:
client_ids = list(self._results)
if not client_ids:
return
error_msg = f"Event loop terminated with error: {error}"
errors = [ErrorResponse(client_id, error_msg, -1) for client_id in client_ids]
logger.error(f"Failing {len(errors)} pending request(s) after event-loop crash: {error}")
for client_id in client_ids:
self._pop_result(client_id)
self._response_queue.put(errors)

async def fetch_responses_async(self, timeout: Optional[float] = None) -> list:
"""Async version of fetch_responses using asyncio.to_thread."""
# Use asyncio.to_thread to avoid blocking the event loop
Expand Down
45 changes: 45 additions & 0 deletions tests/unittest/executor/test_rpc_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
import asyncio
import time

import pytest

from tensorrt_llm.executor.request import GenerationRequest
from tensorrt_llm.executor.rpc_worker import RpcWorker
from tensorrt_llm.executor.utils import ErrorResponse
from tensorrt_llm.llmapi.llm_args import KvCacheConfig, TorchLlmArgs
from tensorrt_llm.sampling_params import SamplingParams

Expand Down Expand Up @@ -82,3 +85,45 @@ def test_fetch_kv_cache_events_async(self):
# Fetch kv cache events
events = asyncio.run(self.worker.fetch_kv_cache_events_async())
assert isinstance(events, list)

def test_fetch_responses_after_event_loop_crash(
self, monkeypatch: pytest.MonkeyPatch) -> None:
engine = self.worker.engine
start_iteration = engine._start_response_staging_iteration
crash_message = "injected RPC executor event-loop crash"
sampling_params = SamplingParams(max_tokens=10, ignore_eos=True)
pending = GenerationRequest(prompt_token_ids=[3, 4, 5],
sampling_params=sampling_params).set_id(1)

def crash_with_pending_request() -> None:
if pending.id in self.worker._client_id_to_request_id:
raise RuntimeError(crash_message)
start_iteration()

monkeypatch.setattr(engine, "_start_response_staging_iteration",
crash_with_pending_request)
self.worker.submit(pending)

deadline = time.monotonic() + 10
responses = []
while not responses and time.monotonic() < deadline:
responses = self.worker.fetch_responses(timeout=0.1)
if not responses:
time.sleep(0.01)
assert responses, "Timed out waiting for the pending request's crash response"
assert responses == [
ErrorResponse(pending.id,
f"Event loop terminated with error: {crash_message}",
-1)
]

late = GenerationRequest(prompt_token_ids=[3, 4, 5],
sampling_params=sampling_params).set_id(2)
self.worker.submit(late)

assert self.worker.fetch_responses(timeout=0.1) == [
ErrorResponse(late.id,
f"Event loop terminated with error: {crash_message}",
-1)
]
assert not engine._event_loop_error_delivered.is_set()
80 changes: 78 additions & 2 deletions tests/unittest/executor/test_rpc_worker_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from collections.abc import Callable
from queue import Queue
from threading import Event
from types import SimpleNamespace

import pytest

from tensorrt_llm.executor.base_worker import AwaitResponseHelper
from tensorrt_llm.executor.base_worker import AwaitResponseHelper, BaseWorker
from tensorrt_llm.executor.rpc_worker_mixin import RpcWorkerMixin
from tensorrt_llm.executor.utils import ErrorResponse

Expand All @@ -27,12 +30,24 @@
class _WorkerBaseStub:
def await_responses(self, timeout):
self.await_responses_timeout = timeout
return self.engine_responses
responses = self.engine_responses
self.engine_responses = []
if self.await_responses_hook is not None:
self.await_responses_hook()
return responses


class _RpcWorkerStub(RpcWorkerMixin, _WorkerBaseStub):
_pop_result = BaseWorker._pop_result

def __init__(self, engine_responses):
self.engine_responses = engine_responses
self.await_responses_hook: Callable[[], None] | None = None
self.engine = SimpleNamespace(
is_shutdown=False, _event_loop_error=None, _event_loop_error_delivered=Event()
)
self._results: dict[int, object] = {}
self._client_id_to_request_id: dict[int, int] = {}
self.rank = 0
self._fetch_timeout = 0.1
self._response_queue = Queue()
Expand All @@ -41,6 +56,10 @@ def __init__(self, engine_responses):
self._await_response_helper.responses_handler = self._responses_handler
self.handler_responses = None

def _register_pending_request(self, client_id: int, request_id: int) -> None:
self._results[client_id] = object()
self._client_id_to_request_id[client_id] = request_id

def _responses_handler(self, responses):
self.handler_responses = responses
if responses:
Expand Down Expand Up @@ -77,3 +96,60 @@ def test_fetch_responses_dispatches_engine_responses_and_submission_errors(
assert responses[-1] is submission_error
assert engine_responses == original_responses
assert worker._await_response_helper.temp_error_responses.empty()


def test_fetch_responses_fails_pending_requests_once_after_loop_crash() -> None:
worker = _RpcWorkerStub([])
worker.engine._event_loop_error = RuntimeError("loop crashed")
worker.engine.is_shutdown = True
worker._register_pending_request(1, 11)
worker._register_pending_request(2, 12)

assert worker.fetch_responses() == [
ErrorResponse(1, "Event loop terminated with error: loop crashed", -1),
ErrorResponse(2, "Event loop terminated with error: loop crashed", -1),
]
assert not worker._results
assert not worker._client_id_to_request_id
assert not worker.engine._event_loop_error_delivered.is_set()
assert worker.fetch_responses() == []


def test_fetch_responses_fails_request_registered_after_loop_crash() -> None:
worker = _RpcWorkerStub([])
worker.engine._event_loop_error = RuntimeError("loop crashed")
worker.engine.is_shutdown = True
assert worker.fetch_responses() == []

worker._register_pending_request(3, 13)

assert worker.fetch_responses() == [
ErrorResponse(3, "Event loop terminated with error: loop crashed", -1),
]


def test_fetch_responses_dispatches_flushed_final_response_before_crash_errors() -> None:
worker = _RpcWorkerStub([])
worker.engine._event_loop_error = RuntimeError("loop crashed")
worker._register_pending_request(1, 11)
worker._register_pending_request(2, 12)
final_response = SimpleNamespace(client_id=1)

def finish_cleanup() -> None:
worker.engine_responses = [final_response]
worker.engine.is_shutdown = True
worker.await_responses_hook = None

def dispatch_final_responses(responses: list[SimpleNamespace]) -> None:
worker._responses_handler(responses)
for response in responses:
worker._pop_result(response.client_id)

worker.await_responses_hook = finish_cleanup
worker._await_response_helper.responses_handler = dispatch_final_responses

assert worker.fetch_responses() == []
assert worker.fetch_responses() == [
final_response,
ErrorResponse(2, "Event loop terminated with error: loop crashed", -1),
]
Loading