From bde62bbd362e9749fcc46aab62ccaad3523c8073 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Tue, 17 Nov 2026 19:54:26 -0800 Subject: [PATCH 01/12] [TRTLLM-17766][refactor] Extract the completion of ScheduledEncoderExecutor A Completion paired with the runner stores each forward's results; PoolingCompletion keeps the pooled-row behaviour. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/encoder_completion.py | 40 +++++++++++++++++++ .../pyexecutor/scheduled_encoder_executor.py | 23 ++++++----- .../test_scheduled_encoder_executor.py | 2 + 3 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 tensorrt_llm/_torch/pyexecutor/encoder_completion.py diff --git a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py new file mode 100644 index 000000000000..a67c7b81f629 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""How one forward of ``ScheduledEncoderExecutor`` completes its requests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from .llm_request import LlmRequest + +__all__ = ["Completion", "PoolingCompletion"] + + +class Completion(Protocol): + """How one forward's outputs complete its requests; paired with the runner at initialization.""" + + accepts_multimodal_inputs: bool + """Whether a request may carry multimodal inputs.""" + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + """Attach one forward's results to its requests before their responses are created.""" + + def retire(self, requests: list[LlmRequest]) -> None: + """Release what ``store`` exported for requests whose responses will not be published.""" + + +class PoolingCompletion: + """Store each request's pooled row as its context logits, when the request asked for them.""" + + accepts_multimodal_inputs = False + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + for request, row in zip(requests, outputs["logits"]): + if request.py_return_context_logits: + request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) + + def retire(self, requests: list[LlmRequest]) -> None: + pass diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index d05b5b7761e8..caa2dcf67a7a 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -11,7 +11,7 @@ import traceback from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -28,6 +28,7 @@ from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import host_profiler_context from ..moe.fused_moe.moe_comm_timeout_guard import set_moe_comm_warmup +from .encoder_completion import Completion, PoolingCompletion from .engine.model_setup import build_encoder_runner from .engine.runners.encoder import EncoderRunner, resolve_scheduled_encoder_args from .engine.runners.interface import ScheduledInputs @@ -61,9 +62,7 @@ __all__ = ["ScheduledEncoderExecutor", "fit_batch", "prompt_cost"] -_MULTIMODAL_INPUT_ERROR = ( - "Scheduled requests to a non-generation model cannot carry multimodal inputs." -) +_MULTIMODAL_INPUT_ERROR = "Requests to a pooling model cannot carry multimodal inputs." # One forward runs while the next is launched; a third waits for the first to be published. _MAX_IN_FLIGHT = 2 @@ -131,6 +130,7 @@ def __init__( self, runner: EncoderRunner, *, + completion: Completion, model_loader: ModelLoader, llm_args: TorchLlmArgs, mapping: Mapping, @@ -143,6 +143,7 @@ def __init__( Args: runner: The model's runner; every forward goes through its scheduled entry point. + completion: How a forward's outputs complete its requests; it fits the runner. model_loader: The loader that loaded the model, cleaned up at shutdown. llm_args: The resolved arguments; the executor reads its timeouts, garbage collection threshold and metrics switches. @@ -153,6 +154,7 @@ def __init__( max_seq_len: The longest prompt a request may have. """ self._runner = runner + self._completion = completion self._model_loader = model_loader self.dist = dist self.max_batch_size = max_batch_size @@ -242,6 +244,7 @@ def create( ) return cls( built.runner, + completion=PoolingCompletion(), model_loader=built.model_loader, llm_args=llm_args, mapping=mapping, @@ -538,7 +541,7 @@ def _receive(self) -> None: def _check_admissible(self, item: RequestQueueItem) -> None: """Raise ValueError if one forward cannot run this request.""" request = item.request - if ( + if not self._completion.accepts_multimodal_inputs and ( getattr(request, "py_multimodal_data", None) is not None or request.mrope_config is not None ): @@ -626,6 +629,7 @@ def _run(self, items: list[RequestQueueItem]) -> None: done.record() except Exception as error: self._slots.release() + self._completion.retire(requests) self._fail(requests, error) return self._in_flight.put( @@ -680,15 +684,14 @@ def _log_iteration(self, flight: _InFlight) -> None: @nvtx_range("_complete") def _complete( - self, requests: list[LlmRequest], outputs: dict[str, torch.Tensor] + self, requests: list[LlmRequest], outputs: dict[str, Any] ) -> list[tuple[int, LlmResponse]]: - """Store each request's pooled row as its context logits and report it finished with reason LENGTH.""" + """Store the forward's results on its requests and report each finished with reason LENGTH.""" + self._completion.store(requests, outputs) batch_token_time = self._perf_manager.get_timestamp() gpu_times_cache = {} responses = [] - for request, row in zip(requests, outputs["logits"]): - if request.py_return_context_logits: - request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) + for request in requests: # The public result reports a finished request with no tokens and reason LENGTH. request.state = LlmRequestState.GENERATION_COMPLETE request.set_finished_reason(FinishReason.LENGTH, 0) diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index f181af9426e8..863f7864939f 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -19,6 +19,7 @@ import torch from tensorrt_llm._torch.pyexecutor import scheduled_encoder_executor +from tensorrt_llm._torch.pyexecutor.encoder_completion import PoolingCompletion from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import ( ScheduledEncoderExecutor, @@ -126,6 +127,7 @@ def __init__( self.runner = _Runner(max_batch_size) self.executor = ScheduledEncoderExecutor( self.runner, + completion=PoolingCompletion(), model_loader=Mock(), llm_args=SimpleNamespace( garbage_collection_gen0_threshold=20000, From e9f5bcc294351bef3c8fdf4f6be9a04ddf074a05 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Tue, 17 Nov 2026 20:14:00 -0800 Subject: [PATCH 02/12] [TRTLLM-17766][refactor] Run mm_encoder_only on ScheduledEncoderExecutor The vision encoder is built from the model-setup functions and paired with MultimodalEmbeddingCompletion, which exports on rank 0 only. PyExecutor no longer receives non-generation models. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/encoder_completion.py | 43 +++- .../_torch/pyexecutor/engine/model_setup.py | 193 +++++++++++++++--- .../pyexecutor/engine/runners/mm_encoder.py | 37 ++-- tensorrt_llm/_torch/pyexecutor/llm_request.py | 26 +++ .../_torch/pyexecutor/py_executor_creator.py | 2 +- .../pyexecutor/scheduled_encoder_executor.py | 27 ++- tensorrt_llm/llmapi/mm_encoder.py | 2 +- .../engine/runners/test_mm_encoder.py | 40 ++-- .../_torch/pyexecutor/test_model_engine.py | 27 --- .../test_multimodal_embedding_lengths.py | 33 ++- ...est_py_executor_creator_no_kv_execution.py | 15 +- .../test_scheduled_encoder_executor.py | 100 +++++++-- 12 files changed, 406 insertions(+), 139 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py index a67c7b81f629..e56b2d535210 100644 --- a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py +++ b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from .llm_request import LlmRequest -__all__ = ["Completion", "PoolingCompletion"] +__all__ = ["Completion", "MultimodalEmbeddingCompletion", "PoolingCompletion"] class Completion(Protocol): @@ -38,3 +38,44 @@ def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: def retire(self, requests: list[LlmRequest]) -> None: pass + + +class MultimodalEmbeddingCompletion: + """Export each request's multimodal embeddings, and its MRoPE positions, as CUDA IPC handles. + + A request without multimodal items in the batch completes without a result. Only the rank + whose responses reach the frontend exports: an exported allocation stays alive until a + consumer opens and releases its handle, or ``retire`` consumes it. + """ + + accepts_multimodal_inputs = True + + def __init__(self, *, exports: bool) -> None: + self._exports = exports + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + if not self._exports: + return + metadata = outputs.get("mm_embedding_metadata") + mrope_position_ids = outputs.get("mrope_position_ids") + mrope_position_deltas = outputs.get("mrope_position_deltas") + for result, (request_index, embeddings, lengths) in enumerate( + zip( + outputs["mm_embedding_request_indices"], + outputs["mm_embeddings"], + outputs["mm_embedding_lengths"], + strict=True, + ) + ): + py_result = requests[request_index].py_result + py_result.append_mm_embeddings( + embeddings, lengths, None if metadata is None else metadata[result] + ) + if mrope_position_ids is not None and mrope_position_deltas is not None: + py_result.set_mrope_position( + mrope_position_ids[result], mrope_position_deltas[result] + ) + + def retire(self, requests: list[LlmRequest]) -> None: + for request in requests: + request.py_result.release_exports() diff --git a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py index b8b8af1170a2..7eb93c20a400 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py @@ -31,12 +31,19 @@ from tensorrt_llm._torch.nccl_window_tensor_scope import install_eager_nccl_window_tensor_scopes from tensorrt_llm._torch.pyexecutor.engine.cuda_graph import plan_cuda_graphs from tensorrt_llm._torch.pyexecutor.engine.model_call import ModelCaller +from tensorrt_llm._torch.pyexecutor.engine.multimodal import ( + setup_mm_encoder_attn_metadata, + setup_mm_encoder_cuda_graphs, +) from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.runner import PrefillCompiledModel from tensorrt_llm._torch.pyexecutor.engine.runners.encoder import EncoderRunner +from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledModelRunner +from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.layerwise_nvtx_marker import LayerwiseNvtxMarker from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader from tensorrt_llm._torch.utils import set_torch_compiling from tensorrt_llm._utils import mpi_disabled +from tensorrt_llm.inputs.registry import InputProcessor, create_input_processor from tensorrt_llm.llmapi.llm_args import ( DecodingBaseConfig, PrefillCudaGraphBackend, @@ -455,6 +462,23 @@ def create_model_loader( ) +def create_engine_input_processor(model_path: str, llm_args: TorchLlmArgs) -> InputProcessor: + """Create the checkpoint's input processor with the multimodal options the engine applies.""" + input_processor_kwargs = {} + video_pruning_rate = llm_args.multimodal_config.video_pruning_rate + if video_pruning_rate is not None: + input_processor_kwargs["video_pruning_rate"] = video_pruning_rate + if llm_args.multimodal_config.vision_gpu_normalize: + input_processor_kwargs["vision_gpu_normalize"] = True + return create_input_processor( + model_path, + tokenizer=None, + checkpoint_format=llm_args.checkpoint_format, + trust_remote_code=llm_args.trust_remote_code, + **input_processor_kwargs, + ) + + def prepare_loaded_model(model: torch.nn.Module, llm_args: TorchLlmArgs) -> None: """Give the model its extra attributes and the requested NVTX markers.""" # AutoModelForCausalLM sets extra_attrs; a model passed in through @@ -492,7 +516,7 @@ def resolve_attention_backend( class EncoderBuild: """An encoder runner built from a checkpoint, with the loader that owns its weights.""" - runner: EncoderRunner + runner: ScheduledModelRunner model_loader: ModelLoader model_config: ModelConfig max_batch_size: int @@ -500,30 +524,27 @@ class EncoderBuild: max_seq_len: int +@dataclass(frozen=True) +class _LoadedEncoder: + model: torch.nn.Module + model_loader: ModelLoader + max_beam_width: int + max_batch_size: int + max_num_tokens: int + max_seq_len: int + prefill_capture_num_tokens: list[int] + + @contextmanager -def build_encoder_runner( +def _load_encoder( llm_args: TorchLlmArgs, *, model_path: str, checkpoint_loader: BaseCheckpointLoader, mapping: Mapping, dist: Distributed, -) -> Iterator[EncoderBuild]: - """Load and compile the checkpoint's model and build an encoder runner for it. - - The loader is cleaned up when building or the ``with`` body raises, so the executor - constructed in the body takes the loader over only once it exists. - - Args: - llm_args: The resolved arguments. - model_path: The checkpoint directory. - checkpoint_loader: The loader for the checkpoint's format. - mapping: The parallel mapping of this rank. - dist: The communicator across the model's ranks. - - Yields: - The runner with its loader, model config and resolved capacity. - """ +) -> Iterator[_LoadedEncoder]: + """Load the checkpoint's model and resolve its capacity; the loader is cleaned up on a raise.""" init_process_setup(mapping, dist) configure_autotuner_nvmmh(llm_args) max_beam_width, max_num_tokens, max_seq_len, max_batch_size = llm_args.get_runtime_sizes() @@ -536,7 +557,7 @@ def build_encoder_runner( ) try: # Encoder executors run without a MoE load balancer: encode_only has one rank, - # and scheduled pooling rejects the setting. + # and scheduled encoders reject the setting. model, _ = model_loader.load(checkpoint_dir=model_path, checkpoint_loader=checkpoint_loader) prepare_loaded_model(model, llm_args) max_seq_len = resolve_max_seq_len(model, max_seq_len) @@ -551,35 +572,139 @@ def build_encoder_runner( max_seq_len=max_seq_len, decode_tokens_per_request=1, ) + yield _LoadedEncoder( + model=model, + model_loader=model_loader, + max_beam_width=max_beam_width, + max_batch_size=max_batch_size, + max_num_tokens=max_num_tokens, + max_seq_len=max_seq_len, + prefill_capture_num_tokens=plan.prefill_num_tokens, + ) + except BaseException: + model_loader.cleanup() + raise + + +@contextmanager +def build_encoder_runner( + llm_args: TorchLlmArgs, + *, + model_path: str, + checkpoint_loader: BaseCheckpointLoader, + mapping: Mapping, + dist: Distributed, +) -> Iterator[EncoderBuild]: + """Load and compile the checkpoint's model and build an encoder runner for it. + + The loader is cleaned up when building or the ``with`` body raises, so the executor + constructed in the body takes the loader over only once it exists. + + Args: + llm_args: The resolved arguments. + model_path: The checkpoint directory. + checkpoint_loader: The loader for the checkpoint's format. + mapping: The parallel mapping of this rank. + dist: The communicator across the model's ranks. + + Yields: + The runner with its loader, model config and resolved capacity. + """ + with _load_encoder( + llm_args, + model_path=model_path, + checkpoint_loader=checkpoint_loader, + mapping=mapping, + dist=dist, + ) as loaded: compiled = compile_model( - model, + loaded.model, llm_args=llm_args, mapping=mapping, dist=dist, - max_num_tokens=max_num_tokens, - prefill_capture_num_tokens=plan.prefill_num_tokens, + max_num_tokens=loaded.max_num_tokens, + prefill_capture_num_tokens=loaded.prefill_capture_num_tokens, ) runner = EncoderRunner.create( compiled.model, llm_args=llm_args, mapping=mapping, dist=dist, - max_batch_size=max_batch_size, - max_num_tokens=max_num_tokens, - max_seq_len=max_seq_len, - max_beam_width=max_beam_width, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, + max_beam_width=loaded.max_beam_width, model_caller=compiled.caller, attention_backend=resolve_attention_backend(compiled.model, llm_args), attention_runtime_features=AttentionRuntimeFeatures(), ) yield EncoderBuild( runner=runner, - model_loader=model_loader, - model_config=model.model_config, - max_batch_size=max_batch_size, - max_num_tokens=max_num_tokens, - max_seq_len=max_seq_len, + model_loader=loaded.model_loader, + model_config=loaded.model.model_config, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, + ) + + +@contextmanager +def build_mm_encoder_runner( + llm_args: TorchLlmArgs, + *, + model_path: str, + checkpoint_loader: BaseCheckpointLoader, + mapping: Mapping, + dist: Distributed, +) -> Iterator[EncoderBuild]: + """Load the checkpoint's vision encoder, set up its towers and build a multimodal encoder runner. + + The towers size their attention metadata, then capture their block CUDA graphs after + compilation. The loader is cleaned up as in ``build_encoder_runner``. + + Args: + llm_args: The resolved arguments, with ``mm_encoder_only`` set. + model_path: The checkpoint directory. + checkpoint_loader: The loader for the checkpoint's format. + mapping: The parallel mapping of this rank. + dist: The communicator across the model's ranks. + + Yields: + The runner with its loader, model config and resolved capacity. + """ + encoder_max_num_tokens = ( + llm_args.encoder_max_num_tokens + if llm_args.encoder_max_num_tokens is not None + else llm_args.max_num_tokens + ) + with _load_encoder( + llm_args, + model_path=model_path, + checkpoint_loader=checkpoint_loader, + mapping=mapping, + dist=dist, + ) as loaded: + input_processor = create_engine_input_processor(model_path, llm_args) + setup_mm_encoder_attn_metadata(loaded.model, input_processor, encoder_max_num_tokens, None) + compiled = compile_model( + loaded.model, + llm_args=llm_args, + mapping=mapping, + dist=dist, + max_num_tokens=loaded.max_num_tokens, + prefill_capture_num_tokens=loaded.prefill_capture_num_tokens, + ) + mm_config = llm_args.multimodal_config + setup_mm_encoder_cuda_graphs( + compiled.model, + mm_config.encoder_cuda_graph if mm_config is not None else None, + max_num_tokens=encoder_max_num_tokens, + ) + yield EncoderBuild( + runner=MultimodalEncoderRunner.create(compiled.model), + model_loader=loaded.model_loader, + model_config=loaded.model.model_config, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, ) - except BaseException: - model_loader.cleanup() - raise diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py index 28a529b3c199..d7a848fae6a4 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py @@ -10,6 +10,7 @@ from torch import nn from typing_extensions import Self +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin from tensorrt_llm._torch.models.modeling_multimodal_mixin import _build_request_multimodal_input from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import ( MoeLoadBalancer, @@ -17,7 +18,7 @@ ) from tensorrt_llm._torch.nccl_window_tensor_scope import discard_nccl_window_tensor_outputs from tensorrt_llm._torch.pyexecutor.llm_request import get_multimodal_embedding_lengths -from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType +from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.utils import with_model_extra_attrs from tensorrt_llm._utils import nvtx_range, prefer_pinned @@ -40,8 +41,8 @@ class MultimodalEncoderRunner(ScheduledModelRunner): """Run an engine whose only job is multimodal encoding. This family is distinct from ``MultimodalItemScheduler``, which schedules vision items - inside a decoder engine. Its graph phase is a no-op because - ``MultimodalEncoderGraphRunner`` belongs to, and is owned by, the vision model layer. + inside a decoder engine. The vision towers capture their block CUDA graphs when they are + built, so warmup has nothing to do; ``release_graphs`` clears them. """ def __init__( @@ -90,17 +91,15 @@ def _prepare_inputs(self, scheduled_requests: ScheduledRequests) -> _PayloadBatc embedding_lengths.append(list(lengths)) return _PayloadBatch(request_indices, params, embedding_lengths) - def _validate_resources(self, resource_manager: ResourceManager) -> None: - kv_cache_manager = resource_manager.get_resource_manager( - ResourceManagerType.KV_CACHE_MANAGER - ) - assert kv_cache_manager is None, ( - "a no-KV-cache runner was initialized, but a KV cache manager was allocated" - ) + def warmup(self, resource_manager: ResourceManager | None = None) -> None: + del resource_manager - def warmup(self, resource_manager: ResourceManager) -> None: - """Validate the resources used by this runner before execution.""" - self._validate_resources(resource_manager) + def release_graphs(self, *, release_nccl_window_owners: bool = True) -> None: + for module in self._model.modules(): + if isinstance(module, MultimodalEncoderMixin): + module.clear_blocks_cuda_graph( + release_nccl_window_owners=release_nccl_window_owners + ) @torch.inference_mode() @with_model_extra_attrs(lambda self: self._model.extra_attrs) @@ -111,11 +110,14 @@ def forward( resource_manager: ResourceManager, is_dummy: bool = False, ) -> dict[str, Any]: - del is_dummy - self._validate_resources(resource_manager) + del resource_manager, is_dummy batch = self._prepare_inputs(inputs.batch) with MoeLoadBalancerIterContext(self._moe_load_balancer): - return self._forward_step(batch) + outputs = self._forward_step(batch) + if inputs.inputs_staged is not None: + # The towers' attention metadata stages through reused host buffers during the forward. + inputs.inputs_staged.record() + return outputs @nvtx_range("_forward_step_mm_encoder_only") def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: @@ -159,7 +161,7 @@ def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: mrope_position_deltas_list.append(mrope_position_deltas) # mrope lists must align 1:1 with multimodal_params (or be empty); - # the sampler indexes them by per-MM-result position into mm_embeddings. + # the completion indexes them by per-MM-result position into mm_embeddings. assert len(mrope_position_ids_list) == len(mrope_position_deltas_list) and len( mrope_position_ids_list ) in (0, len(batch.params)), ( @@ -170,7 +172,6 @@ def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: result = { "mm_embeddings": mm_embeddings, - "logits": None, "mm_embedding_request_indices": batch.request_indices, "mm_embedding_lengths": batch.embedding_lengths, } diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 509173015a46..6037020510bc 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -588,6 +588,16 @@ def set_mrope_position( self.diff.mrope_position_ids = self._mrope_position_ids self.diff.mrope_position_deltas = self._mrope_position_deltas + def release_exports(self) -> None: + """Consume the handles this result exported; no response will deliver them.""" + release_exported_handles(self) + self._mm_embeddings = None + self._mrope_position_ids = None + self._mrope_position_deltas = None + self.diff.mm_embeddings = None + self.diff.mrope_position_ids = None + self.diff.mrope_position_deltas = None + def set_encoder_output(self, encoder_output: torch.Tensor): self._encoder_output = encoder_output self.diff.encoder_output = encoder_output @@ -726,6 +736,22 @@ def encoder_output(self) -> torch.Tensor | None: return self._encoder_output +def release_exported_handles(result: "PyResult | LlmResult") -> None: + """Consume a result's CUDA IPC handles that no consumer will open. + + The producer cannot free an exported allocation until each of its handles is opened + and released, or consumed here. + """ + handles = [ + *(result.mm_embedding_handles or ()), + result.mrope_position_ids_handle, + result.mrope_position_deltas_handle, + ] + for handle in handles: + if handle is not None: + SharedTensorContainer.release_dict(handle) + + class LlmResult: """LlmResult wraps `bindings.executor.Result` but detour some features to Python implementation""" py_result_properties = frozenset( diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index f2ea4985623b..9cb797461e7c 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -737,7 +737,7 @@ def _create_py_executor( llm_args, checkpoint_loader, is_generation = _load_config_and_create_checkpoint_loader( llm_args, checkpoint_dir) - if not llm_args.mm_encoder_only and not is_generation: + if not is_generation: return _create_scheduled_encoder_executor(llm_args, checkpoint_dir, checkpoint_loader, _startup_timer, diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index caa2dcf67a7a..9cf2abb715b4 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -28,10 +28,10 @@ from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import host_profiler_context from ..moe.fused_moe.moe_comm_timeout_guard import set_moe_comm_warmup -from .encoder_completion import Completion, PoolingCompletion -from .engine.model_setup import build_encoder_runner -from .engine.runners.encoder import EncoderRunner, resolve_scheduled_encoder_args -from .engine.runners.interface import ScheduledInputs +from .encoder_completion import Completion, MultimodalEmbeddingCompletion, PoolingCompletion +from .engine.model_setup import build_encoder_runner, build_mm_encoder_runner +from .engine.runners.encoder import resolve_scheduled_encoder_args +from .engine.runners.interface import ScheduledInputs, ScheduledModelRunner from .error_classification import ErrorBudget from .executor_request_queue import ExecutorRequestQueue, RequestQueueItem from .hang_detector import ( @@ -121,6 +121,9 @@ class ScheduledEncoderExecutor: are always empty. With ``print_iter_log`` rank 0 logs one line per batch: its requests and prompt tokens, the requests still waiting when it launched, the host time from taking the batch to publishing it, and the GPU time of the forward. + + A pooling model's runner pairs with ``PoolingCompletion``, a multimodal encoder's + with ``MultimodalEmbeddingCompletion``. """ is_generation_model = False @@ -128,7 +131,7 @@ class ScheduledEncoderExecutor: def __init__( self, - runner: EncoderRunner, + runner: ScheduledModelRunner, *, completion: Completion, model_loader: ModelLoader, @@ -220,6 +223,9 @@ def create( ) -> ScheduledEncoderExecutor: """Load and compile the model, build its runner, and warm it up on every rank. + With ``mm_encoder_only`` the checkpoint's vision encoder is built and rank 0 exports + its embeddings; otherwise the checkpoint's pooling model is built. + Args: llm_args: The arguments; scheduled-encoder constraints are applied to them first. model_path: The checkpoint directory. @@ -231,7 +237,14 @@ def create( The executor, warmed up; its loop starts with ``start_worker``. """ llm_args = resolve_scheduled_encoder_args(llm_args) - with build_encoder_runner( + if llm_args.mm_encoder_only: + build = build_mm_encoder_runner + # Attention DP is rejected, so rank 0 alone answers the frontend. + completion = MultimodalEmbeddingCompletion(exports=dist.rank == 0) + else: + build = build_encoder_runner + completion = PoolingCompletion() + with build( llm_args, model_path=model_path, checkpoint_loader=checkpoint_loader, @@ -244,7 +257,7 @@ def create( ) return cls( built.runner, - completion=PoolingCompletion(), + completion=completion, model_loader=built.model_loader, llm_args=llm_args, mapping=mapping, diff --git a/tensorrt_llm/llmapi/mm_encoder.py b/tensorrt_llm/llmapi/mm_encoder.py index 71cb5cd7f051..f36b75bc3fbe 100644 --- a/tensorrt_llm/llmapi/mm_encoder.py +++ b/tensorrt_llm/llmapi/mm_encoder.py @@ -72,7 +72,7 @@ def _build_model(self): mpi_session=self.mpi_session, reuse_mpi_comm=external_mpi_comm_available( self.args.parallel_config.world_size), - is_llm_executor=True, # TODO: check if this is correct or needed + is_llm_executor=True, hf_model_dir=self._hf_model_dir, tokenizer=self.tokenizer, llm_args=self.args) diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py index 6c3295b687ae..acefc2c06def 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py @@ -7,12 +7,14 @@ import pytest import torch +from torch import nn +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin +from tensorrt_llm._torch.pyexecutor.encoder_completion import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.engine.runners import mm_encoder as mm_encoder_module from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledInputs from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.llm_request import PyResult -from tensorrt_llm._torch.pyexecutor.sampler import EarlyStopWithMMResult from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._torch.utils import get_model_extra_attrs @@ -95,7 +97,6 @@ def test_mm_encoder_runner_forward_skips_metadata_only_and_missing_length_reques to_device.assert_called_once_with(params[0], "multimodal_data", "cuda", pin_memory=True) for request in requests: request.get_tokens.assert_not_called() - assert result["logits"] is None assert result["mm_embedding_request_indices"] == [3] assert result["mm_embedding_lengths"] == [[2]] torch.testing.assert_close(result["mm_embeddings"][0], torch.arange(4).reshape(2, 2)) @@ -145,7 +146,6 @@ def test_mm_encoder_runner_splits_embeddings_and_returns_mrope_metadata() -> Non model.forward.assert_called_once() params = model.forward.call_args.args[0] assert [param.input_ids_start_offset for param in params] == [0, 1] - assert result["logits"] is None assert result["mm_embedding_request_indices"] == [0, 1] assert result["mm_embedding_lengths"] == [[1], [2]] torch.testing.assert_close(result["mm_embeddings"][0], embeddings[:1]) @@ -178,7 +178,6 @@ def test_mm_encoder_handoff_preserves_per_item_metadata_in_sparse_batch() -> Non ] for request in requests: request.py_result = PyResult(prompt_len=1, max_new_tokens=1) - request.set_finished_reason = Mock() request.py_multimodal_params = MultimodalParams(multimodal_data=request.py_multimodal_data) request.get_num_tokens = Mock(return_value=1) scheduled_requests = SimpleNamespace( @@ -202,9 +201,7 @@ def encode(batch_params: list[MultimodalParams]) -> list[torch.Tensor]: resource_manager=SimpleNamespace(get_resource_manager=Mock(return_value=None)), ) assert get_model_extra_attrs() is None - sampler = EarlyStopWithMMResult() - state = sampler.sample_async(scheduled_requests, result, []) - sampler.update_requests(state) + MultimodalEmbeddingCompletion(exports=True).store(requests, result) assert requests[1].py_result.mm_embedding_handles is None image_handles = json.loads(json.dumps(requests[0].py_result.mm_embedding_handles)) @@ -225,20 +222,21 @@ def encode(batch_params: list[MultimodalParams]) -> list[torch.Tensor]: torch.testing.assert_close(torch.cat(restored), embeddings[1:]) -def test_mm_encoder_runner_rejects_kv_manager_before_preparation() -> None: +class _Tower(MultimodalEncoderMixin, nn.Module): + pass + + +def test_mm_encoder_runner_releases_every_tower_graph() -> None: + towers = [_Tower(), _Tower()] + graphs = [Mock(), Mock()] + for tower, graph in zip(towers, graphs): + tower._blocks_graph_runner = graph runner = MultimodalEncoderRunner( - SimpleNamespace(extra_attrs={}), encoder_cache_enabled=False, moe_load_balancer=None + nn.Sequential(*towers), encoder_cache_enabled=False, moe_load_balancer=None ) - runner._prepare_inputs = Mock() - resource_manager = Mock() - resource_manager.get_resource_manager.return_value = object() - - with pytest.raises( - AssertionError, - match="no-KV-cache runner was initialized, but a KV cache manager was allocated", - ): - runner.forward( - ScheduledInputs(batch=ScheduledRequests()), resource_manager=resource_manager - ) - runner._prepare_inputs.assert_not_called() + runner.release_graphs(release_nccl_window_owners=False) + + for tower, graph in zip(towers, graphs): + graph.clear.assert_called_once_with(release_nccl_window_owners=False) + assert tower._blocks_graph_runner is None diff --git a/tests/unittest/_torch/pyexecutor/test_model_engine.py b/tests/unittest/_torch/pyexecutor/test_model_engine.py index b827d48e5441..e2d378958ce6 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_engine.py +++ b/tests/unittest/_torch/pyexecutor/test_model_engine.py @@ -1217,33 +1217,6 @@ def test_allreduce_forward_routes_tuning_and_inference(self) -> None: ) unused.assert_not_called() - @pytest.mark.cpu_only - def test_no_kv_cache_warmup_rejects_allocated_kv_cache(self) -> None: - """Verify a no-KV-cache runner rejects an allocated KV cache before warmup.""" - model_engine = object.__new__(ModelEngine) - model_engine.llm_args = SimpleNamespace(enable_autotuner=False) - model_engine._metrics = {} - model_engine.model = SimpleNamespace(model_config=SimpleNamespace(is_encoder_decoder=False)) - model_engine.moe_load_balancer = None - model_engine._warmup_depth = 0 - model_engine._runner = Mock(spec=MultimodalEncoderRunner) - runner = model_engine._runner - runner._validate_resources = MultimodalEncoderRunner._validate_resources.__get__(runner) - runner.warmup.side_effect = MultimodalEncoderRunner.warmup.__get__(runner) - resource_manager = Mock() - resource_manager.get_resource_manager.return_value = object() - - with self.assertRaisesRegex( - AssertionError, - "no-KV-cache runner was initialized, but a KV cache manager was allocated", - ): - model_engine.warmup(resource_manager) - - model_engine._runner.warmup.assert_called_once_with(resource_manager) - resource_manager.get_resource_manager.assert_called_once_with( - ResourceManagerType.KV_CACHE_MANAGER - ) - @pytest.mark.cpu_only def test_warmup_total_timing_and_kv_cache_cleanup(self) -> None: model_engine = object.__new__(ModelEngine) diff --git a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py index 8a1f99a887d4..6bcae7ccd3a2 100644 --- a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py +++ b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py @@ -7,6 +7,7 @@ import torch from tensorrt_llm._torch.pyexecutor import _util +from tensorrt_llm._torch.pyexecutor.encoder_completion import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.llm_request import ( LlmRequestState, PyResult, @@ -159,7 +160,7 @@ def __init__(self): self.mm_embeddings = [] self.mrope_position = None - def append_mm_embeddings(self, mm_embedding, mm_embedding_lengths): + def append_mm_embeddings(self, mm_embedding, mm_embedding_lengths, mm_embedding_metadata=None): self.mm_embeddings.append((mm_embedding, mm_embedding_lengths)) def set_mrope_position(self, position_ids, position_deltas): @@ -267,6 +268,36 @@ def test_mm_encoder_sampler_exports_results_only_on_response_ranks(mapping, expo assert (request.py_result.mrope_position is not None) is exports_results +@pytest.mark.parametrize("exports", [True, False], ids=["response_rank", "follower_rank"]) +@pytest.mark.cpu_only +def test_mm_embedding_completion_attaches_results_by_request_index(exports): + """A sparse batch's results reach their own requests, only on the rank that exports them.""" + text_request = _FakeRequest() + mm_request = _FakeRequest(multimodal_lengths=[4]) + + MultimodalEmbeddingCompletion(exports=exports).store( + [text_request, mm_request], + { + "mm_embeddings": [torch.ones(4, 2)], + "mm_embedding_request_indices": [1], + "mm_embedding_lengths": [[4]], + "mrope_position_ids": ["mm-pos"], + "mrope_position_deltas": ["mm-delta"], + }, + ) + + assert text_request.py_result.mm_embeddings == [] + assert text_request.py_result.mrope_position is None + if exports: + [(mm_embedding, mm_embedding_lengths)] = mm_request.py_result.mm_embeddings + assert mm_embedding.shape == (4, 2) + assert mm_embedding_lengths == [4] + assert mm_request.py_result.mrope_position == ("mm-pos", "mm-delta") + else: + assert mm_request.py_result.mm_embeddings == [] + assert mm_request.py_result.mrope_position is None + + @pytest.mark.cpu_only def test_py_result_mm_embedding_handles_use_shared_tensor_handles(): """MM encoder result handles should preserve the producer tensor device.""" diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py index 85b7176230ec..ff8ec84be13d 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py @@ -105,23 +105,24 @@ def test_scheduled_encoder_serves_tensor_parallel_rpc_workers(): @pytest.mark.parametrize( - ("config", "model_kwargs", "is_generation"), + ("config", "args_kwargs", "is_generation"), [ - (BertConfig(architectures=["BertForSequenceClassification"]), None, False), - (LlamaConfig(architectures=["LlamaForCausalLM"]), None, True), + (BertConfig(architectures=["BertForSequenceClassification"]), {}, False), + (LlamaConfig(architectures=["LlamaForCausalLM"]), {}, True), ( LlamaConfig(architectures=["LlamaForCausalLM"]), - {"architectures": ["LlamaForTextEmbedding"]}, + {"model_kwargs": {"architectures": ["LlamaForTextEmbedding"]}}, False, ), + (LlamaConfig(architectures=["LlamaForCausalLM"]), {"mm_encoder_only": True}, False), ], - ids=["pooling", "generation", "overridden-to-pooling"], + ids=["pooling", "generation", "overridden-to-pooling", "mm-encoder-only"], ) def test_dispatch_reads_the_architecture_model_loading_will_use( - tmp_path, config, model_kwargs, is_generation + tmp_path, config, args_kwargs, is_generation ): config.save_pretrained(tmp_path) - args = TorchLlmArgs(model=str(tmp_path), model_kwargs=model_kwargs) + args = TorchLlmArgs(model=str(tmp_path), **args_kwargs) _, loaded = ModelLoader.load_config_with_defaults(str(tmp_path), args, HfCheckpointLoader()) diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index 863f7864939f..95d6cc7a711c 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """Scheduled encoder executor behaviour, driven through the members the worker calls. -A fake runner answers each request with a row holding its request id. It writes into one -reused output buffer, as a captured graph does, and can hold or fail a forward. The -executor's queue, admission, batching, completion and delivery are real. +A fake runner answers each request with a row holding its request id. As a pooling model it +writes into one reused output buffer, as a captured graph does; as a multimodal encoder it +returns one CPU embedding per request, which the completion exports through shared memory. +It can hold or fail a forward. The executor's queue, admission, batching, completion and +delivery are real. """ import re @@ -19,12 +21,16 @@ import torch from tensorrt_llm._torch.pyexecutor import scheduled_encoder_executor -from tensorrt_llm._torch.pyexecutor.encoder_completion import PoolingCompletion +from tensorrt_llm._torch.pyexecutor.encoder_completion import ( + MultimodalEmbeddingCompletion, + PoolingCompletion, +) from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import ( ScheduledEncoderExecutor, fit_batch, ) +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm.bindings.executor import FinishReason, OutputConfig, Request, SamplingConfig from tensorrt_llm.executor.utils import is_context_length_exceeded_message from tensorrt_llm.mapping import Mapping @@ -72,9 +78,10 @@ def test_fit_batch_charges_each_child_request_a_full_row(): class _Runner: - """Answer each request with a row of its request id, through one reused output buffer.""" + """Answer each request with a row of its request id.""" - def __init__(self, max_batch_size: int): + def __init__(self, max_batch_size: int, multimodal: bool): + self.multimodal = multimodal self.output = torch.zeros(max_batch_size, WIDTH, device="cuda") self.batches: list[list[int]] = [] self.errors: list[Exception] = [] @@ -115,6 +122,14 @@ def forward(self, inputs, *, resource_manager): self.staged.append(inputs.inputs_staged) if self.gpu_cycles: torch.cuda._sleep(self.gpu_cycles) + if self.multimodal: + return { + "mm_embeddings": [ + torch.full((1, WIDTH), float(request.py_request_id)) for request in requests + ], + "mm_embedding_request_indices": list(range(len(requests))), + "mm_embedding_lengths": [[1]] * len(requests), + } for row, request in enumerate(requests): self.output[row].fill_(request.py_request_id) return {"logits": self.output[: len(requests)]} @@ -122,12 +137,21 @@ def forward(self, inputs, *, resource_manager): class _Harness: def __init__( - self, *, max_batch_size=4, max_num_tokens=64, max_seq_len=32, print_iter_log=False + self, + *, + completion="pooling", + max_batch_size=4, + max_num_tokens=64, + max_seq_len=32, + print_iter_log=False, ): - self.runner = _Runner(max_batch_size) + self.multimodal = multimodal = completion == "multimodal" + self.runner = _Runner(max_batch_size, multimodal) self.executor = ScheduledEncoderExecutor( self.runner, - completion=PoolingCompletion(), + completion=( + MultimodalEmbeddingCompletion(exports=True) if multimodal else PoolingCompletion() + ), model_loader=Mock(), llm_args=SimpleNamespace( garbage_collection_gen0_threshold=20000, @@ -148,7 +172,7 @@ def submit( self, num_tokens: int = 4, *, - context_logits: bool = True, + context_logits: bool | None = None, perf_metrics: bool = False, **attrs, ) -> int: @@ -156,7 +180,9 @@ def submit( input_token_ids=list(range(1, num_tokens + 1)), max_tokens=1, output_config=OutputConfig( - return_context_logits=context_logits, + return_context_logits=( + not self.multimodal if context_logits is None else context_logits + ), exclude_input_from_output=True, return_perf_metrics=perf_metrics, ), @@ -194,8 +220,13 @@ def _harness(**kwargs): h.shutdown() -def test_batches_follow_the_budget_in_arrival_order(): - with _harness(max_batch_size=2, max_num_tokens=64, max_seq_len=64) as h: +@pytest.fixture(params=["pooling", "multimodal"]) +def completion(request): + return request.param + + +def test_batches_follow_the_budget_in_arrival_order(completion): + with _harness(completion=completion, max_batch_size=2, max_num_tokens=64, max_seq_len=64) as h: ids = [h.submit(), h.submit(), h.submit(61), h.submit()] h.executor.start_worker() @@ -221,6 +252,20 @@ def test_results_are_pooled_rows_with_no_tokens_and_reason_length(): assert result.finish_reasons == [FinishReason.LENGTH] +def test_multimodal_results_are_exported_embeddings_with_no_tokens_and_reason_length(): + with _harness(completion="multimodal", max_batch_size=2) as h: + ids = [h.submit(py_multimodal_data={"image": {}}) for _ in range(3)] + h.executor.start_worker() + + for request_id in ids: + result = h.result(request_id) + (handle,) = result.mm_embedding_handles + embedding = SharedTensorContainer.from_dict(handle).get_local_view() + assert torch.equal(embedding, torch.full((1, WIDTH), float(request_id))) + assert result.output_token_ids == [[]] + assert result.finish_reasons == [FinishReason.LENGTH] + + # Long enough that a launched batch is still running when the loop has launched the next one. GPU_CYCLES = 1_000_000_000 @@ -307,8 +352,21 @@ def test_inadmissible_requests_fail_alone(): assert h.runner.batches == [[fitting]] -def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(): - with _harness(max_batch_size=1) as h: +def test_a_multimodal_encoder_admits_multimodal_requests_and_fails_an_oversized_one_alone(): + with _harness(completion="multimodal", max_seq_len=32, max_num_tokens=64) as h: + too_long = h.submit(33, py_multimodal_data={"image": {}}) + fitting = h.submit(32, py_multimodal_data={"image": {}}) + h.executor.start_worker() + + (long_response,) = h.responses(too_long) + h.result(fitting) + + assert is_context_length_exceeded_message(long_response.error_msg) + assert h.runner.batches == [[fitting]] + + +def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() running, queued = h.submit(), h.submit() h.executor.start_worker() @@ -330,8 +388,8 @@ def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(): assert h.runner.batches == [[running], [later]] -def test_a_survivable_forward_error_fails_only_its_batch(): - with _harness(max_batch_size=1) as h: +def test_a_survivable_forward_error_fails_only_its_batch(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.errors.append(RuntimeError("bad batch")) failed, later = h.submit(), h.submit() h.executor.start_worker() @@ -352,8 +410,8 @@ def test_a_failed_forward_answers_a_request_with_children_once(): assert (error.request_id, error.error_msg) == (failed, "bad batch") -def test_a_fatal_forward_error_fails_everything_queued_and_shuts_down(): - with _harness(max_batch_size=1) as h: +def test_a_fatal_forward_error_fails_everything_queued_and_shuts_down(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() h.runner.errors.append(RuntimeError("an illegal memory access was encountered")) running, waiting = h.submit(), h.submit() @@ -409,8 +467,8 @@ def test_print_iter_log_logs_each_batch_with_its_size_and_tokens(monkeypatch): assert "host_step_time = " in line and "device_step_time = " in line -def test_shutdown_runs_queued_requests_first(): - with _harness(max_batch_size=1) as h: +def test_shutdown_runs_queued_requests_first(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() ids = [h.submit(), h.submit(), h.submit()] h.executor.start_worker() From 9ea0d75567da3c4e9f5e4f2b6df46f5cb45bf529 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Tue, 17 Nov 2026 20:37:55 -0800 Subject: [PATCH 03/12] [TRTLLM-17766][fix] Finish requests cancelled after launch as cancelled A cancellation that arrives before the batch is published now retires the request's exports and answers CANCELLED; pooling requests change the same way. Signed-off-by: Lori Ren --- .../pyexecutor/scheduled_encoder_executor.py | 51 ++++++++++++-- .../test_scheduled_encoder_executor.py | 67 ++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index 9cf2abb715b4..ee486207945d 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -190,6 +190,10 @@ def __init__( self._last_staged: torch.cuda.Event | None = None self._completion_thread: threading.Thread | None = None self._completion_error: BaseException | None = None + # Ids of the launched, unpublished requests, and of those among them cancelled since. + self._cancel_lock = threading.Lock() + self._launched_ids: set[int] = set() + self._cancelled_after_launch: set[int] = set() self._response_channel = ResponseChannel() self._iter_counter = 0 self._profiler = ProfileController( @@ -349,7 +353,7 @@ def _await_single(self, request_id: int, timeout: float | None) -> list[LlmRespo ) def cancel_request(self, id: int) -> None: - """Cancel a request; a queued one finishes as cancelled, a finished one is ignored. + """Cancel a request; it finishes as cancelled unless its result was already published. Args: id: The request id. @@ -548,6 +552,8 @@ def _receive(self) -> None: if self.dist.rank == 0: self._respond_with_errors(rejected) if canceled_ids: + with self._cancel_lock: + self._cancelled_after_launch.update(canceled_ids & self._launched_ids) self._cancel_waiting(canceled_ids) @nvtx_range("_check_admissible") @@ -569,7 +575,7 @@ def _check_admissible(self, item: RequestQueueItem) -> None: ) def _cancel_waiting(self, request_ids: set[int]) -> None: - """Finish queued requests as cancelled; ids no longer queued have already completed.""" + """Finish queued requests as cancelled; ids no longer queued are launched or completed.""" items = [item for item in self._waiting_queue if item.id in request_ids] if not items: return @@ -645,6 +651,8 @@ def _run(self, items: list[RequestQueueItem]) -> None: self._completion.retire(requests) self._fail(requests, error) return + with self._cancel_lock: + self._launched_ids.update(item.id for item in items) self._in_flight.put( _InFlight( requests=requests, @@ -666,10 +674,9 @@ def _completion_loop(self) -> None: with nvtx_range("synchronize"): # Responses carry device tensors that other threads and processes read. flight.done.synchronize() + responses = self._cancel_launched(flight) if self.dist.rank == 0: - self._publish( - flight.responses, (request.py_request_id for request in flight.requests) - ) + self._publish(responses, (request.py_request_id for request in flight.requests)) if self._iter_log: self._log_iteration(flight) except Exception as error: @@ -682,6 +689,40 @@ def _completion_loop(self) -> None: finally: self._slots.release() + def _cancel_launched(self, flight: _InFlight) -> list[tuple[int, LlmResponse]]: + """The batch's responses, with those of requests cancelled since its launch replaced. + + A cancelled request's exports are retired and it finishes as cancelled; a + cancellation that arrives after this point finds the request completed. + """ + request_ids = {request.py_request_id for request in flight.requests if not request.is_child} + with self._cancel_lock: + self._launched_ids -= request_ids + cancelled_ids = self._cancelled_after_launch & request_ids + self._cancelled_after_launch -= cancelled_ids + if not cancelled_ids: + return flight.responses + cancelled = [ + request + for request in flight.requests + if (request.parent_request_id if request.is_child else request.py_request_id) + in cancelled_ids + ] + self._completion.retire(cancelled) + # A child's response carries its parent's request id. + responses = [ + (request_id, response) + for request_id, response in flight.responses + if response.request_id not in cancelled_ids + ] + for request in cancelled: + # Replaces the reason LENGTH set by _complete. + request.set_finished_reason(FinishReason.CANCELLED, 0) + response = request.create_response() + if response is not None: + responses.append((request.py_request_id, response)) + return responses + def _log_iteration(self, flight: _InFlight) -> None: """Log one batch: its size, the host time until it was published and the GPU time of its forward.""" host_ms = (time.monotonic() - flight.host_start) * 1000 diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index 95d6cc7a711c..f04b5d1215c4 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -9,6 +9,7 @@ delivery are real. """ +import gc import re import threading import time @@ -82,6 +83,7 @@ class _Runner: def __init__(self, max_batch_size: int, multimodal: bool): self.multimodal = multimodal + self.embedding_device = "cpu" self.output = torch.zeros(max_batch_size, WIDTH, device="cuda") self.batches: list[list[int]] = [] self.errors: list[Exception] = [] @@ -125,7 +127,10 @@ def forward(self, inputs, *, resource_manager): if self.multimodal: return { "mm_embeddings": [ - torch.full((1, WIDTH), float(request.py_request_id)) for request in requests + torch.full( + (1, WIDTH), float(request.py_request_id), device=self.embedding_device + ) + for request in requests ], "mm_embedding_request_indices": list(range(len(requests))), "mm_embedding_lengths": [[1]] * len(requests), @@ -388,6 +393,66 @@ def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): assert h.runner.batches == [[running], [later]] +def test_cancelling_a_launched_request_finishes_it_once_as_cancelled(completion): + with _harness(completion=completion, max_batch_size=1) as h: + h.runner.hold() + # The launched forward is still running when the loop takes the cancellation. + h.runner.gpu_cycles = GPU_CYCLES + launched, later = h.submit(), h.submit() + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + + h.executor.cancel_request(launched) + h.runner.release.set() + + (cancelled,) = h.responses(launched) + assert cancelled.result.is_final + assert cancelled.result.finish_reasons == [FinishReason.CANCELLED] + assert cancelled.result.mm_embedding_handles is None + h.result(later) + assert h.responses(launched, NO_WAIT) == [] + assert h.runner.batches == [[launched], [later]] + + +def _cuda_memory_allocated() -> int: + gc.collect() + torch.cuda.synchronize() + torch.cuda.ipc_collect() + return torch.cuda.memory_allocated() + + +@pytest.mark.parametrize("cause", ["cancelled", "failed"]) +def test_a_launched_request_whose_result_is_not_delivered_frees_its_exports(cause): + with _harness(completion="multimodal", max_batch_size=1) as h: + h.runner.embedding_device = "cuda" + h.runner.gpu_cycles = GPU_CYCLES + if cause == "failed": + + def fail_after_the_export(*args, **kwargs): + raise RuntimeError("bad step") + + h.executor._perf_manager.append_step_metrics = fail_after_the_export + allocated = _cuda_memory_allocated() + h.runner.hold() + launched = h.submit(py_multimodal_data={"image": {}}) + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + if cause == "cancelled": + h.executor.cancel_request(launched) + h.runner.release.set() + + (response,) = h.responses(launched) + undelivered = ( + response.result.finish_reasons == [FinishReason.CANCELLED] + if cause == "cancelled" + else response.error_msg == "bad step" + ) + del response + + assert _cuda_memory_allocated() == allocated + assert undelivered + + def test_a_survivable_forward_error_fails_only_its_batch(completion): with _harness(completion=completion, max_batch_size=1) as h: h.runner.errors.append(RuntimeError("bad batch")) From b6c48f7acc92db06fc43b4f8c9fd710a8716498c Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Tue, 17 Nov 2026 21:04:43 -0800 Subject: [PATCH 04/12] [TRTLLM-17766][fix] Release exported multimodal handles that are never delivered Responses dropped by the worker, proxy or RPC proxy now consume their exports. The encoder route releases the MRoPE exports it does not return. Signed-off-by: Lori Ren --- tensorrt_llm/_torch/pyexecutor/llm_request.py | 11 ++ tensorrt_llm/executor/base_worker.py | 4 +- tensorrt_llm/executor/proxy.py | 2 + tensorrt_llm/executor/rpc_proxy_mixin.py | 2 + tensorrt_llm/serve/openai_server.py | 14 ++- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../executor/test_undelivered_exports.py | 100 ++++++++++++++++++ .../llmapi/apps/test_openai_input_metrics.py | 4 +- .../unittest/serve/test_mm_encoder_exports.py | 91 ++++++++++++++++ 9 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 tests/unittest/executor/test_undelivered_exports.py create mode 100644 tests/unittest/serve/test_mm_encoder_exports.py diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 6037020510bc..50b13dac06bd 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -752,6 +752,17 @@ def release_exported_handles(result: "PyResult | LlmResult") -> None: SharedTensorContainer.release_dict(handle) +def release_undelivered_response(response: Any) -> None: + """Consume the exported handles of a response that no client will receive. + + Takes the executor's responses, also when the worker has wrapped them; error + responses and other records carry no ``LlmResult`` and export nothing. + """ + result = getattr(response, "result", None) + if isinstance(result, LlmResult): + release_exported_handles(result) + + class LlmResult: """LlmResult wraps `bindings.executor.Result` but detour some features to Python implementation""" py_result_properties = frozenset( diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 7c310416dd07..68ff43dda8eb 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -33,7 +33,8 @@ from .._torch.peft.lora.manager import LoraManager from .._torch.pyexecutor.iteration_stats import IterationStatsRecord -from .._torch.pyexecutor.llm_request import LlmResponse +from .._torch.pyexecutor.llm_request import (LlmResponse, + release_undelivered_response) from .._torch.pyexecutor.request_inputs import (ExecutorRequest, PythonRequestInputs) from .._utils import (global_mpi_rank, global_mpi_size, mpi_rank, @@ -1834,6 +1835,7 @@ def _drop_refused_response(worker, response: LlmResponse) -> bool: return False if response.has_error() or response.result.is_final: worker._refused_client_ids.discard(response.client_id) + release_undelivered_response(response) return True diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 428307b9fed3..bb784799f0e8 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -39,6 +39,7 @@ wrap_worker_startup_error) from tensorrt_llm.logger import logger +from .._torch.pyexecutor.llm_request import release_undelivered_response from .._utils import customized_gc_thresholds, mpi_rank, nvtx_range_debug from ..llmapi.llm_args import TorchLlmArgs from ..llmapi.mpi_session import (MpiCommSession, MpiPoolSession, MpiSession, @@ -909,6 +910,7 @@ def process_res(res): # late responses instead of crashing with a KeyError. result = self._results.get(client_id) if result is None: + release_undelivered_response(res) return queue = result.queue if isinstance(queue, _SyncQueue): diff --git a/tensorrt_llm/executor/rpc_proxy_mixin.py b/tensorrt_llm/executor/rpc_proxy_mixin.py index f02a058cc1f0..624f96cdd1d2 100644 --- a/tensorrt_llm/executor/rpc_proxy_mixin.py +++ b/tensorrt_llm/executor/rpc_proxy_mixin.py @@ -19,6 +19,7 @@ import threading from typing import Callable, List, Optional +from .._torch.pyexecutor.llm_request import release_undelivered_response from .._utils import nvtx_range_debug from ..llmapi.tracer import global_tracer from ..llmapi.utils import _SyncQueue @@ -140,6 +141,7 @@ def process_res(res: list): if client_id not in self._results: logger.warning(f"Received response for unknown client_id: {client_id}") + release_undelivered_response(r) continue queue = self._results[client_id].queue diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index b43f511556db..19702ddf7d5d 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -40,6 +40,7 @@ from tensorrt_llm._startup import _StartupTimer from tensorrt_llm._torch.async_llm import AsyncLLM +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._utils import EnergyMonitor from tensorrt_llm.checkpoint_format import CheckpointFormat # yapf: disable @@ -3073,9 +3074,16 @@ async def openai_mm_encoder(self, request: ChatCompletionRequest, async def create_mm_embedding_response(promise: RequestOutput): await promise.aresult() # TODO: Replace mm_embedding_handles with a dedicated OpenAIBaseModel(JSON-safe), when enable multimodal disagg E2E - mm_embedding_handles = ( - promise.disaggregated_params.multimodal_embedding_handles - if promise.disaggregated_params else None) + params = promise.disaggregated_params + mm_embedding_handles = (params.multimodal_embedding_handles + if params else None) + # The response returns only the embedding handles; release the + # exports no caller can open. + if params is not None: + for handle in (params.mrope_position_ids_handle, + params.mrope_position_deltas_handle): + if handle is not None: + SharedTensorContainer.release_dict(handle) if not mm_embedding_handles: return self.create_error_response( message="Multimodal embedding handle missing in response", diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 9e9e052798f8..abe5e74fa477 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -308,6 +308,7 @@ l0_cpu: - unittest/executor/test_ray_stub.py - unittest/executor/test_generation_request_array_prompt.py - unittest/executor/test_llm_result_unpickle.py + - unittest/executor/test_undelivered_exports.py - unittest/grpc/openengine/test_coordinator.py - unittest/_torch/test_shared_tensor_cpu.py - unittest/grpc/openengine/test_frontend_lifecycle.py diff --git a/tests/unittest/executor/test_undelivered_exports.py b/tests/unittest/executor/test_undelivered_exports.py new file mode 100644 index 000000000000..77d7d53816b3 --- /dev/null +++ b/tests/unittest/executor/test_undelivered_exports.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exports of a response that is dropped on its way to the client. + +A multimodal encoder's response carries its embeddings and MRoPE positions as exported +handles, which only a consumer that opens them, or ``release_dict``, consumes. The +exports here are CPU tensors: each lives in a ``/dev/shm`` file until its handles are +consumed. +""" + +import base64 +import gc +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.llm_request import LlmResponse, LlmResult, PyResult +from tensorrt_llm.executor.base_worker import _drop_refused_response +from tensorrt_llm.executor.executor import GenerationExecutor +from tensorrt_llm.executor.proxy import GenerationExecutorProxy +from tensorrt_llm.executor.result import ResponseWrapper +from tensorrt_llm.executor.rpc_proxy_mixin import RpcExecutorMixin + +pytestmark = pytest.mark.cpu_only + +CLIENT_ID = 7 + + +def _exporting_response() -> tuple[LlmResponse, set[Path]]: + py_result = PyResult(prompt_len=1, max_new_tokens=1) + py_result.append_mm_embeddings(torch.ones(2, 1024), [1, 1]) + py_result.set_mrope_position(torch.zeros(3, 1024), torch.zeros(1024)) + handles = [ + *py_result.mm_embedding_handles, + py_result.mrope_position_ids_handle, + py_result.mrope_position_deltas_handle, + ] + files = { + Path("/dev/shm") / base64.b64decode(handle["storage_handle"]).decode().lstrip("/") + for handle in handles + } + response = LlmResponse( + request_id=CLIENT_ID, + result=LlmResult(b"", py_result, is_final=True), + client_id=CLIENT_ID, + ) + return response, files + + +def _drop_at_worker(response): + assert _drop_refused_response(SimpleNamespace(_refused_client_ids={CLIENT_ID}), response) + + +class _ResultQueue: + def __init__(self, item): + self._item = item + + def get(self): + return self._item + + +def _drop_at_proxy(response): + # Without __init__, which spawns workers; the attributes keep teardown a no-op. + proxy = object.__new__(GenerationExecutorProxy) + proxy.workers_started = False + proxy._multi_frontend_ipc_dir = None + proxy._results = {} + proxy.result_queue = _ResultQueue(ResponseWrapper(response)) + assert proxy.dispatch_result_task() + + +class _RpcProxy(RpcExecutorMixin, GenerationExecutor): + pass + + +_RpcProxy.__abstractmethods__ = frozenset() + + +def _drop_at_rpc_proxy(response): + proxy = _RpcProxy.__new__(_RpcProxy) + proxy._results = {} + proxy.handle_responses([[response]]) + + +@pytest.mark.parametrize( + "drop", + [_drop_at_worker, _drop_at_proxy, _drop_at_rpc_proxy], + ids=["refused_at_worker", "late_at_proxy", "unknown_at_rpc_proxy"], +) +def test_a_dropped_response_releases_its_exports(drop): + response, files = _exporting_response() + gc.collect() + assert all(path.exists() for path in files) + + drop(response) + gc.collect() + + assert not any(path.exists() for path in files) diff --git a/tests/unittest/llmapi/apps/test_openai_input_metrics.py b/tests/unittest/llmapi/apps/test_openai_input_metrics.py index 2efba46f036f..47b51e9225dd 100644 --- a/tests/unittest/llmapi/apps/test_openai_input_metrics.py +++ b/tests/unittest/llmapi/apps/test_openai_input_metrics.py @@ -148,7 +148,9 @@ async def complete(): prompt_token_ids=[7, 8], aresult=AsyncMock(side_effect=complete), disaggregated_params=SimpleNamespace( - multimodal_embedding_handles=[{"tensor_size": [4, 8]}] + multimodal_embedding_handles=[{"tensor_size": [4, 8]}], + mrope_position_ids_handle=None, + mrope_position_deltas_handle=None, ), ) diff --git a/tests/unittest/serve/test_mm_encoder_exports.py b/tests/unittest/serve/test_mm_encoder_exports.py new file mode 100644 index 000000000000..98e57dbee050 --- /dev/null +++ b/tests/unittest/serve/test_mm_encoder_exports.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The multimodal encoder route and the exports in the encoder's result. + +The result carries the embeddings and the MRoPE positions as exported handles, which only +a consumer that opens them, or ``release_dict``, consumes. The route answers with the +embedding handles alone. The exports here are CPU tensors: each lives in a ``/dev/shm`` +file until its handles are consumed. +""" + +import asyncio +import base64 +import gc +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +import torch +from test_common.serve.app_factory import AggConfig + +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer +from tensorrt_llm.disaggregated_params import DisaggregatedParams +from tensorrt_llm.llmapi.disagg_utils import ServerRole +from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + +pytestmark = pytest.mark.cpu_only + + +def _export(tensor: torch.Tensor) -> tuple[dict, Path]: + handle = SharedTensorContainer.from_tensor(tensor).dump_to_dict() + path = Path("/dev/shm") / base64.b64decode(handle["storage_handle"]).decode().lstrip("/") + return handle, path + + +@pytest.mark.asyncio +async def test_the_route_delivers_the_embedding_exports_and_releases_the_mrope_exports(serve_env): + server = serve_env.build_openai_server(AggConfig(role=ServerRole.MM_ENCODER)) + server.model = "test-model" + server.allow_request_chat_template = False + server.model_config = None + server.multimodal_server_config = None + server.enable_store = False + server.conversation_store = None + server.use_harmony = False + server.tokenizer = None + server.processor = None + server.chat_template = None + server.tool_parser = None + server.metrics_collector = None + server.await_disconnected = AsyncMock() + server._extract_metrics = AsyncMock() + + embedding, embedding_file = _export(torch.ones(2, 1024)) + position_ids, position_ids_file = _export(torch.zeros(3, 1024)) + position_deltas, position_deltas_file = _export(torch.zeros(1024)) + promise = SimpleNamespace( + request_id="1", + aresult=AsyncMock(), + disaggregated_params=DisaggregatedParams( + multimodal_embedding_handles=[embedding], + mrope_position_ids_handle=position_ids, + mrope_position_deltas_handle=position_deltas, + ), + ) + server.generator = SimpleNamespace(generate_async=lambda **kwargs: promise) + request = ChatCompletionRequest( + model=server.model, + messages=[{"role": "user", "content": "hi"}], + prompt_token_ids=[7, 8], + ) + gc.collect() + assert all(path.exists() for path in (embedding_file, position_ids_file, position_deltas_file)) + + with patch( + "tensorrt_llm.serve.openai_server.parse_chat_messages_coroutines", + side_effect=lambda *args, **kwargs: ([], asyncio.sleep(0, result=(None, None)), {}, []), + ): + response = await server.openai_mm_encoder( + request, server._synthetic_request(request.model_dump(exclude_none=True)) + ) + gc.collect() + + assert response.status_code == 200 + assert not position_ids_file.exists() + assert not position_deltas_file.exists() + delivered = json.loads(response.body)["choices"][0]["mm_embedding_handle"] + assert torch.equal( + SharedTensorContainer.from_dict(delivered).get_local_view(), torch.ones(2, 1024) + ) From 68927d35298775bad7e54d3710626ad5982da83c Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Tue, 17 Nov 2026 22:27:21 -0800 Subject: [PATCH 05/12] [TRTLLM-17766][refactor] Remove the mm_encoder_only path from PyExecutor Non-generation models no longer reach PyExecutor or ModelEngine, so the MM sampler and runner branch go. PyExecutor drops the KV manager None checks that only that path needed. Signed-off-by: Lori Ren --- tensorrt_llm/_torch/pyexecutor/_util.py | 13 +- .../pyexecutor/engine/runners/__init__.py | 11 +- .../_torch/pyexecutor/model_engine.py | 22 +- tensorrt_llm/_torch/pyexecutor/py_executor.py | 121 +++++------ .../_torch/pyexecutor/py_executor_creator.py | 25 +-- .../_torch/pyexecutor/sampler/sampler.py | 190 +----------------- .../_torch/speculative/spec_sampler_base.py | 3 - .../pyexecutor/engine/runners/test_common.py | 12 +- .../pyexecutor/test_iter_stats_populate.py | 12 +- .../_torch/pyexecutor/test_model_engine.py | 32 +-- .../test_multimodal_embedding_lengths.py | 189 +---------------- .../_torch/pyexecutor/test_py_executor.py | 6 - ...est_py_executor_creator_no_kv_execution.py | 7 +- .../pyexecutor/test_remote_tail_adp_phase.py | 2 +- 14 files changed, 88 insertions(+), 557 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index b77df9ef8fe1..691ce9c37193 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -90,7 +90,7 @@ PeftCacheManager, ResourceManager, ResourceManagerType) from .runtime_hooks import runtime_hooks_of -from .sampler import EarlyStopWithMMResult, TorchSampler +from .sampler import TorchSampler from .sampler.penalties import PenaltyStore from .scheduler import (BindCapacityScheduler, BindMicroBatchScheduler, KVCacheV2Scheduler, MultimodalEagerEncoderScheduler, @@ -6012,7 +6012,6 @@ def instantiate_sampler( *, max_batch_size: int, max_beam_width: int, - mm_encoder_only: bool, speculative_config: SpeculativeConfig, max_num_sequences: Optional[int] = None, ): @@ -6052,16 +6051,6 @@ def instantiate_sampler( ) return sampler - if mm_encoder_only: - # NOTE: handle model outputs specially for mm encoder executor/engine. - # Only export results on ranks whose responses reach the frontend - # (see PyExecutor._enqueue_responses): rank 0, plus the attention-DP - # ranks gathered into it. A CUDA IPC handle exported on any other rank - # is never opened, so PyTorch would never release its allocation. - returns_responses = mapping.rank == 0 or (mapping.enable_attention_dp - and 0 in mapping.tp_group) - return EarlyStopWithMMResult(return_mm_results=returns_responses, - max_num_tokens=engine.max_num_tokens) return TorchSampler(sampler_args) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py index 4b4ee6328836..9714c0ab3b25 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py @@ -9,12 +9,9 @@ from torch import nn -from tensorrt_llm.llmapi.llm_args import TorchLlmArgs - from .decoder import DecoderRunner from .encoder import EncoderRunner from .encoder_decoder import EncoderDecoderRunner -from .mm_encoder import MultimodalEncoderRunner if TYPE_CHECKING: from .interface import ModelRunner @@ -27,14 +24,8 @@ ] -def resolve_runner_type( - model: nn.Module, - llm_args: TorchLlmArgs, -) -> type[ModelRunner]: +def resolve_runner_type(model: nn.Module) -> type[ModelRunner]: """Resolve the startup runner class without constructing its dependencies.""" - if llm_args.mm_encoder_only: - return MultimodalEncoderRunner - if model.model_config.is_encoder_decoder: return EncoderDecoderRunner diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 6e70f167e39a..14e1a6a00679 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -64,10 +64,9 @@ from .engine.runners import resolve_runner_type from .engine.runners.common import make_scheduled_inputs, resolve_mrope_position_deltas_cache from .engine.runners.decoder import DecoderRunner, DecoderRunnerConfig -from .engine.runners.encoder import EncoderConfig, resolve_scheduled_encoder_args +from .engine.runners.encoder import EncoderConfig from .engine.runners.encoder_decoder import EncoderDecoderRunner, EncoderDecoderRunnerConfig from .engine.runners.interface import ModelRunner, ScheduledInputs, ScheduledModelRunner -from .engine.runners.mm_encoder import MultimodalEncoderRunner from .guided_decoder import CapturableGuidedDecoder from .llm_request import LlmRequest from .model_loader import ModelLoader @@ -147,12 +146,10 @@ def __init__( model_weights_restore_mode=model_weights_restore_mode, ) if not self.model.model_config.is_generation: - if not self.llm_args.mm_encoder_only: - raise ValueError( - "Non-generation models run through ScheduledEncoderExecutor, which does not " - "use ModelEngine." - ) - self.llm_args = resolve_scheduled_encoder_args(self.llm_args) + raise ValueError( + "Non-generation models run through ScheduledEncoderExecutor, which does not " + "use ModelEngine." + ) self._validate_breakable_cuda_graph_compatibility() self._init_from_model() self._size_cuda_graphs() @@ -455,12 +452,7 @@ def _init_execution_services(self) -> None: self.attn_backend = resolve_attention_backend(self.model, self.llm_args) - runner_cls = resolve_runner_type(self.model, self.llm_args) - if self.is_spec_decode and issubclass(runner_cls, MultimodalEncoderRunner): - raise ValueError( - f"{runner_cls.__name__} does not support speculative decoding; " - "multimodal encoder models must not set speculative_config." - ) + runner_cls = resolve_runner_type(self.model) if self.is_spec_decode: update_spec_config_from_loaded_model(self.spec_config, self.model) @@ -493,8 +485,6 @@ def _initialize_runner(self, runner_cls: type[ModelRunner]) -> ModelRunner: assert self._model_caller is not None if issubclass(runner_cls, EncoderDecoderRunner): return self._initialize_encoder_decoder_runner(runner_cls) - if issubclass(runner_cls, MultimodalEncoderRunner): - return runner_cls.create(self.model, moe_load_balancer=self.moe_load_balancer) if issubclass(runner_cls, DecoderRunner): return self._initialize_decoder_runner(runner_cls) raise TypeError( diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 129faf8ae063..6b4320e5304c 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -690,6 +690,8 @@ class _ExecutorExecutionConfig: class PyExecutor: + # Requests generate tokens, rather than finish after one forward. + is_generation_model = True # True while the overlap loop runs with the execution stream current # (_overlap_loop_stream_scope). _loop_on_execution_stream = False @@ -1162,7 +1164,7 @@ def _initialize_execution(self, resources: PyExecutorResources) -> None: # context resources before transfer or response handling can terminate # a request. self._prefetched_request_ids: set[int] = set() - self.enable_kv_cache_events = self.kv_cache_manager is not None and ( + self.enable_kv_cache_events = ( self.kv_cache_manager.event_buffer_max_size > 0 or getattr( self.kv_cache_manager, "streaming_kv_events_enabled", False)) # Event routing also needs the sum of actual per-rank capacities after @@ -1186,7 +1188,7 @@ def _initialize_execution(self, resources: PyExecutorResources) -> None: self._kv_load_tracker.snapshot = aggregate_load(rank_loads) elif initial_sample: self._kv_load_tracker.record_local(initial_sample) - self.enable_kv_cache_reuse = self.kv_cache_manager is not None and self.kv_cache_manager.enable_block_reuse + self.enable_kv_cache_reuse = self.kv_cache_manager.enable_block_reuse # Router Replay (R3): SharedRouteCache bound, derived once from the KV # pool token capacity on the first non-warmup step (0 = block reuse off, # -1 = capacity unknown -> the capturer keeps its own fallback bound). @@ -1784,11 +1786,6 @@ def _setup_encoder_perf_metrics(self) -> None: and isinstance(model, MultimodalModelMixin)): model._encoder_forward_context = self.perf_manager.record_encoder_forward - @property - def is_generation_model(self) -> bool: - """Whether requests generate tokens, rather than finish after one forward.""" - return self.sampler.is_generation_model() - @property def metrics(self) -> dict[str, float | dict[str, float]]: """Return executor construction and model-engine startup metrics.""" @@ -1861,10 +1858,6 @@ def _maybe_init_kv_connector_manager(self) -> None: raise ValueError( "KV connector and executor attention-DP modes must agree.") - if self.kv_cache_manager is None: - raise ValueError( - "KV Cache Connector requires a KV Cache Manager.") - is_kv_cache_manager_v2 = isinstance(self.kv_cache_manager, KVCacheManagerV2) @@ -3011,8 +3004,6 @@ def get_latest_iteration_stats(self): def _read_kv_cache_capacity(self) -> dict: kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is None: - return {} block_counts = getattr(type(kv_cache_manager), "get_primary_block_counts", None) @@ -3067,7 +3058,7 @@ def get_kv_cache_load(self) -> dict: def get_latest_kv_cache_events(self): kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if not kv_cache_manager or not self.enable_kv_cache_events: + if not self.enable_kv_cache_events: return [] events = kv_cache_manager.get_latest_events(0) @@ -3490,36 +3481,34 @@ def _update_iter_stats( kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is not None: - kv_stats = kv_cache_manager.get_kv_cache_stats() - kv_stats_to_save = KvCacheStats() - kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks - kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks - kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks - kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block - kv_stats_to_save.alloc_total_blocks = kv_stats.alloc_total_blocks - kv_stats_to_save.alloc_new_blocks = kv_stats.alloc_new_blocks - kv_stats_to_save.reused_blocks = kv_stats.reused_blocks - kv_stats_to_save.missed_blocks = kv_stats.missed_blocks - kv_stats_to_save.cache_hit_rate = kv_stats.cache_hit_rate - self._fill_kv_host_stats(kv_stats_to_save, kv_cache_manager) - stats.kv_cache_stats = kv_stats_to_save - - # Collect per-iteration stats (with deltas) at configured interval. - # Between calls, C++ deltas accumulate so the reported values cover multiple iterations. - # Key the interval on the record's construction iteration, not - # the live counter: under the overlap and PP schedulers this runs - # in a later loop, and with iter_perf_stats_interval > 1 only - # sampled construction iterations reach this point, so a check on - # the live counter could never match. - # Guard: only fetch once per record iteration. - if (stats.iter % self._kv_iter_stats_interval == 0 - and self._last_kv_iter_stats_fetch_iter != stats.iter): - self._latest_kv_iter_stats = kv_cache_manager.get_iteration_stats( - ) - self._last_kv_iter_stats_fetch_iter = stats.iter - else: - self._latest_kv_iter_stats = None + kv_stats = kv_cache_manager.get_kv_cache_stats() + kv_stats_to_save = KvCacheStats() + kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks + kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks + kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks + kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block + kv_stats_to_save.alloc_total_blocks = kv_stats.alloc_total_blocks + kv_stats_to_save.alloc_new_blocks = kv_stats.alloc_new_blocks + kv_stats_to_save.reused_blocks = kv_stats.reused_blocks + kv_stats_to_save.missed_blocks = kv_stats.missed_blocks + kv_stats_to_save.cache_hit_rate = kv_stats.cache_hit_rate + self._fill_kv_host_stats(kv_stats_to_save, kv_cache_manager) + stats.kv_cache_stats = kv_stats_to_save + + # Collect per-iteration stats (with deltas) at configured interval. + # Between calls, C++ deltas accumulate so the reported values cover multiple iterations. + # Key the interval on the record's construction iteration, not + # the live counter: under the overlap and PP schedulers this runs + # in a later loop, and with iter_perf_stats_interval > 1 only + # sampled construction iterations reach this point, so a check on + # the live counter could never match. + # Guard: only fetch once per record iteration. + if (stats.iter % self._kv_iter_stats_interval == 0 + and self._last_kv_iter_stats_fetch_iter != stats.iter): + self._latest_kv_iter_stats = kv_cache_manager.get_iteration_stats() + self._last_kv_iter_stats_fetch_iter = stats.iter + else: + self._latest_kv_iter_stats = None rank_metrics = { "rank": self.dist.rank, @@ -3529,20 +3518,19 @@ def _update_iter_stats( "cpuMemUsage": stats.cpu_mem_usage, "pinnedMemUsage": stats.pinned_mem_usage, } - if kv_cache_manager is not None: - rank_metrics["kvCacheStats"] = { - "maxNumBlocks": kv_stats.max_num_blocks, - "freeNumBlocks": kv_stats.free_num_blocks, - "usedNumBlocks": kv_stats.used_num_blocks, - "tokensPerBlock": kv_stats.tokens_per_block, - "allocTotalBlocks": kv_stats.alloc_total_blocks, - "allocNewBlocks": kv_stats.alloc_new_blocks, - "reusedBlocks": kv_stats.reused_blocks, - "missedBlocks": kv_stats.missed_blocks, - "cacheHitRate": kv_stats.cache_hit_rate, - } - append_kv_cache_iteration_stats(rank_metrics, - self._latest_kv_iter_stats) + rank_metrics["kvCacheStats"] = { + "maxNumBlocks": kv_stats.max_num_blocks, + "freeNumBlocks": kv_stats.free_num_blocks, + "usedNumBlocks": kv_stats.used_num_blocks, + "tokensPerBlock": kv_stats.tokens_per_block, + "allocTotalBlocks": kv_stats.alloc_total_blocks, + "allocNewBlocks": kv_stats.alloc_new_blocks, + "reusedBlocks": kv_stats.reused_blocks, + "missedBlocks": kv_stats.missed_blocks, + "cacheHitRate": kv_stats.cache_hit_rate, + } + append_kv_cache_iteration_stats(rank_metrics, + self._latest_kv_iter_stats) self._perf_metrics.collect_rank(rank_metrics, capture_iter=self.iter_counter) @@ -6805,14 +6793,13 @@ def _emit_initial_stats(self) -> None: stats = self._get_init_iter_stats(0, 0) kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is not None: - kv_stats = kv_cache_manager.get_kv_cache_stats() - kv_stats_to_save = KvCacheStats() - kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks - kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block - kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks - kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks - stats.kv_cache_stats = kv_stats_to_save + kv_stats = kv_cache_manager.get_kv_cache_stats() + kv_stats_to_save = KvCacheStats() + kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks + kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block + kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks + kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks + stats.kv_cache_stats = kv_stats_to_save self._append_iter_stats(stats) def _build_step_scope_label(self, @@ -9711,8 +9698,6 @@ def _respond_if_invalid(request: LlmRequest) -> bool: def _add_kv_cache_events(self): kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if not kv_cache_manager: - return # Flush iteration events at each iteration to ensure that events have enough time # to be transferred to main thread when user needs them. kv_cache_manager.flush_iteration_events() diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 9cb797461e7c..4ebf4ad72766 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -465,11 +465,8 @@ def _branch_snapshot_ctx_chunk_config( return (ContextChunkingPolicy.FORCE_CHUNK, chunk_unit_size) -def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping, - is_generation: bool) -> None: +def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping) -> None: """Reject the settings a non-generation model cannot run with; warn on those it ignores.""" - if is_generation: - return if mapping.has_pp(): raise ValueError( "Pipeline parallelism is not supported for non-generation models " @@ -488,7 +485,7 @@ def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping, def _check_scheduled_encoder_support(llm_args: TorchLlmArgs, mapping: Mapping) -> None: """Reject the settings ScheduledEncoderExecutor cannot serve a non-generation model with.""" - _check_no_kv_execution(llm_args, mapping, is_generation=False) + _check_no_kv_execution(llm_args, mapping) if mapping.cp_size > 1: raise ValueError( "Context parallelism is not supported for non-generation models; " @@ -788,7 +785,6 @@ def _create_py_executor( guided_decoding_config = get_guided_decoding_config( llm_args.guided_decoding_backend, tokenizer) - mm_encoder_only = llm_args.mm_encoder_only enable_chunked_context = llm_args.enable_chunked_prefill ( @@ -850,16 +846,6 @@ def _create_py_executor( llm_args.disable_overlap_scheduler, sparse_attention_config=llm_args.sparse_attention_config) - if mm_encoder_only: - llm_args.mm_encoder_only = True - llm_args.disable_overlap_scheduler = True - - # Disable overlap scheduler for multimodal encoder-only mode - logger.warning( - "Disabling overlap scheduler for multimodal encoder-only mode. " - "The overlap scheduler is designed for generation models and is not needed " - "when only processing vision encoder inputs.") - _rank_dspark_on_host_without_overlap(spec_config, llm_args.disable_overlap_scheduler) @@ -1056,12 +1042,6 @@ def allocation_scope(current_stage: ExecutorMemoryType): model_weights_restore_mode=model_weights_restore_mode, ) - is_generation = model_engine.model.model_config.is_generation - _check_no_kv_execution(llm_args, mapping, is_generation) - if not is_generation: - enable_chunked_context = False - model_engine.attn_runtime_features.chunked_prefill = False - # Model loading can disable reuse after runtime features were created. model_engine.attn_runtime_features.cache_reuse = ( kv_cache_config.enable_block_reuse) @@ -1277,7 +1257,6 @@ def allocation_scope(current_stage: ExecutorMemoryType): mapping, max_batch_size=max_batch_size, max_beam_width=max_beam_width, - mm_encoder_only=mm_encoder_only, speculative_config=spec_config, max_num_sequences=max_num_seq_slots, ) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index d31035bd124e..2ec1df6cdf49 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -13,10 +13,8 @@ # limitations under the License. """Sampler orchestration: the entry points the executor drives each step. -Holds the :class:`Sampler` ABC and its ``SampleState`` types, the trivial -early-stop sampler (:class:`EarlyStopWithMMResult` for the -multimodal-encoder-only engine), and :class:`TorchSampler` -- the PyTorch -sampling path. ``TorchSampler`` owns no sampling logic of its own beyond +Holds the :class:`Sampler` ABC and its ``SampleState`` types, and +:class:`TorchSampler` -- the PyTorch sampling path. ``TorchSampler`` owns no sampling logic of its own beyond batching and orchestration: each feature (beam search, penalties, token bans, top-p decay, finish reasons, log-probs, seeds) lives in its own module and is held here as a handler, driven through ``setup_sampler_step`` / @@ -27,7 +25,7 @@ from abc import ABC, abstractmethod from collections.abc import Hashable, Iterable from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, cast import torch @@ -36,7 +34,6 @@ from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.bindings.internal.batch_manager import add_new_tokens_to_requests from tensorrt_llm.executor.result import Logprob -from tensorrt_llm.executor.utils import context_length_exceeded_message from tensorrt_llm.logger import logger from ...utils import torch_multi_arange @@ -288,10 +285,6 @@ def beam_width(requests: Iterable[LlmRequest]) -> int: return req.py_beam_width return 0 - @abstractmethod - def is_generation_model(self) -> bool: - raise NotImplementedError - def validate_request(self, request: LlmRequest) -> None: """Validate that the request can be processed by the sampler. @@ -309,179 +302,6 @@ def validate_request(self, request: LlmRequest) -> None: """ -def _validate_single_forward_request(request: LlmRequest, max_num_tokens: Optional[int]) -> None: - """Reject a prompt that does not fit the one forward a non-generation request runs in.""" - if max_num_tokens is not None and request.prompt_len > max_num_tokens: - raise ValueError( - context_length_exceeded_message( - max_context_length=max_num_tokens, num_prompt_tokens=request.prompt_len - ) - ) - - -@dataclass(kw_only=True) -class MultimodalResult: - mm_embeddings: List[torch.Tensor] - # needed to torch.split the mm_embeddings into item-wise chunks - mm_embedding_lengths: List[List[int]] - # needed when requests mix text-only and multimodal ones - mm_embedding_request_indices: List[int] - # number of context requests in the batch - num_context_requests: int - # Can be used to include e.g. `mrope_position_ids`, etc. - extra_data: Optional[Dict[str, Any]] = None - - def __post_init__(self) -> None: - num_embeddings = len(self.mm_embeddings) - metadata = (self.extra_data or {}).get("mm_embedding_metadata") - if metadata is not None and len(metadata) != num_embeddings: - raise ValueError("mm_embedding_metadata batch size does not match mm_embeddings") - num_lengths = len(self.mm_embedding_lengths) - if num_lengths != num_embeddings: - raise ValueError( - "mm_embedding_lengths batch size does not match mm_embeddings: " - f"{num_lengths} != {num_embeddings}" - ) - num_request_indices = len(self.mm_embedding_request_indices) - if num_request_indices != num_embeddings: - raise ValueError( - "mm_embedding_request_indices batch size does not match " - f"mm_embeddings: {num_request_indices} != {num_embeddings}" - ) - for result_index, (mm_embedding, mm_embedding_lengths) in enumerate( - zip(self.mm_embeddings, self.mm_embedding_lengths, strict=True) - ): - actual_rows = len(mm_embedding) - expected_rows = sum(mm_embedding_lengths) - if actual_rows != expected_rows: - raise ValueError( - f"mm_embedding shape mismatch for result {result_index}: " - f"{actual_rows} != {expected_rows}" - ) - for request_index in self.mm_embedding_request_indices: - if request_index < 0 or request_index >= self.num_context_requests: - raise ValueError( - "mm_embedding_request_indices contains an invalid request " - f"index: {request_index} not in [0, {self.num_context_requests})" - ) - - @classmethod - def from_model_outputs( - cls, model_outputs: Dict[str, Any], num_context_requests: int - ) -> "MultimodalResult": - result_keys = { - "mm_embeddings", - "mm_embedding_lengths", - "mm_embedding_request_indices", - } - return cls( - mm_embeddings=model_outputs["mm_embeddings"], - mm_embedding_lengths=model_outputs["mm_embedding_lengths"], - mm_embedding_request_indices=model_outputs["mm_embedding_request_indices"], - num_context_requests=num_context_requests, - extra_data={ - key: value for key, value in model_outputs.items() if key not in result_keys - }, - ) - - -@dataclass(kw_only=True) -class SampleStateWithMMResult(SampleState[SampleStateTensors, SampleStateTensors]): - data: MultimodalResult - - -class EarlyStopWithMMResult(Sampler[SampleStateWithMMResult]): - """ - Use for skipping decoding step for non generation model, and return the batch_output (such as mm_embeddings) - - Args: - return_mm_results: Whether to attach the multimodal outputs to the - requests' results. Disable it on ranks whose responses never reach - the frontend: the results are exported as CUDA IPC handles, and - PyTorch keeps an exported allocation alive until a consumer opens - and releases its handle. - max_num_tokens: The engine's token budget. A request runs whole in one - forward, so a prompt longer than this is rejected; None skips the check. - """ - - SampleState: TypeAlias = SampleStateWithMMResult - - def __init__( - self, return_mm_results: bool = True, max_num_tokens: Optional[int] = None - ) -> None: - self.return_mm_results = return_mm_results - self.max_num_tokens = max_num_tokens - - @override - def validate_request(self, request: LlmRequest) -> None: - _validate_single_forward_request(request, self.max_num_tokens) - - @override - def sample_async( - self, - scheduled_requests: ScheduledRequests, - model_outputs: dict[str, Any], - num_context_logits_prefix_sum: list[int], - resource_manager: Optional[ResourceManager] = None, - ) -> SampleState: - # from model_outputs to MultimodalResult - assert not scheduled_requests.generation_requests - data = MultimodalResult.from_model_outputs( - model_outputs, scheduled_requests.num_context_requests - ) - return self.SampleState(requests=scheduled_requests.context_requests, data=data) - - @override - def update_requests( - self, - state: SampleState, - resource_manager: Optional[ResourceManager] = None, - ) -> None: - # resource_manager will not be used in this function, just for interface consistency. - assert isinstance(state, SampleState) - requests = state.requests - mm_embeddings = state.data.mm_embeddings - extra_data = state.data.extra_data or {} - mrope_position_ids = extra_data.get("mrope_position_ids", None) - mrope_position_deltas = extra_data.get("mrope_position_deltas", None) - for request in requests: - request.state = LlmRequestState.GENERATION_COMPLETE - # NOTE: This is a hack: set finish reason manually and set the beam 0 - request.set_finished_reason(FinishReason.LENGTH, 0) - - if not self.return_mm_results: - return - - request_indices = state.data.mm_embedding_request_indices - for result_index, (request_index, mm_embedding) in enumerate( - zip(request_indices, mm_embeddings, strict=True) - ): - request = requests[request_index] - mm_embedding_lengths = state.data.mm_embedding_lengths[result_index] - - embedding_metadata = extra_data.get("mm_embedding_metadata") - if embedding_metadata is None: - request.py_result.append_mm_embeddings(mm_embedding, mm_embedding_lengths) - else: - request.py_result.append_mm_embeddings( - mm_embedding, mm_embedding_lengths, embedding_metadata[result_index] - ) - - # Store mrope data if available - if mrope_position_ids is not None and mrope_position_deltas is not None: - mrope_index = ( - request_index if len(mrope_position_ids) == len(requests) else result_index - ) - request.py_result.set_mrope_position( - mrope_position_ids[mrope_index], - mrope_position_deltas[mrope_index], - ) - - @override - def is_generation_model(self) -> bool: - return False - - @dataclass(kw_only=True) class SamplingRequestsMetadata: """Metadata for the sampling requests.""" @@ -555,10 +375,6 @@ def start_beam_histories(self, requests: list[LlmRequest]) -> PendingBeamHistori """ return self._beam_search.start_beam_histories(requests) - @override - def is_generation_model(self) -> bool: - return True - @dataclass(kw_only=True) @dataclass(kw_only=True) class Store: diff --git a/tensorrt_llm/_torch/speculative/spec_sampler_base.py b/tensorrt_llm/_torch/speculative/spec_sampler_base.py index f5c5f6a3a909..a587b1ad08b7 100644 --- a/tensorrt_llm/_torch/speculative/spec_sampler_base.py +++ b/tensorrt_llm/_torch/speculative/spec_sampler_base.py @@ -194,9 +194,6 @@ class SpecSampler(Sampler[SampleStateSpec], AsyncWorkerMixin): SampleState = SampleStateSpec - def is_generation_model(self) -> bool: - return True - def validate_request(self, request: LlmRequest) -> None: """Reject sampling parameters the one-model speculative path cannot honor. diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py index 5208cc6ff496..7300e57b9f2e 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py @@ -26,7 +26,6 @@ token_model_inputs, ) from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ModelRunner -from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.model_engine import ModelEngine from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm.llmapi.llm_args import PrefillCudaGraphBackend @@ -35,22 +34,19 @@ @pytest.mark.parametrize( - ("mm_encoder_only", "is_encoder_decoder", "runner_type"), + ("is_encoder_decoder", "runner_type"), [ - (True, False, MultimodalEncoderRunner), - (False, True, EncoderDecoderRunner), - (False, False, DecoderRunner), + (True, EncoderDecoderRunner), + (False, DecoderRunner), ], ) def test_resolve_runner_dispatches_startup_families( - mm_encoder_only: bool, is_encoder_decoder: bool, runner_type: type[ModelRunner], ) -> None: - args = SimpleNamespace(mm_encoder_only=mm_encoder_only) model = SimpleNamespace(model_config=SimpleNamespace(is_encoder_decoder=is_encoder_decoder)) - assert resolve_runner_type(model, args) is runner_type + assert resolve_runner_type(model) is runner_type @pytest.mark.parametrize( diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py index c7805379152a..22ea6807b5bb 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py @@ -186,8 +186,9 @@ def _build_fake_self(queued_items, model_engine_iter_states, *, enable_attention * ``executor_request_queue.get_request_queue_size()`` — for top-level ``num_queued_requests`` (not one of the fields exercised here) - * ``resource_manager.resource_managers.get(...)`` — returns None so - the KV-cache-stats block is skipped entirely + * ``resource_manager.resource_managers.get(...)`` — a KV cache manager + reporting zeroed ``KvCacheStats`` and no per-iteration KV stats (the + KV fields are not exercised here) * ``drafter`` — None (spec-decode block no-ops when ``stats.specdec_stats`` is None, which is the default on a fresh ``IterationStats``) @@ -209,7 +210,12 @@ def _build_fake_self(queued_items, model_engine_iter_states, *, enable_attention fake.iter_counter = 1 fake.executor_request_queue.get_request_queue_size.return_value = len(queued_items) fake.executor_request_queue.get_request_queue.return_value.queue = queued_items - fake.resource_manager.resource_managers.get.return_value = None + kv_cache_manager = MagicMock() + kv_cache_manager.get_kv_cache_stats.return_value = KvCacheStats() + kv_cache_manager.get_iteration_stats.return_value = None + fake.resource_manager.resource_managers.get.return_value = kv_cache_manager + fake._kv_iter_stats_interval = 1 + fake._last_kv_iter_stats_fetch_iter = None fake.drafter = None fake.model_engine = types.SimpleNamespace(iter_states=model_engine_iter_states) fake.enable_attention_dp = enable_attention_dp diff --git a/tests/unittest/_torch/pyexecutor/test_model_engine.py b/tests/unittest/_torch/pyexecutor/test_model_engine.py index e2d378958ce6..b16d4722336b 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_engine.py +++ b/tests/unittest/_torch/pyexecutor/test_model_engine.py @@ -52,7 +52,7 @@ from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.utils import torch_compiling from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor -from tensorrt_llm.llmapi import CudaGraphConfig, KvCacheConfig, SADecodingConfig +from tensorrt_llm.llmapi import CudaGraphConfig, KvCacheConfig from tensorrt_llm.llmapi.llm_args import MultimodalConfig, PrefillCudaGraphBackend, TorchLlmArgs from tensorrt_llm.mapping import Mapping @@ -656,15 +656,12 @@ def test_model_engine_rejects_encode_only() -> None: @pytest.mark.cpu_only @pytest.mark.parametrize("backend", ["piecewise", "breakable"]) -@pytest.mark.parametrize("mm_encoder_only", [False, True]) def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( - monkeypatch: pytest.MonkeyPatch, backend: str, mm_encoder_only: bool + monkeypatch: pytest.MonkeyPatch, backend: str ) -> None: model = DummyLegacyMultimodalIndexModel() - model.model_config = SimpleNamespace(is_generation=not mm_encoder_only) - llm_args = TorchLlmArgs( - model="dummy", mm_encoder_only=mm_encoder_only, prefill_cuda_graph_backend=backend - ) + model.model_config = SimpleNamespace(is_generation=True) + llm_args = TorchLlmArgs(model="dummy", prefill_cuda_graph_backend=backend) monkeypatch.setattr(model_setup, "_configure_deep_gemm_pdl", Mock()) monkeypatch.setattr( model_engine_module, @@ -678,7 +675,7 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( ) monkeypatch.setattr(ModelEngine, "__del__", lambda self: None) engine = ModelEngine.__new__(ModelEngine) - rejects_breakable = not mm_encoder_only and backend == "breakable" + rejects_breakable = backend == "breakable" expected_error = ValueError if rejects_breakable else RuntimeError message = "multimodal models" if rejects_breakable else "model configuration resolved" with pytest.raises(expected_error, match=message): @@ -686,10 +683,7 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( engine, model_path="dummy", llm_args=llm_args, mapping=Mapping(), model=model ) - expected_backend = "disabled" if mm_encoder_only else backend - assert engine.llm_args.prefill_cuda_graph_backend == PrefillCudaGraphBackend(expected_backend) - if mm_encoder_only: - assert engine.llm_args.torch_compile_config is None + assert engine.llm_args.prefill_cuda_graph_backend == PrefillCudaGraphBackend(backend) class ModelEngineTestCase(unittest.TestCase): @@ -778,20 +772,6 @@ def test_model_engine_rejects_non_generation_models(self) -> None: with self.assertRaisesRegex(ValueError, "ScheduledEncoderExecutor"): DummyModelEngine(llm_args, torch.half, is_generation=False) - def test_no_kv_cache_runners_reject_speculative_config(self) -> None: - llm_args = TorchLlmArgs( - model="dummy", max_batch_size=4, max_num_tokens=64, mm_encoder_only=True - ) - with self.assertRaisesRegex( - ValueError, "MultimodalEncoderRunner does not support speculative decoding" - ): - DummyModelEngine( - llm_args, - torch.half, - spec_config=SADecodingConfig(max_draft_len=1), - is_generation=False, - ) - @pytest.mark.cpu_only def test_runner_engine_shared_entries_skip_decoder_state(self) -> None: runner = Mock(spec=MultimodalEncoderRunner) diff --git a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py index 6bcae7ccd3a2..e78e5cb64283 100644 --- a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py +++ b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py @@ -6,22 +6,14 @@ import pytest import torch -from tensorrt_llm._torch.pyexecutor import _util from tensorrt_llm._torch.pyexecutor.encoder_completion import MultimodalEmbeddingCompletion -from tensorrt_llm._torch.pyexecutor.llm_request import ( - LlmRequestState, - PyResult, - get_multimodal_embedding_lengths, -) -from tensorrt_llm._torch.pyexecutor.sampler import EarlyStopWithMMResult, MultimodalResult +from tensorrt_llm._torch.pyexecutor.llm_request import PyResult, get_multimodal_embedding_lengths from tensorrt_llm._torch.shared_tensor import SharedTensorContainer -from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalItemMetadata, MultimodalParams, ) -from tensorrt_llm.mapping import Mapping @pytest.mark.parametrize( @@ -171,101 +163,6 @@ class _FakeRequest: def __init__(self, multimodal_lengths=None): self.multimodal_lengths = multimodal_lengths self.py_result = _FakePyResult() - self.state = None - self.finished_reason = None - - def set_finished_reason(self, reason, beam): - self.finished_reason = (reason, beam) - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_aligns_mixed_batch_by_request_index(): - """Sparse MM encoder outputs attach to the original request index.""" - text_request = _FakeRequest() - mm_request = _FakeRequest(multimodal_lengths=[4]) - sampler = EarlyStopWithMMResult() - state = sampler.SampleState( - requests=[text_request, mm_request], - data=MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[1], - mm_embedding_lengths=[[4]], - num_context_requests=2, - extra_data={ - "mrope_position_ids": ["text-pos", "mm-pos"], - "mrope_position_deltas": ["text-delta", "mm-delta"], - }, - ), - ) - - sampler.update_requests(state) - - assert text_request.state == LlmRequestState.GENERATION_COMPLETE - assert mm_request.state == LlmRequestState.GENERATION_COMPLETE - assert text_request.finished_reason == (FinishReason.LENGTH, 0) - assert mm_request.finished_reason == (FinishReason.LENGTH, 0) - assert len(text_request.py_result.mm_embeddings) == 0 - [(mm_embedding, mm_embedding_lengths)] = mm_request.py_result.mm_embeddings - assert mm_embedding.shape == (4, 2) - assert mm_embedding_lengths == [4] - assert text_request.py_result.mrope_position is None - assert mm_request.py_result.mrope_position == ("mm-pos", "mm-delta") - - -@pytest.mark.parametrize( - "mapping,exports_results", - [ - pytest.param(Mapping(), True, id="single_rank"), - pytest.param(Mapping(world_size=2, rank=1, tp_size=2), False, id="tp_follower"), - pytest.param( - Mapping(world_size=2, rank=1, tp_size=2, enable_attention_dp=True), - True, - id="attention_dp_follower", - ), - pytest.param( - Mapping(world_size=4, rank=2, tp_size=2, pp_size=2, enable_attention_dp=True), - False, - id="attention_dp_later_pp_stage", - ), - ], -) -@pytest.mark.cpu_only -def test_mm_encoder_sampler_exports_results_only_on_response_ranks(mapping, exports_results): - """Ranks whose responses never reach the frontend must not export MM results.""" - sampler = _util.instantiate_sampler( - SimpleNamespace(max_seq_len=16, max_num_tokens=16, spec_config=None), - SimpleNamespace( - sampler_force_async_worker=False, - disable_overlap_scheduler=True, - enable_speculative_beam_history_d2h=False, - ), - mapping, - max_batch_size=1, - max_beam_width=1, - mm_encoder_only=True, - speculative_config=None, - max_num_sequences=1, - ) - with pytest.raises(ValueError, match="maximum context length is 16 tokens"): - sampler.validate_request(SimpleNamespace(prompt_len=17)) - request = _FakeRequest(multimodal_lengths=[4]) - state = sampler.SampleState( - requests=[request], - data=MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[0], - mm_embedding_lengths=[[4]], - num_context_requests=1, - extra_data={"mrope_position_ids": ["pos"], "mrope_position_deltas": ["delta"]}, - ), - ) - - sampler.update_requests(state) - - assert request.state == LlmRequestState.GENERATION_COMPLETE - assert request.finished_reason == (FinishReason.LENGTH, 0) - assert bool(request.py_result.mm_embeddings) is exports_results - assert (request.py_result.mrope_position is not None) is exports_results @pytest.mark.parametrize("exports", [True, False], ids=["response_rank", "follower_rank"]) @@ -324,87 +221,3 @@ def test_py_result_cuda_mm_embedding_handles_stay_cuda_backed(): handles = result.mm_embedding_handles assert handles is not None assert [handle["method_key"] for handle in handles] == [1, 1] - - -class _FakeScheduledRequests: - def __init__(self, num_context_requests): - self.generation_requests = [] - self.context_requests = [_FakeRequest() for _ in range(num_context_requests)] - - @property - def num_context_requests(self): - return len(self.context_requests) - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_builds_typed_result_from_model_outputs(): - """Sampler converts raw model-output dicts into typed MM results.""" - sampler = EarlyStopWithMMResult() - model_outputs = { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [[4]], - "mrope_position_ids": ["pos"], - } - state = sampler.sample_async( - _FakeScheduledRequests(2), - model_outputs, - [], - ) - - assert state.data.mm_embedding_request_indices == [1] - assert state.data.mm_embedding_lengths == [[4]] - assert state.data.extra_data == {"mrope_position_ids": ["pos"]} - assert set(model_outputs) == { - "mm_embeddings", - "mm_embedding_request_indices", - "mm_embedding_lengths", - "mrope_position_ids", - } - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_rejects_typed_result_batch_mismatch(): - """MM embedding arrays must stay length-aligned with request indices.""" - sampler = EarlyStopWithMMResult() - - with pytest.raises(ValueError, match="batch size"): - sampler.sample_async( - _FakeScheduledRequests(2), - { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [], - }, - [], - ) - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_rejects_invalid_request_index(): - """MM encoder output cannot target a request outside the scheduled batch.""" - sampler = EarlyStopWithMMResult() - - with pytest.raises(ValueError, match="invalid request index"): - sampler.sample_async( - _FakeScheduledRequests(1), - { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [[4]], - }, - [], - ) - - -@pytest.mark.cpu_only -def test_multimodal_result_rejects_embedding_shape_mismatch(): - """Per-item lengths must sum to the attached embedding rows.""" - with pytest.raises(ValueError, match="shape mismatch"): - MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[0], - mm_embedding_lengths=[[3]], - num_context_requests=1, - extra_data={}, - ) diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index 608bf2a4c206..f87996578772 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -1906,12 +1906,6 @@ def _make_executor_with_kv_cache_manager(kv_cache_manager: object | None) -> PyE return executor -def test_get_kv_cache_capacity_without_manager(): - executor = _make_executor_with_kv_cache_manager(None) - - assert executor.get_kv_cache_capacity() == {} - - @pytest.mark.parametrize("streaming_events", [False, True]) def test_get_kv_cache_capacity_from_stats(streaming_events): """KV capacity is available without consuming iteration stats.""" diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py index ff8ec84be13d..b3aef2c4c89f 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py @@ -34,12 +34,7 @@ def _args(): ) def test_non_generation_models_reject_unsupported_loops(mapping, error): with pytest.raises(ValueError, match=error): - _check_no_kv_execution(_args(), Mapping(**mapping), is_generation=False) - - -@pytest.mark.parametrize("mapping", [_PP2, _ADP], ids=["pp", "adp"]) -def test_generation_models_are_not_checked(mapping): - _check_no_kv_execution(_args(), Mapping(**mapping), is_generation=True) + _check_no_kv_execution(_args(), Mapping(**mapping)) def _encoder_args(**overrides): diff --git a/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py b/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py index 76184b321bbd..a0f8a2e55be5 100644 --- a/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py +++ b/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py @@ -389,7 +389,7 @@ def test_rebalance_reserves_execution_row_before_resuming_real_requests( ) -> None: executor = _make_executor() executor._enable_remote_tail_adp = True - executor.resource_manager = SimpleNamespace(resource_managers={}) + executor._read_kv_cache_capacity = Mock(return_value={}) real = SimpleNamespace(py_request_id=1) dummy = SimpleNamespace(py_request_id=2) executor.active_requests = [real] From b90110e287e50c2395d219fcfd15342f6b16647b Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Wed, 18 Nov 2026 19:03:47 -0800 Subject: [PATCH 06/12] [TRTLLM-17766][fix] Release exports on a worker background error and drop a cancelled pooled row Also tests that only rank 0 exports, and removes the generation check that every model reaching the PyExecutor creator now passes. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/encoder_completion.py | 5 +- tensorrt_llm/_torch/pyexecutor/llm_request.py | 5 + .../_torch/pyexecutor/py_executor_creator.py | 146 +++++++++--------- tensorrt_llm/executor/base_worker.py | 1 + .../_torch/modules/dwdp/test_dwdp_creator.py | 8 +- .../test_scheduled_encoder_executor.py | 52 ++++++- .../executor/test_undelivered_exports.py | 32 +++- 7 files changed, 162 insertions(+), 87 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py index e56b2d535210..0de2949534ea 100644 --- a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py +++ b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py @@ -23,7 +23,7 @@ def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: """Attach one forward's results to its requests before their responses are created.""" def retire(self, requests: list[LlmRequest]) -> None: - """Release what ``store`` exported for requests whose responses will not be published.""" + """Withdraw what ``store`` attached, for requests that finish without their result.""" class PoolingCompletion: @@ -37,7 +37,8 @@ def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) def retire(self, requests: list[LlmRequest]) -> None: - pass + for request in requests: + request.py_result.drop_context_logits() class MultimodalEmbeddingCompletion: diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 50b13dac06bd..a5f3c90bb9ee 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -598,6 +598,11 @@ def release_exports(self) -> None: self.diff.mrope_position_ids = None self.diff.mrope_position_deltas = None + def drop_context_logits(self) -> None: + """Drop the stored context logits; no response will deliver them.""" + self._context_logits = None + self.diff.context_logits_list = [] + def set_encoder_output(self, encoder_output: torch.Tensor): self._encoder_output = encoder_output self.diff.encoder_output = encoder_output diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 4ebf4ad72766..618ba92942e1 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -1384,82 +1384,78 @@ def allocation_scope(current_stage: ExecutorMemoryType): logger.info( f"[create_py_executor] Created execution_stream: {execution_stream}") - if model_engine.model.model_config.is_generation: - #NOTE: non-generation models do not have kv cache - - is_disagg = is_disagg_enabled(cache_transceiver_config) - is_hybrid = is_hybrid_linear( - model_engine.model.model_config.pretrained_config) - - if is_disagg and is_hybrid: - # NOTE: TRTLLM_USE_PY_MAMBA is an agg-mode-only override and has - # no effect in disagg. The disagg manager choice is driven by - # get_kv_cache_manager_cls and cache_transceiver_config. - if os.environ.get("TRTLLM_USE_PY_MAMBA", "0") == "1": - logger.warning( - "TRTLLM_USE_PY_MAMBA is ignored in disaggregated serving; " - "configure transceiver_runtime='PYTHON' with backend='NIXL' " - "to select MixedMambaHybridCacheManager.") - else: - logger.info("Disaggregated serving with hybrid model detected. " - "Using the configured Mamba cache manager.") - - # Get draft config for one-engine speculative decoding if available - draft_config = getattr(model_engine.model, 'draft_config', None) - - if not skip_est and is_decode_only(sparse_attention_config): - # The estimation dry run prefills dummy prompts, which a - # decode-only worker cannot execute. Size the KV cache from free - # memory instead, as with TRTLLM_SKIP_KV_CACHE_ESTIMATION=1. - logger.info("KV cache size estimation is not supported for a " - "decode-only worker, disable it.") - skip_est = True - - kv_cache_creator = KvCacheCreator( - model_engine=model_engine, - mapping=mapping, - net_max_seq_len=net_max_seq_len, - kv_connector_manager=kv_connector_manager, - max_num_tokens=max_num_tokens, - max_beam_width=max_beam_width, - tokens_per_block=tokens_per_block, - max_seq_len=max_seq_len, - max_batch_size=max_batch_size, - kv_cache_config=kv_cache_config, - llm_args=llm_args, - speculative_config=spec_config, - profiling_stage_data=profiling_stage_data, - sparse_attention_config=sparse_attention_config, - execution_stream=execution_stream, - draft_config=draft_config, - gen_preemption_headroom=getattr(model_engine, - "gen_preemption_headroom", 0), - skip_est=skip_est, - is_disagg=is_disagg, - ) - - if not skip_est: - log_memory_usage("after loading weights") - - estimating_kv_cache = kv_cache_creator.try_prepare_estimation() - - # Without the estimation, build_managers sizes the final KV cache from - # the device memory free now, so DWDP is set up first: the KV cache - # then gets the replicated dense weights dense DWDP frees, and the - # expert prefetch ring holds its memory before the KV cache takes its - # share of the rest. With the estimation, it is set up below, before - # the estimation run measures the memory. - if not estimating_kv_cache and (dwdp_manager is not None - or dense_dwdp_manager is not None): - manager = (dwdp_manager - if dwdp_manager is not None else dense_dwdp_manager) - manager.setup(model_engine.model) - dwdp_set_up = True + is_disagg = is_disagg_enabled(cache_transceiver_config) + is_hybrid = is_hybrid_linear( + model_engine.model.model_config.pretrained_config) + + if is_disagg and is_hybrid: + # NOTE: TRTLLM_USE_PY_MAMBA is an agg-mode-only override and has + # no effect in disagg. The disagg manager choice is driven by + # get_kv_cache_manager_cls and cache_transceiver_config. + if os.environ.get("TRTLLM_USE_PY_MAMBA", "0") == "1": + logger.warning( + "TRTLLM_USE_PY_MAMBA is ignored in disaggregated serving; " + "configure transceiver_runtime='PYTHON' with backend='NIXL' " + "to select MixedMambaHybridCacheManager.") + else: + logger.info("Disaggregated serving with hybrid model detected. " + "Using the configured Mamba cache manager.") + + # Get draft config for one-engine speculative decoding if available + draft_config = getattr(model_engine.model, 'draft_config', None) + + if not skip_est and is_decode_only(sparse_attention_config): + # The estimation dry run prefills dummy prompts, which a + # decode-only worker cannot execute. Size the KV cache from free + # memory instead, as with TRTLLM_SKIP_KV_CACHE_ESTIMATION=1. + logger.info("KV cache size estimation is not supported for a " + "decode-only worker, disable it.") + skip_est = True + + kv_cache_creator = KvCacheCreator( + model_engine=model_engine, + mapping=mapping, + net_max_seq_len=net_max_seq_len, + kv_connector_manager=kv_connector_manager, + max_num_tokens=max_num_tokens, + max_beam_width=max_beam_width, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + llm_args=llm_args, + speculative_config=spec_config, + profiling_stage_data=profiling_stage_data, + sparse_attention_config=sparse_attention_config, + execution_stream=execution_stream, + draft_config=draft_config, + gen_preemption_headroom=getattr(model_engine, "gen_preemption_headroom", + 0), + skip_est=skip_est, + is_disagg=is_disagg, + ) - with allocation_scope( - ExecutorMemoryType.INIT_KV_CACHE - if estimating_kv_cache else ExecutorMemoryType.KV_CACHE): - kv_cache_creator.build_managers(resources, estimating_kv_cache) + if not skip_est: + log_memory_usage("after loading weights") + + estimating_kv_cache = kv_cache_creator.try_prepare_estimation() + + # Without the estimation, build_managers sizes the final KV cache from + # the device memory free now, so DWDP is set up first: the KV cache + # then gets the replicated dense weights dense DWDP frees, and the + # expert prefetch ring holds its memory before the KV cache takes its + # share of the rest. With the estimation, it is set up below, before + # the estimation run measures the memory. + if not estimating_kv_cache and (dwdp_manager is not None + or dense_dwdp_manager is not None): + manager = (dwdp_manager + if dwdp_manager is not None else dense_dwdp_manager) + manager.setup(model_engine.model) + dwdp_set_up = True + + with allocation_scope(ExecutorMemoryType.INIT_KV_CACHE if + estimating_kv_cache else ExecutorMemoryType.KV_CACHE): + kv_cache_creator.build_managers(resources, estimating_kv_cache) # DWDP setup: MNNVL handle exchange + composite VA weight buffer + # weight manager + MoE backend fixup (single entry point). Dense DWDP is set diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 68ff43dda8eb..9a1daefd1fe9 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -1610,6 +1610,7 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: elif _drop_refused_response(self.worker, response): continue elif self.worker._has_background_error(): + release_undelivered_response(response) response = self.worker._create_error_response(response) elif response.has_error(): # Convert to ErrorResponse, because tllm.Response cannot be diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py index f27fe3e54fd9..e38604eb747a 100644 --- a/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py @@ -196,7 +196,7 @@ def _create( Args: mode: ``"expert"`` (expert DWDP alone), ``"attached"`` (dense DWDP attached to it) or ``"standalone"`` (dense DWDP alone). - kv_cache: ``"estimated"``, ``"not_estimated"`` or ``"none"`` (a model without one). + kv_cache: ``"estimated"`` or ``"not_estimated"``. real_dense_manager: Standalone, the creator builds a real ``DenseDwdpManager``, whose setup only logs; the model engine logs the SM reserve it is built with. copy_mode: Standalone, ``dwdp_config.copy_mode``. @@ -265,7 +265,6 @@ def model_engine(attn_runtime_features: object, **_kwargs: object) -> _DummyMode engine = _DummyModelEngine( attn_runtime_features=attn_runtime_features, kv_cache_quant_algo=QuantAlgo.NO_QUANT ) - engine.model.model_config.is_generation = kv_cache != "none" return engine dense_mode = {"attached": DenseMode.ATTACHED, "standalone": DenseMode.STANDALONE}.get(mode) @@ -320,7 +319,7 @@ def model_engine(attn_runtime_features: object, **_kwargs: object) -> _DummyMode return log, manager -@pytest.mark.parametrize("kv_cache", ["estimated", "not_estimated", "none"]) +@pytest.mark.parametrize("kv_cache", ["estimated", "not_estimated"]) @pytest.mark.parametrize("mode", ["expert", "attached", "standalone"]) def test_only_the_serving_phase_gets_the_manager( monkeypatch: pytest.MonkeyPatch, mode: str, kv_cache: str @@ -348,7 +347,6 @@ def test_only_the_serving_phase_gets_the_manager( "rebind", ], "not_estimated": ["setup", final_kv_cache], - "none": ["setup"], }[kv_cache] if kv_cache != "estimated": expected.append(("executor", *given)) @@ -372,7 +370,7 @@ def test_expert_dwdp_accepts_a_tactic_naming_an_implementation_with_dwdp( monkeypatch: pytest.MonkeyPatch, tactic: str ) -> None: """A tactic leaves moe_config.backend at AUTO, so the tactic is what the launch checks.""" - log, _ = _create(monkeypatch, "expert", "none", moe_backend="AUTO", moe_tactic=tactic) + log, _ = _create(monkeypatch, "expert", "not_estimated", moe_backend="AUTO", moe_tactic=tactic) assert "setup" in log diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index f04b5d1215c4..d83bd30ebf38 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -13,7 +13,7 @@ import re import threading import time -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from datetime import timedelta from types import SimpleNamespace from unittest.mock import Mock @@ -27,6 +27,7 @@ PoolingCompletion, ) from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem +from tensorrt_llm._torch.pyexecutor.llm_request import PyResult from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import ( ScheduledEncoderExecutor, fit_batch, @@ -409,11 +410,60 @@ def test_cancelling_a_launched_request_finishes_it_once_as_cancelled(completion) assert cancelled.result.is_final assert cancelled.result.finish_reasons == [FinishReason.CANCELLED] assert cancelled.result.mm_embedding_handles is None + assert cancelled.result.context_logits is None h.result(later) assert h.responses(launched, NO_WAIT) == [] assert h.runner.batches == [[launched], [later]] +@pytest.mark.parametrize("rank, exports", [(0, True), (1, False)], ids=["rank_0", "tp_follower"]) +def test_only_rank_0_of_a_multimodal_encoder_exports(monkeypatch, rank, exports): + """A follower's responses reach no client, so its exports would pin each batch's memory.""" + built = SimpleNamespace( + runner=_Runner(1, multimodal=True), + model_config=SimpleNamespace(is_generation=False), + model_loader=Mock(), + max_batch_size=1, + max_num_tokens=64, + max_seq_len=32, + ) + monkeypatch.setattr( + scheduled_encoder_executor, "resolve_scheduled_encoder_args", lambda llm_args: llm_args + ) + monkeypatch.setattr( + scheduled_encoder_executor, + "build_mm_encoder_runner", + lambda *args, **kwargs: nullcontext(built), + ) + executor = ScheduledEncoderExecutor.create( + SimpleNamespace( + mm_encoder_only=True, + garbage_collection_gen0_threshold=20000, + batch_wait_timeout_ms=0, + return_perf_metrics=False, + enable_iter_perf_stats=False, + print_iter_log=False, + ), + model_path="", + checkpoint_loader=Mock(), + mapping=Mapping(world_size=2, rank=rank, tp_size=2), + dist=SimpleNamespace(rank=rank, world_size=2, has_pp=False, barrier=lambda: None), + ) + request = SimpleNamespace(py_result=PyResult(prompt_len=1, max_new_tokens=1)) + + executor._completion.store( + [request], + { + "mm_embeddings": [torch.ones(1, WIDTH)], + "mm_embedding_request_indices": [0], + "mm_embedding_lengths": [[1]], + }, + ) + + assert (request.py_result.mm_embedding_handles is not None) is exports + request.py_result.release_exports() + + def _cuda_memory_allocated() -> int: gc.collect() torch.cuda.synchronize() diff --git a/tests/unittest/executor/test_undelivered_exports.py b/tests/unittest/executor/test_undelivered_exports.py index 77d7d53816b3..658fbf5c92c7 100644 --- a/tests/unittest/executor/test_undelivered_exports.py +++ b/tests/unittest/executor/test_undelivered_exports.py @@ -11,17 +11,23 @@ import base64 import gc from pathlib import Path -from types import SimpleNamespace +from queue import Queue +from types import MethodType, SimpleNamespace import pytest import torch from tensorrt_llm._torch.pyexecutor.llm_request import LlmResponse, LlmResult, PyResult -from tensorrt_llm.executor.base_worker import _drop_refused_response +from tensorrt_llm.executor.base_worker import ( + AwaitResponseHelper, + BaseWorker, + _drop_refused_response, +) from tensorrt_llm.executor.executor import GenerationExecutor from tensorrt_llm.executor.proxy import GenerationExecutorProxy from tensorrt_llm.executor.result import ResponseWrapper from tensorrt_llm.executor.rpc_proxy_mixin import RpcExecutorMixin +from tensorrt_llm.executor.utils import ErrorResponse pytestmark = pytest.mark.cpu_only @@ -53,6 +59,24 @@ def _drop_at_worker(response): assert _drop_refused_response(SimpleNamespace(_refused_client_ids={CLIENT_ID}), response) +def _replace_at_worker_on_background_error(response): + errors = Queue() + errors.put(RuntimeError("background error")) + sent = [] + worker = SimpleNamespace( + enable_postprocess_parallel=False, + _refused_client_ids=set(), + _error_queue=errors, + frontend_result_queues=None, + result_queue=SimpleNamespace(put=sent.extend), + _pop_result=lambda client_id: None, + ) + worker._has_background_error = MethodType(BaseWorker._has_background_error, worker) + worker._create_error_response = MethodType(BaseWorker._create_error_response, worker) + AwaitResponseHelper(worker).handle_for_ipc_batched([response]) + assert sent == [ErrorResponse(CLIENT_ID, "background error", CLIENT_ID)] + + class _ResultQueue: def __init__(self, item): self._item = item @@ -86,8 +110,8 @@ def _drop_at_rpc_proxy(response): @pytest.mark.parametrize( "drop", - [_drop_at_worker, _drop_at_proxy, _drop_at_rpc_proxy], - ids=["refused_at_worker", "late_at_proxy", "unknown_at_rpc_proxy"], + [_drop_at_worker, _replace_at_worker_on_background_error, _drop_at_proxy, _drop_at_rpc_proxy], + ids=["refused_at_worker", "errored_at_worker", "late_at_proxy", "unknown_at_rpc_proxy"], ) def test_a_dropped_response_releases_its_exports(drop): response, files = _exporting_response() From 902c194d12a73258fa1bd1a18956223b6184ef53 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Wed, 18 Nov 2026 19:45:37 -0800 Subject: [PATCH 07/12] [TRTLLM-17766][refactor] Move the encoder completions into the scheduled encoder executor Two classes used only by the executor need no module or Protocol of their own; the executor takes either one. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/encoder_completion.py | 82 ------------------- .../pyexecutor/scheduled_encoder_executor.py | 69 +++++++++++++++- .../engine/runners/test_mm_encoder.py | 2 +- .../test_multimodal_embedding_lengths.py | 2 +- .../test_scheduled_encoder_executor.py | 6 +- 5 files changed, 70 insertions(+), 91 deletions(-) delete mode 100644 tensorrt_llm/_torch/pyexecutor/encoder_completion.py diff --git a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py b/tensorrt_llm/_torch/pyexecutor/encoder_completion.py deleted file mode 100644 index 0de2949534ea..000000000000 --- a/tensorrt_llm/_torch/pyexecutor/encoder_completion.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""How one forward of ``ScheduledEncoderExecutor`` completes its requests.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Protocol - -if TYPE_CHECKING: - from .llm_request import LlmRequest - -__all__ = ["Completion", "MultimodalEmbeddingCompletion", "PoolingCompletion"] - - -class Completion(Protocol): - """How one forward's outputs complete its requests; paired with the runner at initialization.""" - - accepts_multimodal_inputs: bool - """Whether a request may carry multimodal inputs.""" - - def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: - """Attach one forward's results to its requests before their responses are created.""" - - def retire(self, requests: list[LlmRequest]) -> None: - """Withdraw what ``store`` attached, for requests that finish without their result.""" - - -class PoolingCompletion: - """Store each request's pooled row as its context logits, when the request asked for them.""" - - accepts_multimodal_inputs = False - - def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: - for request, row in zip(requests, outputs["logits"]): - if request.py_return_context_logits: - request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) - - def retire(self, requests: list[LlmRequest]) -> None: - for request in requests: - request.py_result.drop_context_logits() - - -class MultimodalEmbeddingCompletion: - """Export each request's multimodal embeddings, and its MRoPE positions, as CUDA IPC handles. - - A request without multimodal items in the batch completes without a result. Only the rank - whose responses reach the frontend exports: an exported allocation stays alive until a - consumer opens and releases its handle, or ``retire`` consumes it. - """ - - accepts_multimodal_inputs = True - - def __init__(self, *, exports: bool) -> None: - self._exports = exports - - def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: - if not self._exports: - return - metadata = outputs.get("mm_embedding_metadata") - mrope_position_ids = outputs.get("mrope_position_ids") - mrope_position_deltas = outputs.get("mrope_position_deltas") - for result, (request_index, embeddings, lengths) in enumerate( - zip( - outputs["mm_embedding_request_indices"], - outputs["mm_embeddings"], - outputs["mm_embedding_lengths"], - strict=True, - ) - ): - py_result = requests[request_index].py_result - py_result.append_mm_embeddings( - embeddings, lengths, None if metadata is None else metadata[result] - ) - if mrope_position_ids is not None and mrope_position_deltas is not None: - py_result.set_mrope_position( - mrope_position_ids[result], mrope_position_deltas[result] - ) - - def retire(self, requests: list[LlmRequest]) -> None: - for request in requests: - request.py_result.release_exports() diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index ee486207945d..3a131fc82386 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -28,7 +28,6 @@ from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import host_profiler_context from ..moe.fused_moe.moe_comm_timeout_guard import set_moe_comm_warmup -from .encoder_completion import Completion, MultimodalEmbeddingCompletion, PoolingCompletion from .engine.model_setup import build_encoder_runner, build_mm_encoder_runner from .engine.runners.encoder import resolve_scheduled_encoder_args from .engine.runners.interface import ScheduledInputs, ScheduledModelRunner @@ -60,7 +59,13 @@ from .llm_request import ExecutorRequest from .model_loader import ModelLoader -__all__ = ["ScheduledEncoderExecutor", "fit_batch", "prompt_cost"] +__all__ = [ + "MultimodalEmbeddingCompletion", + "PoolingCompletion", + "ScheduledEncoderExecutor", + "fit_batch", + "prompt_cost", +] _MULTIMODAL_INPUT_ERROR = "Requests to a pooling model cannot carry multimodal inputs." @@ -106,6 +111,62 @@ def fit_batch( return len(items) +class PoolingCompletion: + """Store each request's pooled row as its context logits, when the request asked for them.""" + + accepts_multimodal_inputs = False + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + for request, row in zip(requests, outputs["logits"]): + if request.py_return_context_logits: + request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) + + def retire(self, requests: list[LlmRequest]) -> None: + for request in requests: + request.py_result.drop_context_logits() + + +class MultimodalEmbeddingCompletion: + """Export each request's multimodal embeddings, and its MRoPE positions, as CUDA IPC handles. + + A request without multimodal items in the batch completes without a result. Only the rank + whose responses reach the frontend exports: an exported allocation stays alive until a + consumer opens and releases its handle, or ``retire`` consumes it. + """ + + accepts_multimodal_inputs = True + + def __init__(self, *, exports: bool) -> None: + self._exports = exports + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + if not self._exports: + return + metadata = outputs.get("mm_embedding_metadata") + mrope_position_ids = outputs.get("mrope_position_ids") + mrope_position_deltas = outputs.get("mrope_position_deltas") + for result, (request_index, embeddings, lengths) in enumerate( + zip( + outputs["mm_embedding_request_indices"], + outputs["mm_embeddings"], + outputs["mm_embedding_lengths"], + strict=True, + ) + ): + py_result = requests[request_index].py_result + py_result.append_mm_embeddings( + embeddings, lengths, None if metadata is None else metadata[result] + ) + if mrope_position_ids is not None and mrope_position_deltas is not None: + py_result.set_mrope_position( + mrope_position_ids[result], mrope_position_deltas[result] + ) + + def retire(self, requests: list[LlmRequest]) -> None: + for request in requests: + request.py_result.release_exports() + + class ScheduledEncoderExecutor: """Serve scheduled requests to a non-generation model: queue, admission, batching, one forward. @@ -133,7 +194,7 @@ def __init__( self, runner: ScheduledModelRunner, *, - completion: Completion, + completion: PoolingCompletion | MultimodalEmbeddingCompletion, model_loader: ModelLoader, llm_args: TorchLlmArgs, mapping: Mapping, @@ -147,6 +208,8 @@ def __init__( Args: runner: The model's runner; every forward goes through its scheduled entry point. completion: How a forward's outputs complete its requests; it fits the runner. + ``store`` attaches a forward's results before their responses are created; + ``retire`` withdraws them from requests that finish without their result. model_loader: The loader that loaded the model, cleaned up at shutdown. llm_args: The resolved arguments; the executor reads its timeouts, garbage collection threshold and metrics switches. diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py index acefc2c06def..bcac1651ac38 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py @@ -10,11 +10,11 @@ from torch import nn from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin -from tensorrt_llm._torch.pyexecutor.encoder_completion import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.engine.runners import mm_encoder as mm_encoder_module from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledInputs from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.llm_request import PyResult +from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._torch.utils import get_model_extra_attrs diff --git a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py index e78e5cb64283..009f5d449e81 100644 --- a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py +++ b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py @@ -6,8 +6,8 @@ import pytest import torch -from tensorrt_llm._torch.pyexecutor.encoder_completion import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.llm_request import PyResult, get_multimodal_embedding_lengths +from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import MultimodalEmbeddingCompletion from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index d83bd30ebf38..fa755ee112c6 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -22,13 +22,11 @@ import torch from tensorrt_llm._torch.pyexecutor import scheduled_encoder_executor -from tensorrt_llm._torch.pyexecutor.encoder_completion import ( - MultimodalEmbeddingCompletion, - PoolingCompletion, -) from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem from tensorrt_llm._torch.pyexecutor.llm_request import PyResult from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import ( + MultimodalEmbeddingCompletion, + PoolingCompletion, ScheduledEncoderExecutor, fit_batch, ) From 74baf0c045d313c8489fa3fa57754ae34131cce9 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Wed, 18 Nov 2026 21:31:24 -0800 Subject: [PATCH 08/12] [TRTLLM-17766][fix] Size the multimodal encoder towers from the resolved token budget Without encoder_max_num_tokens the towers took the raw max_num_tokens, which can be None. Also test the runner's staging event and the token-budget admission limit, and order the cancel tests by a signal. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/engine/model_setup.py | 10 ++--- .../engine/runners/test_mm_encoder.py | 11 +++-- .../pyexecutor/engine/test_multimodal.py | 40 +++++++++++++++++++ .../test_scheduled_encoder_executor.py | 35 +++++++++++++--- 4 files changed, 83 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py index 7eb93c20a400..647470b566d0 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py @@ -672,11 +672,6 @@ def build_mm_encoder_runner( Yields: The runner with its loader, model config and resolved capacity. """ - encoder_max_num_tokens = ( - llm_args.encoder_max_num_tokens - if llm_args.encoder_max_num_tokens is not None - else llm_args.max_num_tokens - ) with _load_encoder( llm_args, model_path=model_path, @@ -684,6 +679,11 @@ def build_mm_encoder_runner( mapping=mapping, dist=dist, ) as loaded: + encoder_max_num_tokens = ( + llm_args.encoder_max_num_tokens + if llm_args.encoder_max_num_tokens is not None + else loaded.max_num_tokens + ) input_processor = create_engine_input_processor(model_path, llm_args) setup_mm_encoder_attn_metadata(loaded.model, input_processor, encoder_max_num_tokens, None) compiled = compile_model( diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py index bcac1651ac38..66d15a6442b0 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py @@ -103,6 +103,7 @@ def test_mm_encoder_runner_forward_skips_metadata_only_and_missing_length_reques def test_mm_encoder_runner_splits_embeddings_and_returns_mrope_metadata() -> None: + """The runner also marks its inputs staged once the towers have run.""" requests = [ SimpleNamespace( py_multimodal_data={ @@ -132,18 +133,22 @@ def test_mm_encoder_runner_splits_embeddings_and_returns_mrope_metadata() -> Non ), ] embeddings = torch.arange(6).reshape(3, 2) - model = SimpleNamespace(extra_attrs={}, forward=Mock(return_value=[embeddings])) + calls = Mock() + calls.forward.return_value = [embeddings] + model = SimpleNamespace(extra_attrs={}, forward=calls.forward) runner = MultimodalEncoderRunner(model, encoder_cache_enabled=False, moe_load_balancer=None) for index, request in enumerate(requests): request.py_multimodal_params = MultimodalParams(multimodal_data=request.py_multimodal_data) request.get_num_tokens = Mock(return_value=index + 1) result = runner.forward( - ScheduledInputs(batch=SimpleNamespace(context_requests=requests)), + ScheduledInputs( + batch=SimpleNamespace(context_requests=requests), inputs_staged=calls.inputs_staged + ), resource_manager=SimpleNamespace(get_resource_manager=Mock(return_value=None)), ) - model.forward.assert_called_once() + assert [name for name, _, _ in calls.mock_calls] == ["forward", "inputs_staged.record"] params = model.forward.call_args.args[0] assert [param.input_ids_start_offset for param in params] == [0, 1] assert result["mm_embedding_request_indices"] == [0, 1] diff --git a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py index f7c580cec0b8..4878b46c4178 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py @@ -344,6 +344,46 @@ def _build_blocks_graph_runner(self, config: MultimodalEncoderCudaGraphConfig) - model.encoder._blocks_graph_runner.capture_all.assert_called_once_with(torch.device("cpu")) +@pytest.mark.parametrize("encoder_max_num_tokens", [None, 1024]) +def test_mm_encoder_builder_sizes_the_towers_with_the_resolved_token_budget( + monkeypatch: pytest.MonkeyPatch, encoder_max_num_tokens: int | None +) -> None: + from tensorrt_llm._torch.pyexecutor.engine import model_setup + + model = SimpleNamespace(model_config=SimpleNamespace()) + loader = Mock() + loader.load.return_value = (model, None) + for name in ("init_process_setup", "configure_autotuner_nvmmh", "prepare_loaded_model"): + monkeypatch.setattr(model_setup, name, Mock()) + monkeypatch.setattr(model_setup, "create_model_loader", Mock(return_value=loader)) + monkeypatch.setattr(model_setup, "resolve_max_seq_len", lambda model, max_seq_len: max_seq_len) + monkeypatch.setattr(model_setup, "create_engine_input_processor", Mock()) + monkeypatch.setattr( + model_setup, "compile_model", Mock(return_value=SimpleNamespace(model=model)) + ) + monkeypatch.setattr(model_setup.MultimodalEncoderRunner, "create", Mock()) + setup_metadata, setup_graphs = Mock(), Mock() + monkeypatch.setattr(model_setup, "setup_mm_encoder_attn_metadata", setup_metadata) + monkeypatch.setattr(model_setup, "setup_mm_encoder_cuda_graphs", setup_graphs) + llm_args = SimpleNamespace( + max_num_tokens=None, + encoder_max_num_tokens=encoder_max_num_tokens, + get_runtime_sizes=lambda: (1, None, 128, 4), + multimodal_config=None, + cuda_graph_config=None, + prefill_capture_num_tokens=None, + ) + + with model_setup.build_mm_encoder_runner( + llm_args, model_path="", checkpoint_loader=Mock(), mapping=Mapping(), dist=None + ) as built: + assert built.max_num_tokens == 128 * 4 + + expected = built.max_num_tokens if encoder_max_num_tokens is None else encoder_max_num_tokens + assert setup_metadata.call_args.args[2] == expected + assert setup_graphs.call_args.kwargs["max_num_tokens"] == expected + + @pytest.mark.parametrize("enable_padding", [False, True]) @pytest.mark.parametrize("automatic", [False, True], ids=["explicit", "automatic"]) def test_encoder_graphs_preserve_eager_metadata_capacity( diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index fa755ee112c6..e3aa5479fb9f 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -208,6 +208,24 @@ def result(self, request_id: int): assert response.result.is_final return response.result + def complete_after_the_loop_takes_the_cancellation(self, request_id: int) -> None: + """Hold each launched batch's completion until the loop has taken this cancellation.""" + executor = self.executor + taken = threading.Event() + receive, cancel_launched = executor._receive, executor._cancel_launched + + def receive_and_report(): + receive() + if request_id in executor._cancelled_after_launch: + taken.set() + + def cancel_once_taken(flight): + assert taken.wait(timeout=30) + return cancel_launched(flight) + + executor._receive = receive_and_report + executor._cancel_launched = cancel_once_taken + def shutdown(self) -> None: if not self.closed: self.closed = True @@ -356,8 +374,15 @@ def test_inadmissible_requests_fail_alone(): assert h.runner.batches == [[fitting]] -def test_a_multimodal_encoder_admits_multimodal_requests_and_fails_an_oversized_one_alone(): - with _harness(completion="multimodal", max_seq_len=32, max_num_tokens=64) as h: +@pytest.mark.parametrize( + ("max_seq_len", "max_num_tokens"), [(32, 64), (64, 32)], ids=["max_seq_len", "max_num_tokens"] +) +def test_a_multimodal_encoder_admits_multimodal_requests_and_fails_an_oversized_one_alone( + max_seq_len, max_num_tokens +): + with _harness( + completion="multimodal", max_seq_len=max_seq_len, max_num_tokens=max_num_tokens + ) as h: too_long = h.submit(33, py_multimodal_data={"image": {}}) fitting = h.submit(32, py_multimodal_data={"image": {}}) h.executor.start_worker() @@ -395,9 +420,8 @@ def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): def test_cancelling_a_launched_request_finishes_it_once_as_cancelled(completion): with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() - # The launched forward is still running when the loop takes the cancellation. - h.runner.gpu_cycles = GPU_CYCLES launched, later = h.submit(), h.submit() + h.complete_after_the_loop_takes_the_cancellation(launched) h.executor.start_worker() assert h.runner.entered.wait(timeout=30) @@ -473,7 +497,6 @@ def _cuda_memory_allocated() -> int: def test_a_launched_request_whose_result_is_not_delivered_frees_its_exports(cause): with _harness(completion="multimodal", max_batch_size=1) as h: h.runner.embedding_device = "cuda" - h.runner.gpu_cycles = GPU_CYCLES if cause == "failed": def fail_after_the_export(*args, **kwargs): @@ -483,6 +506,8 @@ def fail_after_the_export(*args, **kwargs): allocated = _cuda_memory_allocated() h.runner.hold() launched = h.submit(py_multimodal_data={"image": {}}) + if cause == "cancelled": + h.complete_after_the_loop_takes_the_cancellation(launched) h.executor.start_worker() assert h.runner.entered.wait(timeout=30) if cause == "cancelled": From 4ea7bc7ad96372989efdc550d5040987b719dec5 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Wed, 18 Nov 2026 23:28:49 -0800 Subject: [PATCH 09/12] [TRTLLM-17766][fix] Deliver every sequence of a request cancelled after launch Responses were created at launch, so the cancelled first sequence came out final and the client dropped the rest. Create each response once, after cancellations are settled. Signed-off-by: Lori Ren --- .../pyexecutor/scheduled_encoder_executor.py | 48 +++++++++---------- .../test_scheduled_encoder_executor.py | 26 ++++++++++ 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index 3a131fc82386..71e76f7d946f 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -78,7 +78,6 @@ class _InFlight: """A launched batch whose forward the completion thread still has to wait for and publish.""" requests: list[LlmRequest] - responses: list[tuple[int, LlmResponse]] done: torch.cuda.Event iteration: int host_start: float @@ -706,7 +705,7 @@ def _run(self, items: list[RequestQueueItem]) -> None: timing.end_time, timing.end_time, ) - responses = self._complete(requests, outputs) + self._complete(requests, outputs) done = torch.cuda.Event() done.record() except Exception as error: @@ -719,7 +718,6 @@ def _run(self, items: list[RequestQueueItem]) -> None: self._in_flight.put( _InFlight( requests=requests, - responses=responses, done=done, iteration=self._iter_counter, host_start=host_start, @@ -737,9 +735,12 @@ def _completion_loop(self) -> None: with nvtx_range("synchronize"): # Responses carry device tensors that other threads and processes read. flight.done.synchronize() - responses = self._cancel_launched(flight) + self._cancel_launched(flight) if self.dist.rank == 0: - self._publish(responses, (request.py_request_id for request in flight.requests)) + self._publish( + self._create_responses(flight.requests), + (request.py_request_id for request in flight.requests), + ) if self._iter_log: self._log_iteration(flight) except Exception as error: @@ -752,11 +753,11 @@ def _completion_loop(self) -> None: finally: self._slots.release() - def _cancel_launched(self, flight: _InFlight) -> list[tuple[int, LlmResponse]]: - """The batch's responses, with those of requests cancelled since its launch replaced. + def _cancel_launched(self, flight: _InFlight) -> None: + """Finish the batch's requests cancelled since its launch as cancelled, without results. - A cancelled request's exports are retired and it finishes as cancelled; a - cancellation that arrives after this point finds the request completed. + A cancelled request's exports are retired; a cancellation that arrives after this + point finds the request completed. """ request_ids = {request.py_request_id for request in flight.requests if not request.is_child} with self._cancel_lock: @@ -764,7 +765,7 @@ def _cancel_launched(self, flight: _InFlight) -> list[tuple[int, LlmResponse]]: cancelled_ids = self._cancelled_after_launch & request_ids self._cancelled_after_launch -= cancelled_ids if not cancelled_ids: - return flight.responses + return cancelled = [ request for request in flight.requests @@ -772,19 +773,9 @@ def _cancel_launched(self, flight: _InFlight) -> list[tuple[int, LlmResponse]]: in cancelled_ids ] self._completion.retire(cancelled) - # A child's response carries its parent's request id. - responses = [ - (request_id, response) - for request_id, response in flight.responses - if response.request_id not in cancelled_ids - ] for request in cancelled: # Replaces the reason LENGTH set by _complete. request.set_finished_reason(FinishReason.CANCELLED, 0) - response = request.create_response() - if response is not None: - responses.append((request.py_request_id, response)) - return responses def _log_iteration(self, flight: _InFlight) -> None: """Log one batch: its size, the host time until it was published and the GPU time of its forward.""" @@ -800,14 +791,11 @@ def _log_iteration(self, flight: _InFlight) -> None: ) @nvtx_range("_complete") - def _complete( - self, requests: list[LlmRequest], outputs: dict[str, Any] - ) -> list[tuple[int, LlmResponse]]: - """Store the forward's results on its requests and report each finished with reason LENGTH.""" + def _complete(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + """Store the forward's results on its requests and finish each with reason LENGTH.""" self._completion.store(requests, outputs) batch_token_time = self._perf_manager.get_timestamp() gpu_times_cache = {} - responses = [] for request in requests: # The public result reports a finished request with no tokens and reason LENGTH. request.state = LlmRequestState.GENERATION_COMPLETE @@ -819,6 +807,16 @@ def _complete( self._perf_manager.compute_batch_gpu_times( [request], gpu_times_cache=gpu_times_cache ) + + @staticmethod + def _create_responses(requests: list[LlmRequest]) -> list[tuple[int, LlmResponse]]: + """The batch's responses, created once each request's finish reason is settled. + + Creating a response marks its sequence final, so a second one for a sequence could + end the request before its other sequences arrive. + """ + responses = [] + for request in requests: response = request.create_response() if response is not None: responses.append((request.py_request_id, response)) diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index e3aa5479fb9f..fb3829ffac8d 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -438,6 +438,32 @@ def test_cancelling_a_launched_request_finishes_it_once_as_cancelled(completion) assert h.runner.batches == [[launched], [later]] +def test_cancelling_a_launched_request_with_two_sequences_delivers_both_and_ends_with_the_last( + completion, +): + with _harness(completion=completion) as h: + h.runner.hold() + launched = h.submit(sampling_config=SamplingConfig(num_return_sequences=2)) + h.complete_after_the_loop_takes_the_cancellation(launched) + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + + h.executor.cancel_request(launched) + h.runner.release.set() + + # The worker takes every response; a child's is stored under its own id. + responses = [] + while len(responses) < 2 and (ready := h.executor.await_responses(timeout=WAIT)): + responses.extend(ready) + for response in responses: + response.result.deserialize() + assert [response.request_id for response in responses] == [launched, launched] + assert [response.result.sequence_index for response in responses] == [0, 1] + assert [response.result.is_final for response in responses] == [False, True] + for response in responses: + assert response.result.finish_reasons == [FinishReason.CANCELLED] + + @pytest.mark.parametrize("rank, exports", [(0, True), (1, False)], ids=["rank_0", "tp_follower"]) def test_only_rank_0_of_a_multimodal_encoder_exports(monkeypatch, rank, exports): """A follower's responses reach no client, so its exports would pin each batch's memory.""" From 1219b667b14a5e03624a97c906fe1b80efe7eb23 Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Thu, 19 Nov 2026 20:47:56 -0800 Subject: [PATCH 10/12] [TRTLLM-17766][fix] Keep the configured tower budget and honour a cancellation sent before publication The towers count patches, so the clamped max_num_tokens could cost an image its CUDA graph. A batch completing while the loop waits to launch now sees a cancellation sent before its publication. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/engine/model_setup.py | 14 +++++-- .../pyexecutor/scheduled_encoder_executor.py | 11 ++++- .../pyexecutor/engine/test_multimodal.py | 25 +++++++---- .../test_scheduled_encoder_executor.py | 41 +++++++++++++++++++ 4 files changed, 76 insertions(+), 15 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py index 647470b566d0..b36dca931850 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py @@ -679,10 +679,16 @@ def build_mm_encoder_runner( mapping=mapping, dist=dist, ) as loaded: - encoder_max_num_tokens = ( - llm_args.encoder_max_num_tokens - if llm_args.encoder_max_num_tokens is not None - else loaded.max_num_tokens + # The towers count patches, which a merged item needs more of than prompt tokens, + # so they keep the configured budget rather than the clamped one. + encoder_max_num_tokens = next( + budget + for budget in ( + llm_args.encoder_max_num_tokens, + llm_args.max_num_tokens, + loaded.max_num_tokens, + ) + if budget is not None ) input_processor = create_engine_input_processor(model_path, llm_args) setup_mm_encoder_attn_metadata(loaded.model, input_processor, encoder_max_num_tokens, None) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index 71e76f7d946f..028aae342599 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -252,10 +252,12 @@ def __init__( self._last_staged: torch.cuda.Event | None = None self._completion_thread: threading.Thread | None = None self._completion_error: BaseException | None = None - # Ids of the launched, unpublished requests, and of those among them cancelled since. + # Ids of the launched, unpublished requests, and of those among them cancelled since; + # and ids whose cancellation was sent but not yet taken by the loop. self._cancel_lock = threading.Lock() self._launched_ids: set[int] = set() self._cancelled_after_launch: set[int] = set() + self._cancel_sent: set[int] = set() self._response_channel = ResponseChannel() self._iter_counter = 0 self._profiler = ProfileController( @@ -420,6 +422,10 @@ def cancel_request(self, id: int) -> None: Args: id: The request id. """ + # A launched batch may complete before the loop takes the queued cancellation; + # rank 0, which publishes, checks this record then. + with self._cancel_lock: + self._cancel_sent.add(id) self._request_queue.enqueue_cancel_request(id) def start_profile( @@ -616,6 +622,7 @@ def _receive(self) -> None: if canceled_ids: with self._cancel_lock: self._cancelled_after_launch.update(canceled_ids & self._launched_ids) + self._cancel_sent -= canceled_ids self._cancel_waiting(canceled_ids) @nvtx_range("_check_admissible") @@ -762,7 +769,7 @@ def _cancel_launched(self, flight: _InFlight) -> None: request_ids = {request.py_request_id for request in flight.requests if not request.is_child} with self._cancel_lock: self._launched_ids -= request_ids - cancelled_ids = self._cancelled_after_launch & request_ids + cancelled_ids = (self._cancelled_after_launch | self._cancel_sent) & request_ids self._cancelled_after_launch -= cancelled_ids if not cancelled_ids: return diff --git a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py index 4878b46c4178..d86ea80c9ad6 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py @@ -344,10 +344,18 @@ def _build_blocks_graph_runner(self, config: MultimodalEncoderCudaGraphConfig) - model.encoder._blocks_graph_runner.capture_all.assert_called_once_with(torch.device("cpu")) -@pytest.mark.parametrize("encoder_max_num_tokens", [None, 1024]) -def test_mm_encoder_builder_sizes_the_towers_with_the_resolved_token_budget( - monkeypatch: pytest.MonkeyPatch, encoder_max_num_tokens: int | None +@pytest.mark.parametrize( + ("encoder_max_num_tokens", "max_num_tokens", "tower_budget"), + [(None, 4096, 4096), (1024, 4096, 1024), (None, None, 128)], + ids=["configured", "encoder_budget", "resolved"], +) +def test_mm_encoder_builder_sizes_the_towers_with_the_configured_token_budget( + monkeypatch: pytest.MonkeyPatch, + encoder_max_num_tokens: int | None, + max_num_tokens: int | None, + tower_budget: int, ) -> None: + """A merged item fits a prompt shorter than its patches, so the clamped budget is too small.""" from tensorrt_llm._torch.pyexecutor.engine import model_setup model = SimpleNamespace(model_config=SimpleNamespace()) @@ -366,9 +374,9 @@ def test_mm_encoder_builder_sizes_the_towers_with_the_resolved_token_budget( monkeypatch.setattr(model_setup, "setup_mm_encoder_attn_metadata", setup_metadata) monkeypatch.setattr(model_setup, "setup_mm_encoder_cuda_graphs", setup_graphs) llm_args = SimpleNamespace( - max_num_tokens=None, + max_num_tokens=max_num_tokens, encoder_max_num_tokens=encoder_max_num_tokens, - get_runtime_sizes=lambda: (1, None, 128, 4), + get_runtime_sizes=lambda: (1, max_num_tokens, 128, 1), multimodal_config=None, cuda_graph_config=None, prefill_capture_num_tokens=None, @@ -377,11 +385,10 @@ def test_mm_encoder_builder_sizes_the_towers_with_the_resolved_token_budget( with model_setup.build_mm_encoder_runner( llm_args, model_path="", checkpoint_loader=Mock(), mapping=Mapping(), dist=None ) as built: - assert built.max_num_tokens == 128 * 4 + assert built.max_num_tokens == 128 - expected = built.max_num_tokens if encoder_max_num_tokens is None else encoder_max_num_tokens - assert setup_metadata.call_args.args[2] == expected - assert setup_graphs.call_args.kwargs["max_num_tokens"] == expected + assert setup_metadata.call_args.args[2] == tower_budget + assert setup_graphs.call_args.kwargs["max_num_tokens"] == tower_budget @pytest.mark.parametrize("enable_padding", [False, True]) diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index fb3829ffac8d..1bfa3745bcff 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -464,6 +464,47 @@ def test_cancelling_a_launched_request_with_two_sequences_delivers_both_and_ends assert response.result.finish_reasons == [FinishReason.CANCELLED] +def test_a_cancellation_sent_while_the_loop_waits_for_a_slot_still_cancels_a_launched_request( + completion, +): + with _harness(completion=completion, max_batch_size=1) as h: + slots, waiting = h.executor._slots, threading.Event() + + class Slots: + def acquire(self): + if not slots.acquire(blocking=False): + waiting.set() + slots.acquire() + + def release(self): + slots.release() + + h.executor._slots = Slots() + sent = threading.Event() + cancel_launched = h.executor._cancel_launched + + def cancel_once_sent(flight): + assert sent.wait(timeout=30) + return cancel_launched(flight) + + h.executor._cancel_launched = cancel_once_sent + first, second, third = h.submit(), h.submit(), h.submit() + h.executor.start_worker() + # Two batches are in flight, and the loop waits in the third launch, not in _receive. + assert waiting.wait(timeout=30) + + h.executor.cancel_request(first) + sent.set() + + (cancelled,) = h.responses(first) + assert cancelled.result.finish_reasons == [FinishReason.CANCELLED] + assert cancelled.result.mm_embedding_handles is None + assert cancelled.result.context_logits is None + h.result(second) + h.result(third) + assert h.runner.batches == [[first], [second], [third]] + + @pytest.mark.parametrize("rank, exports", [(0, True), (1, False)], ids=["rank_0", "tp_follower"]) def test_only_rank_0_of_a_multimodal_encoder_exports(monkeypatch, rank, exports): """A follower's responses reach no client, so its exports would pin each batch's memory.""" From 80fb7d91a1b8aa2d57e2e19151ea2b0e4a03fd9c Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Thu, 19 Nov 2026 22:56:02 -0800 Subject: [PATCH 11/12] [TRTLLM-17766][fix] Honour the waiting-queue policy in the scheduled encoder and reject client IO during GPU wait LLM(mm_encoder_only=True) passes request priorities, so the queue now follows scheduler_config, wait cap included. The worker would bind wait_for_intake_window, which this executor lacks. Signed-off-by: Lori Ren --- .../_torch/pyexecutor/py_executor_creator.py | 2 ++ .../pyexecutor/scheduled_encoder_executor.py | 25 ++++++++++++++----- ...est_py_executor_creator_no_kv_execution.py | 2 ++ .../test_scheduled_encoder_executor.py | 24 ++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 618ba92942e1..5ff101ff639d 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -502,6 +502,8 @@ def _check_scheduled_encoder_support(llm_args: TorchLlmArgs, ("enable_resource_governor", llm_args.enable_resource_governor), ("sleep_config", llm_args.sleep_config is not None), ("orchestrator_type='ray'", llm_args.orchestrator_type == "ray"), + ("enable_client_io_during_gpu_wait", + llm_args.enable_client_io_during_gpu_wait), ) settings = [name for name, used in checks if used] if settings: diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index 028aae342599..b23de2e197f0 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -175,9 +175,10 @@ class ScheduledEncoderExecutor: forwards are in flight, so the next batch is prepared and launched while the previous one runs, and a completion thread publishes each batch as soon as its forward finishes. A forward that fails on the device after its launch is not recoverable and ends the - loop like any other crash. Settings that only tune generation are ignored: the overlap - scheduler, chunked prefill, guided decoding, ``scheduler_config``, the waiting-queue - policy and the iteration-based batch waits. Iteration statistics and KV cache events + loop like any other crash. Requests wait in the queue that ``scheduler_config``'s + waiting-queue policy selects. Settings that only tune generation are ignored: the overlap + scheduler, chunked prefill, guided decoding, the rest of ``scheduler_config`` and the + iteration-based batch waits. Iteration statistics and KV cache events are always empty. With ``print_iter_log`` rank 0 logs one line per batch: its requests and prompt tokens, the requests still waiting when it launched, the host time from taking the batch to publishing it, and the GPU time of the forward. @@ -236,14 +237,22 @@ def __init__( self._gc_gen0_threshold = llm_args.garbage_collection_gen0_threshold self._hang_detector = HangDetector(on_detected=self._on_hang_detected) + scheduler_config = llm_args.scheduler_config + # The wait cap ages requests by rank 0's arrival stamps on rank 0's clock, so that + # every rank pops the same order. + aging = scheduler_config.waiting_queue_max_wait_s is not None self._request_queue = ExecutorRequestQueue( dist, max_batch_size, enable_iter_perf_stats=False, batch_wait_timeout_ms=llm_args.batch_wait_timeout_ms, + stamp_admission_times=aging, + ) + self._broadcaster = RequestBroadcaster(dist, self._hang_detector, sync_clock=aging) + self._waiting_queue = create_waiting_queue( + scheduler_config.waiting_queue_policy, + max_wait_s=scheduler_config.waiting_queue_max_wait_s, ) - self._broadcaster = RequestBroadcaster(dist, self._hang_detector) - self._waiting_queue = create_waiting_queue() self._error_budget = ErrorBudget() self._perf_manager = PerfMetricsManager(enabled=llm_args.return_perf_metrics) self._iter_log = llm_args.print_iter_log and dist.rank == 0 @@ -417,7 +426,9 @@ def _await_single(self, request_id: int, timeout: float | None) -> list[LlmRespo ) def cancel_request(self, id: int) -> None: - """Cancel a request; it finishes as cancelled unless its result was already published. + """Cancel a request; it finishes as cancelled unless its batch's completion was settled. + + The completion thread settles a batch in ``_cancel_launched``, just before publishing it. Args: id: The request id. @@ -594,6 +605,8 @@ def _receive(self) -> None: with self._hang_detector.pause(): items = self._request_queue.get_from_request_queue(timeout) items = self._broadcaster.broadcast(items, prefer_cpu=idle) + if self._broadcaster.sync_clock: + self._waiting_queue.set_now(self._broadcaster.rank0_now) canceled_ids = set() rejected = [] diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py index b3aef2c4c89f..c1e3e72f5b2a 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py @@ -49,6 +49,7 @@ def _encoder_args(**overrides): enable_resource_governor=False, sleep_config=None, orchestrator_type=None, + enable_client_io_during_gpu_wait=False, ) args.update(overrides) return SimpleNamespace(**args) @@ -83,6 +84,7 @@ def test_scheduled_encoder_rejects_unsupported_parallelism(mapping, error): (dict(enable_resource_governor=True), "enable_resource_governor"), (dict(sleep_config=object()), "sleep_config"), (dict(orchestrator_type="ray"), "orchestrator_type='ray'"), + (dict(enable_client_io_during_gpu_wait=True), "enable_client_io_during_gpu_wait"), ], ) def test_scheduled_encoder_rejects_settings_by_name(overrides, setting): diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index 1bfa3745bcff..754317ddd40e 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -33,6 +33,7 @@ from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm.bindings.executor import FinishReason, OutputConfig, Request, SamplingConfig from tensorrt_llm.executor.utils import is_context_length_exceeded_message +from tensorrt_llm.llmapi.llm_args import SchedulerConfig, WaitingQueuePolicy from tensorrt_llm.mapping import Mapping WAIT = timedelta(seconds=30) @@ -148,6 +149,7 @@ def __init__( max_num_tokens=64, max_seq_len=32, print_iter_log=False, + scheduler_config=None, ): self.multimodal = multimodal = completion == "multimodal" self.runner = _Runner(max_batch_size, multimodal) @@ -163,6 +165,7 @@ def __init__( return_perf_metrics=False, enable_iter_perf_stats=False, print_iter_log=print_iter_log, + scheduler_config=scheduler_config or SchedulerConfig(), ), mapping=Mapping(), dist=SimpleNamespace(rank=0, world_size=1, has_pp=False, barrier=lambda: None), @@ -394,6 +397,26 @@ def test_a_multimodal_encoder_admits_multimodal_requests_and_fails_an_oversized_ assert h.runner.batches == [[fitting]] +@pytest.mark.parametrize("max_wait_s", [None, 60.0], ids=["priority", "aging"]) +def test_a_priority_queue_admits_the_higher_priority_request_first(max_wait_s): + with _harness( + max_batch_size=1, + scheduler_config=SchedulerConfig( + waiting_queue_policy=WaitingQueuePolicy.PRIORITY, waiting_queue_max_wait_s=max_wait_s + ), + ) as h: + h.runner.hold() + running = h.submit() + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + low, high = h.submit(priority=0.1), h.submit(priority=0.9) + h.runner.release.set() + + for request_id in (running, low, high): + h.result(request_id) + assert h.runner.batches == [[running], [high], [low]] + + def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() @@ -532,6 +555,7 @@ def test_only_rank_0_of_a_multimodal_encoder_exports(monkeypatch, rank, exports) return_perf_metrics=False, enable_iter_perf_stats=False, print_iter_log=False, + scheduler_config=SchedulerConfig(), ), model_path="", checkpoint_loader=Mock(), From 591ab0ee030375db442eea690efee677336d1d6c Mon Sep 17 00:00:00 2001 From: Lori Ren Date: Thu, 19 Nov 2026 23:14:16 -0800 Subject: [PATCH 12/12] [TRTLLM-17766][fix] Mark each popped encoder batch admitted in the waiting queue The aging queue keeps a request's arrival sequence until it is marked admitted, so it grew with every request served. Also test a request crossing the wait cap. Signed-off-by: Lori Ren --- .../pyexecutor/scheduled_encoder_executor.py | 4 ++- .../test_scheduled_encoder_executor.py | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index b23de2e197f0..b655a3dd88fd 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -573,7 +573,9 @@ def _loop(self) -> None: max_num_tokens=self.max_num_tokens, ) if count: - self._run([self._waiting_queue.pop_request() for _ in range(count)]) + items = [self._waiting_queue.pop_request() for _ in range(count)] + self._waiting_queue.mark_admitted(items) + self._run(items) # Every pass counts, idle ones too, so a stop requested while idle fires. self._iter_counter += 1 finally: diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index 754317ddd40e..e99ae5319df3 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -417,6 +417,31 @@ def test_a_priority_queue_admits_the_higher_priority_request_first(max_wait_s): assert h.runner.batches == [[running], [high], [low]] +def test_a_request_past_the_wait_cap_goes_ahead_of_a_higher_priority_one(): + max_wait_s = 0.2 + with _harness( + max_batch_size=1, + scheduler_config=SchedulerConfig( + waiting_queue_policy=WaitingQueuePolicy.PRIORITY, waiting_queue_max_wait_s=max_wait_s + ), + ) as h: + h.runner.hold() + running = h.submit() + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + # Arrivals are stamped at enqueue, so the older request is past the cap when popped. + old = h.submit(priority=0.1) + time.sleep(2 * max_wait_s) + new = h.submit(priority=0.9) + h.runner.release.set() + + for request_id in (running, old, new): + h.result(request_id) + assert h.runner.batches == [[running], [old], [new]] + # Admitted requests leave no bookkeeping behind in the queue. + assert not h.executor._waiting_queue._seq + + def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold()