diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index b77df9ef8fe1..691ce9c37193 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -90,7 +90,7 @@ PeftCacheManager, ResourceManager, ResourceManagerType) from .runtime_hooks import runtime_hooks_of -from .sampler import EarlyStopWithMMResult, TorchSampler +from .sampler import TorchSampler from .sampler.penalties import PenaltyStore from .scheduler import (BindCapacityScheduler, BindMicroBatchScheduler, KVCacheV2Scheduler, MultimodalEagerEncoderScheduler, @@ -6012,7 +6012,6 @@ def instantiate_sampler( *, max_batch_size: int, max_beam_width: int, - mm_encoder_only: bool, speculative_config: SpeculativeConfig, max_num_sequences: Optional[int] = None, ): @@ -6052,16 +6051,6 @@ def instantiate_sampler( ) return sampler - if mm_encoder_only: - # NOTE: handle model outputs specially for mm encoder executor/engine. - # Only export results on ranks whose responses reach the frontend - # (see PyExecutor._enqueue_responses): rank 0, plus the attention-DP - # ranks gathered into it. A CUDA IPC handle exported on any other rank - # is never opened, so PyTorch would never release its allocation. - returns_responses = mapping.rank == 0 or (mapping.enable_attention_dp - and 0 in mapping.tp_group) - return EarlyStopWithMMResult(return_mm_results=returns_responses, - max_num_tokens=engine.max_num_tokens) return TorchSampler(sampler_args) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py index b8b8af1170a2..b36dca931850 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/model_setup.py @@ -31,12 +31,19 @@ from tensorrt_llm._torch.nccl_window_tensor_scope import install_eager_nccl_window_tensor_scopes from tensorrt_llm._torch.pyexecutor.engine.cuda_graph import plan_cuda_graphs from tensorrt_llm._torch.pyexecutor.engine.model_call import ModelCaller +from tensorrt_llm._torch.pyexecutor.engine.multimodal import ( + setup_mm_encoder_attn_metadata, + setup_mm_encoder_cuda_graphs, +) from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.runner import PrefillCompiledModel from tensorrt_llm._torch.pyexecutor.engine.runners.encoder import EncoderRunner +from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledModelRunner +from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.layerwise_nvtx_marker import LayerwiseNvtxMarker from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader from tensorrt_llm._torch.utils import set_torch_compiling from tensorrt_llm._utils import mpi_disabled +from tensorrt_llm.inputs.registry import InputProcessor, create_input_processor from tensorrt_llm.llmapi.llm_args import ( DecodingBaseConfig, PrefillCudaGraphBackend, @@ -455,6 +462,23 @@ def create_model_loader( ) +def create_engine_input_processor(model_path: str, llm_args: TorchLlmArgs) -> InputProcessor: + """Create the checkpoint's input processor with the multimodal options the engine applies.""" + input_processor_kwargs = {} + video_pruning_rate = llm_args.multimodal_config.video_pruning_rate + if video_pruning_rate is not None: + input_processor_kwargs["video_pruning_rate"] = video_pruning_rate + if llm_args.multimodal_config.vision_gpu_normalize: + input_processor_kwargs["vision_gpu_normalize"] = True + return create_input_processor( + model_path, + tokenizer=None, + checkpoint_format=llm_args.checkpoint_format, + trust_remote_code=llm_args.trust_remote_code, + **input_processor_kwargs, + ) + + def prepare_loaded_model(model: torch.nn.Module, llm_args: TorchLlmArgs) -> None: """Give the model its extra attributes and the requested NVTX markers.""" # AutoModelForCausalLM sets extra_attrs; a model passed in through @@ -492,7 +516,7 @@ def resolve_attention_backend( class EncoderBuild: """An encoder runner built from a checkpoint, with the loader that owns its weights.""" - runner: EncoderRunner + runner: ScheduledModelRunner model_loader: ModelLoader model_config: ModelConfig max_batch_size: int @@ -500,30 +524,27 @@ class EncoderBuild: max_seq_len: int +@dataclass(frozen=True) +class _LoadedEncoder: + model: torch.nn.Module + model_loader: ModelLoader + max_beam_width: int + max_batch_size: int + max_num_tokens: int + max_seq_len: int + prefill_capture_num_tokens: list[int] + + @contextmanager -def build_encoder_runner( +def _load_encoder( llm_args: TorchLlmArgs, *, model_path: str, checkpoint_loader: BaseCheckpointLoader, mapping: Mapping, dist: Distributed, -) -> Iterator[EncoderBuild]: - """Load and compile the checkpoint's model and build an encoder runner for it. - - The loader is cleaned up when building or the ``with`` body raises, so the executor - constructed in the body takes the loader over only once it exists. - - Args: - llm_args: The resolved arguments. - model_path: The checkpoint directory. - checkpoint_loader: The loader for the checkpoint's format. - mapping: The parallel mapping of this rank. - dist: The communicator across the model's ranks. - - Yields: - The runner with its loader, model config and resolved capacity. - """ +) -> Iterator[_LoadedEncoder]: + """Load the checkpoint's model and resolve its capacity; the loader is cleaned up on a raise.""" init_process_setup(mapping, dist) configure_autotuner_nvmmh(llm_args) max_beam_width, max_num_tokens, max_seq_len, max_batch_size = llm_args.get_runtime_sizes() @@ -536,7 +557,7 @@ def build_encoder_runner( ) try: # Encoder executors run without a MoE load balancer: encode_only has one rank, - # and scheduled pooling rejects the setting. + # and scheduled encoders reject the setting. model, _ = model_loader.load(checkpoint_dir=model_path, checkpoint_loader=checkpoint_loader) prepare_loaded_model(model, llm_args) max_seq_len = resolve_max_seq_len(model, max_seq_len) @@ -551,35 +572,145 @@ def build_encoder_runner( max_seq_len=max_seq_len, decode_tokens_per_request=1, ) + yield _LoadedEncoder( + model=model, + model_loader=model_loader, + max_beam_width=max_beam_width, + max_batch_size=max_batch_size, + max_num_tokens=max_num_tokens, + max_seq_len=max_seq_len, + prefill_capture_num_tokens=plan.prefill_num_tokens, + ) + except BaseException: + model_loader.cleanup() + raise + + +@contextmanager +def build_encoder_runner( + llm_args: TorchLlmArgs, + *, + model_path: str, + checkpoint_loader: BaseCheckpointLoader, + mapping: Mapping, + dist: Distributed, +) -> Iterator[EncoderBuild]: + """Load and compile the checkpoint's model and build an encoder runner for it. + + The loader is cleaned up when building or the ``with`` body raises, so the executor + constructed in the body takes the loader over only once it exists. + + Args: + llm_args: The resolved arguments. + model_path: The checkpoint directory. + checkpoint_loader: The loader for the checkpoint's format. + mapping: The parallel mapping of this rank. + dist: The communicator across the model's ranks. + + Yields: + The runner with its loader, model config and resolved capacity. + """ + with _load_encoder( + llm_args, + model_path=model_path, + checkpoint_loader=checkpoint_loader, + mapping=mapping, + dist=dist, + ) as loaded: compiled = compile_model( - model, + loaded.model, llm_args=llm_args, mapping=mapping, dist=dist, - max_num_tokens=max_num_tokens, - prefill_capture_num_tokens=plan.prefill_num_tokens, + max_num_tokens=loaded.max_num_tokens, + prefill_capture_num_tokens=loaded.prefill_capture_num_tokens, ) runner = EncoderRunner.create( compiled.model, llm_args=llm_args, mapping=mapping, dist=dist, - max_batch_size=max_batch_size, - max_num_tokens=max_num_tokens, - max_seq_len=max_seq_len, - max_beam_width=max_beam_width, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, + max_beam_width=loaded.max_beam_width, model_caller=compiled.caller, attention_backend=resolve_attention_backend(compiled.model, llm_args), attention_runtime_features=AttentionRuntimeFeatures(), ) yield EncoderBuild( runner=runner, - model_loader=model_loader, - model_config=model.model_config, - max_batch_size=max_batch_size, - max_num_tokens=max_num_tokens, - max_seq_len=max_seq_len, + model_loader=loaded.model_loader, + model_config=loaded.model.model_config, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, + ) + + +@contextmanager +def build_mm_encoder_runner( + llm_args: TorchLlmArgs, + *, + model_path: str, + checkpoint_loader: BaseCheckpointLoader, + mapping: Mapping, + dist: Distributed, +) -> Iterator[EncoderBuild]: + """Load the checkpoint's vision encoder, set up its towers and build a multimodal encoder runner. + + The towers size their attention metadata, then capture their block CUDA graphs after + compilation. The loader is cleaned up as in ``build_encoder_runner``. + + Args: + llm_args: The resolved arguments, with ``mm_encoder_only`` set. + model_path: The checkpoint directory. + checkpoint_loader: The loader for the checkpoint's format. + mapping: The parallel mapping of this rank. + dist: The communicator across the model's ranks. + + Yields: + The runner with its loader, model config and resolved capacity. + """ + with _load_encoder( + llm_args, + model_path=model_path, + checkpoint_loader=checkpoint_loader, + mapping=mapping, + dist=dist, + ) as loaded: + # The towers count patches, which a merged item needs more of than prompt tokens, + # so they keep the configured budget rather than the clamped one. + encoder_max_num_tokens = next( + budget + for budget in ( + llm_args.encoder_max_num_tokens, + llm_args.max_num_tokens, + loaded.max_num_tokens, + ) + if budget is not None + ) + input_processor = create_engine_input_processor(model_path, llm_args) + setup_mm_encoder_attn_metadata(loaded.model, input_processor, encoder_max_num_tokens, None) + compiled = compile_model( + loaded.model, + llm_args=llm_args, + mapping=mapping, + dist=dist, + max_num_tokens=loaded.max_num_tokens, + prefill_capture_num_tokens=loaded.prefill_capture_num_tokens, + ) + mm_config = llm_args.multimodal_config + setup_mm_encoder_cuda_graphs( + compiled.model, + mm_config.encoder_cuda_graph if mm_config is not None else None, + max_num_tokens=encoder_max_num_tokens, + ) + yield EncoderBuild( + runner=MultimodalEncoderRunner.create(compiled.model), + model_loader=loaded.model_loader, + model_config=loaded.model.model_config, + max_batch_size=loaded.max_batch_size, + max_num_tokens=loaded.max_num_tokens, + max_seq_len=loaded.max_seq_len, ) - except BaseException: - model_loader.cleanup() - raise diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py index 4b4ee6328836..9714c0ab3b25 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py @@ -9,12 +9,9 @@ from torch import nn -from tensorrt_llm.llmapi.llm_args import TorchLlmArgs - from .decoder import DecoderRunner from .encoder import EncoderRunner from .encoder_decoder import EncoderDecoderRunner -from .mm_encoder import MultimodalEncoderRunner if TYPE_CHECKING: from .interface import ModelRunner @@ -27,14 +24,8 @@ ] -def resolve_runner_type( - model: nn.Module, - llm_args: TorchLlmArgs, -) -> type[ModelRunner]: +def resolve_runner_type(model: nn.Module) -> type[ModelRunner]: """Resolve the startup runner class without constructing its dependencies.""" - if llm_args.mm_encoder_only: - return MultimodalEncoderRunner - if model.model_config.is_encoder_decoder: return EncoderDecoderRunner diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py index 28a529b3c199..d7a848fae6a4 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/mm_encoder.py @@ -10,6 +10,7 @@ from torch import nn from typing_extensions import Self +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin from tensorrt_llm._torch.models.modeling_multimodal_mixin import _build_request_multimodal_input from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import ( MoeLoadBalancer, @@ -17,7 +18,7 @@ ) from tensorrt_llm._torch.nccl_window_tensor_scope import discard_nccl_window_tensor_outputs from tensorrt_llm._torch.pyexecutor.llm_request import get_multimodal_embedding_lengths -from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType +from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.utils import with_model_extra_attrs from tensorrt_llm._utils import nvtx_range, prefer_pinned @@ -40,8 +41,8 @@ class MultimodalEncoderRunner(ScheduledModelRunner): """Run an engine whose only job is multimodal encoding. This family is distinct from ``MultimodalItemScheduler``, which schedules vision items - inside a decoder engine. Its graph phase is a no-op because - ``MultimodalEncoderGraphRunner`` belongs to, and is owned by, the vision model layer. + inside a decoder engine. The vision towers capture their block CUDA graphs when they are + built, so warmup has nothing to do; ``release_graphs`` clears them. """ def __init__( @@ -90,17 +91,15 @@ def _prepare_inputs(self, scheduled_requests: ScheduledRequests) -> _PayloadBatc embedding_lengths.append(list(lengths)) return _PayloadBatch(request_indices, params, embedding_lengths) - def _validate_resources(self, resource_manager: ResourceManager) -> None: - kv_cache_manager = resource_manager.get_resource_manager( - ResourceManagerType.KV_CACHE_MANAGER - ) - assert kv_cache_manager is None, ( - "a no-KV-cache runner was initialized, but a KV cache manager was allocated" - ) + def warmup(self, resource_manager: ResourceManager | None = None) -> None: + del resource_manager - def warmup(self, resource_manager: ResourceManager) -> None: - """Validate the resources used by this runner before execution.""" - self._validate_resources(resource_manager) + def release_graphs(self, *, release_nccl_window_owners: bool = True) -> None: + for module in self._model.modules(): + if isinstance(module, MultimodalEncoderMixin): + module.clear_blocks_cuda_graph( + release_nccl_window_owners=release_nccl_window_owners + ) @torch.inference_mode() @with_model_extra_attrs(lambda self: self._model.extra_attrs) @@ -111,11 +110,14 @@ def forward( resource_manager: ResourceManager, is_dummy: bool = False, ) -> dict[str, Any]: - del is_dummy - self._validate_resources(resource_manager) + del resource_manager, is_dummy batch = self._prepare_inputs(inputs.batch) with MoeLoadBalancerIterContext(self._moe_load_balancer): - return self._forward_step(batch) + outputs = self._forward_step(batch) + if inputs.inputs_staged is not None: + # The towers' attention metadata stages through reused host buffers during the forward. + inputs.inputs_staged.record() + return outputs @nvtx_range("_forward_step_mm_encoder_only") def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: @@ -159,7 +161,7 @@ def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: mrope_position_deltas_list.append(mrope_position_deltas) # mrope lists must align 1:1 with multimodal_params (or be empty); - # the sampler indexes them by per-MM-result position into mm_embeddings. + # the completion indexes them by per-MM-result position into mm_embeddings. assert len(mrope_position_ids_list) == len(mrope_position_deltas_list) and len( mrope_position_ids_list ) in (0, len(batch.params)), ( @@ -170,7 +172,6 @@ def _forward_step(self, batch: _PayloadBatch) -> dict[str, Any]: result = { "mm_embeddings": mm_embeddings, - "logits": None, "mm_embedding_request_indices": batch.request_indices, "mm_embedding_lengths": batch.embedding_lengths, } diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 509173015a46..a5f3c90bb9ee 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -588,6 +588,21 @@ def set_mrope_position( self.diff.mrope_position_ids = self._mrope_position_ids self.diff.mrope_position_deltas = self._mrope_position_deltas + def release_exports(self) -> None: + """Consume the handles this result exported; no response will deliver them.""" + release_exported_handles(self) + self._mm_embeddings = None + self._mrope_position_ids = None + self._mrope_position_deltas = None + self.diff.mm_embeddings = None + self.diff.mrope_position_ids = None + self.diff.mrope_position_deltas = None + + def drop_context_logits(self) -> None: + """Drop the stored context logits; no response will deliver them.""" + self._context_logits = None + self.diff.context_logits_list = [] + def set_encoder_output(self, encoder_output: torch.Tensor): self._encoder_output = encoder_output self.diff.encoder_output = encoder_output @@ -726,6 +741,33 @@ def encoder_output(self) -> torch.Tensor | None: return self._encoder_output +def release_exported_handles(result: "PyResult | LlmResult") -> None: + """Consume a result's CUDA IPC handles that no consumer will open. + + The producer cannot free an exported allocation until each of its handles is opened + and released, or consumed here. + """ + handles = [ + *(result.mm_embedding_handles or ()), + result.mrope_position_ids_handle, + result.mrope_position_deltas_handle, + ] + for handle in handles: + if handle is not None: + SharedTensorContainer.release_dict(handle) + + +def release_undelivered_response(response: Any) -> None: + """Consume the exported handles of a response that no client will receive. + + Takes the executor's responses, also when the worker has wrapped them; error + responses and other records carry no ``LlmResult`` and export nothing. + """ + result = getattr(response, "result", None) + if isinstance(result, LlmResult): + release_exported_handles(result) + + class LlmResult: """LlmResult wraps `bindings.executor.Result` but detour some features to Python implementation""" py_result_properties = frozenset( diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 6e70f167e39a..14e1a6a00679 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -64,10 +64,9 @@ from .engine.runners import resolve_runner_type from .engine.runners.common import make_scheduled_inputs, resolve_mrope_position_deltas_cache from .engine.runners.decoder import DecoderRunner, DecoderRunnerConfig -from .engine.runners.encoder import EncoderConfig, resolve_scheduled_encoder_args +from .engine.runners.encoder import EncoderConfig from .engine.runners.encoder_decoder import EncoderDecoderRunner, EncoderDecoderRunnerConfig from .engine.runners.interface import ModelRunner, ScheduledInputs, ScheduledModelRunner -from .engine.runners.mm_encoder import MultimodalEncoderRunner from .guided_decoder import CapturableGuidedDecoder from .llm_request import LlmRequest from .model_loader import ModelLoader @@ -147,12 +146,10 @@ def __init__( model_weights_restore_mode=model_weights_restore_mode, ) if not self.model.model_config.is_generation: - if not self.llm_args.mm_encoder_only: - raise ValueError( - "Non-generation models run through ScheduledEncoderExecutor, which does not " - "use ModelEngine." - ) - self.llm_args = resolve_scheduled_encoder_args(self.llm_args) + raise ValueError( + "Non-generation models run through ScheduledEncoderExecutor, which does not " + "use ModelEngine." + ) self._validate_breakable_cuda_graph_compatibility() self._init_from_model() self._size_cuda_graphs() @@ -455,12 +452,7 @@ def _init_execution_services(self) -> None: self.attn_backend = resolve_attention_backend(self.model, self.llm_args) - runner_cls = resolve_runner_type(self.model, self.llm_args) - if self.is_spec_decode and issubclass(runner_cls, MultimodalEncoderRunner): - raise ValueError( - f"{runner_cls.__name__} does not support speculative decoding; " - "multimodal encoder models must not set speculative_config." - ) + runner_cls = resolve_runner_type(self.model) if self.is_spec_decode: update_spec_config_from_loaded_model(self.spec_config, self.model) @@ -493,8 +485,6 @@ def _initialize_runner(self, runner_cls: type[ModelRunner]) -> ModelRunner: assert self._model_caller is not None if issubclass(runner_cls, EncoderDecoderRunner): return self._initialize_encoder_decoder_runner(runner_cls) - if issubclass(runner_cls, MultimodalEncoderRunner): - return runner_cls.create(self.model, moe_load_balancer=self.moe_load_balancer) if issubclass(runner_cls, DecoderRunner): return self._initialize_decoder_runner(runner_cls) raise TypeError( diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 1ecc8df33166..bfe3fd80aa48 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -694,6 +694,8 @@ class _ExecutorExecutionConfig: class PyExecutor: + # Requests generate tokens, rather than finish after one forward. + is_generation_model = True # True while the overlap loop runs with the execution stream current # (_overlap_loop_stream_scope). _loop_on_execution_stream = False @@ -1181,7 +1183,7 @@ def _initialize_execution(self, resources: PyExecutorResources) -> None: # context resources before transfer or response handling can terminate # a request. self._prefetched_request_ids: set[int] = set() - self.enable_kv_cache_events = self.kv_cache_manager is not None and ( + self.enable_kv_cache_events = ( self.kv_cache_manager.event_buffer_max_size > 0 or getattr( self.kv_cache_manager, "streaming_kv_events_enabled", False)) # Event routing also needs the sum of actual per-rank capacities after @@ -1205,7 +1207,7 @@ def _initialize_execution(self, resources: PyExecutorResources) -> None: self._kv_load_tracker.snapshot = aggregate_load(rank_loads) elif initial_sample: self._kv_load_tracker.record_local(initial_sample) - self.enable_kv_cache_reuse = self.kv_cache_manager is not None and self.kv_cache_manager.enable_block_reuse + self.enable_kv_cache_reuse = self.kv_cache_manager.enable_block_reuse # Router Replay (R3): SharedRouteCache bound, derived once from the KV # pool token capacity on the first non-warmup step (0 = block reuse off, # -1 = capacity unknown -> the capturer keeps its own fallback bound). @@ -1803,11 +1805,6 @@ def _setup_encoder_perf_metrics(self) -> None: and isinstance(model, MultimodalModelMixin)): model._encoder_forward_context = self.perf_manager.record_encoder_forward - @property - def is_generation_model(self) -> bool: - """Whether requests generate tokens, rather than finish after one forward.""" - return self.sampler.is_generation_model() - @property def metrics(self) -> dict[str, float | dict[str, float]]: """Return executor construction and model-engine startup metrics.""" @@ -1880,10 +1877,6 @@ def _maybe_init_kv_connector_manager(self) -> None: raise ValueError( "KV connector and executor attention-DP modes must agree.") - if self.kv_cache_manager is None: - raise ValueError( - "KV Cache Connector requires a KV Cache Manager.") - is_kv_cache_manager_v2 = isinstance(self.kv_cache_manager, KVCacheManagerV2) @@ -3030,8 +3023,6 @@ def get_latest_iteration_stats(self): def _read_kv_cache_capacity(self) -> dict: kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is None: - return {} block_counts = getattr(type(kv_cache_manager), "get_primary_block_counts", None) @@ -3086,7 +3077,7 @@ def get_kv_cache_load(self) -> dict: def get_latest_kv_cache_events(self): kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if not kv_cache_manager or not self.enable_kv_cache_events: + if not self.enable_kv_cache_events: return [] events = kv_cache_manager.get_latest_events(0) @@ -3511,41 +3502,40 @@ def _update_iter_stats( kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is not None: - kv_stats = kv_cache_manager.get_kv_cache_stats() - kv_stats_to_save = KvCacheStats() - kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks - kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks - kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks - kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block - kv_stats_to_save.alloc_total_blocks = kv_stats.alloc_total_blocks - kv_stats_to_save.alloc_new_blocks = kv_stats.alloc_new_blocks - kv_stats_to_save.reused_blocks = kv_stats.reused_blocks - kv_stats_to_save.missed_blocks = kv_stats.missed_blocks - kv_stats_to_save.cache_hit_rate = kv_stats.cache_hit_rate - self._fill_kv_host_stats(kv_stats_to_save, kv_cache_manager, - consume_deltas) - stats.kv_cache_stats = kv_stats_to_save - - # Collect per-iteration stats (with deltas) at configured interval. - # Between calls, C++ deltas accumulate so the reported values cover multiple iterations. - # Key the interval on the record's construction iteration, not - # the live counter: under the overlap and PP schedulers this runs - # in a later loop, and with iter_perf_stats_interval > 1 only - # sampled construction iterations reach this point, so a check on - # the live counter could never match. - # Guard: only fetch once per record iteration. - # Queue-only snapshots pass consume_deltas=False: the C++ delta - # fetch and the two cached fields below belong to measured - # iterations and must not be consumed or cleared here. - if consume_deltas: - if (stats.iter % self._kv_iter_stats_interval == 0 - and self._last_kv_iter_stats_fetch_iter != stats.iter): - self._latest_kv_iter_stats = kv_cache_manager.get_iteration_stats( - ) - self._last_kv_iter_stats_fetch_iter = stats.iter - else: - self._latest_kv_iter_stats = None + kv_stats = kv_cache_manager.get_kv_cache_stats() + kv_stats_to_save = KvCacheStats() + kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks + kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks + kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks + kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block + kv_stats_to_save.alloc_total_blocks = kv_stats.alloc_total_blocks + kv_stats_to_save.alloc_new_blocks = kv_stats.alloc_new_blocks + kv_stats_to_save.reused_blocks = kv_stats.reused_blocks + kv_stats_to_save.missed_blocks = kv_stats.missed_blocks + kv_stats_to_save.cache_hit_rate = kv_stats.cache_hit_rate + self._fill_kv_host_stats(kv_stats_to_save, kv_cache_manager, + consume_deltas) + stats.kv_cache_stats = kv_stats_to_save + + # Collect per-iteration stats (with deltas) at configured interval. + # Between calls, C++ deltas accumulate so the reported values cover multiple iterations. + # Key the interval on the record's construction iteration, not + # the live counter: under the overlap and PP schedulers this runs + # in a later loop, and with iter_perf_stats_interval > 1 only + # sampled construction iterations reach this point, so a check on + # the live counter could never match. + # Guard: only fetch once per record iteration. + # Queue-only snapshots pass consume_deltas=False: the C++ delta + # fetch and the two cached fields below belong to measured + # iterations and must not be consumed or cleared here. + if consume_deltas: + if (stats.iter % self._kv_iter_stats_interval == 0 + and self._last_kv_iter_stats_fetch_iter != stats.iter): + self._latest_kv_iter_stats = kv_cache_manager.get_iteration_stats( + ) + self._last_kv_iter_stats_fetch_iter = stats.iter + else: + self._latest_kv_iter_stats = None rank_metrics = { "rank": self.dist.rank, @@ -3555,20 +3545,19 @@ def _update_iter_stats( "cpuMemUsage": stats.cpu_mem_usage, "pinnedMemUsage": stats.pinned_mem_usage, } - if kv_cache_manager is not None: - rank_metrics["kvCacheStats"] = { - "maxNumBlocks": kv_stats.max_num_blocks, - "freeNumBlocks": kv_stats.free_num_blocks, - "usedNumBlocks": kv_stats.used_num_blocks, - "tokensPerBlock": kv_stats.tokens_per_block, - "allocTotalBlocks": kv_stats.alloc_total_blocks, - "allocNewBlocks": kv_stats.alloc_new_blocks, - "reusedBlocks": kv_stats.reused_blocks, - "missedBlocks": kv_stats.missed_blocks, - "cacheHitRate": kv_stats.cache_hit_rate, - } - append_kv_cache_iteration_stats(rank_metrics, - self._latest_kv_iter_stats) + rank_metrics["kvCacheStats"] = { + "maxNumBlocks": kv_stats.max_num_blocks, + "freeNumBlocks": kv_stats.free_num_blocks, + "usedNumBlocks": kv_stats.used_num_blocks, + "tokensPerBlock": kv_stats.tokens_per_block, + "allocTotalBlocks": kv_stats.alloc_total_blocks, + "allocNewBlocks": kv_stats.alloc_new_blocks, + "reusedBlocks": kv_stats.reused_blocks, + "missedBlocks": kv_stats.missed_blocks, + "cacheHitRate": kv_stats.cache_hit_rate, + } + append_kv_cache_iteration_stats(rank_metrics, + self._latest_kv_iter_stats) # Queue-only snapshots keep the rank snapshot of the last measured # iteration: a new one would carry that iteration's KV deltas again. if consume_deltas: @@ -6945,14 +6934,13 @@ def _emit_initial_stats(self) -> None: stats = self._get_init_iter_stats(0, 0) kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is not None: - kv_stats = kv_cache_manager.get_kv_cache_stats() - kv_stats_to_save = KvCacheStats() - kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks - kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block - kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks - kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks - stats.kv_cache_stats = kv_stats_to_save + kv_stats = kv_cache_manager.get_kv_cache_stats() + kv_stats_to_save = KvCacheStats() + kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks + kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block + kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks + kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks + stats.kv_cache_stats = kv_stats_to_save self._append_iter_stats(stats) def _build_step_scope_label(self, @@ -10112,8 +10100,6 @@ def _respond_if_invalid(request: LlmRequest) -> bool: def _add_kv_cache_events(self): kv_cache_manager = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - if not kv_cache_manager: - return # Flush iteration events at each iteration to ensure that events have enough time # to be transferred to main thread when user needs them. kv_cache_manager.flush_iteration_events() diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index f2ea4985623b..5ff101ff639d 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -465,11 +465,8 @@ def _branch_snapshot_ctx_chunk_config( return (ContextChunkingPolicy.FORCE_CHUNK, chunk_unit_size) -def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping, - is_generation: bool) -> None: +def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping) -> None: """Reject the settings a non-generation model cannot run with; warn on those it ignores.""" - if is_generation: - return if mapping.has_pp(): raise ValueError( "Pipeline parallelism is not supported for non-generation models " @@ -488,7 +485,7 @@ def _check_no_kv_execution(llm_args: TorchLlmArgs, mapping: Mapping, def _check_scheduled_encoder_support(llm_args: TorchLlmArgs, mapping: Mapping) -> None: """Reject the settings ScheduledEncoderExecutor cannot serve a non-generation model with.""" - _check_no_kv_execution(llm_args, mapping, is_generation=False) + _check_no_kv_execution(llm_args, mapping) if mapping.cp_size > 1: raise ValueError( "Context parallelism is not supported for non-generation models; " @@ -505,6 +502,8 @@ def _check_scheduled_encoder_support(llm_args: TorchLlmArgs, ("enable_resource_governor", llm_args.enable_resource_governor), ("sleep_config", llm_args.sleep_config is not None), ("orchestrator_type='ray'", llm_args.orchestrator_type == "ray"), + ("enable_client_io_during_gpu_wait", + llm_args.enable_client_io_during_gpu_wait), ) settings = [name for name, used in checks if used] if settings: @@ -737,7 +736,7 @@ def _create_py_executor( llm_args, checkpoint_loader, is_generation = _load_config_and_create_checkpoint_loader( llm_args, checkpoint_dir) - if not llm_args.mm_encoder_only and not is_generation: + if not is_generation: return _create_scheduled_encoder_executor(llm_args, checkpoint_dir, checkpoint_loader, _startup_timer, @@ -788,7 +787,6 @@ def _create_py_executor( guided_decoding_config = get_guided_decoding_config( llm_args.guided_decoding_backend, tokenizer) - mm_encoder_only = llm_args.mm_encoder_only enable_chunked_context = llm_args.enable_chunked_prefill ( @@ -850,16 +848,6 @@ def _create_py_executor( llm_args.disable_overlap_scheduler, sparse_attention_config=llm_args.sparse_attention_config) - if mm_encoder_only: - llm_args.mm_encoder_only = True - llm_args.disable_overlap_scheduler = True - - # Disable overlap scheduler for multimodal encoder-only mode - logger.warning( - "Disabling overlap scheduler for multimodal encoder-only mode. " - "The overlap scheduler is designed for generation models and is not needed " - "when only processing vision encoder inputs.") - _rank_dspark_on_host_without_overlap(spec_config, llm_args.disable_overlap_scheduler) @@ -1056,12 +1044,6 @@ def allocation_scope(current_stage: ExecutorMemoryType): model_weights_restore_mode=model_weights_restore_mode, ) - is_generation = model_engine.model.model_config.is_generation - _check_no_kv_execution(llm_args, mapping, is_generation) - if not is_generation: - enable_chunked_context = False - model_engine.attn_runtime_features.chunked_prefill = False - # Model loading can disable reuse after runtime features were created. model_engine.attn_runtime_features.cache_reuse = ( kv_cache_config.enable_block_reuse) @@ -1277,7 +1259,6 @@ def allocation_scope(current_stage: ExecutorMemoryType): mapping, max_batch_size=max_batch_size, max_beam_width=max_beam_width, - mm_encoder_only=mm_encoder_only, speculative_config=spec_config, max_num_sequences=max_num_seq_slots, ) @@ -1405,82 +1386,78 @@ def allocation_scope(current_stage: ExecutorMemoryType): logger.info( f"[create_py_executor] Created execution_stream: {execution_stream}") - if model_engine.model.model_config.is_generation: - #NOTE: non-generation models do not have kv cache - - is_disagg = is_disagg_enabled(cache_transceiver_config) - is_hybrid = is_hybrid_linear( - model_engine.model.model_config.pretrained_config) - - if is_disagg and is_hybrid: - # NOTE: TRTLLM_USE_PY_MAMBA is an agg-mode-only override and has - # no effect in disagg. The disagg manager choice is driven by - # get_kv_cache_manager_cls and cache_transceiver_config. - if os.environ.get("TRTLLM_USE_PY_MAMBA", "0") == "1": - logger.warning( - "TRTLLM_USE_PY_MAMBA is ignored in disaggregated serving; " - "configure transceiver_runtime='PYTHON' with backend='NIXL' " - "to select MixedMambaHybridCacheManager.") - else: - logger.info("Disaggregated serving with hybrid model detected. " - "Using the configured Mamba cache manager.") - - # Get draft config for one-engine speculative decoding if available - draft_config = getattr(model_engine.model, 'draft_config', None) - - if not skip_est and is_decode_only(sparse_attention_config): - # The estimation dry run prefills dummy prompts, which a - # decode-only worker cannot execute. Size the KV cache from free - # memory instead, as with TRTLLM_SKIP_KV_CACHE_ESTIMATION=1. - logger.info("KV cache size estimation is not supported for a " - "decode-only worker, disable it.") - skip_est = True - - kv_cache_creator = KvCacheCreator( - model_engine=model_engine, - mapping=mapping, - net_max_seq_len=net_max_seq_len, - kv_connector_manager=kv_connector_manager, - max_num_tokens=max_num_tokens, - max_beam_width=max_beam_width, - tokens_per_block=tokens_per_block, - max_seq_len=max_seq_len, - max_batch_size=max_batch_size, - kv_cache_config=kv_cache_config, - llm_args=llm_args, - speculative_config=spec_config, - profiling_stage_data=profiling_stage_data, - sparse_attention_config=sparse_attention_config, - execution_stream=execution_stream, - draft_config=draft_config, - gen_preemption_headroom=getattr(model_engine, - "gen_preemption_headroom", 0), - skip_est=skip_est, - is_disagg=is_disagg, - ) - - if not skip_est: - log_memory_usage("after loading weights") - - estimating_kv_cache = kv_cache_creator.try_prepare_estimation() + is_disagg = is_disagg_enabled(cache_transceiver_config) + is_hybrid = is_hybrid_linear( + model_engine.model.model_config.pretrained_config) - # Without the estimation, build_managers sizes the final KV cache from - # the device memory free now, so DWDP is set up first: the KV cache - # then gets the replicated dense weights dense DWDP frees, and the - # expert prefetch ring holds its memory before the KV cache takes its - # share of the rest. With the estimation, it is set up below, before - # the estimation run measures the memory. - if not estimating_kv_cache and (dwdp_manager is not None - or dense_dwdp_manager is not None): - manager = (dwdp_manager - if dwdp_manager is not None else dense_dwdp_manager) - manager.setup(model_engine.model) - dwdp_set_up = True + if is_disagg and is_hybrid: + # NOTE: TRTLLM_USE_PY_MAMBA is an agg-mode-only override and has + # no effect in disagg. The disagg manager choice is driven by + # get_kv_cache_manager_cls and cache_transceiver_config. + if os.environ.get("TRTLLM_USE_PY_MAMBA", "0") == "1": + logger.warning( + "TRTLLM_USE_PY_MAMBA is ignored in disaggregated serving; " + "configure transceiver_runtime='PYTHON' with backend='NIXL' " + "to select MixedMambaHybridCacheManager.") + else: + logger.info("Disaggregated serving with hybrid model detected. " + "Using the configured Mamba cache manager.") + + # Get draft config for one-engine speculative decoding if available + draft_config = getattr(model_engine.model, 'draft_config', None) + + if not skip_est and is_decode_only(sparse_attention_config): + # The estimation dry run prefills dummy prompts, which a + # decode-only worker cannot execute. Size the KV cache from free + # memory instead, as with TRTLLM_SKIP_KV_CACHE_ESTIMATION=1. + logger.info("KV cache size estimation is not supported for a " + "decode-only worker, disable it.") + skip_est = True + + kv_cache_creator = KvCacheCreator( + model_engine=model_engine, + mapping=mapping, + net_max_seq_len=net_max_seq_len, + kv_connector_manager=kv_connector_manager, + max_num_tokens=max_num_tokens, + max_beam_width=max_beam_width, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + llm_args=llm_args, + speculative_config=spec_config, + profiling_stage_data=profiling_stage_data, + sparse_attention_config=sparse_attention_config, + execution_stream=execution_stream, + draft_config=draft_config, + gen_preemption_headroom=getattr(model_engine, "gen_preemption_headroom", + 0), + skip_est=skip_est, + is_disagg=is_disagg, + ) - with allocation_scope( - ExecutorMemoryType.INIT_KV_CACHE - if estimating_kv_cache else ExecutorMemoryType.KV_CACHE): - kv_cache_creator.build_managers(resources, estimating_kv_cache) + if not skip_est: + log_memory_usage("after loading weights") + + estimating_kv_cache = kv_cache_creator.try_prepare_estimation() + + # Without the estimation, build_managers sizes the final KV cache from + # the device memory free now, so DWDP is set up first: the KV cache + # then gets the replicated dense weights dense DWDP frees, and the + # expert prefetch ring holds its memory before the KV cache takes its + # share of the rest. With the estimation, it is set up below, before + # the estimation run measures the memory. + if not estimating_kv_cache and (dwdp_manager is not None + or dense_dwdp_manager is not None): + manager = (dwdp_manager + if dwdp_manager is not None else dense_dwdp_manager) + manager.setup(model_engine.model) + dwdp_set_up = True + + with allocation_scope(ExecutorMemoryType.INIT_KV_CACHE if + estimating_kv_cache else ExecutorMemoryType.KV_CACHE): + kv_cache_creator.build_managers(resources, estimating_kv_cache) # DWDP setup: MNNVL handle exchange + composite VA weight buffer + # weight manager + MoE backend fixup (single entry point). Dense DWDP is set diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index d31035bd124e..2ec1df6cdf49 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -13,10 +13,8 @@ # limitations under the License. """Sampler orchestration: the entry points the executor drives each step. -Holds the :class:`Sampler` ABC and its ``SampleState`` types, the trivial -early-stop sampler (:class:`EarlyStopWithMMResult` for the -multimodal-encoder-only engine), and :class:`TorchSampler` -- the PyTorch -sampling path. ``TorchSampler`` owns no sampling logic of its own beyond +Holds the :class:`Sampler` ABC and its ``SampleState`` types, and +:class:`TorchSampler` -- the PyTorch sampling path. ``TorchSampler`` owns no sampling logic of its own beyond batching and orchestration: each feature (beam search, penalties, token bans, top-p decay, finish reasons, log-probs, seeds) lives in its own module and is held here as a handler, driven through ``setup_sampler_step`` / @@ -27,7 +25,7 @@ from abc import ABC, abstractmethod from collections.abc import Hashable, Iterable from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, cast import torch @@ -36,7 +34,6 @@ from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.bindings.internal.batch_manager import add_new_tokens_to_requests from tensorrt_llm.executor.result import Logprob -from tensorrt_llm.executor.utils import context_length_exceeded_message from tensorrt_llm.logger import logger from ...utils import torch_multi_arange @@ -288,10 +285,6 @@ def beam_width(requests: Iterable[LlmRequest]) -> int: return req.py_beam_width return 0 - @abstractmethod - def is_generation_model(self) -> bool: - raise NotImplementedError - def validate_request(self, request: LlmRequest) -> None: """Validate that the request can be processed by the sampler. @@ -309,179 +302,6 @@ def validate_request(self, request: LlmRequest) -> None: """ -def _validate_single_forward_request(request: LlmRequest, max_num_tokens: Optional[int]) -> None: - """Reject a prompt that does not fit the one forward a non-generation request runs in.""" - if max_num_tokens is not None and request.prompt_len > max_num_tokens: - raise ValueError( - context_length_exceeded_message( - max_context_length=max_num_tokens, num_prompt_tokens=request.prompt_len - ) - ) - - -@dataclass(kw_only=True) -class MultimodalResult: - mm_embeddings: List[torch.Tensor] - # needed to torch.split the mm_embeddings into item-wise chunks - mm_embedding_lengths: List[List[int]] - # needed when requests mix text-only and multimodal ones - mm_embedding_request_indices: List[int] - # number of context requests in the batch - num_context_requests: int - # Can be used to include e.g. `mrope_position_ids`, etc. - extra_data: Optional[Dict[str, Any]] = None - - def __post_init__(self) -> None: - num_embeddings = len(self.mm_embeddings) - metadata = (self.extra_data or {}).get("mm_embedding_metadata") - if metadata is not None and len(metadata) != num_embeddings: - raise ValueError("mm_embedding_metadata batch size does not match mm_embeddings") - num_lengths = len(self.mm_embedding_lengths) - if num_lengths != num_embeddings: - raise ValueError( - "mm_embedding_lengths batch size does not match mm_embeddings: " - f"{num_lengths} != {num_embeddings}" - ) - num_request_indices = len(self.mm_embedding_request_indices) - if num_request_indices != num_embeddings: - raise ValueError( - "mm_embedding_request_indices batch size does not match " - f"mm_embeddings: {num_request_indices} != {num_embeddings}" - ) - for result_index, (mm_embedding, mm_embedding_lengths) in enumerate( - zip(self.mm_embeddings, self.mm_embedding_lengths, strict=True) - ): - actual_rows = len(mm_embedding) - expected_rows = sum(mm_embedding_lengths) - if actual_rows != expected_rows: - raise ValueError( - f"mm_embedding shape mismatch for result {result_index}: " - f"{actual_rows} != {expected_rows}" - ) - for request_index in self.mm_embedding_request_indices: - if request_index < 0 or request_index >= self.num_context_requests: - raise ValueError( - "mm_embedding_request_indices contains an invalid request " - f"index: {request_index} not in [0, {self.num_context_requests})" - ) - - @classmethod - def from_model_outputs( - cls, model_outputs: Dict[str, Any], num_context_requests: int - ) -> "MultimodalResult": - result_keys = { - "mm_embeddings", - "mm_embedding_lengths", - "mm_embedding_request_indices", - } - return cls( - mm_embeddings=model_outputs["mm_embeddings"], - mm_embedding_lengths=model_outputs["mm_embedding_lengths"], - mm_embedding_request_indices=model_outputs["mm_embedding_request_indices"], - num_context_requests=num_context_requests, - extra_data={ - key: value for key, value in model_outputs.items() if key not in result_keys - }, - ) - - -@dataclass(kw_only=True) -class SampleStateWithMMResult(SampleState[SampleStateTensors, SampleStateTensors]): - data: MultimodalResult - - -class EarlyStopWithMMResult(Sampler[SampleStateWithMMResult]): - """ - Use for skipping decoding step for non generation model, and return the batch_output (such as mm_embeddings) - - Args: - return_mm_results: Whether to attach the multimodal outputs to the - requests' results. Disable it on ranks whose responses never reach - the frontend: the results are exported as CUDA IPC handles, and - PyTorch keeps an exported allocation alive until a consumer opens - and releases its handle. - max_num_tokens: The engine's token budget. A request runs whole in one - forward, so a prompt longer than this is rejected; None skips the check. - """ - - SampleState: TypeAlias = SampleStateWithMMResult - - def __init__( - self, return_mm_results: bool = True, max_num_tokens: Optional[int] = None - ) -> None: - self.return_mm_results = return_mm_results - self.max_num_tokens = max_num_tokens - - @override - def validate_request(self, request: LlmRequest) -> None: - _validate_single_forward_request(request, self.max_num_tokens) - - @override - def sample_async( - self, - scheduled_requests: ScheduledRequests, - model_outputs: dict[str, Any], - num_context_logits_prefix_sum: list[int], - resource_manager: Optional[ResourceManager] = None, - ) -> SampleState: - # from model_outputs to MultimodalResult - assert not scheduled_requests.generation_requests - data = MultimodalResult.from_model_outputs( - model_outputs, scheduled_requests.num_context_requests - ) - return self.SampleState(requests=scheduled_requests.context_requests, data=data) - - @override - def update_requests( - self, - state: SampleState, - resource_manager: Optional[ResourceManager] = None, - ) -> None: - # resource_manager will not be used in this function, just for interface consistency. - assert isinstance(state, SampleState) - requests = state.requests - mm_embeddings = state.data.mm_embeddings - extra_data = state.data.extra_data or {} - mrope_position_ids = extra_data.get("mrope_position_ids", None) - mrope_position_deltas = extra_data.get("mrope_position_deltas", None) - for request in requests: - request.state = LlmRequestState.GENERATION_COMPLETE - # NOTE: This is a hack: set finish reason manually and set the beam 0 - request.set_finished_reason(FinishReason.LENGTH, 0) - - if not self.return_mm_results: - return - - request_indices = state.data.mm_embedding_request_indices - for result_index, (request_index, mm_embedding) in enumerate( - zip(request_indices, mm_embeddings, strict=True) - ): - request = requests[request_index] - mm_embedding_lengths = state.data.mm_embedding_lengths[result_index] - - embedding_metadata = extra_data.get("mm_embedding_metadata") - if embedding_metadata is None: - request.py_result.append_mm_embeddings(mm_embedding, mm_embedding_lengths) - else: - request.py_result.append_mm_embeddings( - mm_embedding, mm_embedding_lengths, embedding_metadata[result_index] - ) - - # Store mrope data if available - if mrope_position_ids is not None and mrope_position_deltas is not None: - mrope_index = ( - request_index if len(mrope_position_ids) == len(requests) else result_index - ) - request.py_result.set_mrope_position( - mrope_position_ids[mrope_index], - mrope_position_deltas[mrope_index], - ) - - @override - def is_generation_model(self) -> bool: - return False - - @dataclass(kw_only=True) class SamplingRequestsMetadata: """Metadata for the sampling requests.""" @@ -555,10 +375,6 @@ def start_beam_histories(self, requests: list[LlmRequest]) -> PendingBeamHistori """ return self._beam_search.start_beam_histories(requests) - @override - def is_generation_model(self) -> bool: - return True - @dataclass(kw_only=True) @dataclass(kw_only=True) class Store: diff --git a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py index d05b5b7761e8..b655a3dd88fd 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduled_encoder_executor.py @@ -11,7 +11,7 @@ import traceback from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -28,9 +28,9 @@ from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import host_profiler_context from ..moe.fused_moe.moe_comm_timeout_guard import set_moe_comm_warmup -from .engine.model_setup import build_encoder_runner -from .engine.runners.encoder import EncoderRunner, resolve_scheduled_encoder_args -from .engine.runners.interface import ScheduledInputs +from .engine.model_setup import build_encoder_runner, build_mm_encoder_runner +from .engine.runners.encoder import resolve_scheduled_encoder_args +from .engine.runners.interface import ScheduledInputs, ScheduledModelRunner from .error_classification import ErrorBudget from .executor_request_queue import ExecutorRequestQueue, RequestQueueItem from .hang_detector import ( @@ -59,11 +59,15 @@ from .llm_request import ExecutorRequest from .model_loader import ModelLoader -__all__ = ["ScheduledEncoderExecutor", "fit_batch", "prompt_cost"] +__all__ = [ + "MultimodalEmbeddingCompletion", + "PoolingCompletion", + "ScheduledEncoderExecutor", + "fit_batch", + "prompt_cost", +] -_MULTIMODAL_INPUT_ERROR = ( - "Scheduled requests to a non-generation model cannot carry multimodal inputs." -) +_MULTIMODAL_INPUT_ERROR = "Requests to a pooling model cannot carry multimodal inputs." # One forward runs while the next is launched; a third waits for the first to be published. _MAX_IN_FLIGHT = 2 @@ -74,7 +78,6 @@ class _InFlight: """A launched batch whose forward the completion thread still has to wait for and publish.""" requests: list[LlmRequest] - responses: list[tuple[int, LlmResponse]] done: torch.cuda.Event iteration: int host_start: float @@ -107,6 +110,62 @@ def fit_batch( return len(items) +class PoolingCompletion: + """Store each request's pooled row as its context logits, when the request asked for them.""" + + accepts_multimodal_inputs = False + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + for request, row in zip(requests, outputs["logits"]): + if request.py_return_context_logits: + request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) + + def retire(self, requests: list[LlmRequest]) -> None: + for request in requests: + request.py_result.drop_context_logits() + + +class MultimodalEmbeddingCompletion: + """Export each request's multimodal embeddings, and its MRoPE positions, as CUDA IPC handles. + + A request without multimodal items in the batch completes without a result. Only the rank + whose responses reach the frontend exports: an exported allocation stays alive until a + consumer opens and releases its handle, or ``retire`` consumes it. + """ + + accepts_multimodal_inputs = True + + def __init__(self, *, exports: bool) -> None: + self._exports = exports + + def store(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + if not self._exports: + return + metadata = outputs.get("mm_embedding_metadata") + mrope_position_ids = outputs.get("mrope_position_ids") + mrope_position_deltas = outputs.get("mrope_position_deltas") + for result, (request_index, embeddings, lengths) in enumerate( + zip( + outputs["mm_embedding_request_indices"], + outputs["mm_embeddings"], + outputs["mm_embedding_lengths"], + strict=True, + ) + ): + py_result = requests[request_index].py_result + py_result.append_mm_embeddings( + embeddings, lengths, None if metadata is None else metadata[result] + ) + if mrope_position_ids is not None and mrope_position_deltas is not None: + py_result.set_mrope_position( + mrope_position_ids[result], mrope_position_deltas[result] + ) + + def retire(self, requests: list[LlmRequest]) -> None: + for request in requests: + request.py_result.release_exports() + + class ScheduledEncoderExecutor: """Serve scheduled requests to a non-generation model: queue, admission, batching, one forward. @@ -116,12 +175,16 @@ class ScheduledEncoderExecutor: forwards are in flight, so the next batch is prepared and launched while the previous one runs, and a completion thread publishes each batch as soon as its forward finishes. A forward that fails on the device after its launch is not recoverable and ends the - loop like any other crash. Settings that only tune generation are ignored: the overlap - scheduler, chunked prefill, guided decoding, ``scheduler_config``, the waiting-queue - policy and the iteration-based batch waits. Iteration statistics and KV cache events + loop like any other crash. Requests wait in the queue that ``scheduler_config``'s + waiting-queue policy selects. Settings that only tune generation are ignored: the overlap + scheduler, chunked prefill, guided decoding, the rest of ``scheduler_config`` and the + iteration-based batch waits. Iteration statistics and KV cache events are always empty. With ``print_iter_log`` rank 0 logs one line per batch: its requests and prompt tokens, the requests still waiting when it launched, the host time from taking the batch to publishing it, and the GPU time of the forward. + + A pooling model's runner pairs with ``PoolingCompletion``, a multimodal encoder's + with ``MultimodalEmbeddingCompletion``. """ is_generation_model = False @@ -129,8 +192,9 @@ class ScheduledEncoderExecutor: def __init__( self, - runner: EncoderRunner, + runner: ScheduledModelRunner, *, + completion: PoolingCompletion | MultimodalEmbeddingCompletion, model_loader: ModelLoader, llm_args: TorchLlmArgs, mapping: Mapping, @@ -143,6 +207,9 @@ def __init__( Args: runner: The model's runner; every forward goes through its scheduled entry point. + completion: How a forward's outputs complete its requests; it fits the runner. + ``store`` attaches a forward's results before their responses are created; + ``retire`` withdraws them from requests that finish without their result. model_loader: The loader that loaded the model, cleaned up at shutdown. llm_args: The resolved arguments; the executor reads its timeouts, garbage collection threshold and metrics switches. @@ -153,6 +220,7 @@ def __init__( max_seq_len: The longest prompt a request may have. """ self._runner = runner + self._completion = completion self._model_loader = model_loader self.dist = dist self.max_batch_size = max_batch_size @@ -169,14 +237,22 @@ def __init__( self._gc_gen0_threshold = llm_args.garbage_collection_gen0_threshold self._hang_detector = HangDetector(on_detected=self._on_hang_detected) + scheduler_config = llm_args.scheduler_config + # The wait cap ages requests by rank 0's arrival stamps on rank 0's clock, so that + # every rank pops the same order. + aging = scheduler_config.waiting_queue_max_wait_s is not None self._request_queue = ExecutorRequestQueue( dist, max_batch_size, enable_iter_perf_stats=False, batch_wait_timeout_ms=llm_args.batch_wait_timeout_ms, + stamp_admission_times=aging, + ) + self._broadcaster = RequestBroadcaster(dist, self._hang_detector, sync_clock=aging) + self._waiting_queue = create_waiting_queue( + scheduler_config.waiting_queue_policy, + max_wait_s=scheduler_config.waiting_queue_max_wait_s, ) - self._broadcaster = RequestBroadcaster(dist, self._hang_detector) - self._waiting_queue = create_waiting_queue() self._error_budget = ErrorBudget() self._perf_manager = PerfMetricsManager(enabled=llm_args.return_perf_metrics) self._iter_log = llm_args.print_iter_log and dist.rank == 0 @@ -185,6 +261,12 @@ def __init__( self._last_staged: torch.cuda.Event | None = None self._completion_thread: threading.Thread | None = None self._completion_error: BaseException | None = None + # Ids of the launched, unpublished requests, and of those among them cancelled since; + # and ids whose cancellation was sent but not yet taken by the loop. + self._cancel_lock = threading.Lock() + self._launched_ids: set[int] = set() + self._cancelled_after_launch: set[int] = set() + self._cancel_sent: set[int] = set() self._response_channel = ResponseChannel() self._iter_counter = 0 self._profiler = ProfileController( @@ -218,6 +300,9 @@ def create( ) -> ScheduledEncoderExecutor: """Load and compile the model, build its runner, and warm it up on every rank. + With ``mm_encoder_only`` the checkpoint's vision encoder is built and rank 0 exports + its embeddings; otherwise the checkpoint's pooling model is built. + Args: llm_args: The arguments; scheduled-encoder constraints are applied to them first. model_path: The checkpoint directory. @@ -229,7 +314,14 @@ def create( The executor, warmed up; its loop starts with ``start_worker``. """ llm_args = resolve_scheduled_encoder_args(llm_args) - with build_encoder_runner( + if llm_args.mm_encoder_only: + build = build_mm_encoder_runner + # Attention DP is rejected, so rank 0 alone answers the frontend. + completion = MultimodalEmbeddingCompletion(exports=dist.rank == 0) + else: + build = build_encoder_runner + completion = PoolingCompletion() + with build( llm_args, model_path=model_path, checkpoint_loader=checkpoint_loader, @@ -242,6 +334,7 @@ def create( ) return cls( built.runner, + completion=completion, model_loader=built.model_loader, llm_args=llm_args, mapping=mapping, @@ -333,11 +426,17 @@ def _await_single(self, request_id: int, timeout: float | None) -> list[LlmRespo ) def cancel_request(self, id: int) -> None: - """Cancel a request; a queued one finishes as cancelled, a finished one is ignored. + """Cancel a request; it finishes as cancelled unless its batch's completion was settled. + + The completion thread settles a batch in ``_cancel_launched``, just before publishing it. Args: id: The request id. """ + # A launched batch may complete before the loop takes the queued cancellation; + # rank 0, which publishes, checks this record then. + with self._cancel_lock: + self._cancel_sent.add(id) self._request_queue.enqueue_cancel_request(id) def start_profile( @@ -474,7 +573,9 @@ def _loop(self) -> None: max_num_tokens=self.max_num_tokens, ) if count: - self._run([self._waiting_queue.pop_request() for _ in range(count)]) + items = [self._waiting_queue.pop_request() for _ in range(count)] + self._waiting_queue.mark_admitted(items) + self._run(items) # Every pass counts, idle ones too, so a stop requested while idle fires. self._iter_counter += 1 finally: @@ -506,6 +607,8 @@ def _receive(self) -> None: with self._hang_detector.pause(): items = self._request_queue.get_from_request_queue(timeout) items = self._broadcaster.broadcast(items, prefer_cpu=idle) + if self._broadcaster.sync_clock: + self._waiting_queue.set_now(self._broadcaster.rank0_now) canceled_ids = set() rejected = [] @@ -532,13 +635,16 @@ def _receive(self) -> None: if self.dist.rank == 0: self._respond_with_errors(rejected) if canceled_ids: + with self._cancel_lock: + self._cancelled_after_launch.update(canceled_ids & self._launched_ids) + self._cancel_sent -= canceled_ids self._cancel_waiting(canceled_ids) @nvtx_range("_check_admissible") def _check_admissible(self, item: RequestQueueItem) -> None: """Raise ValueError if one forward cannot run this request.""" request = item.request - if ( + if not self._completion.accepts_multimodal_inputs and ( getattr(request, "py_multimodal_data", None) is not None or request.mrope_config is not None ): @@ -553,7 +659,7 @@ def _check_admissible(self, item: RequestQueueItem) -> None: ) def _cancel_waiting(self, request_ids: set[int]) -> None: - """Finish queued requests as cancelled; ids no longer queued have already completed.""" + """Finish queued requests as cancelled; ids no longer queued are launched or completed.""" items = [item for item in self._waiting_queue if item.id in request_ids] if not items: return @@ -621,17 +727,19 @@ def _run(self, items: list[RequestQueueItem]) -> None: timing.end_time, timing.end_time, ) - responses = self._complete(requests, outputs) + self._complete(requests, outputs) done = torch.cuda.Event() done.record() except Exception as error: self._slots.release() + self._completion.retire(requests) self._fail(requests, error) return + with self._cancel_lock: + self._launched_ids.update(item.id for item in items) self._in_flight.put( _InFlight( requests=requests, - responses=responses, done=done, iteration=self._iter_counter, host_start=host_start, @@ -649,9 +757,11 @@ def _completion_loop(self) -> None: with nvtx_range("synchronize"): # Responses carry device tensors that other threads and processes read. flight.done.synchronize() + self._cancel_launched(flight) if self.dist.rank == 0: self._publish( - flight.responses, (request.py_request_id for request in flight.requests) + self._create_responses(flight.requests), + (request.py_request_id for request in flight.requests), ) if self._iter_log: self._log_iteration(flight) @@ -665,6 +775,30 @@ def _completion_loop(self) -> None: finally: self._slots.release() + def _cancel_launched(self, flight: _InFlight) -> None: + """Finish the batch's requests cancelled since its launch as cancelled, without results. + + A cancelled request's exports are retired; a cancellation that arrives after this + point finds the request completed. + """ + request_ids = {request.py_request_id for request in flight.requests if not request.is_child} + with self._cancel_lock: + self._launched_ids -= request_ids + cancelled_ids = (self._cancelled_after_launch | self._cancel_sent) & request_ids + self._cancelled_after_launch -= cancelled_ids + if not cancelled_ids: + return + cancelled = [ + request + for request in flight.requests + if (request.parent_request_id if request.is_child else request.py_request_id) + in cancelled_ids + ] + self._completion.retire(cancelled) + for request in cancelled: + # Replaces the reason LENGTH set by _complete. + request.set_finished_reason(FinishReason.CANCELLED, 0) + def _log_iteration(self, flight: _InFlight) -> None: """Log one batch: its size, the host time until it was published and the GPU time of its forward.""" host_ms = (time.monotonic() - flight.host_start) * 1000 @@ -679,16 +813,12 @@ def _log_iteration(self, flight: _InFlight) -> None: ) @nvtx_range("_complete") - def _complete( - self, requests: list[LlmRequest], outputs: dict[str, torch.Tensor] - ) -> list[tuple[int, LlmResponse]]: - """Store each request's pooled row as its context logits and report it finished with reason LENGTH.""" + def _complete(self, requests: list[LlmRequest], outputs: dict[str, Any]) -> None: + """Store the forward's results on its requests and finish each with reason LENGTH.""" + self._completion.store(requests, outputs) batch_token_time = self._perf_manager.get_timestamp() gpu_times_cache = {} - responses = [] - for request, row in zip(requests, outputs["logits"]): - if request.py_return_context_logits: - request.py_result.append_context_logits(row.unsqueeze(0) if row.ndim == 1 else row) + for request in requests: # The public result reports a finished request with no tokens and reason LENGTH. request.state = LlmRequestState.GENERATION_COMPLETE request.set_finished_reason(FinishReason.LENGTH, 0) @@ -699,6 +829,16 @@ def _complete( self._perf_manager.compute_batch_gpu_times( [request], gpu_times_cache=gpu_times_cache ) + + @staticmethod + def _create_responses(requests: list[LlmRequest]) -> list[tuple[int, LlmResponse]]: + """The batch's responses, created once each request's finish reason is settled. + + Creating a response marks its sequence final, so a second one for a sequence could + end the request before its other sequences arrive. + """ + responses = [] + for request in requests: response = request.create_response() if response is not None: responses.append((request.py_request_id, response)) diff --git a/tensorrt_llm/_torch/speculative/spec_sampler_base.py b/tensorrt_llm/_torch/speculative/spec_sampler_base.py index f5c5f6a3a909..a587b1ad08b7 100644 --- a/tensorrt_llm/_torch/speculative/spec_sampler_base.py +++ b/tensorrt_llm/_torch/speculative/spec_sampler_base.py @@ -194,9 +194,6 @@ class SpecSampler(Sampler[SampleStateSpec], AsyncWorkerMixin): SampleState = SampleStateSpec - def is_generation_model(self) -> bool: - return True - def validate_request(self, request: LlmRequest) -> None: """Reject sampling parameters the one-model speculative path cannot honor. diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 7c310416dd07..9a1daefd1fe9 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -33,7 +33,8 @@ from .._torch.peft.lora.manager import LoraManager from .._torch.pyexecutor.iteration_stats import IterationStatsRecord -from .._torch.pyexecutor.llm_request import LlmResponse +from .._torch.pyexecutor.llm_request import (LlmResponse, + release_undelivered_response) from .._torch.pyexecutor.request_inputs import (ExecutorRequest, PythonRequestInputs) from .._utils import (global_mpi_rank, global_mpi_size, mpi_rank, @@ -1609,6 +1610,7 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: elif _drop_refused_response(self.worker, response): continue elif self.worker._has_background_error(): + release_undelivered_response(response) response = self.worker._create_error_response(response) elif response.has_error(): # Convert to ErrorResponse, because tllm.Response cannot be @@ -1834,6 +1836,7 @@ def _drop_refused_response(worker, response: LlmResponse) -> bool: return False if response.has_error() or response.result.is_final: worker._refused_client_ids.discard(response.client_id) + release_undelivered_response(response) return True diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 428307b9fed3..bb784799f0e8 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -39,6 +39,7 @@ wrap_worker_startup_error) from tensorrt_llm.logger import logger +from .._torch.pyexecutor.llm_request import release_undelivered_response from .._utils import customized_gc_thresholds, mpi_rank, nvtx_range_debug from ..llmapi.llm_args import TorchLlmArgs from ..llmapi.mpi_session import (MpiCommSession, MpiPoolSession, MpiSession, @@ -909,6 +910,7 @@ def process_res(res): # late responses instead of crashing with a KeyError. result = self._results.get(client_id) if result is None: + release_undelivered_response(res) return queue = result.queue if isinstance(queue, _SyncQueue): diff --git a/tensorrt_llm/executor/rpc_proxy_mixin.py b/tensorrt_llm/executor/rpc_proxy_mixin.py index f02a058cc1f0..624f96cdd1d2 100644 --- a/tensorrt_llm/executor/rpc_proxy_mixin.py +++ b/tensorrt_llm/executor/rpc_proxy_mixin.py @@ -19,6 +19,7 @@ import threading from typing import Callable, List, Optional +from .._torch.pyexecutor.llm_request import release_undelivered_response from .._utils import nvtx_range_debug from ..llmapi.tracer import global_tracer from ..llmapi.utils import _SyncQueue @@ -140,6 +141,7 @@ def process_res(res: list): if client_id not in self._results: logger.warning(f"Received response for unknown client_id: {client_id}") + release_undelivered_response(r) continue queue = self._results[client_id].queue diff --git a/tensorrt_llm/llmapi/mm_encoder.py b/tensorrt_llm/llmapi/mm_encoder.py index 71cb5cd7f051..f36b75bc3fbe 100644 --- a/tensorrt_llm/llmapi/mm_encoder.py +++ b/tensorrt_llm/llmapi/mm_encoder.py @@ -72,7 +72,7 @@ def _build_model(self): mpi_session=self.mpi_session, reuse_mpi_comm=external_mpi_comm_available( self.args.parallel_config.world_size), - is_llm_executor=True, # TODO: check if this is correct or needed + is_llm_executor=True, hf_model_dir=self._hf_model_dir, tokenizer=self.tokenizer, llm_args=self.args) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index b43f511556db..19702ddf7d5d 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -40,6 +40,7 @@ from tensorrt_llm._startup import _StartupTimer from tensorrt_llm._torch.async_llm import AsyncLLM +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._utils import EnergyMonitor from tensorrt_llm.checkpoint_format import CheckpointFormat # yapf: disable @@ -3073,9 +3074,16 @@ async def openai_mm_encoder(self, request: ChatCompletionRequest, async def create_mm_embedding_response(promise: RequestOutput): await promise.aresult() # TODO: Replace mm_embedding_handles with a dedicated OpenAIBaseModel(JSON-safe), when enable multimodal disagg E2E - mm_embedding_handles = ( - promise.disaggregated_params.multimodal_embedding_handles - if promise.disaggregated_params else None) + params = promise.disaggregated_params + mm_embedding_handles = (params.multimodal_embedding_handles + if params else None) + # The response returns only the embedding handles; release the + # exports no caller can open. + if params is not None: + for handle in (params.mrope_position_ids_handle, + params.mrope_position_deltas_handle): + if handle is not None: + SharedTensorContainer.release_dict(handle) if not mm_embedding_handles: return self.create_error_response( message="Multimodal embedding handle missing in response", diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 570e6030e5c7..bc130003c6d5 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -311,6 +311,7 @@ l0_cpu: - unittest/executor/test_ray_stub.py - unittest/executor/test_generation_request_array_prompt.py - unittest/executor/test_llm_result_unpickle.py + - unittest/executor/test_undelivered_exports.py - unittest/grpc/openengine/test_coordinator.py - unittest/_torch/test_shared_tensor_cpu.py - unittest/grpc/openengine/test_frontend_lifecycle.py diff --git a/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py b/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py index f27fe3e54fd9..e38604eb747a 100644 --- a/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py +++ b/tests/unittest/_torch/modules/dwdp/test_dwdp_creator.py @@ -196,7 +196,7 @@ def _create( Args: mode: ``"expert"`` (expert DWDP alone), ``"attached"`` (dense DWDP attached to it) or ``"standalone"`` (dense DWDP alone). - kv_cache: ``"estimated"``, ``"not_estimated"`` or ``"none"`` (a model without one). + kv_cache: ``"estimated"`` or ``"not_estimated"``. real_dense_manager: Standalone, the creator builds a real ``DenseDwdpManager``, whose setup only logs; the model engine logs the SM reserve it is built with. copy_mode: Standalone, ``dwdp_config.copy_mode``. @@ -265,7 +265,6 @@ def model_engine(attn_runtime_features: object, **_kwargs: object) -> _DummyMode engine = _DummyModelEngine( attn_runtime_features=attn_runtime_features, kv_cache_quant_algo=QuantAlgo.NO_QUANT ) - engine.model.model_config.is_generation = kv_cache != "none" return engine dense_mode = {"attached": DenseMode.ATTACHED, "standalone": DenseMode.STANDALONE}.get(mode) @@ -320,7 +319,7 @@ def model_engine(attn_runtime_features: object, **_kwargs: object) -> _DummyMode return log, manager -@pytest.mark.parametrize("kv_cache", ["estimated", "not_estimated", "none"]) +@pytest.mark.parametrize("kv_cache", ["estimated", "not_estimated"]) @pytest.mark.parametrize("mode", ["expert", "attached", "standalone"]) def test_only_the_serving_phase_gets_the_manager( monkeypatch: pytest.MonkeyPatch, mode: str, kv_cache: str @@ -348,7 +347,6 @@ def test_only_the_serving_phase_gets_the_manager( "rebind", ], "not_estimated": ["setup", final_kv_cache], - "none": ["setup"], }[kv_cache] if kv_cache != "estimated": expected.append(("executor", *given)) @@ -372,7 +370,7 @@ def test_expert_dwdp_accepts_a_tactic_naming_an_implementation_with_dwdp( monkeypatch: pytest.MonkeyPatch, tactic: str ) -> None: """A tactic leaves moe_config.backend at AUTO, so the tactic is what the launch checks.""" - log, _ = _create(monkeypatch, "expert", "none", moe_backend="AUTO", moe_tactic=tactic) + log, _ = _create(monkeypatch, "expert", "not_estimated", moe_backend="AUTO", moe_tactic=tactic) assert "setup" in log diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py index 5208cc6ff496..7300e57b9f2e 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_common.py @@ -26,7 +26,6 @@ token_model_inputs, ) from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ModelRunner -from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.model_engine import ModelEngine from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm.llmapi.llm_args import PrefillCudaGraphBackend @@ -35,22 +34,19 @@ @pytest.mark.parametrize( - ("mm_encoder_only", "is_encoder_decoder", "runner_type"), + ("is_encoder_decoder", "runner_type"), [ - (True, False, MultimodalEncoderRunner), - (False, True, EncoderDecoderRunner), - (False, False, DecoderRunner), + (True, EncoderDecoderRunner), + (False, DecoderRunner), ], ) def test_resolve_runner_dispatches_startup_families( - mm_encoder_only: bool, is_encoder_decoder: bool, runner_type: type[ModelRunner], ) -> None: - args = SimpleNamespace(mm_encoder_only=mm_encoder_only) model = SimpleNamespace(model_config=SimpleNamespace(is_encoder_decoder=is_encoder_decoder)) - assert resolve_runner_type(model, args) is runner_type + assert resolve_runner_type(model) is runner_type @pytest.mark.parametrize( diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py index 6c3295b687ae..66d15a6442b0 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_mm_encoder.py @@ -7,12 +7,14 @@ import pytest import torch +from torch import nn +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin from tensorrt_llm._torch.pyexecutor.engine.runners import mm_encoder as mm_encoder_module from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledInputs from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner from tensorrt_llm._torch.pyexecutor.llm_request import PyResult -from tensorrt_llm._torch.pyexecutor.sampler import EarlyStopWithMMResult +from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import MultimodalEmbeddingCompletion from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm._torch.utils import get_model_extra_attrs @@ -95,13 +97,13 @@ def test_mm_encoder_runner_forward_skips_metadata_only_and_missing_length_reques to_device.assert_called_once_with(params[0], "multimodal_data", "cuda", pin_memory=True) for request in requests: request.get_tokens.assert_not_called() - assert result["logits"] is None assert result["mm_embedding_request_indices"] == [3] assert result["mm_embedding_lengths"] == [[2]] torch.testing.assert_close(result["mm_embeddings"][0], torch.arange(4).reshape(2, 2)) def test_mm_encoder_runner_splits_embeddings_and_returns_mrope_metadata() -> None: + """The runner also marks its inputs staged once the towers have run.""" requests = [ SimpleNamespace( py_multimodal_data={ @@ -131,21 +133,24 @@ def test_mm_encoder_runner_splits_embeddings_and_returns_mrope_metadata() -> Non ), ] embeddings = torch.arange(6).reshape(3, 2) - model = SimpleNamespace(extra_attrs={}, forward=Mock(return_value=[embeddings])) + calls = Mock() + calls.forward.return_value = [embeddings] + model = SimpleNamespace(extra_attrs={}, forward=calls.forward) runner = MultimodalEncoderRunner(model, encoder_cache_enabled=False, moe_load_balancer=None) for index, request in enumerate(requests): request.py_multimodal_params = MultimodalParams(multimodal_data=request.py_multimodal_data) request.get_num_tokens = Mock(return_value=index + 1) result = runner.forward( - ScheduledInputs(batch=SimpleNamespace(context_requests=requests)), + ScheduledInputs( + batch=SimpleNamespace(context_requests=requests), inputs_staged=calls.inputs_staged + ), resource_manager=SimpleNamespace(get_resource_manager=Mock(return_value=None)), ) - model.forward.assert_called_once() + assert [name for name, _, _ in calls.mock_calls] == ["forward", "inputs_staged.record"] params = model.forward.call_args.args[0] assert [param.input_ids_start_offset for param in params] == [0, 1] - assert result["logits"] is None assert result["mm_embedding_request_indices"] == [0, 1] assert result["mm_embedding_lengths"] == [[1], [2]] torch.testing.assert_close(result["mm_embeddings"][0], embeddings[:1]) @@ -178,7 +183,6 @@ def test_mm_encoder_handoff_preserves_per_item_metadata_in_sparse_batch() -> Non ] for request in requests: request.py_result = PyResult(prompt_len=1, max_new_tokens=1) - request.set_finished_reason = Mock() request.py_multimodal_params = MultimodalParams(multimodal_data=request.py_multimodal_data) request.get_num_tokens = Mock(return_value=1) scheduled_requests = SimpleNamespace( @@ -202,9 +206,7 @@ def encode(batch_params: list[MultimodalParams]) -> list[torch.Tensor]: resource_manager=SimpleNamespace(get_resource_manager=Mock(return_value=None)), ) assert get_model_extra_attrs() is None - sampler = EarlyStopWithMMResult() - state = sampler.sample_async(scheduled_requests, result, []) - sampler.update_requests(state) + MultimodalEmbeddingCompletion(exports=True).store(requests, result) assert requests[1].py_result.mm_embedding_handles is None image_handles = json.loads(json.dumps(requests[0].py_result.mm_embedding_handles)) @@ -225,20 +227,21 @@ def encode(batch_params: list[MultimodalParams]) -> list[torch.Tensor]: torch.testing.assert_close(torch.cat(restored), embeddings[1:]) -def test_mm_encoder_runner_rejects_kv_manager_before_preparation() -> None: +class _Tower(MultimodalEncoderMixin, nn.Module): + pass + + +def test_mm_encoder_runner_releases_every_tower_graph() -> None: + towers = [_Tower(), _Tower()] + graphs = [Mock(), Mock()] + for tower, graph in zip(towers, graphs): + tower._blocks_graph_runner = graph runner = MultimodalEncoderRunner( - SimpleNamespace(extra_attrs={}), encoder_cache_enabled=False, moe_load_balancer=None + nn.Sequential(*towers), encoder_cache_enabled=False, moe_load_balancer=None ) - runner._prepare_inputs = Mock() - resource_manager = Mock() - resource_manager.get_resource_manager.return_value = object() - - with pytest.raises( - AssertionError, - match="no-KV-cache runner was initialized, but a KV cache manager was allocated", - ): - runner.forward( - ScheduledInputs(batch=ScheduledRequests()), resource_manager=resource_manager - ) - runner._prepare_inputs.assert_not_called() + runner.release_graphs(release_nccl_window_owners=False) + + for tower, graph in zip(towers, graphs): + graph.clear.assert_called_once_with(release_nccl_window_owners=False) + assert tower._blocks_graph_runner is None diff --git a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py index f7c580cec0b8..d86ea80c9ad6 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_multimodal.py @@ -344,6 +344,53 @@ def _build_blocks_graph_runner(self, config: MultimodalEncoderCudaGraphConfig) - model.encoder._blocks_graph_runner.capture_all.assert_called_once_with(torch.device("cpu")) +@pytest.mark.parametrize( + ("encoder_max_num_tokens", "max_num_tokens", "tower_budget"), + [(None, 4096, 4096), (1024, 4096, 1024), (None, None, 128)], + ids=["configured", "encoder_budget", "resolved"], +) +def test_mm_encoder_builder_sizes_the_towers_with_the_configured_token_budget( + monkeypatch: pytest.MonkeyPatch, + encoder_max_num_tokens: int | None, + max_num_tokens: int | None, + tower_budget: int, +) -> None: + """A merged item fits a prompt shorter than its patches, so the clamped budget is too small.""" + from tensorrt_llm._torch.pyexecutor.engine import model_setup + + model = SimpleNamespace(model_config=SimpleNamespace()) + loader = Mock() + loader.load.return_value = (model, None) + for name in ("init_process_setup", "configure_autotuner_nvmmh", "prepare_loaded_model"): + monkeypatch.setattr(model_setup, name, Mock()) + monkeypatch.setattr(model_setup, "create_model_loader", Mock(return_value=loader)) + monkeypatch.setattr(model_setup, "resolve_max_seq_len", lambda model, max_seq_len: max_seq_len) + monkeypatch.setattr(model_setup, "create_engine_input_processor", Mock()) + monkeypatch.setattr( + model_setup, "compile_model", Mock(return_value=SimpleNamespace(model=model)) + ) + monkeypatch.setattr(model_setup.MultimodalEncoderRunner, "create", Mock()) + setup_metadata, setup_graphs = Mock(), Mock() + monkeypatch.setattr(model_setup, "setup_mm_encoder_attn_metadata", setup_metadata) + monkeypatch.setattr(model_setup, "setup_mm_encoder_cuda_graphs", setup_graphs) + llm_args = SimpleNamespace( + max_num_tokens=max_num_tokens, + encoder_max_num_tokens=encoder_max_num_tokens, + get_runtime_sizes=lambda: (1, max_num_tokens, 128, 1), + multimodal_config=None, + cuda_graph_config=None, + prefill_capture_num_tokens=None, + ) + + with model_setup.build_mm_encoder_runner( + llm_args, model_path="", checkpoint_loader=Mock(), mapping=Mapping(), dist=None + ) as built: + assert built.max_num_tokens == 128 + + assert setup_metadata.call_args.args[2] == tower_budget + assert setup_graphs.call_args.kwargs["max_num_tokens"] == tower_budget + + @pytest.mark.parametrize("enable_padding", [False, True]) @pytest.mark.parametrize("automatic", [False, True], ids=["explicit", "automatic"]) def test_encoder_graphs_preserve_eager_metadata_capacity( diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py index ff4331974ede..6891609bc07f 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py @@ -216,8 +216,9 @@ def _build_fake_self( * ``executor_request_queue.get_request_queue_size()`` — for top-level ``num_queued_requests`` (not one of the fields exercised here) - * ``resource_manager.resource_managers.get(...)`` — returns None so - the KV-cache-stats block is skipped entirely + * ``resource_manager.resource_managers.get(...)`` — a KV cache manager + reporting zeroed ``KvCacheStats`` and no per-iteration KV stats (the + KV fields are not exercised here) * ``drafter`` — None (spec-decode block no-ops when ``stats.specdec_stats`` is None, which is the default on a fresh ``IterationStats``) @@ -239,7 +240,12 @@ def _build_fake_self( fake.iter_counter = 1 fake.executor_request_queue.get_request_queue_size.return_value = len(queued_items) fake.executor_request_queue.get_request_queue.return_value.queue = queued_items - fake.resource_manager.resource_managers.get.return_value = None + kv_cache_manager = MagicMock() + kv_cache_manager.get_kv_cache_stats.return_value = KvCacheStats() + kv_cache_manager.get_iteration_stats.return_value = None + fake.resource_manager.resource_managers.get.return_value = kv_cache_manager + fake._kv_iter_stats_interval = 1 + fake._last_kv_iter_stats_fetch_iter = None fake.drafter = None fake.model_engine = types.SimpleNamespace(iter_states=model_engine_iter_states) fake.enable_attention_dp = enable_attention_dp diff --git a/tests/unittest/_torch/pyexecutor/test_model_engine.py b/tests/unittest/_torch/pyexecutor/test_model_engine.py index b827d48e5441..b16d4722336b 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_engine.py +++ b/tests/unittest/_torch/pyexecutor/test_model_engine.py @@ -52,7 +52,7 @@ from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.utils import torch_compiling from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor -from tensorrt_llm.llmapi import CudaGraphConfig, KvCacheConfig, SADecodingConfig +from tensorrt_llm.llmapi import CudaGraphConfig, KvCacheConfig from tensorrt_llm.llmapi.llm_args import MultimodalConfig, PrefillCudaGraphBackend, TorchLlmArgs from tensorrt_llm.mapping import Mapping @@ -656,15 +656,12 @@ def test_model_engine_rejects_encode_only() -> None: @pytest.mark.cpu_only @pytest.mark.parametrize("backend", ["piecewise", "breakable"]) -@pytest.mark.parametrize("mm_encoder_only", [False, True]) def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( - monkeypatch: pytest.MonkeyPatch, backend: str, mm_encoder_only: bool + monkeypatch: pytest.MonkeyPatch, backend: str ) -> None: model = DummyLegacyMultimodalIndexModel() - model.model_config = SimpleNamespace(is_generation=not mm_encoder_only) - llm_args = TorchLlmArgs( - model="dummy", mm_encoder_only=mm_encoder_only, prefill_cuda_graph_backend=backend - ) + model.model_config = SimpleNamespace(is_generation=True) + llm_args = TorchLlmArgs(model="dummy", prefill_cuda_graph_backend=backend) monkeypatch.setattr(model_setup, "_configure_deep_gemm_pdl", Mock()) monkeypatch.setattr( model_engine_module, @@ -678,7 +675,7 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( ) monkeypatch.setattr(ModelEngine, "__del__", lambda self: None) engine = ModelEngine.__new__(ModelEngine) - rejects_breakable = not mm_encoder_only and backend == "breakable" + rejects_breakable = backend == "breakable" expected_error = ValueError if rejects_breakable else RuntimeError message = "multimodal models" if rejects_breakable else "model configuration resolved" with pytest.raises(expected_error, match=message): @@ -686,10 +683,7 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( engine, model_path="dummy", llm_args=llm_args, mapping=Mapping(), model=model ) - expected_backend = "disabled" if mm_encoder_only else backend - assert engine.llm_args.prefill_cuda_graph_backend == PrefillCudaGraphBackend(expected_backend) - if mm_encoder_only: - assert engine.llm_args.torch_compile_config is None + assert engine.llm_args.prefill_cuda_graph_backend == PrefillCudaGraphBackend(backend) class ModelEngineTestCase(unittest.TestCase): @@ -778,20 +772,6 @@ def test_model_engine_rejects_non_generation_models(self) -> None: with self.assertRaisesRegex(ValueError, "ScheduledEncoderExecutor"): DummyModelEngine(llm_args, torch.half, is_generation=False) - def test_no_kv_cache_runners_reject_speculative_config(self) -> None: - llm_args = TorchLlmArgs( - model="dummy", max_batch_size=4, max_num_tokens=64, mm_encoder_only=True - ) - with self.assertRaisesRegex( - ValueError, "MultimodalEncoderRunner does not support speculative decoding" - ): - DummyModelEngine( - llm_args, - torch.half, - spec_config=SADecodingConfig(max_draft_len=1), - is_generation=False, - ) - @pytest.mark.cpu_only def test_runner_engine_shared_entries_skip_decoder_state(self) -> None: runner = Mock(spec=MultimodalEncoderRunner) @@ -1217,33 +1197,6 @@ def test_allreduce_forward_routes_tuning_and_inference(self) -> None: ) unused.assert_not_called() - @pytest.mark.cpu_only - def test_no_kv_cache_warmup_rejects_allocated_kv_cache(self) -> None: - """Verify a no-KV-cache runner rejects an allocated KV cache before warmup.""" - model_engine = object.__new__(ModelEngine) - model_engine.llm_args = SimpleNamespace(enable_autotuner=False) - model_engine._metrics = {} - model_engine.model = SimpleNamespace(model_config=SimpleNamespace(is_encoder_decoder=False)) - model_engine.moe_load_balancer = None - model_engine._warmup_depth = 0 - model_engine._runner = Mock(spec=MultimodalEncoderRunner) - runner = model_engine._runner - runner._validate_resources = MultimodalEncoderRunner._validate_resources.__get__(runner) - runner.warmup.side_effect = MultimodalEncoderRunner.warmup.__get__(runner) - resource_manager = Mock() - resource_manager.get_resource_manager.return_value = object() - - with self.assertRaisesRegex( - AssertionError, - "no-KV-cache runner was initialized, but a KV cache manager was allocated", - ): - model_engine.warmup(resource_manager) - - model_engine._runner.warmup.assert_called_once_with(resource_manager) - resource_manager.get_resource_manager.assert_called_once_with( - ResourceManagerType.KV_CACHE_MANAGER - ) - @pytest.mark.cpu_only def test_warmup_total_timing_and_kv_cache_cleanup(self) -> None: model_engine = object.__new__(ModelEngine) diff --git a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py index 8a1f99a887d4..009f5d449e81 100644 --- a/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py +++ b/tests/unittest/_torch/pyexecutor/test_multimodal_embedding_lengths.py @@ -6,21 +6,14 @@ import pytest import torch -from tensorrt_llm._torch.pyexecutor import _util -from tensorrt_llm._torch.pyexecutor.llm_request import ( - LlmRequestState, - PyResult, - get_multimodal_embedding_lengths, -) -from tensorrt_llm._torch.pyexecutor.sampler import EarlyStopWithMMResult, MultimodalResult +from tensorrt_llm._torch.pyexecutor.llm_request import PyResult, get_multimodal_embedding_lengths +from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import MultimodalEmbeddingCompletion from tensorrt_llm._torch.shared_tensor import SharedTensorContainer -from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalItemMetadata, MultimodalParams, ) -from tensorrt_llm.mapping import Mapping @pytest.mark.parametrize( @@ -159,7 +152,7 @@ def __init__(self): self.mm_embeddings = [] self.mrope_position = None - def append_mm_embeddings(self, mm_embedding, mm_embedding_lengths): + def append_mm_embeddings(self, mm_embedding, mm_embedding_lengths, mm_embedding_metadata=None): self.mm_embeddings.append((mm_embedding, mm_embedding_lengths)) def set_mrope_position(self, position_ids, position_deltas): @@ -170,101 +163,36 @@ class _FakeRequest: def __init__(self, multimodal_lengths=None): self.multimodal_lengths = multimodal_lengths self.py_result = _FakePyResult() - self.state = None - self.finished_reason = None - - def set_finished_reason(self, reason, beam): - self.finished_reason = (reason, beam) +@pytest.mark.parametrize("exports", [True, False], ids=["response_rank", "follower_rank"]) @pytest.mark.cpu_only -def test_mm_encoder_sampler_aligns_mixed_batch_by_request_index(): - """Sparse MM encoder outputs attach to the original request index.""" +def test_mm_embedding_completion_attaches_results_by_request_index(exports): + """A sparse batch's results reach their own requests, only on the rank that exports them.""" text_request = _FakeRequest() mm_request = _FakeRequest(multimodal_lengths=[4]) - sampler = EarlyStopWithMMResult() - state = sampler.SampleState( - requests=[text_request, mm_request], - data=MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[1], - mm_embedding_lengths=[[4]], - num_context_requests=2, - extra_data={ - "mrope_position_ids": ["text-pos", "mm-pos"], - "mrope_position_deltas": ["text-delta", "mm-delta"], - }, - ), - ) - - sampler.update_requests(state) - assert text_request.state == LlmRequestState.GENERATION_COMPLETE - assert mm_request.state == LlmRequestState.GENERATION_COMPLETE - assert text_request.finished_reason == (FinishReason.LENGTH, 0) - assert mm_request.finished_reason == (FinishReason.LENGTH, 0) - assert len(text_request.py_result.mm_embeddings) == 0 - [(mm_embedding, mm_embedding_lengths)] = mm_request.py_result.mm_embeddings - assert mm_embedding.shape == (4, 2) - assert mm_embedding_lengths == [4] - assert text_request.py_result.mrope_position is None - assert mm_request.py_result.mrope_position == ("mm-pos", "mm-delta") - - -@pytest.mark.parametrize( - "mapping,exports_results", - [ - pytest.param(Mapping(), True, id="single_rank"), - pytest.param(Mapping(world_size=2, rank=1, tp_size=2), False, id="tp_follower"), - pytest.param( - Mapping(world_size=2, rank=1, tp_size=2, enable_attention_dp=True), - True, - id="attention_dp_follower", - ), - pytest.param( - Mapping(world_size=4, rank=2, tp_size=2, pp_size=2, enable_attention_dp=True), - False, - id="attention_dp_later_pp_stage", - ), - ], -) -@pytest.mark.cpu_only -def test_mm_encoder_sampler_exports_results_only_on_response_ranks(mapping, exports_results): - """Ranks whose responses never reach the frontend must not export MM results.""" - sampler = _util.instantiate_sampler( - SimpleNamespace(max_seq_len=16, max_num_tokens=16, spec_config=None), - SimpleNamespace( - sampler_force_async_worker=False, - disable_overlap_scheduler=True, - enable_speculative_beam_history_d2h=False, - ), - mapping, - max_batch_size=1, - max_beam_width=1, - mm_encoder_only=True, - speculative_config=None, - max_num_sequences=1, - ) - with pytest.raises(ValueError, match="maximum context length is 16 tokens"): - sampler.validate_request(SimpleNamespace(prompt_len=17)) - request = _FakeRequest(multimodal_lengths=[4]) - state = sampler.SampleState( - requests=[request], - data=MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[0], - mm_embedding_lengths=[[4]], - num_context_requests=1, - extra_data={"mrope_position_ids": ["pos"], "mrope_position_deltas": ["delta"]}, - ), + MultimodalEmbeddingCompletion(exports=exports).store( + [text_request, mm_request], + { + "mm_embeddings": [torch.ones(4, 2)], + "mm_embedding_request_indices": [1], + "mm_embedding_lengths": [[4]], + "mrope_position_ids": ["mm-pos"], + "mrope_position_deltas": ["mm-delta"], + }, ) - sampler.update_requests(state) - - assert request.state == LlmRequestState.GENERATION_COMPLETE - assert request.finished_reason == (FinishReason.LENGTH, 0) - assert bool(request.py_result.mm_embeddings) is exports_results - assert (request.py_result.mrope_position is not None) is exports_results + assert text_request.py_result.mm_embeddings == [] + assert text_request.py_result.mrope_position is None + if exports: + [(mm_embedding, mm_embedding_lengths)] = mm_request.py_result.mm_embeddings + assert mm_embedding.shape == (4, 2) + assert mm_embedding_lengths == [4] + assert mm_request.py_result.mrope_position == ("mm-pos", "mm-delta") + else: + assert mm_request.py_result.mm_embeddings == [] + assert mm_request.py_result.mrope_position is None @pytest.mark.cpu_only @@ -293,87 +221,3 @@ def test_py_result_cuda_mm_embedding_handles_stay_cuda_backed(): handles = result.mm_embedding_handles assert handles is not None assert [handle["method_key"] for handle in handles] == [1, 1] - - -class _FakeScheduledRequests: - def __init__(self, num_context_requests): - self.generation_requests = [] - self.context_requests = [_FakeRequest() for _ in range(num_context_requests)] - - @property - def num_context_requests(self): - return len(self.context_requests) - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_builds_typed_result_from_model_outputs(): - """Sampler converts raw model-output dicts into typed MM results.""" - sampler = EarlyStopWithMMResult() - model_outputs = { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [[4]], - "mrope_position_ids": ["pos"], - } - state = sampler.sample_async( - _FakeScheduledRequests(2), - model_outputs, - [], - ) - - assert state.data.mm_embedding_request_indices == [1] - assert state.data.mm_embedding_lengths == [[4]] - assert state.data.extra_data == {"mrope_position_ids": ["pos"]} - assert set(model_outputs) == { - "mm_embeddings", - "mm_embedding_request_indices", - "mm_embedding_lengths", - "mrope_position_ids", - } - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_rejects_typed_result_batch_mismatch(): - """MM embedding arrays must stay length-aligned with request indices.""" - sampler = EarlyStopWithMMResult() - - with pytest.raises(ValueError, match="batch size"): - sampler.sample_async( - _FakeScheduledRequests(2), - { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [], - }, - [], - ) - - -@pytest.mark.cpu_only -def test_mm_encoder_sampler_rejects_invalid_request_index(): - """MM encoder output cannot target a request outside the scheduled batch.""" - sampler = EarlyStopWithMMResult() - - with pytest.raises(ValueError, match="invalid request index"): - sampler.sample_async( - _FakeScheduledRequests(1), - { - "mm_embeddings": [torch.ones(4, 2)], - "mm_embedding_request_indices": [1], - "mm_embedding_lengths": [[4]], - }, - [], - ) - - -@pytest.mark.cpu_only -def test_multimodal_result_rejects_embedding_shape_mismatch(): - """Per-item lengths must sum to the attached embedding rows.""" - with pytest.raises(ValueError, match="shape mismatch"): - MultimodalResult( - mm_embeddings=[torch.ones(4, 2)], - mm_embedding_request_indices=[0], - mm_embedding_lengths=[[3]], - num_context_requests=1, - extra_data={}, - ) diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index 18d78cb99c7d..5bb7edbe6a56 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -1906,12 +1906,6 @@ def _make_executor_with_kv_cache_manager(kv_cache_manager: object | None) -> PyE return executor -def test_get_kv_cache_capacity_without_manager(): - executor = _make_executor_with_kv_cache_manager(None) - - assert executor.get_kv_cache_capacity() == {} - - @pytest.mark.parametrize("streaming_events", [False, True]) def test_get_kv_cache_capacity_from_stats(streaming_events): """KV capacity is available without consuming iteration stats.""" diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py index 85b7176230ec..c1e3e72f5b2a 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor_creator_no_kv_execution.py @@ -34,12 +34,7 @@ def _args(): ) def test_non_generation_models_reject_unsupported_loops(mapping, error): with pytest.raises(ValueError, match=error): - _check_no_kv_execution(_args(), Mapping(**mapping), is_generation=False) - - -@pytest.mark.parametrize("mapping", [_PP2, _ADP], ids=["pp", "adp"]) -def test_generation_models_are_not_checked(mapping): - _check_no_kv_execution(_args(), Mapping(**mapping), is_generation=True) + _check_no_kv_execution(_args(), Mapping(**mapping)) def _encoder_args(**overrides): @@ -54,6 +49,7 @@ def _encoder_args(**overrides): enable_resource_governor=False, sleep_config=None, orchestrator_type=None, + enable_client_io_during_gpu_wait=False, ) args.update(overrides) return SimpleNamespace(**args) @@ -88,6 +84,7 @@ def test_scheduled_encoder_rejects_unsupported_parallelism(mapping, error): (dict(enable_resource_governor=True), "enable_resource_governor"), (dict(sleep_config=object()), "sleep_config"), (dict(orchestrator_type="ray"), "orchestrator_type='ray'"), + (dict(enable_client_io_during_gpu_wait=True), "enable_client_io_during_gpu_wait"), ], ) def test_scheduled_encoder_rejects_settings_by_name(overrides, setting): @@ -105,23 +102,24 @@ def test_scheduled_encoder_serves_tensor_parallel_rpc_workers(): @pytest.mark.parametrize( - ("config", "model_kwargs", "is_generation"), + ("config", "args_kwargs", "is_generation"), [ - (BertConfig(architectures=["BertForSequenceClassification"]), None, False), - (LlamaConfig(architectures=["LlamaForCausalLM"]), None, True), + (BertConfig(architectures=["BertForSequenceClassification"]), {}, False), + (LlamaConfig(architectures=["LlamaForCausalLM"]), {}, True), ( LlamaConfig(architectures=["LlamaForCausalLM"]), - {"architectures": ["LlamaForTextEmbedding"]}, + {"model_kwargs": {"architectures": ["LlamaForTextEmbedding"]}}, False, ), + (LlamaConfig(architectures=["LlamaForCausalLM"]), {"mm_encoder_only": True}, False), ], - ids=["pooling", "generation", "overridden-to-pooling"], + ids=["pooling", "generation", "overridden-to-pooling", "mm-encoder-only"], ) def test_dispatch_reads_the_architecture_model_loading_will_use( - tmp_path, config, model_kwargs, is_generation + tmp_path, config, args_kwargs, is_generation ): config.save_pretrained(tmp_path) - args = TorchLlmArgs(model=str(tmp_path), model_kwargs=model_kwargs) + args = TorchLlmArgs(model=str(tmp_path), **args_kwargs) _, loaded = ModelLoader.load_config_with_defaults(str(tmp_path), args, HfCheckpointLoader()) diff --git a/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py b/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py index 76184b321bbd..a0f8a2e55be5 100644 --- a/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py +++ b/tests/unittest/_torch/pyexecutor/test_remote_tail_adp_phase.py @@ -389,7 +389,7 @@ def test_rebalance_reserves_execution_row_before_resuming_real_requests( ) -> None: executor = _make_executor() executor._enable_remote_tail_adp = True - executor.resource_manager = SimpleNamespace(resource_managers={}) + executor._read_kv_cache_capacity = Mock(return_value={}) real = SimpleNamespace(py_request_id=1) dummy = SimpleNamespace(py_request_id=2) executor.active_requests = [real] diff --git a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py index f181af9426e8..e99ae5319df3 100644 --- a/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_scheduled_encoder_executor.py @@ -2,15 +2,18 @@ # SPDX-License-Identifier: Apache-2.0 """Scheduled encoder executor behaviour, driven through the members the worker calls. -A fake runner answers each request with a row holding its request id. It writes into one -reused output buffer, as a captured graph does, and can hold or fail a forward. The -executor's queue, admission, batching, completion and delivery are real. +A fake runner answers each request with a row holding its request id. As a pooling model it +writes into one reused output buffer, as a captured graph does; as a multimodal encoder it +returns one CPU embedding per request, which the completion exports through shared memory. +It can hold or fail a forward. The executor's queue, admission, batching, completion and +delivery are real. """ +import gc import re import threading import time -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from datetime import timedelta from types import SimpleNamespace from unittest.mock import Mock @@ -20,12 +23,17 @@ from tensorrt_llm._torch.pyexecutor import scheduled_encoder_executor from tensorrt_llm._torch.pyexecutor.executor_request_queue import RequestQueueItem +from tensorrt_llm._torch.pyexecutor.llm_request import PyResult from tensorrt_llm._torch.pyexecutor.scheduled_encoder_executor import ( + MultimodalEmbeddingCompletion, + PoolingCompletion, ScheduledEncoderExecutor, fit_batch, ) +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer from tensorrt_llm.bindings.executor import FinishReason, OutputConfig, Request, SamplingConfig from tensorrt_llm.executor.utils import is_context_length_exceeded_message +from tensorrt_llm.llmapi.llm_args import SchedulerConfig, WaitingQueuePolicy from tensorrt_llm.mapping import Mapping WAIT = timedelta(seconds=30) @@ -71,9 +79,11 @@ def test_fit_batch_charges_each_child_request_a_full_row(): class _Runner: - """Answer each request with a row of its request id, through one reused output buffer.""" + """Answer each request with a row of its request id.""" - def __init__(self, max_batch_size: int): + def __init__(self, max_batch_size: int, multimodal: bool): + self.multimodal = multimodal + self.embedding_device = "cpu" self.output = torch.zeros(max_batch_size, WIDTH, device="cuda") self.batches: list[list[int]] = [] self.errors: list[Exception] = [] @@ -114,6 +124,17 @@ def forward(self, inputs, *, resource_manager): self.staged.append(inputs.inputs_staged) if self.gpu_cycles: torch.cuda._sleep(self.gpu_cycles) + if self.multimodal: + return { + "mm_embeddings": [ + torch.full( + (1, WIDTH), float(request.py_request_id), device=self.embedding_device + ) + for request in requests + ], + "mm_embedding_request_indices": list(range(len(requests))), + "mm_embedding_lengths": [[1]] * len(requests), + } for row, request in enumerate(requests): self.output[row].fill_(request.py_request_id) return {"logits": self.output[: len(requests)]} @@ -121,11 +142,22 @@ def forward(self, inputs, *, resource_manager): class _Harness: def __init__( - self, *, max_batch_size=4, max_num_tokens=64, max_seq_len=32, print_iter_log=False + self, + *, + completion="pooling", + max_batch_size=4, + max_num_tokens=64, + max_seq_len=32, + print_iter_log=False, + scheduler_config=None, ): - self.runner = _Runner(max_batch_size) + self.multimodal = multimodal = completion == "multimodal" + self.runner = _Runner(max_batch_size, multimodal) self.executor = ScheduledEncoderExecutor( self.runner, + completion=( + MultimodalEmbeddingCompletion(exports=True) if multimodal else PoolingCompletion() + ), model_loader=Mock(), llm_args=SimpleNamespace( garbage_collection_gen0_threshold=20000, @@ -133,6 +165,7 @@ def __init__( return_perf_metrics=False, enable_iter_perf_stats=False, print_iter_log=print_iter_log, + scheduler_config=scheduler_config or SchedulerConfig(), ), mapping=Mapping(), dist=SimpleNamespace(rank=0, world_size=1, has_pp=False, barrier=lambda: None), @@ -146,7 +179,7 @@ def submit( self, num_tokens: int = 4, *, - context_logits: bool = True, + context_logits: bool | None = None, perf_metrics: bool = False, **attrs, ) -> int: @@ -154,7 +187,9 @@ def submit( input_token_ids=list(range(1, num_tokens + 1)), max_tokens=1, output_config=OutputConfig( - return_context_logits=context_logits, + return_context_logits=( + not self.multimodal if context_logits is None else context_logits + ), exclude_input_from_output=True, return_perf_metrics=perf_metrics, ), @@ -176,6 +211,24 @@ def result(self, request_id: int): assert response.result.is_final return response.result + def complete_after_the_loop_takes_the_cancellation(self, request_id: int) -> None: + """Hold each launched batch's completion until the loop has taken this cancellation.""" + executor = self.executor + taken = threading.Event() + receive, cancel_launched = executor._receive, executor._cancel_launched + + def receive_and_report(): + receive() + if request_id in executor._cancelled_after_launch: + taken.set() + + def cancel_once_taken(flight): + assert taken.wait(timeout=30) + return cancel_launched(flight) + + executor._receive = receive_and_report + executor._cancel_launched = cancel_once_taken + def shutdown(self) -> None: if not self.closed: self.closed = True @@ -192,8 +245,13 @@ def _harness(**kwargs): h.shutdown() -def test_batches_follow_the_budget_in_arrival_order(): - with _harness(max_batch_size=2, max_num_tokens=64, max_seq_len=64) as h: +@pytest.fixture(params=["pooling", "multimodal"]) +def completion(request): + return request.param + + +def test_batches_follow_the_budget_in_arrival_order(completion): + with _harness(completion=completion, max_batch_size=2, max_num_tokens=64, max_seq_len=64) as h: ids = [h.submit(), h.submit(), h.submit(61), h.submit()] h.executor.start_worker() @@ -219,6 +277,20 @@ def test_results_are_pooled_rows_with_no_tokens_and_reason_length(): assert result.finish_reasons == [FinishReason.LENGTH] +def test_multimodal_results_are_exported_embeddings_with_no_tokens_and_reason_length(): + with _harness(completion="multimodal", max_batch_size=2) as h: + ids = [h.submit(py_multimodal_data={"image": {}}) for _ in range(3)] + h.executor.start_worker() + + for request_id in ids: + result = h.result(request_id) + (handle,) = result.mm_embedding_handles + embedding = SharedTensorContainer.from_dict(handle).get_local_view() + assert torch.equal(embedding, torch.full((1, WIDTH), float(request_id))) + assert result.output_token_ids == [[]] + assert result.finish_reasons == [FinishReason.LENGTH] + + # Long enough that a launched batch is still running when the loop has launched the next one. GPU_CYCLES = 1_000_000_000 @@ -305,8 +377,73 @@ def test_inadmissible_requests_fail_alone(): assert h.runner.batches == [[fitting]] -def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(): - with _harness(max_batch_size=1) as h: +@pytest.mark.parametrize( + ("max_seq_len", "max_num_tokens"), [(32, 64), (64, 32)], ids=["max_seq_len", "max_num_tokens"] +) +def test_a_multimodal_encoder_admits_multimodal_requests_and_fails_an_oversized_one_alone( + max_seq_len, max_num_tokens +): + with _harness( + completion="multimodal", max_seq_len=max_seq_len, max_num_tokens=max_num_tokens + ) as h: + too_long = h.submit(33, py_multimodal_data={"image": {}}) + fitting = h.submit(32, py_multimodal_data={"image": {}}) + h.executor.start_worker() + + (long_response,) = h.responses(too_long) + h.result(fitting) + + assert is_context_length_exceeded_message(long_response.error_msg) + assert h.runner.batches == [[fitting]] + + +@pytest.mark.parametrize("max_wait_s", [None, 60.0], ids=["priority", "aging"]) +def test_a_priority_queue_admits_the_higher_priority_request_first(max_wait_s): + with _harness( + max_batch_size=1, + scheduler_config=SchedulerConfig( + waiting_queue_policy=WaitingQueuePolicy.PRIORITY, waiting_queue_max_wait_s=max_wait_s + ), + ) as h: + h.runner.hold() + running = h.submit() + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + low, high = h.submit(priority=0.1), h.submit(priority=0.9) + h.runner.release.set() + + for request_id in (running, low, high): + h.result(request_id) + assert h.runner.batches == [[running], [high], [low]] + + +def test_a_request_past_the_wait_cap_goes_ahead_of_a_higher_priority_one(): + max_wait_s = 0.2 + with _harness( + max_batch_size=1, + scheduler_config=SchedulerConfig( + waiting_queue_policy=WaitingQueuePolicy.PRIORITY, waiting_queue_max_wait_s=max_wait_s + ), + ) as h: + h.runner.hold() + running = h.submit() + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + # Arrivals are stamped at enqueue, so the older request is past the cap when popped. + old = h.submit(priority=0.1) + time.sleep(2 * max_wait_s) + new = h.submit(priority=0.9) + h.runner.release.set() + + for request_id in (running, old, new): + h.result(request_id) + assert h.runner.batches == [[running], [old], [new]] + # Admitted requests leave no bookkeeping behind in the queue. + assert not h.executor._waiting_queue._seq + + +def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() running, queued = h.submit(), h.submit() h.executor.start_worker() @@ -328,8 +465,185 @@ def test_cancelling_a_queued_request_finishes_it_once_as_cancelled(): assert h.runner.batches == [[running], [later]] -def test_a_survivable_forward_error_fails_only_its_batch(): - with _harness(max_batch_size=1) as h: +def test_cancelling_a_launched_request_finishes_it_once_as_cancelled(completion): + with _harness(completion=completion, max_batch_size=1) as h: + h.runner.hold() + launched, later = h.submit(), h.submit() + h.complete_after_the_loop_takes_the_cancellation(launched) + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + + h.executor.cancel_request(launched) + h.runner.release.set() + + (cancelled,) = h.responses(launched) + assert cancelled.result.is_final + assert cancelled.result.finish_reasons == [FinishReason.CANCELLED] + assert cancelled.result.mm_embedding_handles is None + assert cancelled.result.context_logits is None + h.result(later) + assert h.responses(launched, NO_WAIT) == [] + assert h.runner.batches == [[launched], [later]] + + +def test_cancelling_a_launched_request_with_two_sequences_delivers_both_and_ends_with_the_last( + completion, +): + with _harness(completion=completion) as h: + h.runner.hold() + launched = h.submit(sampling_config=SamplingConfig(num_return_sequences=2)) + h.complete_after_the_loop_takes_the_cancellation(launched) + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + + h.executor.cancel_request(launched) + h.runner.release.set() + + # The worker takes every response; a child's is stored under its own id. + responses = [] + while len(responses) < 2 and (ready := h.executor.await_responses(timeout=WAIT)): + responses.extend(ready) + for response in responses: + response.result.deserialize() + assert [response.request_id for response in responses] == [launched, launched] + assert [response.result.sequence_index for response in responses] == [0, 1] + assert [response.result.is_final for response in responses] == [False, True] + for response in responses: + assert response.result.finish_reasons == [FinishReason.CANCELLED] + + +def test_a_cancellation_sent_while_the_loop_waits_for_a_slot_still_cancels_a_launched_request( + completion, +): + with _harness(completion=completion, max_batch_size=1) as h: + slots, waiting = h.executor._slots, threading.Event() + + class Slots: + def acquire(self): + if not slots.acquire(blocking=False): + waiting.set() + slots.acquire() + + def release(self): + slots.release() + + h.executor._slots = Slots() + sent = threading.Event() + cancel_launched = h.executor._cancel_launched + + def cancel_once_sent(flight): + assert sent.wait(timeout=30) + return cancel_launched(flight) + + h.executor._cancel_launched = cancel_once_sent + first, second, third = h.submit(), h.submit(), h.submit() + h.executor.start_worker() + # Two batches are in flight, and the loop waits in the third launch, not in _receive. + assert waiting.wait(timeout=30) + + h.executor.cancel_request(first) + sent.set() + + (cancelled,) = h.responses(first) + assert cancelled.result.finish_reasons == [FinishReason.CANCELLED] + assert cancelled.result.mm_embedding_handles is None + assert cancelled.result.context_logits is None + h.result(second) + h.result(third) + assert h.runner.batches == [[first], [second], [third]] + + +@pytest.mark.parametrize("rank, exports", [(0, True), (1, False)], ids=["rank_0", "tp_follower"]) +def test_only_rank_0_of_a_multimodal_encoder_exports(monkeypatch, rank, exports): + """A follower's responses reach no client, so its exports would pin each batch's memory.""" + built = SimpleNamespace( + runner=_Runner(1, multimodal=True), + model_config=SimpleNamespace(is_generation=False), + model_loader=Mock(), + max_batch_size=1, + max_num_tokens=64, + max_seq_len=32, + ) + monkeypatch.setattr( + scheduled_encoder_executor, "resolve_scheduled_encoder_args", lambda llm_args: llm_args + ) + monkeypatch.setattr( + scheduled_encoder_executor, + "build_mm_encoder_runner", + lambda *args, **kwargs: nullcontext(built), + ) + executor = ScheduledEncoderExecutor.create( + SimpleNamespace( + mm_encoder_only=True, + garbage_collection_gen0_threshold=20000, + batch_wait_timeout_ms=0, + return_perf_metrics=False, + enable_iter_perf_stats=False, + print_iter_log=False, + scheduler_config=SchedulerConfig(), + ), + model_path="", + checkpoint_loader=Mock(), + mapping=Mapping(world_size=2, rank=rank, tp_size=2), + dist=SimpleNamespace(rank=rank, world_size=2, has_pp=False, barrier=lambda: None), + ) + request = SimpleNamespace(py_result=PyResult(prompt_len=1, max_new_tokens=1)) + + executor._completion.store( + [request], + { + "mm_embeddings": [torch.ones(1, WIDTH)], + "mm_embedding_request_indices": [0], + "mm_embedding_lengths": [[1]], + }, + ) + + assert (request.py_result.mm_embedding_handles is not None) is exports + request.py_result.release_exports() + + +def _cuda_memory_allocated() -> int: + gc.collect() + torch.cuda.synchronize() + torch.cuda.ipc_collect() + return torch.cuda.memory_allocated() + + +@pytest.mark.parametrize("cause", ["cancelled", "failed"]) +def test_a_launched_request_whose_result_is_not_delivered_frees_its_exports(cause): + with _harness(completion="multimodal", max_batch_size=1) as h: + h.runner.embedding_device = "cuda" + if cause == "failed": + + def fail_after_the_export(*args, **kwargs): + raise RuntimeError("bad step") + + h.executor._perf_manager.append_step_metrics = fail_after_the_export + allocated = _cuda_memory_allocated() + h.runner.hold() + launched = h.submit(py_multimodal_data={"image": {}}) + if cause == "cancelled": + h.complete_after_the_loop_takes_the_cancellation(launched) + h.executor.start_worker() + assert h.runner.entered.wait(timeout=30) + if cause == "cancelled": + h.executor.cancel_request(launched) + h.runner.release.set() + + (response,) = h.responses(launched) + undelivered = ( + response.result.finish_reasons == [FinishReason.CANCELLED] + if cause == "cancelled" + else response.error_msg == "bad step" + ) + del response + + assert _cuda_memory_allocated() == allocated + assert undelivered + + +def test_a_survivable_forward_error_fails_only_its_batch(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.errors.append(RuntimeError("bad batch")) failed, later = h.submit(), h.submit() h.executor.start_worker() @@ -350,8 +664,8 @@ def test_a_failed_forward_answers_a_request_with_children_once(): assert (error.request_id, error.error_msg) == (failed, "bad batch") -def test_a_fatal_forward_error_fails_everything_queued_and_shuts_down(): - with _harness(max_batch_size=1) as h: +def test_a_fatal_forward_error_fails_everything_queued_and_shuts_down(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() h.runner.errors.append(RuntimeError("an illegal memory access was encountered")) running, waiting = h.submit(), h.submit() @@ -407,8 +721,8 @@ def test_print_iter_log_logs_each_batch_with_its_size_and_tokens(monkeypatch): assert "host_step_time = " in line and "device_step_time = " in line -def test_shutdown_runs_queued_requests_first(): - with _harness(max_batch_size=1) as h: +def test_shutdown_runs_queued_requests_first(completion): + with _harness(completion=completion, max_batch_size=1) as h: h.runner.hold() ids = [h.submit(), h.submit(), h.submit()] h.executor.start_worker() diff --git a/tests/unittest/executor/test_undelivered_exports.py b/tests/unittest/executor/test_undelivered_exports.py new file mode 100644 index 000000000000..658fbf5c92c7 --- /dev/null +++ b/tests/unittest/executor/test_undelivered_exports.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exports of a response that is dropped on its way to the client. + +A multimodal encoder's response carries its embeddings and MRoPE positions as exported +handles, which only a consumer that opens them, or ``release_dict``, consumes. The +exports here are CPU tensors: each lives in a ``/dev/shm`` file until its handles are +consumed. +""" + +import base64 +import gc +from pathlib import Path +from queue import Queue +from types import MethodType, SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.llm_request import LlmResponse, LlmResult, PyResult +from tensorrt_llm.executor.base_worker import ( + AwaitResponseHelper, + BaseWorker, + _drop_refused_response, +) +from tensorrt_llm.executor.executor import GenerationExecutor +from tensorrt_llm.executor.proxy import GenerationExecutorProxy +from tensorrt_llm.executor.result import ResponseWrapper +from tensorrt_llm.executor.rpc_proxy_mixin import RpcExecutorMixin +from tensorrt_llm.executor.utils import ErrorResponse + +pytestmark = pytest.mark.cpu_only + +CLIENT_ID = 7 + + +def _exporting_response() -> tuple[LlmResponse, set[Path]]: + py_result = PyResult(prompt_len=1, max_new_tokens=1) + py_result.append_mm_embeddings(torch.ones(2, 1024), [1, 1]) + py_result.set_mrope_position(torch.zeros(3, 1024), torch.zeros(1024)) + handles = [ + *py_result.mm_embedding_handles, + py_result.mrope_position_ids_handle, + py_result.mrope_position_deltas_handle, + ] + files = { + Path("/dev/shm") / base64.b64decode(handle["storage_handle"]).decode().lstrip("/") + for handle in handles + } + response = LlmResponse( + request_id=CLIENT_ID, + result=LlmResult(b"", py_result, is_final=True), + client_id=CLIENT_ID, + ) + return response, files + + +def _drop_at_worker(response): + assert _drop_refused_response(SimpleNamespace(_refused_client_ids={CLIENT_ID}), response) + + +def _replace_at_worker_on_background_error(response): + errors = Queue() + errors.put(RuntimeError("background error")) + sent = [] + worker = SimpleNamespace( + enable_postprocess_parallel=False, + _refused_client_ids=set(), + _error_queue=errors, + frontend_result_queues=None, + result_queue=SimpleNamespace(put=sent.extend), + _pop_result=lambda client_id: None, + ) + worker._has_background_error = MethodType(BaseWorker._has_background_error, worker) + worker._create_error_response = MethodType(BaseWorker._create_error_response, worker) + AwaitResponseHelper(worker).handle_for_ipc_batched([response]) + assert sent == [ErrorResponse(CLIENT_ID, "background error", CLIENT_ID)] + + +class _ResultQueue: + def __init__(self, item): + self._item = item + + def get(self): + return self._item + + +def _drop_at_proxy(response): + # Without __init__, which spawns workers; the attributes keep teardown a no-op. + proxy = object.__new__(GenerationExecutorProxy) + proxy.workers_started = False + proxy._multi_frontend_ipc_dir = None + proxy._results = {} + proxy.result_queue = _ResultQueue(ResponseWrapper(response)) + assert proxy.dispatch_result_task() + + +class _RpcProxy(RpcExecutorMixin, GenerationExecutor): + pass + + +_RpcProxy.__abstractmethods__ = frozenset() + + +def _drop_at_rpc_proxy(response): + proxy = _RpcProxy.__new__(_RpcProxy) + proxy._results = {} + proxy.handle_responses([[response]]) + + +@pytest.mark.parametrize( + "drop", + [_drop_at_worker, _replace_at_worker_on_background_error, _drop_at_proxy, _drop_at_rpc_proxy], + ids=["refused_at_worker", "errored_at_worker", "late_at_proxy", "unknown_at_rpc_proxy"], +) +def test_a_dropped_response_releases_its_exports(drop): + response, files = _exporting_response() + gc.collect() + assert all(path.exists() for path in files) + + drop(response) + gc.collect() + + assert not any(path.exists() for path in files) diff --git a/tests/unittest/llmapi/apps/test_openai_input_metrics.py b/tests/unittest/llmapi/apps/test_openai_input_metrics.py index 2efba46f036f..47b51e9225dd 100644 --- a/tests/unittest/llmapi/apps/test_openai_input_metrics.py +++ b/tests/unittest/llmapi/apps/test_openai_input_metrics.py @@ -148,7 +148,9 @@ async def complete(): prompt_token_ids=[7, 8], aresult=AsyncMock(side_effect=complete), disaggregated_params=SimpleNamespace( - multimodal_embedding_handles=[{"tensor_size": [4, 8]}] + multimodal_embedding_handles=[{"tensor_size": [4, 8]}], + mrope_position_ids_handle=None, + mrope_position_deltas_handle=None, ), ) diff --git a/tests/unittest/serve/test_mm_encoder_exports.py b/tests/unittest/serve/test_mm_encoder_exports.py new file mode 100644 index 000000000000..98e57dbee050 --- /dev/null +++ b/tests/unittest/serve/test_mm_encoder_exports.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The multimodal encoder route and the exports in the encoder's result. + +The result carries the embeddings and the MRoPE positions as exported handles, which only +a consumer that opens them, or ``release_dict``, consumes. The route answers with the +embedding handles alone. The exports here are CPU tensors: each lives in a ``/dev/shm`` +file until its handles are consumed. +""" + +import asyncio +import base64 +import gc +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +import torch +from test_common.serve.app_factory import AggConfig + +from tensorrt_llm._torch.shared_tensor import SharedTensorContainer +from tensorrt_llm.disaggregated_params import DisaggregatedParams +from tensorrt_llm.llmapi.disagg_utils import ServerRole +from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + +pytestmark = pytest.mark.cpu_only + + +def _export(tensor: torch.Tensor) -> tuple[dict, Path]: + handle = SharedTensorContainer.from_tensor(tensor).dump_to_dict() + path = Path("/dev/shm") / base64.b64decode(handle["storage_handle"]).decode().lstrip("/") + return handle, path + + +@pytest.mark.asyncio +async def test_the_route_delivers_the_embedding_exports_and_releases_the_mrope_exports(serve_env): + server = serve_env.build_openai_server(AggConfig(role=ServerRole.MM_ENCODER)) + server.model = "test-model" + server.allow_request_chat_template = False + server.model_config = None + server.multimodal_server_config = None + server.enable_store = False + server.conversation_store = None + server.use_harmony = False + server.tokenizer = None + server.processor = None + server.chat_template = None + server.tool_parser = None + server.metrics_collector = None + server.await_disconnected = AsyncMock() + server._extract_metrics = AsyncMock() + + embedding, embedding_file = _export(torch.ones(2, 1024)) + position_ids, position_ids_file = _export(torch.zeros(3, 1024)) + position_deltas, position_deltas_file = _export(torch.zeros(1024)) + promise = SimpleNamespace( + request_id="1", + aresult=AsyncMock(), + disaggregated_params=DisaggregatedParams( + multimodal_embedding_handles=[embedding], + mrope_position_ids_handle=position_ids, + mrope_position_deltas_handle=position_deltas, + ), + ) + server.generator = SimpleNamespace(generate_async=lambda **kwargs: promise) + request = ChatCompletionRequest( + model=server.model, + messages=[{"role": "user", "content": "hi"}], + prompt_token_ids=[7, 8], + ) + gc.collect() + assert all(path.exists() for path in (embedding_file, position_ids_file, position_deltas_file)) + + with patch( + "tensorrt_llm.serve.openai_server.parse_chat_messages_coroutines", + side_effect=lambda *args, **kwargs: ([], asyncio.sleep(0, result=(None, None)), {}, []), + ): + response = await server.openai_mm_encoder( + request, server._synthetic_request(request.model_dump(exclude_none=True)) + ) + gc.collect() + + assert response.status_code == 200 + assert not position_ids_file.exists() + assert not position_deltas_file.exists() + delivered = json.loads(response.body)["choices"][0]["mm_embedding_handle"] + assert torch.equal( + SharedTensorContainer.from_dict(delivered).get_local_view(), torch.ones(2, 1024) + )