From dbf58663d4d21177562331ba0cac31ef82647583 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:23 +0000 Subject: [PATCH 1/6] [TRTLLM-17780][feat] Reject media at admission on text-only Modeling V2 targets Problem The Kimi K3, MiniMax-M3 and GLM-5.3-Flash Modeling V2 targets serve the text half of a vision-language checkpoint. Each one treats an image request in its own way: Kimi and MiniMax raise in forward, which fails every request in the batch; GLM checks a fixed key list at admission, and its FP8 target uses a different hook. The MiniMax forward check also counts an M-RoPE-only payload as media, which a disaggregated generation request carries. Solution - Add has_media_or_embeddings() in tensorrt_llm/inputs/multimodal.py: true when multimodal_data has any non-None entry other than mrope_config. Unknown keys count, so it fails closed. - All 14 text-only targets (6 Kimi K3, 5 MiniMax-M3, 3 GLM-5.3-Flash) define validate_multimodal_request_data(), which the executor already calls at admission. It raises when the helper is true, so only that request fails and no budget is charged. The GLM targets move their own checks to the same hook and message. - The Kimi and MiniMax forward checks stay as a backstop and use the same helper (Kimi SM107 through check_text_only), so a payload that passes admission, such as M-RoPE positions alone, also passes forward. The lists of stock calls (UNCERTIFIED_GENERIC_CALLS) name the helper. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs main: 8636 vs 8623 cases, 0 changed outcomes; the 13 new tests pass (the helper, executor admission, the GLM-5.3-Flash FP8 hook). The GLM-5.3-Flash NVFP4 admission tests pass with the shared message, and the Kimi K3 common test checks the forward backstop on media, embeddings and M-RoPE positions. - The GLM-5.3-Flash admission tests run on CPU and pass: media is refused and an M-RoPE-only payload passes. No GPU run: the Kimi K3 and MiniMax-M3 targets need sm_100, sm_103, sm_107 or 16 GPUs. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- .../modeling.py | 16 ++++--- .../modeling.py | 20 ++++----- .../modeling.py | 20 ++++----- .../models/kimi_k3_vl/common/weight_io.py | 9 +++- .../modeling.py | 17 +++++++- .../modeling.py | 17 +++++++- .../kimi_k3_nvfp4__sm_107__dep16/modeling.py | 11 ++++- .../modeling.py | 11 ++++- .../modeling.py | 11 ++++- .../kimi_k3_nvfp4__sm_107__tp1/modeling.py | 11 ++++- .../minimax_m3_nvfp4__sm_103__tp4/modeling.py | 14 ++++++- .../minimax_m3_nvfp4__sm_103__tp8/modeling.py | 14 ++++++- .../minimax_m3_nvfp4__sm_107__tp2/modeling.py | 14 ++++++- .../minimax_m3_nvfp4__sm_107__tp4/modeling.py | 14 ++++++- .../minimax_m3_nvfp4__sm_107__tp8/modeling.py | 14 ++++++- tensorrt_llm/inputs/multimodal.py | 13 ++++++ .../test_glm53_flash_default_text_only.py | 18 ++++---- ...glm53_flash_default_mtp_fused_text_only.py | 18 ++++---- .../glm5_next/test_glm5_next_routing.py | 21 +++++----- .../kimi_k3_vl/common/test_k3_common.py | 8 +++- .../_torch/pyexecutor/test_py_executor.py | 30 +++++++++++++ tests/unittest/inputs/test_multimodal.py | 42 +++++++++++++++++++ 22 files changed, 285 insertions(+), 78 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py index f1a5567008af..3e091bbd8328 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py @@ -52,6 +52,7 @@ from tensorrt_llm._torch.speculative import get_spec_worker, should_use_separate_draft_kv_cache from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.functional import AllReduceFusionOp, AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping as ParallelMapping @@ -73,7 +74,6 @@ if TYPE_CHECKING: from tensorrt_llm._torch.pyexecutor.guided_decoder import CapturableGuidedDecoder - from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.speculative.interface import SpecMetadata #: The TP all-reduce of more rows than this runs on NCCL; fewer on the one-shot kernel, @@ -746,15 +746,13 @@ def infer_max_seq_len(self) -> int: """The text config's context limit; the model has no RoPE (NoPE MLA, KDA).""" return int(self.text_config.max_position_embeddings) - def validate_request(self, request: LlmRequest) -> None: - """Reject media on admission, as that request's own error: the vision tower is not built, - and the forward would assert on the whole step.""" - data = request.py_multimodal_data or {} - media = [kind for kind in ("image", "video") if data.get(kind)] - if media: + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"this GLM-5.3-Flash deployment serves text only (disable_mm_encoder); the " - f"request carries {' and '.join(media)} input" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) def set_guided_decoder(self, guided_decoder: CapturableGuidedDecoder) -> bool: diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py index f8795c714634..000cde915130 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py @@ -68,6 +68,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import unwrap_glm5_next_text_config from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm._utils import async_tensor_h2d +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -84,9 +85,6 @@ # and the certification is per arch. _SM = (10, 0) -#: ``multimodal_data`` entries that carry media for an encoder, or encoder output in place of it. -_MEDIA_KEYS = frozenset({"audio", "image", "video", "multimodal_embedding"}) - #: Stock code the generic path runs outside the catalog, declared so it is not consumed silently: every #: tensorrt_llm import of this module that computes (this target's unit tests check the list both ways, with #: ``_glm5_next_generic_calls.py``). An entry leaves the list when a catalog entry replaces it. @@ -128,6 +126,8 @@ "tensorrt_llm._torch.modules.embedding.Embedding", "tensorrt_llm._torch.distributed.AllReduce", "tensorrt_llm._torch.modules.multi_stream_utils.maybe_execute_in_parallel", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) @@ -2087,14 +2087,10 @@ def load_weights(self, weights: Any, weight_mapper: BaseWeightMapper | None = No _weights.load(self, weights, weight_mapper) def validate_multimodal_request_data(self, mm_data: dict) -> None: - """Refuse, at admission, a request whose multimodal payload carries media: this target runs no encoder. - - Refusing here fails that request alone; the forward would otherwise serve its text without the media. - A payload without media is accepted. - """ - media = sorted(key for key in _MEDIA_KEYS if mm_data.get(key) is not None) - if media: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"{type(self).__name__} serves this checkpoint's language model only and does not accept " - f"multimodal input; the request carries {', '.join(media)}" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py index 6a78ccb11edf..fcf9249fe215 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py @@ -76,6 +76,7 @@ from tensorrt_llm._torch.speculative.spec_tail_kernels import strict_accept from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm._utils import async_tensor_h2d +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -93,9 +94,6 @@ # and the certification is per arch. _SM = (10, 0) -#: ``multimodal_data`` entries that carry media for an encoder, or encoder output in place of it. -_MEDIA_KEYS = frozenset({"audio", "image", "video", "multimodal_embedding"}) - #: Stock code the generic path runs outside the catalog, declared so it is not consumed silently: every #: tensorrt_llm import of this module that computes (this target's unit tests check the list both ways, with #: ``_glm5_next_generic_calls.py``). An entry leaves the list when a catalog entry replaces it. @@ -144,6 +142,8 @@ "tensorrt_llm._torch.distributed.ops.allgather", "tensorrt_llm._torch.speculative.interface.INVALID_PROMPT_LOOKAHEAD_TOKEN", "tensorrt_llm._torch.moe.fused_moe.moe_load_balancer_set_repeated_for_next_layer", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) @@ -2479,14 +2479,10 @@ def load_weights(self, weights: Any, weight_mapper: BaseWeightMapper | None = No _weights.load(self, weights, weight_mapper) def validate_multimodal_request_data(self, mm_data: dict) -> None: - """Refuse, at admission, a request whose multimodal payload carries media: this target runs no encoder. - - Refusing here fails that request alone; the forward would otherwise serve its text without the media. - A payload without media is accepted. - """ - media = sorted(key for key in _MEDIA_KEYS if mm_data.get(key) is not None) - if media: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"{type(self).__name__} serves this checkpoint's language model only and does not accept " - f"multimodal input; the request carries {', '.join(media)}" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py index ed5c5c9205a9..fcf64283d8d8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py @@ -18,6 +18,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional from tensorrt_llm._torch.models.checkpoints.base_weight_loader import ConsumableWeightsDict +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger __all__ = [ @@ -89,8 +90,12 @@ def text_model_weights(weights) -> Any: def check_text_only(multimodal_params: Optional[List[Any]]) -> None: - """Refuse a step carrying multimodal input (the engine passes only requests with content).""" - if multimodal_params: + """Refuse a step carrying media or embeddings. M-RoPE positions alone pass, as they pass the + targets' admission check (``has_media_or_embeddings``).""" + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) + for p in multimodal_params or [] + ): raise ValueError( "this Kimi K3 target is text only: it builds no vision tower, and a request carried " "multimodal input" diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py index 4537d92b0e17..d43964df3576 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py @@ -140,6 +140,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import is_mla from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -249,6 +250,8 @@ "tensorrt_llm._torch.models.modeling_dspark.draft_is_embedded_in_target", "tensorrt_llm._torch.models.modeling_dflash.DFlashForCausalLM", "tensorrt_llm._torch.pyexecutor.config_utils.is_mla", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) # The K3 decode kernels' bounds: the token-count kernels take one tile of DECODE_MAX_TOKENS rows; the request-aware @@ -3481,6 +3484,15 @@ def _check_step_contract(self, attn_metadata: AttentionMetadata) -> None: ) self._step_checked = True + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -3492,7 +3504,10 @@ def forward( resource_manager=None, **kwargs, ) -> torch.Tensor: - if kwargs.pop("multimodal_params", None): + multimodal_params = kwargs.pop("multimodal_params", None) or [] + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this Kimi K3 target is text only: it loads no vision tower, and a request carried image input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py index c0cbe46118eb..274eeb2da65b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py @@ -143,6 +143,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import is_mla from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -252,6 +253,8 @@ "tensorrt_llm._torch.models.modeling_dspark.draft_is_embedded_in_target", "tensorrt_llm._torch.models.modeling_dflash.DFlashForCausalLM", "tensorrt_llm._torch.pyexecutor.config_utils.is_mla", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) # The K3 decode kernels' bounds: the token-count kernels take one tile of DECODE_MAX_TOKENS rows; the request-aware @@ -3595,6 +3598,15 @@ def _check_step_contract(self, attn_metadata: AttentionMetadata) -> None: ) self._step_checked = True + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -3606,7 +3618,10 @@ def forward( resource_manager=None, **kwargs, ) -> torch.Tensor: - if kwargs.pop("multimodal_params", None): + multimodal_params = kwargs.pop("multimodal_params", None) or [] + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this Kimi K3 target is text only: it loads no vision tower, and a request carried image input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py index 876c17dfcf03..379fad49f19a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py @@ -120,7 +120,7 @@ maybe_compiled_copy_, ) from tensorrt_llm._utils import get_sm_version, is_sm_100f -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -5640,6 +5640,15 @@ def set_guided_decoder(self, *args, **kwargs): def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + @torch.inference_mode() def forward( self, diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py index 5f5a478601c9..fc501c343890 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py @@ -130,7 +130,7 @@ ) from tensorrt_llm._utils import get_sm_version, is_sm_100f from tensorrt_llm.functional import AllReduceStrategy -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -7710,5 +7710,14 @@ def runtime_hooks(self) -> RuntimeHooks: """The family's hooks, with ``KimiK3Sm107Helix8CacheManager`` as the KV cache manager.""" return replace(super().runtime_hooks, kv_cache_manager_cls=KimiK3Sm107Helix8CacheManager) + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + __all__ = ["ModelingV2KimiK3Nvfp4Sm107Helix8Moeep8"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py index 42501c8f9179..1377cc89661d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py @@ -130,7 +130,7 @@ ) from tensorrt_llm._utils import get_sm_version, is_sm_100f from tensorrt_llm.functional import AllReduceStrategy -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -7710,5 +7710,14 @@ def runtime_hooks(self) -> RuntimeHooks: """The family's hooks, with ``KimiK3Sm107Helix8CacheManager`` as the KV cache manager.""" return replace(super().runtime_hooks, kv_cache_manager_cls=KimiK3Sm107Helix8CacheManager) + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + __all__ = ["ModelingV2KimiK3Nvfp4Sm107Helix8Moetp2ep4"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py index c94888750224..26fcd2282233 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py @@ -97,7 +97,7 @@ maybe_compiled_copy_, ) from tensorrt_llm._utils import get_sm_version, is_sm_100f -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -4701,6 +4701,15 @@ def set_guided_decoder(self, *args, **kwargs): def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + @torch.inference_mode() def forward( self, diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py index 101a3660b623..fee4863d2dc4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py @@ -137,6 +137,7 @@ ) from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -4585,6 +4586,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -4599,7 +4609,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py index da29e313753a..315f035b388f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py @@ -139,6 +139,7 @@ ) from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -4254,6 +4255,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -4268,7 +4278,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py index d8a6afc8b7e6..9b8409b1375d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py @@ -240,6 +240,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen, get_model_extra_attrs from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -6183,6 +6184,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -6197,7 +6207,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft, and on a decode step the spec fold's store of its outputs). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py index a0c944ab0dd7..49ba2ad7dfaa 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py @@ -218,6 +218,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen, is_torch_compiling from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -5647,6 +5648,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -5661,7 +5671,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py index ddb3f5f54c6c..d2f2e4c7517b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py @@ -235,6 +235,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -5855,6 +5856,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -5869,7 +5879,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft, and on a decode step the spec fold's store of its outputs). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index 68b7857cb10b..6bf851b86cd0 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -603,6 +603,19 @@ def strip_mm_encoder_inputs(mm_data: Dict[str, Any]) -> None: mm_data.pop(modality, None) +def has_media_or_embeddings(mm_data: Optional[Dict[str, Any]]) -> bool: + """Whether a request's `multimodal_data` carries more than M-RoPE positions. + + Media, precomputed embeddings and placeholder bookkeeping count. A payload + with only `mrope_config` (a disaggregated generation request, or a + text-only warmup request) does not. Any other non-None entry counts, also + one this function does not know, so a text-only model that refuses on it + fails closed. + """ + return any(value is not None for key, value in (mm_data or {}).items() + if key != "mrope_config") + + @dataclass class MultimodalInput: """Per-logical-unit multimodal metadata for KV-cache hashing (C++ layer). diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py index f724e184f318..eb0f3b64e387 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py @@ -24,21 +24,19 @@ def test_the_target_defines_the_admission_hook(): @pytest.mark.parametrize( - "mm_data, carried", + "mm_data", [ - ({"image": {"pixel_values": None, "image_grid_thw": None}}, "image"), - ({"video": {"pixel_values_videos": None, "video_grid_thw": None}}, "video"), - ({"audio": object()}, "audio"), - ({"multimodal_embedding": object()}, "multimodal_embedding"), - ({"image": object(), "video": object()}, "image, video"), + {"image": {"pixel_values": None, "image_grid_thw": None}}, + {"video": {"pixel_values_videos": None, "video_grid_thw": None}}, + {"audio": object()}, + {"multimodal_embedding": object()}, + {"image": object(), "video": object()}, ], ids=["image", "video", "audio", "embedding", "image-and-video"], ) -def test_media_is_refused_at_admission(mm_data, carried): +def test_media_is_refused_at_admission(mm_data): # The hook reads no model state, so it runs without building the model. - with pytest.raises( - ValueError, match=f"does not accept multimodal input; the request carries {carried}$" - ): + with pytest.raises(ValueError, match="serves text only; send no images, videos or embeddings$"): getattr(Target, _HOOK)(types.SimpleNamespace(), mm_data) diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py index 919720b00cda..34d054e5d557 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py @@ -24,21 +24,19 @@ def test_the_target_defines_the_admission_hook(): @pytest.mark.parametrize( - "mm_data, carried", + "mm_data", [ - ({"image": {"pixel_values": None, "image_grid_thw": None}}, "image"), - ({"video": {"pixel_values_videos": None, "video_grid_thw": None}}, "video"), - ({"audio": object()}, "audio"), - ({"multimodal_embedding": object()}, "multimodal_embedding"), - ({"image": object(), "video": object()}, "image, video"), + {"image": {"pixel_values": None, "image_grid_thw": None}}, + {"video": {"pixel_values_videos": None, "video_grid_thw": None}}, + {"audio": object()}, + {"multimodal_embedding": object()}, + {"image": object(), "video": object()}, ], ids=["image", "video", "audio", "embedding", "image-and-video"], ) -def test_media_is_refused_at_admission(mm_data, carried): +def test_media_is_refused_at_admission(mm_data): # The hook reads no model state, so it runs without building the model. - with pytest.raises( - ValueError, match=f"does not accept multimodal input; the request carries {carried}$" - ): + with pytest.raises(ValueError, match="serves text only; send no images, videos or embeddings$"): getattr(Target, _HOOK)(types.SimpleNamespace(), mm_data) diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py index da8de5a6dba3..89cdb001bf4b 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py @@ -588,12 +588,13 @@ def test_fp8_constructor_refuses_what_route_declines(): @pytest.mark.parametrize( "multimodal_data, refused", [ - (None, None), - ({}, None), - ({"image": {"pixel_values": torch.zeros(1)}}, "image"), - ({"video": {"pixel_values_videos": torch.zeros(1)}}, "video"), + (None, False), + ({}, False), + ({"mrope_config": {"mrope_position_deltas": None}}, False), + ({"image": {"pixel_values": torch.zeros(1)}}, True), + ({"video": {"pixel_values_videos": torch.zeros(1)}}, True), ], - ids=["text", "empty-payload", "image", "video"], + ids=["text", "empty-payload", "mrope-only", "image", "video"], ) def test_fp8_target_refuses_media_on_admission(multimodal_data, refused): """The text-only target fails a media request on admission, as that request's own error.""" @@ -603,9 +604,9 @@ def test_fp8_target_refuses_media_on_admission(multimodal_data, refused): ModelingV2Glm53FlashFp8Sm10xTp4, ) - request = SimpleNamespace(py_multimodal_data=multimodal_data) - if refused is None: - ModelingV2Glm53FlashFp8Sm10xTp4.validate_request(None, request) + validate = ModelingV2Glm53FlashFp8Sm10xTp4.validate_multimodal_request_data + if not refused: + validate(SimpleNamespace(), multimodal_data) return - with pytest.raises(ValueError, match=f"carries {refused} input"): - ModelingV2Glm53FlashFp8Sm10xTp4.validate_request(None, request) + with pytest.raises(ValueError, match="serves text only"): + validate(SimpleNamespace(), multimodal_data) diff --git a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py index 067d96d789e1..e56b33b49b9b 100644 --- a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py +++ b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py @@ -13,6 +13,7 @@ import importlib from pathlib import Path +from types import SimpleNamespace import pytest @@ -132,11 +133,14 @@ def test_text_model_weights_skip_the_vision_tower(): def test_check_text_only(): + """The forward refuses what the targets' admission check refuses, and passes the rest.""" fam = _common("weight_io") fam.check_text_only(None) fam.check_text_only([]) - with pytest.raises(ValueError, match="text only"): - fam.check_text_only([object()]) + fam.check_text_only([SimpleNamespace(multimodal_data={"mrope_config": {}})]) + for data in ({"image": {"pixel_values": 1}}, {"multimodal_embedding": 1}): + with pytest.raises(ValueError, match="text only"): + fam.check_text_only([SimpleNamespace(multimodal_data=data)]) def test_common_imports_no_target_code(): diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index 18d78cb99c7d..ad81541b05a6 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -791,6 +791,36 @@ def test_encoder_input_above_max_num_tokens_fails_only_that_request() -> None: assert kwargs == {"requests": [requests[1]], "charge_budget": False} +def test_text_only_model_fails_only_the_media_request() -> None: + """A text-only model's admission hook refuses media alone: the image request fails with + its own error and no budget charge, while a text request and an M-RoPE-only generation + request (disaggregated serving) are admitted.""" + from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + + def validate_multimodal_request_data(mm_data) -> None: + if has_media_or_embeddings(mm_data): + raise ValueError("serves text only") + + requests = [_make_admission_request(i) for i in range(3)] + requests[0].py_multimodal_data = None + requests[1].py_multimodal_data = {"image": {"pixel_values": torch.zeros(1)}} + requests[2].py_multimodal_data = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + model = types.SimpleNamespace( + validate_multimodal_request_data=Mock(side_effect=validate_multimodal_request_data) + ) + executor = _make_admission_executor(model, requests) + + admitted = PyExecutor._fetch_and_activate_new_requests(executor) + + assert admitted == [requests[0], requests[2]] + assert executor.active_requests == [requests[0], requests[2]] + executor._handle_errors.assert_called_once_with( + "serves text only", requests=[requests[1]], charge_budget=False + ) + # The hook runs only for a request that carries multimodal data. + assert model.validate_multimodal_request_data.call_count == 2 + + def test_model_validation_runs_after_the_sampler_and_before_the_budget() -> None: order = [] model = types.SimpleNamespace(validate_request=lambda request: order.append("model")) diff --git a/tests/unittest/inputs/test_multimodal.py b/tests/unittest/inputs/test_multimodal.py index 01bcd83e5372..ecf842c5dec4 100644 --- a/tests/unittest/inputs/test_multimodal.py +++ b/tests/unittest/inputs/test_multimodal.py @@ -20,6 +20,7 @@ MultimodalRuntimeData, _find_mm_embedding_lengths_from_masks, find_mm_token_lengths, + has_media_or_embeddings, select_multimodal_fields, ) from tensorrt_llm.inputs.multimodal_data import MEDIA_MODALITIES, AudioData, VideoData @@ -1225,3 +1226,44 @@ def test_params_do_not_infer_item_layout_from_raw_buckets(): params = MultimodalParams(multimodal_data={"audio": {}, "image": {}}) assert params.item_metadata is None assert params.get_item_layout() == ((), ()) + + +_MROPE = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + + +@pytest.mark.parametrize( + "mm_data,expected", + [ + (None, False), + ({}, False), + (_MROPE, False), + ({"mrope_config": None, "image": None}, False), + ({"image": {"pixel_values": torch.zeros(1)}}, True), + ({"video": {"pixel_values_videos": torch.zeros(1)}}, True), + ({"audio": {"input_features": torch.zeros(1)}}, True), + ({"multimodal_embedding": torch.zeros(1, 4)}, True), + ({"multimodal_embed_mask_cumsum": torch.arange(3)}, True), + ({**_MROPE, "image": {"pixel_values": torch.zeros(1)}}, True), + ({"some_future_key": 1}, True), + ], + ids=[ + "none", + "empty", + "mrope_only", + "none_values", + "image", + "video", + "audio", + "embedding", + "cumsum_only", + "mrope_and_image", + "unknown_key", + ], +) +def test_has_media_or_embeddings_ignores_only_mrope_positions(mm_data, expected): + """Only M-RoPE positions are not media. + + Text-only models refuse everything else, including keys they do not know, so an + M-RoPE-only disaggregated generation request still passes. + """ + assert has_media_or_embeddings(mm_data) is expected From 8ce7cd26e8aae3939b1bda67afa27ef37b4a8ff9 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:23 +0000 Subject: [PATCH 2/6] [TRTLLM-17780][feat] Honor disable_mm_encoder in the qwen4_exp Modeling V2 target Problem The qwen4_exp NVFP4 Modeling V2 target ignores disable_mm_encoder. It still declares, loads and warms up the full vision tower on every rank, and an image request with the switch on would reach a tower the user asked not to build. Solution - Move the 21 vision weight entries out of the language model table into VISION_WEIGHTS, a second table with the same declare / manifest / read passes and its own load. The load checks coverage both ways: every tower parameter is written and every model.visual key is read. The language model table now always leaves model.visual keys unread. - VisionTower is an nn.Module that declares and loads its own table and builds its views and warms up in post_load_weights. The shell builds it only when disable_mm_encoder is off. - Without a tower, validate_request skips the geometry checks and validate_multimodal_request_data refuses media or embeddings, so only that request fails. M-RoPE-only payloads still pass. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs the one before: 8641 vs 8636 cases, 0 changed outcomes; the 5 new tests pass (table split, vision keys left by the language table, tower load with two-way coverage, admission without a tower). - The checkpoint-backed coverage tests are updated for the second table; they skip without the checkpoint, as before. - Pre-commit hooks pass. qwen4_exp GPU runs need GB300 (sm_103); not run. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- .../modeling.py | 32 +- .../vision.py | 43 ++- .../weights.py | 359 ++++++++++-------- .../test_vision.py | 55 +-- .../test_vision_warm_up.py | 33 +- .../test_weight_shards.py | 112 +++++- 6 files changed, 411 insertions(+), 223 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 2383908a6d17..64b890379671 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -431,6 +431,7 @@ from tensorrt_llm._torch.utils import ActivationType, is_gdn_replay_enabled from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.functional import AllReduceFusionOp +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -1225,7 +1226,7 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig # device-address view that owns none of those pages. self._ple_table_host = None - # ---- the vision half (the weight table declares the tower) --------- + # ---- the vision half (the tower declares its own weights) ---------- vcfg = getattr(composite_config, "vision_config", None) assert vcfg is not None, "the composite config carries no vision_config" self.vision_config = vcfg @@ -7050,8 +7051,13 @@ def __init__(self, model_config: ModelConfig): # init). self._mm_token_ids_value = (self.model.image_token_id, self.model.video_token_id) self._mm_token_ids_cache: torch.Tensor | None = None - # Replicated on every rank; built once the weights are real (`post_load_weights`). - self.vision_tower: _vision.VisionTower | None = None + # Replicated on every rank. None with `disable_mm_encoder`: then no vision weight is + # declared or loaded and a media request is refused at admission. + self.vision_tower: _vision.VisionTower | None = ( + None + if model_config.disable_mm_encoder + else _vision.VisionTower(self.model.vision_config, model_config.torch_dtype) + ) @property def multimodal_token_ids(self) -> torch.Tensor: @@ -7071,8 +7077,19 @@ def multimodal_data_device_paths(self) -> list[str]: def validate_request(self, request) -> None: """Admission check, run by the executor on every new request: a media request the vision tower would assert on fails here as its own error (`vision.validate_request`); - a text request returns at once.""" - _vision.validate_request(self.vision_tower, request) + a text request returns at once. Without a tower `validate_multimodal_request_data` + refuses media instead.""" + if self.vision_tower is not None: + _vision.validate_request(self.vision_tower, request) + + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: with `disable_mm_encoder` the target serves text only, so a + request with media or embeddings fails alone. M-RoPE positions alone pass.""" + if self.vision_tower is None and has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} runs with disable_mm_encoder and serves text only; " + "send no images, videos or embeddings" + ) def forward( self, @@ -7169,12 +7186,13 @@ def forward( def load_weights(self, weights, *args, **kwargs): _weights.MODEL_WEIGHTS.load(self, weights) + if self.vision_tower is not None: + self.vision_tower.load_weights(weights) def post_load_weights(self): + # The engine's post-load walk calls the vision tower's own hook. super().post_load_weights() self.model.post_load_weights() - self.vision_tower = _vision.VisionTower(self.model.w, self.model.vision_config) - self.vision_tower.warm_up() if self.one_rank: self._build_heads(self.lm_head.weight.data) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py index f14708d1110b..1a3f74270a73 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py @@ -21,6 +21,7 @@ from typing import NamedTuple import torch +from torch import nn from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm @@ -38,6 +39,8 @@ ) from tensorrt_llm.inputs.multimodal import get_multimodal_item_metadata +from . import weights as _weights + #: Tower rope base (`vision_config` has no rope field); unrelated to the text 1e7. VISION_ROPE_THETA = 10000.0 @@ -292,7 +295,7 @@ def _rope_tables( return angles.cos().contiguous(), angles.sin().contiguous() -class VisionTower: +class VisionTower(nn.Module): """One request's images or videos through the 27-block ViT and its patch merger. Not a target: runs only for a context request carrying media, from the shell, before @@ -306,11 +309,15 @@ class VisionTower: frames: attention is per frame, the position embedding and rope are per grid row and repeat per frame, the norms and MLPs are row-wise, and a frame's `h * w` is a multiple of `merge**2`, so a cut keeps every merger block whole. One frame past the bound cannot - be served; `validate_request` refuses it at admission.""" + be served; `validate_request` refuses it at admission. + + The tower declares and loads its own weights (`weights.VISION_WEIGHTS`), so a model + built without it (`disable_mm_encoder`) allocates no vision weight.""" - def __init__(self, w: dict, vision_config) -> None: - """`w` is the core's weight table after load; `vision_config` the composite - config's vision half, which `check_deployment` matched against this checkpoint's.""" + def __init__(self, vision_config, dtype: torch.dtype) -> None: + """`vision_config` is the composite config's vision half, which `check_deployment` + matched against this checkpoint's; `dtype` the activation dtype the tower runs at.""" + super().__init__() vcfg = vision_config self.depth = vcfg.depth self.hidden = vcfg.hidden_size @@ -324,12 +331,28 @@ def __init__(self, w: dict, vision_config) -> None: self.merge_dim = self.hidden * self.merge * self.merge # The position table is a square grid of this side. self.grid_side = math.isqrt(vcfg.num_position_embeddings) - self.dtype = w["embed"].dtype - self._w = _vision_views(w, self.depth) + self.dtype = dtype + # Every weight, declared here (meta-init intercepts `torch.empty`) and filled by + # `load_weights`; the views the ops read are built after load. + self.dims = _weights.vision_dims(vcfg, dtype) + self.w = _weights.VISION_WEIGHTS.declare(self.dims) + self._w: _VisionWeights | None = None # Persistent scratch: `thop_attention` grows `workspace` in place; never the # engine's own workspace. self._ws: torch.Tensor | None = None + def load_weights(self, weights) -> None: + """Fill the tower's weights from the checkpoint's `model.visual.` keys.""" + _weights.VISION_WEIGHTS.load(self, weights) + + def post_load_weights(self) -> None: + """Build the views the ops read, then warm the kernels up; once, though the + engine's post-load walk may reach this module more than once.""" + if self._w is not None: + return + self._w = _vision_views(self.w, self.depth) + self.warm_up() + def warm_up(self) -> None: """Run the tower once per `WARM_UP_GRIDS` grid on zero pixels, so every JIT kernel it launches is compiled at load rather than on the first media request. The rope and @@ -340,7 +363,7 @@ def warm_up(self) -> None: rows = sum(t * h * w for t, h, w in grid) self._run(torch.zeros(rows, self.pixel_width, dtype=self.dtype, device=device), grid) - def __call__(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: """`pixel_values` is `[sum(t*h*w), in_channels*kt*kh*kw]`, one flat patch per row, channel-slowest. `grid_thw` is one `(frames, height, width)` row per image or video and stays on the host (no synchronization). Returns @@ -498,7 +521,9 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): if runtime is not None and int(runtime.num_mm_tokens_in_chunk) == 0: # Every placeholder row of this request lies in another chunk. continue - assert tower is not None, "post_load_weights builds the vision tower before any step" + assert tower is not None, ( + "no vision tower (disable_mm_encoder), so admission refuses media requests" + ) towers = {} if image is not None: towers["image"] = run_tower(tower, image, *MEDIA_KEYS["image"]) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index a2c2a81932e3..b6be1bf1ee56 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -90,7 +90,8 @@ `load(model, weights)` fills every declared parameter once and asserts coverage both ways: every parameter written, and every checkpoint key consumed (a key read through a cut is consumed) or predicted by `expected_unconsumed` -- the experts outside this -rank's windows, and the MTP module when there is no speculation. +rank's windows, the MTP module when there is no speculation, and the vision tower, whose +weights are a second table (`VISION_WEIGHTS`) that the tower module declares and loads. """ import mmap @@ -1271,134 +1272,6 @@ class Qwen4ExpDefaultWeights: src=lambda d, i: [(f"{_ngram_prefix(d.ple_layer)}.weight_scale", None)], layers=None, ), - # --- the vision tower: all bf16, replicated, one-to-one, no transform --- - W( - "vis_patch_w", - shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), - src=_visual("patch_embed.proj.weight"), - layers=None, - ), - W( - "vis_patch_b", - shape=lambda d: (d.v_hidden,), - src=_visual("patch_embed.proj.bias"), - layers=None, - ), - W( - "vis_pos", - shape=lambda d: (d.v_pos_rows, d.v_hidden), - src=_visual("pos_embed.weight"), - layers=None, - ), - # LayerNorm **with a bias**, unlike every norm in the language tower. - W( - "vis_n1_w", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm1.weight"), - layers=_vision_blocks, - ), - W( - "vis_n1_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm1.bias"), - layers=_vision_blocks, - ), - # One fused `[q | k | v]` projection, MHA, with a bias. - W( - "vis_qkv_w", - shape=lambda d: (3 * d.v_hidden, d.v_hidden), - src=_vision_block("attn.qkv.weight"), - layers=_vision_blocks, - ), - W( - "vis_qkv_b", - shape=lambda d: (3 * d.v_hidden,), - src=_vision_block("attn.qkv.bias"), - layers=_vision_blocks, - ), - W( - "vis_proj_w", - shape=lambda d: (d.v_hidden, d.v_hidden), - src=_vision_block("attn.proj.weight"), - layers=_vision_blocks, - ), - W( - "vis_proj_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("attn.proj.bias"), - layers=_vision_blocks, - ), - W( - "vis_n2_w", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm2.weight"), - layers=_vision_blocks, - ), - W( - "vis_n2_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm2.bias"), - layers=_vision_blocks, - ), - # A **non-gated** MLP: nothing here is split in half, unlike every SwiGLU in the - # language tower. - W( - "vis_fc1_w", - shape=lambda d: (d.v_inter, d.v_hidden), - src=_vision_block("mlp.linear_fc1.weight"), - layers=_vision_blocks, - ), - W( - "vis_fc1_b", - shape=lambda d: (d.v_inter,), - src=_vision_block("mlp.linear_fc1.bias"), - layers=_vision_blocks, - ), - W( - "vis_fc2_w", - shape=lambda d: (d.v_hidden, d.v_inter), - src=_vision_block("mlp.linear_fc2.weight"), - layers=_vision_blocks, - ), - W( - "vis_fc2_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("mlp.linear_fc2.bias"), - layers=_vision_blocks, - ), - # The patch merger. Its norm is applied **before** the 4-to-1 shuffle, so it is - # `hidden` wide and not `merge_dim` wide. - W( - "vis_m_n_w", - shape=lambda d: (d.v_hidden,), - src=_visual("merger.norm.weight"), - layers=None, - ), - W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), - W( - "vis_m_fc1_w", - shape=lambda d: (d.v_merge_dim, d.v_merge_dim), - src=_visual("merger.linear_fc1.weight"), - layers=None, - ), - W( - "vis_m_fc1_b", - shape=lambda d: (d.v_merge_dim,), - src=_visual("merger.linear_fc1.bias"), - layers=None, - ), - W( - "vis_m_fc2_w", - shape=lambda d: (d.hidden, d.v_merge_dim), - src=_visual("merger.linear_fc2.weight"), - layers=None, - ), - W( - "vis_m_fc2_b", - shape=lambda d: (d.hidden,), - src=_visual("merger.linear_fc2.bias"), - layers=None, - ), # --- the MTP draft module, at layer index num_hidden_layers, only when live --- # # Structurally one more decoder layer of this architecture with a two-branch @@ -1516,15 +1389,13 @@ class Qwen4ExpDefaultWeights: def dims(self, core) -> SimpleNamespace: """This rank's dimension bundle, from the core's normalized text config, the - composite's vision config (the core holds it as `vision_config` before it - declares anything), the engine's `Mapping` and the core's engine choices.""" + engine's `Mapping` and the core's engine choices.""" quant_config = ( moe_experts_quant_config(core.model_config, f"{_MTP_LAYER}.mlp.experts") or core.model_config.quant_config ) return rank_dims( core.model_config.pretrained_config, - core.vision_config, core.model_config.mapping, mtp_enabled=getattr(core.model_config, "spec_config", None) is not None, trunk_moe_backend=core.trunk_moe_backend, @@ -1538,9 +1409,11 @@ def expected_unconsumed(self, d: SimpleNamespace, keys) -> set: routed expert outside its window (its per-tensor scalars are read on every rank; under the intermediate split the window is every expert, each read at this rank's slice), and, with MTP live, the draft experts outside its draft window; with MTP off, the - whole `mtp.` subtree. The windows are enumerated from `d`, not read off the + whole `mtp.` subtree. The `model.visual.` subtree is always left: the vision tower + loads it with its own table (`VISION_WEIGHTS`), or nothing does with + `disable_mm_encoder`. The windows are enumerated from `d`, not read off the checkpoint, so a key that belongs to a real weight can never land here.""" - out = set() + out = {k for k in keys if k.startswith(f"{_VISUAL}.")} start, count = _trunk_window(d) for i in range(d.num_layers): for e in range(d.num_experts): @@ -1614,6 +1487,18 @@ def read(self, weights, ckpt_key, cut, transform, d, device=None) -> torch.Tenso assert len(src_all) == 1, "a multi-key row needs a transform" return src_all[0] + def fill(self, weights, d, consumed: set, param, ckpt_key, index, cut, transform) -> None: + """Copy one manifest source into `param` (or its row `index`) and record the + checkpoint keys it read in `consumed`.""" + dst = param.data if index is None else param.data[index] + # Materialize where the destination lives: the expert relayouts are row + # gathers and the scale swizzle is a kernel, both on the device. + src = self.read(weights, ckpt_key, cut, transform, d, dst.device) + assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) + assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + def load(self, model, weights) -> None: """Fill every declared parameter. The n-gram table is declared here first, each source is read through its cut, multi-key rows feed their transform several @@ -1640,14 +1525,7 @@ def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int consumed: set = set() def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: - dst = param.data if index is None else param.data[index] - # Materialize where the destination lives: the expert relayouts are row - # gathers and the scale swizzle is a kernel, both on the device. - src = self.read(weights, ckpt_key, cut, transform, d, dst.device) - assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) - assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) - dst.copy_(src, non_blocking=True) - consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + self.fill(weights, d, consumed, param, ckpt_key, index, cut, transform) assert set(manifest.keys()) == set(core.w.keys()), ( "manifest/parameter drift", @@ -1672,8 +1550,12 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: fill(model.lm_head.weight, "lm_head.weight", None, cut, pad) # Parameter-side coverage, the other direction: name every registered parameter # outside the target's own ParameterDict instead of assuming lm_head is the only - # one. - shell = {n for n, _ in model.named_parameters() if not n.startswith("model.w.")} + # one. The vision tower loads and checks its own table. + shell = { + n + for n, _ in model.named_parameters() + if not n.startswith(("model.w.", "vision_tower.")) + } assert shell == {"lm_head.weight"}, f"unfed shell parameters: {sorted(shell)}" torch.cuda.synchronize() @@ -1681,7 +1563,7 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: expected = self.expected_unconsumed(d, weights.keys()) assert leftover == expected, ( "checkpoint coverage: leftover keys are not exactly this rank's off-window " - f"experts{'' if d.mtp_enabled else ' and the MTP module'}; unexpected " + f"experts{'' if d.mtp_enabled else ', the MTP module'} and the vision tower; unexpected " f"{sorted(leftover - expected)[:8]}, missed {sorted(expected - leftover)[:8]}" ) @@ -1689,6 +1571,168 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: MODEL_WEIGHTS = Qwen4ExpDefaultWeights() +class Qwen4ExpVisionWeights(Qwen4ExpDefaultWeights): + """The vision tower's weights: all bf16, replicated, one-to-one, no transform. + + The same `declare`, `manifest`, `read` and `fill` passes over a table of its own, which + the tower module (`vision.VisionTower`) declares and loads, so the language model's table + holds no vision weight and `disable_mm_encoder` allocates none. Its dimension bundle is + `vision_dims`. + """ + + WEIGHTS: tuple[W, ...] = ( + W( + "vis_patch_w", + shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), + src=_visual("patch_embed.proj.weight"), + layers=None, + ), + W( + "vis_patch_b", + shape=lambda d: (d.v_hidden,), + src=_visual("patch_embed.proj.bias"), + layers=None, + ), + W( + "vis_pos", + shape=lambda d: (d.v_pos_rows, d.v_hidden), + src=_visual("pos_embed.weight"), + layers=None, + ), + # LayerNorm **with a bias**, unlike every norm in the language tower. + W( + "vis_n1_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.weight"), + layers=_vision_blocks, + ), + W( + "vis_n1_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.bias"), + layers=_vision_blocks, + ), + # One fused `[q | k | v]` projection, MHA, with a bias. + W( + "vis_qkv_w", + shape=lambda d: (3 * d.v_hidden, d.v_hidden), + src=_vision_block("attn.qkv.weight"), + layers=_vision_blocks, + ), + W( + "vis_qkv_b", + shape=lambda d: (3 * d.v_hidden,), + src=_vision_block("attn.qkv.bias"), + layers=_vision_blocks, + ), + W( + "vis_proj_w", + shape=lambda d: (d.v_hidden, d.v_hidden), + src=_vision_block("attn.proj.weight"), + layers=_vision_blocks, + ), + W( + "vis_proj_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("attn.proj.bias"), + layers=_vision_blocks, + ), + W( + "vis_n2_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.weight"), + layers=_vision_blocks, + ), + W( + "vis_n2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.bias"), + layers=_vision_blocks, + ), + # A **non-gated** MLP: nothing here is split in half, unlike every SwiGLU in the + # language tower. + W( + "vis_fc1_w", + shape=lambda d: (d.v_inter, d.v_hidden), + src=_vision_block("mlp.linear_fc1.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc1_b", + shape=lambda d: (d.v_inter,), + src=_vision_block("mlp.linear_fc1.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc2_w", + shape=lambda d: (d.v_hidden, d.v_inter), + src=_vision_block("mlp.linear_fc2.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("mlp.linear_fc2.bias"), + layers=_vision_blocks, + ), + # The patch merger. Its norm is applied **before** the 4-to-1 shuffle, so it is + # `hidden` wide and not `merge_dim` wide. + W( + "vis_m_n_w", + shape=lambda d: (d.v_hidden,), + src=_visual("merger.norm.weight"), + layers=None, + ), + W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), + W( + "vis_m_fc1_w", + shape=lambda d: (d.v_merge_dim, d.v_merge_dim), + src=_visual("merger.linear_fc1.weight"), + layers=None, + ), + W( + "vis_m_fc1_b", + shape=lambda d: (d.v_merge_dim,), + src=_visual("merger.linear_fc1.bias"), + layers=None, + ), + W( + "vis_m_fc2_w", + shape=lambda d: (d.hidden, d.v_merge_dim), + src=_visual("merger.linear_fc2.weight"), + layers=None, + ), + W( + "vis_m_fc2_b", + shape=lambda d: (d.hidden,), + src=_visual("merger.linear_fc2.bias"), + layers=None, + ), + ) + + def load(self, tower, weights) -> None: + """Fill every parameter of `tower.w` and assert coverage both ways: every + parameter written, and every `model.visual.` checkpoint key read.""" + d = tower.dims + manifest = self.manifest(d) + assert set(manifest.keys()) == set(tower.w.keys()), ( + "manifest/parameter drift", + set(manifest.keys()) ^ set(tower.w.keys()), + ) + consumed: set = set() + for param_key, sources in manifest.items(): + for source in sources: + self.fill(weights, d, consumed, tower.w[param_key], *source) + visual = {k for k in weights.keys() if k.startswith(f"{_VISUAL}.")} + assert consumed == visual, ( + "vision checkpoint coverage: unexpected " + f"{sorted(visual - consumed)[:8]}, missed {sorted(consumed - visual)[:8]}" + ) + + +VISION_WEIGHTS = Qwen4ExpVisionWeights() + + def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: """(heads on `rank`, its first global head) when `total` heads are split over `ranks`: an even split, or each head repeated on `ranks // total` neighbouring ranks when there @@ -1700,7 +1744,6 @@ def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: def rank_dims( cfg, - vcfg, mapping, *, mtp_enabled: bool, @@ -1713,7 +1756,6 @@ def rank_dims( Args: cfg: the engine-normalized text config (`Qwen4ExpTextConfig`). - vcfg: the composite's vision config. mapping: this rank's `Mapping`. mtp_enabled: whether the engine has a `speculative_config` (the MTP module is live). trunk_moe_backend: the routed backend the trunk runs, which picks the expert layout. @@ -1762,7 +1804,6 @@ def rank_dims( expert_inter = -(-expert_inter_share // EXPERT_INTER_ALIGN) * EXPERT_INTER_ALIGN ngram_heads = cfg.heads_per_ngram * (cfg.ngram_size - 1) - v_hidden = vcfg.hidden_size return SimpleNamespace( num_layers=cfg.num_hidden_layers, is_attn=[t == "full_attention" for t in cfg.layer_types], @@ -1824,7 +1865,22 @@ def rank_dims( ple_head_dim=cfg.ple_embed_dim // ngram_heads, ple_shards=cfg.split_ngram_parts, ple_table_dtype=PLE_TABLE_DTYPE, - # --- the vision tower --- + # --- engine choices --- + mtp_enabled=mtp_enabled, + trunk_moe_backend=trunk_moe_backend, + moe_prefill_copy=moe_prefill_copy, + dtype=dtype, + ) + + +def vision_dims(vcfg, dtype: torch.dtype) -> SimpleNamespace: + """The vision table's dimension bundle, from the composite's vision config. The tower + is replicated, so every rank declares it whole. `hidden` is the merger's output width, + the text hidden size of the rows it replaces.""" + v_hidden = vcfg.hidden_size + return SimpleNamespace( + hidden=vcfg.out_hidden_size, + dtype=dtype, v_depth=vcfg.depth, v_hidden=v_hidden, v_inter=vcfg.intermediate_size, @@ -1833,11 +1889,6 @@ def rank_dims( v_patch=vcfg.patch_size, v_pos_rows=vcfg.num_position_embeddings, v_merge_dim=v_hidden * vcfg.spatial_merge_size * vcfg.spatial_merge_size, - # --- engine choices --- - mtp_enabled=mtp_enabled, - trunk_moe_backend=trunk_moe_backend, - moe_prefill_copy=moe_prefill_copy, - dtype=dtype, ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py index 29c546ac3e11..c3ab21b0043f 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py @@ -16,6 +16,7 @@ import pytest import torch +from torch import nn from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 @@ -24,9 +25,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 vision as V, ) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 - weights as W, -) from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalItemMetadata, @@ -57,28 +55,12 @@ def _tower_over_the_weight_table() -> V.VisionTower: - """The tower over every `vis_*` weight the table declares, at the checkpoint's shapes, - on meta; `embed` stands in for the activation dtype the tower reads off it.""" - vcfg = Qwen4ExpVisionConfig(**_VISION) - d = SimpleNamespace( - hidden=HIDDEN_OUT, - dtype=torch.bfloat16, - v_depth=vcfg.depth, - v_hidden=vcfg.hidden_size, - v_inter=vcfg.intermediate_size, - v_channels=vcfg.in_channels, - v_tpatch=vcfg.temporal_patch_size, - v_patch=vcfg.patch_size, - v_pos_rows=vcfg.num_position_embeddings, - v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, - ) - w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="meta")} - for entry in W.MODEL_WEIGHTS.WEIGHTS: - if not entry.name.startswith("vis_"): - continue - for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): - w[key] = torch.empty(*entry.shape(d), dtype=entry.dtype or d.dtype, device="meta") - return V.VisionTower(w, vcfg) + """The tower at the checkpoint's shapes, on meta: it declares its own `vis_*` table + (`weights.VISION_WEIGHTS`), and the views are what `post_load_weights` builds.""" + with torch.device("meta"): + tower = V.VisionTower(Qwen4ExpVisionConfig(**_VISION), torch.bfloat16) + tower._w = V._vision_views(tower.w, tower.depth) + return tower def test_the_tower_reads_the_weight_tables_vision_rows_at_the_ops_layouts(): @@ -212,6 +194,7 @@ def _stub_tower(monkeypatch, cap): the request.""" monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", cap) tower = object.__new__(V.VisionTower) + nn.Module.__init__(tower) tower.dtype = torch.float32 tower.pixel_width = PIXEL_WIDTH tower.merge = MERGE @@ -572,3 +555,25 @@ def test_the_placeholder_count_is_the_sum_over_both_modalities(): _admit(request) del request.py_multimodal_data["multimodal_embed_mask_cumsum"] _admit(request) + + +def test_without_a_tower_media_is_refused_and_mrope_positions_pass(): + """With `disable_mm_encoder` the shell builds no tower: a media request fails at + admission as its own error, and an M-RoPE-only payload (a disaggregated generation + request) passes. With a tower the geometry check owns media requests.""" + no_tower = SimpleNamespace(vision_tower=None) + request = _request(image=IMAGES) + # No geometry check without a tower: it would read the tower's geometry. + Shell.validate_request(no_tower, request) + with pytest.raises(ValueError, match="disable_mm_encoder and serves text only"): + Shell.validate_multimodal_request_data(no_tower, request.py_multimodal_data) + with pytest.raises(ValueError, match="serves text only"): + Shell.validate_multimodal_request_data( + no_tower, {"multimodal_embedding": torch.zeros(4, HIDDEN)} + ) + mrope_only = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + Shell.validate_request(no_tower, SimpleNamespace(py_multimodal_data=mrope_only)) + Shell.validate_multimodal_request_data(no_tower, mrope_only) + Shell.validate_multimodal_request_data( + SimpleNamespace(vision_tower=_GEOMETRY), request.py_multimodal_data + ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py index fddaf5b036af..9abb4eec46fc 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py @@ -4,22 +4,17 @@ `post_load_weights` runs, a media request of any grid compiles no Triton kernel. The tower is one block deep (the block's kernels are every block's) over random weights at the -checkpoint's shapes, built from the weight table's own `vis_*` declarations. +checkpoint's shapes, declared by the tower itself (`weights.VISION_WEIGHTS`). """ from __future__ import annotations -from types import SimpleNamespace - import torch from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 vision as V, ) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 - weights as W, -) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.ops.attention.vision_rope_2d_packed_qk import ( # noqa: E501 _rotary_neox_inplace_kernel, ) @@ -58,27 +53,13 @@ def _tower() -> V.VisionTower: vcfg = Qwen4ExpVisionConfig(**_VISION) - d = SimpleNamespace( - hidden=HIDDEN_OUT, - dtype=torch.bfloat16, - v_depth=vcfg.depth, - v_hidden=vcfg.hidden_size, - v_inter=vcfg.intermediate_size, - v_channels=vcfg.in_channels, - v_tpatch=vcfg.temporal_patch_size, - v_patch=vcfg.patch_size, - v_pos_rows=vcfg.num_position_embeddings, - v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, - ) + with torch.device("cuda"): + tower = V.VisionTower(vcfg, torch.bfloat16) gen = torch.Generator(device="cuda").manual_seed(7) - w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="cuda")} - for entry in W.MODEL_WEIGHTS.WEIGHTS: - if not entry.name.startswith("vis_"): - continue - for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): - shape, dtype = entry.shape(d), entry.dtype or d.dtype - w[key] = (torch.randn(*shape, generator=gen, device="cuda") * 0.02).to(dtype) - return V.VisionTower(w, vcfg) + for param in tower.w.values(): + param.data.copy_(torch.randn(param.shape, generator=gen, device="cuda") * 0.02) + tower._w = V._vision_views(tower.w, tower.depth) + return tower def _variants() -> dict: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py index 70d77773904a..eb28f6fca4a7 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py @@ -44,6 +44,9 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 pool as P, ) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + vision as V, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 weights as WT, ) @@ -131,10 +134,9 @@ def _mapping(member: str, rank: int) -> Mapping: def _dims(configs, member: str, rank: int, mtp: bool = True, prefill_copy: bool = False): - text, vision = configs + text, _ = configs return WT.rank_dims( text, - vision, _mapping(member, rank), mtp_enabled=mtp, trunk_moe_backend="TRTLLM", @@ -169,6 +171,10 @@ def _rank_manifest(stored, d, member: str) -> dict[str, list[tuple]]: return manifest +def _vision_dims(configs): + return WT.vision_dims(configs[1], torch.bfloat16) + + def _keys_and_cuts(ckpt_key, cut): if isinstance(ckpt_key, tuple): return list(zip(ckpt_key, cut if cut is not None else (None,) * len(ckpt_key))) @@ -253,6 +259,10 @@ def test_every_checkpoint_tensor_is_claimed(stored, configs, member, mtp): f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" ) read_by.update(claimed) + # The tower's own table reads the `model.visual.` keys the language table leaves. + vision_claimed = _claimed(WT.VISION_WEIGHTS.manifest(_vision_dims(configs))) + assert vision_claimed == {k for k in everything if k.startswith("model.visual.")} + read_by.update(vision_claimed) mtp_keys = set() if mtp else {k for k in everything if k.startswith("mtp.")} assert set(read_by) == everything - mtp_keys, "tensors no rank reads: " + str( @@ -278,8 +288,11 @@ def test_rank_operands_have_their_parameter_shapes(stored, configs, member, rank fills its parameter exactly; the rank reads each tensor at its share, and once per layout it is kept in (the routed experts twice with the CuteDSL prefill copy).""" d = _dims(configs, member, rank, mtp=True, prefill_copy=prefill_copy) + vd = _vision_dims(configs) with torch.device("meta"): params = dict(WT.MODEL_WEIGHTS.declare(d)) + # The vision tower's table, which every rank declares whole. + params.update(WT.VISION_WEIGHTS.declare(vd)) # What `load` adds beside the table: the n-gram table and the shell's lm_head. params["ple_table"] = torch.empty( d.ple_shards * _rows_per_shard(stored, d), d.ple_head_dim, dtype=d.ple_table_dtype @@ -289,6 +302,7 @@ def test_rank_operands_have_their_parameter_shapes(stored, configs, member, rank ) entries = {key: _HeaderEntry(*spec) for key, spec in stored.items()} manifest = _rank_manifest(stored, d, member) + manifest.update(WT.VISION_WEIGHTS.manifest(vd)) assert manifest.keys() == params.keys() read = collections.Counter() @@ -507,6 +521,9 @@ def test_load_fills_every_parameter_and_covers_the_checkpoint( model = _stub_model(configs, member, rank, mtp) WT.MODEL_WEIGHTS.load(model, _meta_checkpoint(stored)) assert tuple(model.model.w["ple_table"].shape) == (320_001_536, 160) + with torch.device("meta"): + tower = V.VisionTower(configs[1], torch.bfloat16) + tower.load_weights(_meta_checkpoint(stored)) @pytest.mark.parametrize("expert_override", (None, QuantAlgo.NVFP4)) @@ -1044,6 +1061,7 @@ def test_dep2_ranks_read_the_checkpoint_in_two_256_expert_windows(stored, config f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" ) read_by.update(claimed) + read_by.update(_claimed(WT.VISION_WEIGHTS.manifest(_vision_dims(configs)))) dep4 = _dims(configs, "dep4", rank, mtp=False) non_expert = {k for k in claimed if not _TRUNK_EXPERT_TENSOR.match(k)} @@ -1095,3 +1113,93 @@ def test_dep2_load_fills_every_parameter_and_covers_the_checkpoint(stored, confi model = _stub_model(configs, "dep2", 1, False) WT.MODEL_WEIGHTS.load(model, _meta_checkpoint(stored)) assert model.lm_head.weight.shape[0] == 248320 + + +# ------------------------------------------------------- the vision table -- + +#: Every vision-tower weight, as the language table declared them before the tower took +#: its own table: the split loses none. +_VISION_ENTRIES = { + "vis_patch_w", "vis_patch_b", "vis_pos", + "vis_n1_w", "vis_n1_b", "vis_qkv_w", "vis_qkv_b", "vis_proj_w", "vis_proj_b", + "vis_n2_w", "vis_n2_b", "vis_fc1_w", "vis_fc1_b", "vis_fc2_w", "vis_fc2_b", + "vis_m_n_w", "vis_m_n_b", "vis_m_fc1_w", "vis_m_fc1_b", "vis_m_fc2_w", "vis_m_fc2_b", +} # fmt: skip + +#: A small vision config: two blocks, every width shrunk. +_SMALL_VISION = SimpleNamespace( + depth=2, + hidden_size=8, + num_heads=2, + intermediate_size=12, + in_channels=3, + temporal_patch_size=2, + patch_size=2, + num_position_embeddings=16, + spatial_merge_size=2, + out_hidden_size=6, +) + + +def test_the_vision_weights_are_a_table_of_their_own(): + """The language table declares no vision weight, so `disable_mm_encoder` allocates + none; the tower's table declares exactly the entries the language table had.""" + assert not [e.name for e in WT.MODEL_WEIGHTS.WEIGHTS if e.name.startswith("vis_")] + assert {e.name for e in WT.VISION_WEIGHTS.WEIGHTS} == _VISION_ENTRIES + + +@pytest.mark.parametrize("mtp", [False, True], ids=["no-mtp", "mtp"]) +def test_the_language_table_leaves_every_vision_key(mtp): + d = SimpleNamespace( + num_layers=1, + num_experts=2, + expert_offset=0, + local_experts=2, + mtp_enabled=mtp, + mtp_expert_offset=0, + mtp_local_experts=2, + ) + keys = { + "model.visual.patch_embed.proj.weight", + "model.visual.blocks.0.norm1.weight", + "model.visual.merger.linear_fc2.bias", + "model.language_model.embed_tokens.weight", + "mtp.fc_hidden.weight", + } + unread = WT.MODEL_WEIGHTS.expected_unconsumed(d, keys) + assert {k for k in unread if not k.startswith("mtp.")} == { + k for k in keys if k.startswith("model.visual.") + } + + +def _small_vision_checkpoint(tower) -> dict: + """One distinct tensor per key the tower's table reads, at its declared shape.""" + checkpoint = {} + for value, (key, sources) in enumerate(WT.VISION_WEIGHTS.manifest(tower.dims).items(), 1): + ((ckpt_key, index, cut, transform),) = sources + assert index is None and cut is None and transform is None, ckpt_key + param = tower.w[key] + checkpoint[ckpt_key] = torch.full(param.shape, float(value), dtype=param.dtype) + return checkpoint + + +def test_the_tower_loads_its_table_and_covers_the_vision_keys(): + """The tower module declares its table at a small config, `load_weights` fills every + parameter from `model.visual.` keys and reads them all, and the language model's keys + are not its to read. An unknown or missing vision key fails the load.""" + tower = V.VisionTower(_SMALL_VISION, torch.float32) + # Twelve entries per block, at two blocks. + assert sum(k.startswith(("l0_", "l1_")) for k in tower.w) == 24 + checkpoint = _small_vision_checkpoint(tower) + assert tuple(tower.w["vis_m_fc2_w"].shape) == (6, 32) + tower.load_weights({**checkpoint, "model.language_model.embed_tokens.weight": None}) + manifest = WT.VISION_WEIGHTS.manifest(tower.dims) + for key, param in tower.w.items(): + assert torch.equal(param.data, checkpoint[manifest[key][0][0]]), key + + with pytest.raises(AssertionError, match="vision checkpoint coverage"): + tower.load_weights({**checkpoint, "model.visual.extra.weight": torch.zeros(1)}) + missing = dict(checkpoint) + del missing["model.visual.pos_embed.weight"] + with pytest.raises(AssertionError, match="checkpoint key missing"): + tower.load_weights(missing) From 8f6dca333eabe63ec1943966e99728c8d8484c56 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:24 +0000 Subject: [PATCH 3/6] [TRTLLM-17780][refactor] Split multimodal row preparation from embedding fusion Problem MultimodalModelMixin.prepare_multimodal_inputs always fuses the encoder rows into text embeddings. A language model that places the rows itself, at the step's mm_token_indices, and must keep input_ids (for example to hash token ids) cannot reuse the mixin's selection, encoder cache and chunk slicing. The Modeling V2 README also does not say how a vision-language target may use the shared multimodal mixins. Solution - Add prepare_multimodal_rows(): the existing steps of prepare_multimodal_inputs up to the fuse call (select params, cache or encode, the two embedding hooks, this chunk's rows). It returns (input_ids, active_embeddings, extra_embeds), or None for a text step. - prepare_multimodal_inputs() calls it, then fuses as before. V1 models keep the same calls in the same order. - Modeling V2 README: MultimodalModelMixin and MultimodalEncoderMixin are runtime interfaces a vision-language target may inherit; it still uses no Modeling V1 model code. Model lifecycle names the two target shapes. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs the one before: 8643 vs 8641 cases, 0 changed outcomes; the 2 new mixin tests pass (fusing the rows equals prepare_multimodal_inputs; a text step has no rows). - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- tensorrt_llm/_torch/modeling_v2/README.md | 9 ++- .../models/modeling_multimodal_mixin.py | 49 ++++++++++++--- .../multimodal/test_multimodal_mixin.py | 60 +++++++++++++++++++ 3 files changed, 107 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/README.md b/tensorrt_llm/_torch/modeling_v2/README.md index 3a00db3e0066..25642bf13647 100644 --- a/tensorrt_llm/_torch/modeling_v2/README.md +++ b/tensorrt_llm/_torch/modeling_v2/README.md @@ -134,6 +134,9 @@ family's and its own operations and kernels. From the rest of TensorRT-LLM, it uses only runtime interfaces, for example: * `DecoderModel`, `DecoderModelForCausalLM`, `ModelConfig` and `Mapping`. +* `MultimodalModelMixin` and `MultimodalEncoderMixin`. A vision-language target + may inherit these. It does not subclass or call a Modeling V1 model, vision + tower, layer, attention backend or MoE. * Attention and speculative metadata, and the KV cache managers. A target does not import, subclass or call Modeling V1 model code: the model @@ -172,8 +175,10 @@ example that uses no Modeling V1 model code. ## Model lifecycle A target plugs into TensorRT-LLM through the existing `DecoderModel` and -`DecoderModelForCausalLM` interfaces. It inherits these classes directly, not -a Modeling V1 model class, and can override any of these steps: +`DecoderModelForCausalLM` interfaces. A language model target inherits these +classes directly, not a Modeling V1 model class. A vision-language target is a +composite `MultimodalModelMixin` model that holds such a language model target +as `llm`. Either can override any of these steps: * **Constructor:** builds the model structure and parameters. * **`load_weights`:** loads checkpoint weights into the model. Put weight diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index 113ac19c2d83..ef56ca61156b 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -1051,8 +1051,7 @@ def _slice_per_item_sibling_fields( sliced[key] = [value[i] for i in item_indices] return sliced - # A future optional mixin-owned forward can build on the same template method. - def prepare_multimodal_inputs( + def prepare_multimodal_rows( self, *, input_ids: torch.Tensor, @@ -1060,12 +1059,15 @@ def prepare_multimodal_inputs( multimodal_params: Optional[Sequence[MultimodalParams]], num_context_requests: int, **forward_kwargs: Any, - ) -> PreparedLlmInputs: - """Prepare multimodal inputs for a concrete model forward. - - This method owns the common framework sequence around a model-specific - encoder hook: retrieve/cache embeddings, select active chunk rows if - needed, run optional model hooks, and fuse rows into text embeds. + ) -> Optional[tuple[torch.Tensor, list[torch.Tensor], Sequence[torch.Tensor]]]: + """Return this step's multimodal rows, before any fusion. + + Selects the params, retrieves or encodes their embeddings, runs the two + embedding hooks and keeps the rows of this chunk. Returns + ``(input_ids, active_embeddings, extra_embeds)``, or None when the step + has no multimodal params. A model whose language model places the rows + itself (for example at the step's ``mm_token_indices``) calls this + instead of `prepare_multimodal_inputs`. """ context_params = list( self.select_multimodal_params( @@ -1074,7 +1076,7 @@ def prepare_multimodal_inputs( ) ) if not context_params: - return PreparedLlmInputs(input_ids=input_ids, inputs_embeds=None) + return None embeddings = self._get_or_encode_multimodal_embeddings(context_params) @@ -1091,6 +1093,35 @@ def prepare_multimodal_inputs( multimodal_params=context_params, **forward_kwargs, ) + return input_ids, active_embeddings, extra_embeds + + # A future optional mixin-owned forward can build on the same template method. + def prepare_multimodal_inputs( + self, + *, + input_ids: torch.Tensor, + positions: Optional[torch.Tensor], + multimodal_params: Optional[Sequence[MultimodalParams]], + num_context_requests: int, + **forward_kwargs: Any, + ) -> PreparedLlmInputs: + """Prepare multimodal inputs for a concrete model forward. + + This method owns the common framework sequence around a model-specific + encoder hook: retrieve/cache embeddings, select active chunk rows if + needed, run optional model hooks (`prepare_multimodal_rows`), and fuse + rows into text embeds. + """ + rows = self.prepare_multimodal_rows( + input_ids=input_ids, + positions=positions, + multimodal_params=multimodal_params, + num_context_requests=num_context_requests, + **forward_kwargs, + ) + if rows is None: + return PreparedLlmInputs(input_ids=input_ids, inputs_embeds=None) + input_ids, active_embeddings, extra_embeds = rows fused_input_ids, inputs_embeds, fused_extra_embeds = self._fuse_multimodal_embeddings( input_ids=input_ids, diff --git a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py index 7d00f8daa299..1b0d61da20e8 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py @@ -274,6 +274,66 @@ def test_prepare_multimodal_inputs_accepts_tensor_encoder_output_cpu(): _test_prepare_multimodal_inputs_accepts_tensor_encoder_output("cpu") +@pytest.mark.cpu_only +def test_prepare_multimodal_rows_returns_the_rows_that_inputs_fuses(): + """`prepare_multimodal_inputs` is `prepare_multimodal_rows` plus the fuse: fusing the + rows gives the same inputs, and the rows keep `input_ids` for a model that places them + itself.""" + hidden, mm_token_id = 8, 7 + emb = make_embedding(num_embeddings=40, hidden_size=hidden) + input_ids = torch.tensor([0, mm_token_id, mm_token_id, 1], dtype=torch.long) + text_idx = torch.tensor([0, 3], dtype=torch.long) + mm_idx = torch.tensor([1, 2], dtype=torch.long) + mm_emb = torch.randn(mm_idx.shape[0], hidden) + model = TensorEncoderMultimodalModel(emb, torch.tensor([mm_token_id]), mm_emb) + kwargs = dict( + input_ids=input_ids, + positions=None, + multimodal_params=[make_raw_multimodal_param()], + num_context_requests=1, + text_token_indices=text_idx, + mm_token_indices=mm_idx, + ) + + rows = model.prepare_multimodal_rows(**kwargs) + assert rows is not None + row_input_ids, active_embeddings, extra_embeds = rows + assert row_input_ids is input_ids + assert tuple(extra_embeds) == () + torch.testing.assert_close(torch.cat(active_embeddings), mm_emb) + + fused_ids, fused_embeds, fused_extra = model._fuse_multimodal_embeddings( + input_ids=row_input_ids, + multimodal_embeddings=active_embeddings, + mm_token_ids=model.multimodal_token_ids, + embedding_layer=model.text_embedding_layer, + extra_embeds=extra_embeds, + text_token_indices=text_idx, + mm_token_indices=mm_idx, + ) + prepared = model.prepare_multimodal_inputs(**kwargs) + assert prepared.input_ids is None and fused_ids is None + torch.testing.assert_close(prepared.inputs_embeds, fused_embeds) + assert tuple(prepared.extra_embeds) == tuple(fused_extra) == () + + +@pytest.mark.cpu_only +def test_a_text_step_has_no_multimodal_rows(): + emb = make_embedding(num_embeddings=40, hidden_size=8) + model = DummyMultimodalModel(emb, torch.tensor([7])) + input_ids = torch.tensor([0, 1, 2], dtype=torch.long) + for params in ([], None, [MultimodalParams(multimodal_data={})]): + kwargs = dict( + input_ids=input_ids, + positions=None, + multimodal_params=params, + num_context_requests=1, + ) + assert model.prepare_multimodal_rows(**kwargs) is None + prepared = model.prepare_multimodal_inputs(**kwargs) + assert prepared.input_ids is input_ids and prepared.inputs_embeds is None + + def test_prepare_multimodal_inputs_accepts_tensor_encoder_output_cuda(): _test_prepare_multimodal_inputs_accepts_tensor_encoder_output("cuda") From 1f44eecfd694fefee185ad6626a996dc7001a1c2 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:24 +0000 Subject: [PATCH 4/6] [TRTLLM-17780][feat] Add a shared ForConditionalGeneration base for Modeling V2 vision-language models Problem A Modeling V2 vision-language target needs the composite shape a V1 vision-language model has: the language model as llm and the tower as mm_encoder behind MultimodalModelMixin. Written by hand, every target repeats the code that hands llm what the engine reads, and a new target can miss a piece. Nothing checks that a class routed for a vision-language architecture serves media or refuses it. Solution - Add ModelingV2ForConditionalGeneration in modeling_v2/multimodal.py: MultimodalModelMixin plus PreTrainedModel, the same composite shape as a Modeling V1 vision-language model (for example Qwen3VLModelBase). It holds only the code every V2 target needs. A target subclasses it as ModelingV2ForConditionalGeneration and declares its language model class, its tower class, the tower's weight prefix, its encoder hook and how rows reach the language model. The base does the rest once: - builds llm on a config copy (skip_create_weights_in_init) and keeps the engine's ModelConfig, which ends with the text config; - builds the tower as mm_encoder_class(vision_config, dtype), as it builds llm from language_model_class; a tower built another way overrides build_mm_encoder. No tower with disable_mm_encoder; - forwards lm_head, draft_config, draft_model, spec_worker, multimodal_token_ids and the KV cache manager hook, and copies the language model flags that the router (MoE tactics) and the loader (NVFP4 KV cache) read off the routed class, or the readers' defaults when the language model has none, so a subclass never keeps its parent's; - compiles only llm.model; - at admission: checks the token range (the executor does this for decoder models only) and an optional tower hook, refuses precomputed embeddings, and refuses media without a tower; - selects only requests with media, so M-RoPE-only params never reach the tower; - gives the tower the keys under its prefix and llm the rest, and leaves those keys unread without a tower. llm gets the engine's weight_mapper when its load_weights takes one (call_load_weights), bound to llm first, as the V1 composite models do, since the keys name llm's modules. The base imports no routing module. - forward passes the rows fused into inputs_embeds (default), or, with language_model_takes_rows, as multimodal_rows=(rows, mm_token_indices) beside unchanged input_ids. In both modes it calls the mixin's get_language_model_extra_forward_kwargs hook as the mixin's own forward does and passes what it returns to llm (V1 Qwen3-VL passes its DeepStack rows this way). The fused mode takes the mixin's inputs as they are, also with caller inputs_embeds; the rows mode keeps the extra rows also when the chunk has no multimodal row. The default hook returns nothing. - The router checks every class a route returns. When the architecture's input processor takes media, the class must be a MultimodalModelMixin model (the base is one) or set the supports_multimodal marker, define validate_multimodal_request_data (text only), or define validate_request in its own class (a target that encodes media in its own forward; an inherited validate_request does not count). Otherwise startup fails with a ValueError, under auto too. The router reads the input processor through a new public accessor, InputProcessorRegistry.get_input_processor_cls, which create_input_processor now uses too. - The router declines a base subclass that builds its tower while encoder_side_stream_max_ahead > 0 (NoTarget, so auto uses the built-in model). An encoder-only worker (mm_encoder_only) is not declined: it builds the architecture's encoder model, not this one. - The breakable prefill gate also accepts a composite whose llm is a decoder and that sets language_model_takes_rows: llm takes input_ids and places the rows itself, as a decoder routed alone does. A composite that fuses rows into inputs_embeds is still refused. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs the one before: 8691 vs 8643 cases, 0 changed outcomes; the 48 new tests pass: the base (construction, with the tower built once from vision_config at the activation dtype and never with disable_mm_encoder, mm_encoder_class and the build_mm_encoder override, the copied language model flags, runtime contract, admission also through the executor, forward in both row modes through the real mixin, no tower call for a step without media or a chunk after its placeholders, the extra forward arguments hook, compile scope, weight split, and weight_mapper pass-through bound to llm), the router check (marker True and False, an inherited validate_request), the side-stream decline under auto and require (also with the encoder in the step and on an encoder-only worker), the breakable gate in both row modes and the registry accessor. - One GB200 GPU, all markers, on the series head (the later commits change none of these files): multimodal mixin, engine multimodal, model engine and the base tests: 237 passed, main 42c12879 188, 0 changed outcomes. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- .../_torch/modeling_v2/_router_index.py | 70 +- tensorrt_llm/_torch/modeling_v2/multimodal.py | 323 ++++++++ .../_torch/pyexecutor/model_engine.py | 11 +- tensorrt_llm/inputs/registry.py | 9 +- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../test_modeling_v2_multimodal.py | 755 ++++++++++++++++++ .../_torch/pyexecutor/test_model_engine.py | 23 + .../inputs/test_create_input_processor.py | 31 + 8 files changed, 1216 insertions(+), 7 deletions(-) create mode 100644 tensorrt_llm/_torch/modeling_v2/multimodal.py create mode 100644 tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py diff --git a/tensorrt_llm/_torch/modeling_v2/_router_index.py b/tensorrt_llm/_torch/modeling_v2/_router_index.py index 56570da39a4e..0d8cf89ebf38 100644 --- a/tensorrt_llm/_torch/modeling_v2/_router_index.py +++ b/tensorrt_llm/_torch/modeling_v2/_router_index.py @@ -306,6 +306,71 @@ def _refuse_unhonored_moe_tactic( ) +def _architecture_takes_media(architecture: str) -> bool: + """Whether requests for ``architecture`` can carry media: the input processor registered + for its built-in class, which the LLM API serves every deployment of it with, is a + multimodal one.""" + from tensorrt_llm._torch.models.modeling_utils import get_registered_model_class + from tensorrt_llm.inputs.registry import INPUT_PROCESSOR_REGISTRY, BaseMultimodalInputProcessor + + model_cls = get_registered_model_class(architecture) + processor_cls = INPUT_PROCESSOR_REGISTRY.get_input_processor_cls(model_cls) + return processor_cls is not None and issubclass(processor_cls, BaseMultimodalInputProcessor) + + +def _check_multimodal_contract(target: type, architecture: str) -> None: + """Raise ``ValueError`` when a target of an architecture whose requests can carry media + neither runs media through the multimodal runtime nor refuses it at admission. + + The first is a ``ModelingV2ForConditionalGeneration`` (``multimodal.py``), or any other + ``MultimodalModelMixin`` model or class that sets the ``supports_multimodal`` marker; the + second defines ``validate_multimodal_request_data`` and raises there on media. A target that + encodes media in its own forward refuses what its tower cannot take in a ``validate_request`` + of its own class; an inherited one, such as a token range check, does not count. A target + that does none of these would run placeholder tokens as text or fail a whole batch + in ``forward``: a bug in the target, so this raises in every mode (a plain ``ValueError``, + not a ``NoTarget``). + """ + from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin + + if ( + issubclass(target, MultimodalModelMixin) + or getattr(target, "supports_multimodal", False) is True + or callable(getattr(target, "validate_multimodal_request_data", None)) + or callable(vars(target).get("validate_request")) + ): + return + if _architecture_takes_media(architecture): + raise ValueError( + f"{target.__qualname__} is the target for {architecture!r}, whose requests can " + "carry images, videos or audio. Make it a ModelingV2ForConditionalGeneration " + "subclass to serve media, or define validate_multimodal_request_data to refuse " + "media at admission and serve text only." + ) + + +def _refuse_encoder_side_stream(config: "ModelConfig", target: type) -> None: + """Raise ``NoTarget`` when a ``ModelingV2ForConditionalGeneration`` target would build its + tower and the engine runs the encoder on a side stream: the base runs its encoder inside + the step only. ``auto`` then uses the built-in implementation. An encoder-only deployment + (``mm_encoder_only``) builds the architecture's encoder model, not this one.""" + from .multimodal import ModelingV2ForConditionalGeneration + + if ( + not issubclass(target, ModelingV2ForConditionalGeneration) + or config.disable_mm_encoder + or config.mm_encoder_only + ): + return + mm_config = config.multimodal_config + if mm_config is not None and mm_config.encoder_side_stream_max_ahead > 0: + raise NoTarget( + "multimodal_config.encoder_side_stream_max_ahead=" + f"{mm_config.encoder_side_stream_max_ahead}; {target.__name__} does not run its " + "encoder on a side stream" + ) + + def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: """Decide which modeling_v2 target, if any, this deployment builds. @@ -314,7 +379,8 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti for the architecture, or when that ``route`` raises ``NoTarget``. In ``require`` every one of those but ``off`` raises ``ValueError`` instead. It also returns None for a ``@disable_v2_by_default`` target when - ``modeling_v2`` was left at its default. + ``modeling_v2`` was left at its default. A target that breaks the multimodal + contract (:func:`_check_multimodal_contract`) raises in every mode. """ mode = ModelingV2Mode.of(llm_args) @@ -334,6 +400,8 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti target = route(config, llm_args, _device_compute_capability()) if target is not None: _refuse_unhonored_moe_tactics(config, target, arch) + _check_multimodal_contract(target, arch) + _refuse_encoder_side_stream(config, target) except NoTarget as why: if mode is ModelingV2Mode.REQUIRE: raise ValueError(f"modeling_v2='require', but no target fits {arch!r}: {why}") from None diff --git a/tensorrt_llm/_torch/modeling_v2/multimodal.py b/tensorrt_llm/_torch/modeling_v2/multimodal.py new file mode 100644 index 000000000000..b6eafa20ba19 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/multimodal.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The shared base class of Modeling V2 vision-language models. + +A vision-language target is composite, as a Modeling V1 vision-language model is: a +:class:`ModelingV2ForConditionalGeneration` holds a Modeling V2 language model target as +``llm`` and its encoder tower as ``mm_encoder``. The shared multimodal runtime +(``MultimodalModelMixin``) selects the step's media requests, runs or reuses the tower +(encoder cache) and keeps the current chunk's rows, so the executor sees one multimodal model +shape for V1 and V2. + +A target subclasses this base as ``ModelingV2ForConditionalGeneration`` and declares +what differs: + +* ``language_model_class``: the language model target. It is built on a copy of the + ``ModelConfig`` and adopts the text config on that copy, as it does when it is routed alone. +* ``mm_encoder_class``: the tower, an ``nn.Module`` and ``MultimodalEncoderMixin`` that + declares and loads its own weights. The base builds it as + ``mm_encoder_class(vision_config, dtype)``: the composite config's ``vision_config`` and the + activation dtype. A tower built another way overrides ``build_mm_encoder`` instead. No tower + is built with ``disable_mm_encoder``. +* ``mm_encoder_weight_prefix``: the checkpoint prefix of the tower's weights. The tower loads + those keys and the language model every other key. +* ``encode_multimodal_inputs``: the tower's rows for the selected requests, in request order. +* ``language_model_takes_rows``: how the rows reach the language model. False (the default): + the mixin fuses them into ``inputs_embeds``. True: ``input_ids`` stay as they are and the + language model takes ``multimodal_rows=(rows, mm_token_indices)`` and places the rows itself; + the engine then allows the breakable prefill CUDA graph, as for the language model alone. + +Optional: ``validate_media_request`` (admission checks of the tower) and the mixin's feature +flags such as ``supports_encoder_cache``. To pass extra inputs to the language model, as V1 +Qwen3-VL does for DeepStack, a target overrides ``get_language_model_extra_forward_kwargs``. +""" + +from __future__ import annotations + +import inspect +from typing import TYPE_CHECKING, Any, ClassVar + +import torch +from torch import nn +from transformers import PretrainedConfig, PreTrainedModel + +from tensorrt_llm._torch.models.checkpoints.model_hooks import call_load_weights +from tensorrt_llm._torch.models.modeling_multimodal_mixin import ( + MultimodalModelMixin, + PreparedLlmInputs, +) +from tensorrt_llm._torch.models.modeling_utils import DecoderModelForCausalLM +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + +if TYPE_CHECKING: + from tensorrt_llm._torch.model_config import ModelConfig + +__all__ = ["ModelingV2ForConditionalGeneration"] + +#: `multimodal_data` entries the tower encodes, or rows an earlier step already made. +_ENCODED_KEYS = ("image", "video", "audio", "multimodal_embedding") + +#: Language model flags that are read off the routed class, which is the composite, before any +#: model is built: `moe_impls_honored` and `draft_moe_impls_honored` by the router's MoE tactic +#: check (`_router_index._refuse_unhonored_moe_tactics`), `supports_nvfp4_kv_cache` by the +#: model loader's SM107 KV cache check. Each maps to the value its reader takes when a class has +#: none. A subclass that sets `language_model_class` takes that class's values, or these, unless +#: it sets its own, so it never keeps a parent's; without them the router would refuse a set MoE +#: tactic and the loader would switch an NVFP4 KV cache to FP8. Both read the class, so a +#: property cannot do this. +_LANGUAGE_MODEL_CLASS_ATTRIBUTES = { + "moe_impls_honored": frozenset(), + "draft_moe_impls_honored": frozenset(), + "supports_nvfp4_kv_cache": False, +} + + +class ModelingV2ForConditionalGeneration(MultimodalModelMixin, PreTrainedModel): + """A Modeling V2 language model target as ``llm`` and its encoder tower as ``mm_encoder``, + behind the shared multimodal runtime. See the module docstring for what a target declares. + + Same composite shape as a Modeling V1 vision-language model (for example + ``Qwen3VLModelBase``), holding only the code every V2 target needs. + """ + + language_model_class: ClassVar[type[DecoderModelForCausalLM]] + mm_encoder_class: ClassVar[type[nn.Module] | None] = None + mm_encoder_weight_prefix: ClassVar[str] + language_model_takes_rows: ClassVar[bool] = False + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + llm_class = cls.__dict__.get("language_model_class") + for name, default in _LANGUAGE_MODEL_CLASS_ATTRIBUTES.items(): + if llm_class is not None and name not in cls.__dict__: + setattr(cls, name, getattr(llm_class, name, default)) + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None): + return cls.language_model_class.get_preferred_kv_cache_manager_version(pretrained_config) + + def _check_and_adjust_experts_implementation(self, *args, **kwargs): + """No-op: the model holds no MoE layer of its own for the transformers check.""" + return None + + def __init__(self, model_config: ModelConfig): + composite = model_config.pretrained_config + super().__init__(composite) + # The engine's own `ModelConfig`: the loader, the engine and KV sizing read it here. + self.model_config = model_config + # Built as the engine builds a decoder model (`skip_create_weights_in_init`), on its own + # shallow copy, where it adopts the text config. + self.llm = self.language_model_class( + model_config.copy_with(skip_create_weights_in_init=True) + ) + # Everything that sizes the pools reads the text config. + model_config.pretrained_config = self.llm.config + self.config = self.llm.config + # None with `disable_mm_encoder`: no encoder weight is declared or loaded, and a + # media request is refused at admission. + self.mm_encoder = ( + None if model_config.disable_mm_encoder else self.build_mm_encoder(composite) + ) + + def build_mm_encoder(self, pretrained_config: PretrainedConfig) -> nn.Module | None: + """The encoder tower of the composite ``pretrained_config``, at ``self.dtype``: + ``mm_encoder_class(pretrained_config.vision_config, self.dtype)``. Override it for a + tower built another way. None builds no tower, and the model serves text only.""" + if self.mm_encoder_class is None: + return None + return self.mm_encoder_class(pretrained_config.vision_config, self.dtype) + + @property + def language_model(self) -> nn.Module: + return self.llm + + @property + def dtype(self) -> torch.dtype: + """The activation dtype. Set, not read off the first parameter, which may be a + quantized one.""" + return self.model_config.torch_dtype + + @property + def embedding_dim(self) -> int: + return self.config.hidden_size + + @property + def embedding_dtype(self) -> torch.dtype: + return self.dtype + + @property + def text_embedding_layer(self) -> nn.Module: + """The token embedding the fused path embeds text with.""" + return self.llm.model.embed_tokens + + # What the engine reads off the routed model is the language model's. + + @property + def multimodal_token_ids(self): + return getattr(self.llm, "multimodal_token_ids", None) + + @property + def lm_head(self) -> nn.Module: + return self.llm.lm_head + + @property + def draft_config(self): + return self.llm.draft_config + + @property + def draft_model(self): + return self.llm.draft_model + + @property + def spec_worker(self): + return self.llm.spec_worker + + def apply_llm_torch_compile(self, *, backend, fullgraph: bool) -> None: + """Compile only the language model's core; the tower stays eager.""" + self.llm.model = torch.compile(self.llm.model, backend=backend, fullgraph=fullgraph) + + def validate_request(self, request) -> None: + """Admission check, run by the executor on every new request. The executor checks the + token range of a decoder model only, so it is checked here against the language + model's vocabulary; then, with a tower, `validate_media_request`.""" + num_embeddings = self.llm.lm_head.num_embeddings + token_bound = request.py_prompt_token_bound + if token_bound is not None: + if token_bound >= num_embeddings: + raise ValueError("Token ID out of range") + elif not request.check_token_id_range(num_embeddings): + raise ValueError("Token ID out of range") + if self.mm_encoder is not None: + self.validate_media_request(request) + + def validate_media_request(self, request) -> None: + """Refuse, as its own error, a request whose media the tower cannot encode.""" + + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: with no tower the model serves text only and refuses media or + embeddings; with one it encodes its own media and refuses precomputed embeddings. + M-RoPE positions alone pass.""" + if self.mm_encoder is None: + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} builds no multimodal encoder (disable_mm_encoder) " + "and serves text only; send no images, videos or embeddings" + ) + elif mm_data.get("multimodal_embedding") is not None: + raise ValueError( + f"{type(self).__name__} encodes its own media; precomputed multimodal " + "embeddings are not supported" + ) + + def select_multimodal_params(self, multimodal_params, num_context_requests): + """The mixin's selection, then only requests with media or their rows: an M-RoPE-only + param (a disaggregated generation request, or a generation request's position + deltas) has nothing to encode.""" + return [ + param + for param in super().select_multimodal_params(multimodal_params, num_context_requests) + if any(param.multimodal_data.get(key) is not None for key in _ENCODED_KEYS) + ] + + @torch.inference_mode() + def forward( + self, + attn_metadata, + input_ids: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + return_context_logits: bool = False, + spec_metadata=None, + **kwargs, + ): + """The step's multimodal rows from the mixin, then the language model with every + other argument the engine passed. The rows go in fused `inputs_embeds`, or with + `language_model_takes_rows` as `multimodal_rows=(rows, mm_token_indices)` beside + unchanged `input_ids`. + + The language model also takes what `get_language_model_extra_forward_kwargs` returns; + the hook gets the same arguments as in the mixin's own forward. In the rows mode its + `mm_inputs` holds `input_ids` and `inputs_embeds` as the language model gets them and + the chunk's extra rows from `after_active_multimodal_embeddings` (one per multimodal + row, not spread over the tokens), and `multimodal_rows` is among its keyword + arguments.""" + raw_input_ids = input_ids + multimodal_params = kwargs.pop("multimodal_params", []) + prepare_kwargs = dict( + input_ids=input_ids, + positions=position_ids, + multimodal_params=multimodal_params, + num_context_requests=attn_metadata.num_contexts, + attn_metadata=attn_metadata, + **kwargs, + ) + if self.language_model_takes_rows: + rows = self.prepare_multimodal_rows(**prepare_kwargs) + # The extra rows are kept even when the chunk has no multimodal row, as fusion does. + input_ids, active_embeddings, extra_embeds = rows or (input_ids, [], ()) + multimodal_rows = None + if active_embeddings: + embeds = ( + active_embeddings[0] + if len(active_embeddings) == 1 + else torch.cat(active_embeddings) + ) + mm_token_indices = kwargs.get("mm_token_indices") + assert ( + mm_token_indices is not None and mm_token_indices.shape[0] == embeds.shape[0] + ), ( + f"the step has {embeds.shape[0]} multimodal rows and " + f"{None if mm_token_indices is None else mm_token_indices.shape[0]} " + "placeholder rows (mm_token_indices)" + ) + multimodal_rows = (embeds, mm_token_indices) + kwargs["multimodal_rows"] = multimodal_rows + mm_inputs = PreparedLlmInputs(input_ids, inputs_embeds, extra_embeds) + else: + mm_inputs = self.prepare_multimodal_inputs(**prepare_kwargs) + if inputs_embeds is not None: + # As in the mixin's forward: the caller's embeddings replace input_ids. + assert mm_inputs.inputs_embeds is None, ( + "the engine passed inputs_embeds for a media step" + ) + mm_inputs = PreparedLlmInputs(None, inputs_embeds, mm_inputs.extra_embeds) + # As in the mixin's forward: the language model arguments and the target's extra ones + # (for example DeepStack rows). They go over every argument the engine passed. + llm_kwargs = self.get_language_model_forward_kwargs( + attn_metadata=attn_metadata, + input_ids=mm_inputs.input_ids, + raw_input_ids=raw_input_ids, + position_ids=position_ids, + inputs_embeds=mm_inputs.inputs_embeds, + mm_inputs=mm_inputs, + return_context_logits=return_context_logits, + multimodal_params=multimodal_params, + num_generation_requests=attn_metadata.num_generations, + spec_metadata=spec_metadata, + **kwargs, + ) + return self.llm.forward(**{"spec_metadata": spec_metadata, **kwargs, **llm_kwargs}) + + def load_weights(self, weights, weight_mapper=None) -> None: + """The tower loads the keys under `mm_encoder_weight_prefix` and the language model + every other key, with the engine's `weight_mapper` when its `load_weights` takes one; + each checks its own coverage. The base renames no key. Without a tower the encoder keys + are left unread. + + The engine binds its mapper to this model. The mapper is bound to the language model + before it gets it, as in the Modeling V1 composite models, since the language model's + keys name its own modules.""" + encoder_weights, text_weights = {}, {} + for key in weights.keys(): + if not key.startswith(self.mm_encoder_weight_prefix): + text_weights[key] = weights[key] + elif self.mm_encoder is not None: + encoder_weights[key] = weights[key] + if self.mm_encoder is not None: + self.mm_encoder.load_weights(encoder_weights) + if ( + weight_mapper is not None + and "weight_mapper" in inspect.getfullargspec(self.llm.load_weights).args + ): + weight_mapper.init_model_and_config(self.llm, self.llm.model_config) + call_load_weights(self.llm.load_weights, text_weights, weight_mapper) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 6e70f167e39a..0ff9b32ac377 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -905,10 +905,13 @@ def _validate_breakable_cuda_graph_compatibility(self) -> None: if isinstance(self.model, DecoderModelForCausalLM): return decoder_model = getattr(self.model, "llm", None) - if ( - self.llm_args.disable_mm_encoder - and isinstance(decoder_model, DecoderModelForCausalLM) - and getattr(self.model, "mm_encoder", None) is None + if isinstance(decoder_model, DecoderModelForCausalLM) and ( + # The language model takes `input_ids` and places the rows itself, so its graph + # runs as for the decoder alone. + getattr(self.model, "language_model_takes_rows", False) + or ( + self.llm_args.disable_mm_encoder and getattr(self.model, "mm_encoder", None) is None + ) ): return if isinstance(self.model, MultimodalModelMixin) or isinstance( diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 62192567483e..f757865c1e9f 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -1140,6 +1140,11 @@ def __init__(self) -> None: self._input_processors_cls_by_model_type: Dict[ Type[nn.Module], Type[InputProcessor]] = {} + def get_input_processor_cls( + self, model_cls: Type[nn.Module]) -> Optional[Type[InputProcessor]]: + """Return the input processor class registered for ``model_cls``, or None.""" + return self._input_processors_cls_by_model_type.get(model_cls) + INPUT_PROCESSOR_REGISTRY = InputProcessorRegistry() @@ -1294,8 +1299,8 @@ def create_input_processor( if config is not None: try: model_cls, _ = get_model_architecture(config) - input_processor_cls = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type \ - .get(model_cls) + input_processor_cls = INPUT_PROCESSOR_REGISTRY.get_input_processor_cls( + model_cls) except (RuntimeError, ConfigInputError, UnsupportedError): # unregistered model logger.info("Unregistered model, using DefaultInputProcessor") diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 6cc2692f57a4..e1cc77102dc9 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -130,6 +130,7 @@ l0_cpu: - unittest/_torch/modeling_v2/models/beam/test_component_contracts.py - unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py - unittest/_torch/modeling_v2/test_modeling_v2_contracts.py + - unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py - unittest/_torch/modeling_v2/test_modeling_v2_package_location.py - unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py new file mode 100644 index 000000000000..dcd1b75253a9 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py @@ -0,0 +1,755 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The shared base of Modeling V2 vision-language targets, and the router's multimodal check. + +A tiny language model and a tiny tower stand in for a target's: construction, the runtime +contract, admission (also through the executor), the forward's hand-off in both row modes +through the real multimodal mixin, the compile scope and the weight split. Then the check the +router runs on every class a route returns. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2 import NoTarget +from tensorrt_llm._torch.modeling_v2 import _router_index as router +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration +from tensorrt_llm._torch.models.checkpoints.model_hooks import call_load_weights +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin +from tensorrt_llm._torch.pyexecutor.engine import model_setup +from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor +from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData +from tensorrt_llm.llmapi.llm_args import MultimodalConfig + +pytestmark = pytest.mark.cpu_only + +VOCAB, HIDDEN = 32, 4 +IMAGE_TOKEN = VOCAB - 1 # in vocabulary, so the fused path embeds the whole prompt once +MAX_ITEM = 16 # the most rows the tiny tower takes in one item +ARCH = "TinyForConditionalGeneration" + + +# ------------------------------------------------------------------ stubs --- + + +class _TinyLlm(nn.Module): + """A language model target in miniature: it adopts the text config on its own config + copy, as a target does, and records what it is handed.""" + + moe_impls_honored = frozenset({"tiny_moe"}) + + def __init__(self, model_config): + super().__init__() + self.got = model_config + model_config.pretrained_config = model_config.pretrained_config.text_config + self.model_config = model_config + self.model = nn.Module() + self.model.embed_tokens = nn.Embedding(VOCAB, HIDDEN) + self.lm_head = nn.Embedding(VOCAB, HIDDEN) + self.draft_config = None + self.draft_model = object() + self.spec_worker = object() + self.multimodal_token_ids = torch.tensor([IMAGE_TOKEN], dtype=torch.int32) + self.calls = [] + self.loaded = None + + @property + def config(self): + return self.model_config.pretrained_config + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None): + return "V2" + + def forward( + self, + attn_metadata, + input_ids=None, + position_ids=None, + inputs_embeds=None, + return_context_logits=False, + spec_metadata=None, + **kwargs, + ): + """Records the call: the first six arguments in the order a language model takes them + (`DecoderModelForCausalLM.forward`), and the others by name.""" + args = ( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + ) + self.calls.append((args, kwargs)) + return "logits" + + def load_weights(self, weights): + self.loaded = weights + + +class _TinyTower(nn.Module): + """Each row it returns holds its input pixel's value; it refuses an item of more than + `MAX_ITEM` rows, as a tower with a per-call bound does.""" + + def __init__(self, vision_config, dtype): + super().__init__() + self.vision_config = vision_config + self.scale = nn.Parameter(torch.ones((), dtype=dtype)) + self.calls = 0 + self.loaded = None + + def forward(self, pixels): + self.calls += 1 + assert pixels.shape[0] <= MAX_ITEM, f"an item of {pixels.shape[0]} rows" + return pixels.expand(-1, HIDDEN) * self.scale + + def load_weights(self, weights): + self.loaded = weights + + +class _TinyForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = _TinyLlm + mm_encoder_class = _TinyTower + mm_encoder_weight_prefix = "model.visual." + + def validate_media_request(self, request): + image = (request.py_multimodal_data or {}).get("image") + if image is not None and image["pixel_values"].shape[0] > MAX_ITEM: + raise ValueError("an image larger than the tower takes") + + def encode_multimodal_inputs(self, multimodal_params): + if self.mm_encoder is None: + raise ValueError("no tower") + return torch.cat( + [self.mm_encoder(p.multimodal_data["image"]["pixel_values"]) for p in multimodal_params] + ) + + +class _TinyRowsForConditionalGeneration(_TinyForConditionalGeneration): + language_model_takes_rows = True + + +def _model_config(disable=False, side_stream=0, encoder_only=False): + composite = PretrainedConfig( + architectures=[ARCH], + text_config=PretrainedConfig(hidden_size=HIDDEN, torch_dtype=torch.float32), + vision_config=PretrainedConfig(), + torch_dtype=torch.float32, + ) + return ModelConfig( + pretrained_config=composite, + disable_mm_encoder=disable, + mm_encoder_only=encoder_only, + multimodal_config=MultimodalConfig(encoder_side_stream_max_ahead=side_stream), + ) + + +def _model(cls=_TinyForConditionalGeneration, disable=False): + model = cls(_model_config(disable=disable)) + with torch.no_grad(): + # Token `i` embeds to `i` in every column, so a fused row says where it came from. + model.llm.model.embed_tokens.weight.copy_( + torch.arange(VOCAB, dtype=torch.float32)[:, None].expand(VOCAB, HIDDEN) + ) + return model + + +def _image(*values): + """One image whose rows hold `values`.""" + pixels = torch.tensor(values, dtype=torch.float32)[:, None] + return {"image": {"pixel_values": pixels}} + + +def _params(*values, runtime=None, mrope=False): + data = _image(*values) if values else {} + if mrope: + data["mrope_config"] = {"mrope_position_deltas": torch.zeros(1, 1, dtype=torch.int32)} + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +# ----------------------------------------------------------- construction --- + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_construction_keeps_the_engine_config_and_builds_the_tower_only_when_asked( + monkeypatch, disable +): + model_config = _model_config(disable=disable) + composite = model_config.pretrained_config + # The tower takes the activation dtype: the text config's, not the composite's. + composite.text_config.torch_dtype = torch.bfloat16 + built = [] + + class Tower(_TinyTower): + def __init__(self, vision_config, dtype): + built.append(dtype) + super().__init__(vision_config, dtype) + + monkeypatch.setattr(_TinyForConditionalGeneration, "mm_encoder_class", Tower) + model = _TinyForConditionalGeneration(model_config) + + # The loader's own ModelConfig, now carrying the text config the pools are sized from. + assert model.model_config is model_config + assert model_config.pretrained_config is composite.text_config + assert model.config is composite.text_config + # The language model was built on its own copy, as the engine builds a decoder model. + assert model.llm.got is not model_config + assert model.llm.got.skip_create_weights_in_init is True + assert model.language_model is model.llm + if disable: + # Never built, not built and dropped. + assert built == [] + assert model.mm_encoder is None + assert not any(name.startswith("mm_encoder.") for name, _ in model.named_parameters()) + else: + assert built == [torch.bfloat16] + assert isinstance(model.mm_encoder, Tower) + assert model.mm_encoder.vision_config is composite.vision_config + assert model.mm_encoder.scale.dtype == torch.bfloat16 + + +def test_build_mm_encoder_overrides_mm_encoder_class(): + """A tower not built as `mm_encoder_class(vision_config, dtype)` comes from an override of + `build_mm_encoder`. With neither, the model builds no tower and serves text only.""" + + class Built(_TinyForConditionalGeneration): + def build_mm_encoder(self, pretrained_config): + return nn.Identity() + + class NoTower(_TinyForConditionalGeneration): + mm_encoder_class = None + + assert isinstance(_model(Built).mm_encoder, nn.Identity) + assert _model(NoTower).mm_encoder is None + + +@pytest.mark.parametrize("mode", ["auto", "require"]) +@pytest.mark.parametrize( + "disable, side_stream, encoder_only", + [(False, 2, False), (False, 0, False), (True, 2, False), (False, 2, True)], + ids=["tower", "tower_encoder_inline", "disable_mm_encoder", "mm_encoder_only"], +) +def test_the_route_declines_the_encoder_side_stream_while_a_tower_is_built( + monkeypatch, disable, side_stream, encoder_only, mode +): + """A `NoTarget` from the router, not the constructor: `auto` falls back to the built-in + model and `require` fails. Nothing is declined without a tower, with the encoder in the + step (`encoder_side_stream_max_ahead=0`, the default), or on an encoder-only worker, which + builds the architecture's encoder model and not this one.""" + monkeypatch.setattr( + router, "registered_routes", lambda: {ARCH: lambda *args: _TinyForConditionalGeneration} + ) + monkeypatch.setattr(router, "_device_compute_capability", lambda: (10, 0)) + config = _model_config(disable=disable, side_stream=side_stream, encoder_only=encoder_only) + llm_args = SimpleNamespace(modeling_v2=mode) + if disable or side_stream == 0 or encoder_only: + assert router.modeling_v2_resolve(config, llm_args) is _TinyForConditionalGeneration + elif mode == "auto": + assert router.modeling_v2_resolve(config, llm_args) is None + else: + with pytest.raises(ValueError, match="encoder_side_stream_max_ahead=2"): + router.modeling_v2_resolve(config, llm_args) + + +# --------------------------------------------------------------- contract --- + + +def test_the_model_declares_the_runtime_contract(): + model = _model() + assert isinstance(model, MultimodalModelMixin) + assert model.dtype == torch.float32 + assert (model.embedding_dim, model.embedding_dtype) == (HIDDEN, torch.float32) + assert model.text_embedding_layer is model.llm.model.embed_tokens + assert model.samples_from_vocab_shards is False + # What the engine reads off the routed model is the language model's. + llm = model.llm + assert model.lm_head is llm.lm_head + assert model.draft_config is None + assert (model.draft_model, model.spec_worker) == (llm.draft_model, llm.spec_worker) + assert model.multimodal_token_ids is llm.multimodal_token_ids + # So are the class-level hooks the router and the loader read off the routed class. + assert _TinyForConditionalGeneration.get_preferred_kv_cache_manager_version() == "V2" + assert _TinyForConditionalGeneration.moe_impls_honored == frozenset({"tiny_moe"}) + assert _TinyRowsForConditionalGeneration.moe_impls_honored == frozenset({"tiny_moe"}) + # A flag the language model does not set takes the value its reader takes for none. + assert _TinyForConditionalGeneration.draft_moe_impls_honored == frozenset() + assert _TinyForConditionalGeneration.supports_nvfp4_kv_cache is False + + class Own(_TinyForConditionalGeneration): + language_model_class = _TinyLlm + moe_impls_honored = frozenset({"own"}) + + class FlaglessLlm(nn.Module): + pass + + class Swapped(_TinyForConditionalGeneration): + language_model_class = FlaglessLlm + + assert Own.moe_impls_honored == frozenset({"own"}) + # A subclass that swaps in another language model does not keep its parent's flags. + assert Swapped.moe_impls_honored == frozenset() + + +# -------------------------------------------------------------- admission --- + + +def _request(tokens=(1, 2, 3), bound=None, **data): + return SimpleNamespace( + py_multimodal_data=data or None, + py_prompt_token_bound=bound, + check_token_id_range=lambda vocab: max(tokens) < vocab, + ) + + +def _admit(model, request): + """Both admission hooks, in the executor's order.""" + model.validate_request(request) + if request.py_multimodal_data: + model.validate_multimodal_request_data(request.py_multimodal_data) + + +MROPE_ONLY = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + + +def test_admission_checks_the_token_range_and_the_towers_media(): + model = _model() + _admit(model, _request()) + _admit(model, _request(tokens=(1, IMAGE_TOKEN), **_image(1.0, 2.0))) + _admit(model, _request(bound=VOCAB - 1)) + # M-RoPE positions alone (a disaggregated generation request) are not media. + _admit(model, _request(**MROPE_ONLY)) + with pytest.raises(ValueError, match="Token ID out of range"): + _admit(model, _request(tokens=(1, VOCAB))) + with pytest.raises(ValueError, match="Token ID out of range"): + _admit(model, _request(bound=VOCAB)) + with pytest.raises(ValueError, match="larger than the tower takes"): + _admit(model, _request(**_image(*range(MAX_ITEM + 1)))) + with pytest.raises(ValueError, match="precomputed multimodal embeddings"): + _admit(model, _request(multimodal_embedding=torch.zeros(2, HIDDEN))) + + +def test_without_a_tower_media_is_refused_and_mrope_positions_pass(): + model = _model(disable=True) + _admit(model, _request()) + _admit(model, _request(**MROPE_ONLY)) + # No tower check without a tower; the media check refuses the request instead. + model.validate_request(_request(**_image(*range(MAX_ITEM + 1)))) + for data in (_image(1.0), {"multimodal_embedding": torch.zeros(2, HIDDEN)}): + with pytest.raises(ValueError, match="disable_mm_encoder.*serves text only"): + _admit(model, _request(**data)) + + +def test_the_executor_fails_only_the_request_the_model_refuses(): + """Through the executor's own admission (`_fetch_and_activate_new_requests`): the bad + request fails with its own error and no budget charge; the others are admitted.""" + model = _model() + requests = [ + _request(), + _request(**_image(*range(MAX_ITEM + 1))), + _request(**MROPE_ONLY), + _request(tokens=(1, VOCAB)), + _request(**_image(1.0)), + _request(multimodal_embedding=torch.zeros(2, HIDDEN)), + ] + for request_id, request in enumerate(requests): + request.request_id = request_id + request.py_disaggregated_params = None + request.sampling_config = None + request.kv_cache_retention_config = None + executor = SimpleNamespace( + model_engine=SimpleNamespace(model=model), + _is_kv_manager_v2=False, + kv_cache_manager=None, + kv_cache_transceiver=None, + max_beam_width=1, + sampler=Mock(), + waiting_queue=object(), + active_requests=[], + _mm_encoder_item_scheduling_enabled=False, + _fetch_new_requests=Mock(return_value=requests), + _configure_csa2_remote_tail=Mock(), + _validate_token_id_range=Mock(), + _validate_request_budget=Mock(), + _handle_errors=Mock(), + ) + executor._validate_request = PyExecutor._validate_request.__get__(executor) + + admitted = PyExecutor._fetch_and_activate_new_requests(executor) + + assert admitted == [requests[0], requests[2], requests[4]] + failed = [ + (call.kwargs["requests"], call.kwargs["charge_budget"]) + for call in executor._handle_errors.call_args_list + ] + assert failed == [([requests[1]], False), ([requests[3]], False), ([requests[5]], False)] + + +# ---------------------------------------------------------------- forward --- + + +def _forward(model, params, num_contexts=1, mm_rows=None, inputs_embeds=None): + """One step over a 6-token prompt whose tokens 1..`mm_rows` are placeholders.""" + input_ids = torch.tensor([1, IMAGE_TOKEN, IMAGE_TOKEN, IMAGE_TOKEN, 4, 5], dtype=torch.int32) + kwargs = {"multimodal_params": params, "resource_manager": "rm"} + if mm_rows is not None: + kwargs["mm_token_indices"] = torch.arange(1, 1 + mm_rows, dtype=torch.int64) + kwargs["text_token_indices"] = torch.tensor([0, *range(1 + mm_rows, 6)], dtype=torch.int64) + out = model.forward( + SimpleNamespace(num_contexts=num_contexts, num_generations=0), + input_ids, + "position_ids", + inputs_embeds, + False, + "spec_metadata", + **kwargs, + ) + assert out == "logits" + ((args, llm_kwargs),) = model.llm.calls + assert args[0].num_contexts == num_contexts + assert args[2] == "position_ids" and args[4:] == (False, "spec_metadata") + assert "multimodal_params" not in llm_kwargs and llm_kwargs["resource_manager"] == "rm" + return input_ids, args, llm_kwargs + + +def test_rows_mode_hands_the_rows_and_their_placeholder_rows_to_the_language_model(): + model = _model(_TinyRowsForConditionalGeneration) + input_ids, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)], mm_rows=3) + # `input_ids` reach the language model unchanged, and `inputs_embeds` stays None. + assert args[1] is input_ids and args[3] is None + rows, mm_token_indices = llm_kwargs["multimodal_rows"] + assert rows[:, 0].tolist() == [7.0, 8.0, 9.0] + assert mm_token_indices.tolist() == [1, 2, 3] + + +def test_fused_mode_hands_fused_embeddings_to_the_language_model(): + model = _model() + _, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)], mm_rows=3) + assert args[1] is None + assert args[3][:, 0].tolist() == [1.0, 7.0, 8.0, 9.0, 4.0, 5.0] + assert "multimodal_rows" not in llm_kwargs + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_a_chunk_hands_over_its_slice_of_the_rows(cls): + """The mixin keeps the chunk's rows: 2 of 5 reused, 3 in this chunk.""" + model = _model(cls) + runtime = MultimodalRuntimeData( + embed_mask_cumsum=torch.arange(1, 6, dtype=torch.int64), + past_seen_token_num=2, + chunk_end_pos=5, + ) + _, args, llm_kwargs = _forward( + model, [_params(5.0, 6.0, 7.0, 8.0, 9.0, runtime=runtime)], mm_rows=3 + ) + if cls is _TinyRowsForConditionalGeneration: + assert llm_kwargs["multimodal_rows"][0][:, 0].tolist() == [7.0, 8.0, 9.0] + else: + assert args[3][1:4, 0].tolist() == [7.0, 8.0, 9.0] + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +@pytest.mark.parametrize( + "params, num_contexts", + [ + ([], 1), + ([MultimodalParams(multimodal_data={})], 1), + # A text context mixed with a generation request's position deltas. + ([_params(mrope=True)], 1), + # A generation-only step. + ([_params(7.0)], 0), + # A chunk after the image: its 3 placeholders were all in earlier chunks. + ( + [ + _params( + 7.0, + 8.0, + 9.0, + runtime=MultimodalRuntimeData( + embed_mask_cumsum=torch.tensor([0, 1, 2, 3, 3, 3]), + past_seen_token_num=4, + chunk_end_pos=6, + ), + ) + ], + 1, + ), + ], + ids=["no_params", "empty", "mrope_only", "generation", "chunk_after_the_placeholders"], +) +def test_a_step_without_media_runs_no_tower(params, num_contexts, cls): + model = _model(cls) + input_ids, args, llm_kwargs = _forward(model, params, num_contexts=num_contexts) + assert args[1] is input_ids and args[3] is None + assert llm_kwargs.get("multimodal_rows") is None + assert model.mm_encoder.calls == 0 + + +def test_the_rows_must_match_the_placeholder_rows(): + with pytest.raises(AssertionError, match="3 multimodal rows and 2 placeholder rows"): + _forward(_model(_TinyRowsForConditionalGeneration), [_params(7.0, 8.0, 9.0)], mm_rows=2) + + +class _DeepStack: + """V1 Qwen3-VL's DeepStack pattern in miniature: an extra feature for each of the chunk's + rows (10 times the row), handed to the language model by the mixin's hook.""" + + def after_active_multimodal_embeddings(self, *, active_embeddings, **kwargs): + return active_embeddings, (torch.cat(active_embeddings) * 10,) + + def get_language_model_extra_forward_kwargs(self, *, mm_inputs, spec_metadata, **kwargs): + self.hook_got = dict(kwargs, mm_inputs=mm_inputs) + # A V1 hook also returns `spec_metadata`; the language model still takes it once. + return {"deepstack_embeds": list(mm_inputs.extra_embeds), "spec_metadata": spec_metadata} + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_what_the_extra_forward_kwargs_hook_returns_reaches_the_language_model(cls): + model = _model(type("DeepStackForConditionalGeneration", (_DeepStack, cls), {})) + params = [_params(7.0, 8.0, 9.0)] + input_ids, args, llm_kwargs = _forward(model, params, mm_rows=3) + (deepstack,) = llm_kwargs["deepstack_embeds"] + got = model.hook_got + # The hook gets the same arguments as in the mixin's own forward. + assert got["raw_input_ids"] is input_ids and got["position_ids"] == "position_ids" + assert got["multimodal_params"] is params and got["num_generation_requests"] == 0 + assert got["resource_manager"] == "rm" + if cls is _TinyRowsForConditionalGeneration: + # The rows mode: the unchanged input_ids, no inputs_embeds, one extra row per row. + assert got["mm_inputs"].input_ids is input_ids and got["mm_inputs"].inputs_embeds is None + assert got["multimodal_rows"] is llm_kwargs["multimodal_rows"] + assert deepstack[:, 0].tolist() == [70.0, 80.0, 90.0] + else: + # The fused mode: the fused inputs, the extra rows spread over the tokens. + assert got["mm_inputs"].input_ids is None and got["mm_inputs"].inputs_embeds is args[3] + assert deepstack[:, 0].tolist() == [0.0, 70.0, 80.0, 90.0, 0.0, 0.0] + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_by_default_the_language_model_takes_no_extra_argument(cls): + _, _, llm_kwargs = _forward(_model(cls), [_params(7.0, 8.0, 9.0)], mm_rows=3) + expected = {"resource_manager", "mm_token_indices", "text_token_indices"} + if cls is _TinyRowsForConditionalGeneration: + expected.add("multimodal_rows") + assert set(llm_kwargs) == expected + + +class _ExtraRowsOnly(_DeepStack): + """A hook that keeps no multimodal row of the chunk and returns extra rows.""" + + def after_active_multimodal_embeddings(self, *, active_embeddings, **kwargs): + return [], (torch.cat(active_embeddings) * 10,) + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_extra_rows_reach_the_hook_when_the_chunk_has_no_multimodal_row(cls): + model = _model(type("ExtraRowsOnlyForConditionalGeneration", (_ExtraRowsOnly, cls), {})) + input_ids, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)]) + # Nothing is fused or handed over as rows, and the extra rows still reach the hook. + assert args[1] is input_ids and args[3] is None + assert llm_kwargs.get("multimodal_rows") is None + (extra,) = llm_kwargs["deepstack_embeds"] + assert extra[:, 0].tolist() == [70.0, 80.0, 90.0] + + +def test_in_the_fused_mode_caller_inputs_embeds_replace_input_ids_as_in_the_mixin(): + model = _model( + type("DeepStackForConditionalGeneration", (_DeepStack, _TinyForConditionalGeneration), {}) + ) + inputs_embeds = torch.ones(6, HIDDEN) + _, args, _ = _forward(model, [], inputs_embeds=inputs_embeds) + assert args[1] is None and args[3] is inputs_embeds + mm_inputs = model.hook_got["mm_inputs"] + assert mm_inputs.input_ids is None and mm_inputs.inputs_embeds is inputs_embeds + + +# ---------------------------------------------------- compile and weights --- + + +def test_torch_compile_reaches_only_the_language_models_core(monkeypatch): + compiled = [] + + def spy(module, **kwargs): + compiled.append(module) + return module + + monkeypatch.setattr(torch, "compile", spy) + model = _model() + core = model.llm.model + returned, prefill_only = model_setup.apply_torch_compile( + model, "inductor", llm_args=SimpleNamespace(), fullgraph=False + ) + assert returned is model and prefill_only is False + assert compiled == [core] + + +class _Checkpoint(Mapping): + """Checkpoint weights that record which keys are read.""" + + def __init__(self, weights): + self.weights, self.read = weights, set() + + def __getitem__(self, key): + self.read.add(key) + return self.weights[key] + + def __iter__(self): + return iter(self.weights) + + def __len__(self): + return len(self.weights) + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_the_tower_and_the_language_model_each_load_their_own_keys(disable): + model = _model(disable=disable) + weights = _Checkpoint( + {"model.visual.patch.weight": 1, "model.layers.0.weight": 2, "lm_head.weight": 3} + ) + model.load_weights(weights) + assert model.llm.loaded == {"model.layers.0.weight": 2, "lm_head.weight": 3} + if not disable: + assert model.mm_encoder.loaded == {"model.visual.patch.weight": 1} + # Without a tower, the tower's keys stay unread. + tower_keys = set() if disable else {"model.visual.patch.weight"} + assert weights.read == {"model.layers.0.weight", "lm_head.weight"} | tower_keys + + +class _MappedLlm(_TinyLlm): + def load_weights(self, weights, weight_mapper=None): + self.loaded = weights + self.weight_mapper = weight_mapper + + +class _MappedForConditionalGeneration(_TinyForConditionalGeneration): + language_model_class = _MappedLlm + + +class _Mapper: + """The model a weight mapper is bound to, as `BaseWeightMapper` binds it.""" + + def init_model_and_config(self, model, config): + self.model, self.config = model, config + + +def _bound_mapper(model): + """A mapper the engine bound to the routed model.""" + mapper = _Mapper() + mapper.init_model_and_config(model, model.model_config) + return mapper + + +def test_the_engines_weight_mapper_reaches_a_language_model_that_takes_one(): + """The engine passes `weight_mapper` only to a `load_weights` that names it + (`call_load_weights`); the base names it and hands it on the same way, bound to the + language model, whose modules the keys name.""" + model = _model(_MappedForConditionalGeneration) + mapper = _bound_mapper(model) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, mapper) + assert model.llm.weight_mapper is mapper + assert (mapper.model, mapper.config) == (model.llm, model.llm.model_config) + # Without a mapper, the language model gets none. + model = _model(_MappedForConditionalGeneration) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, None) + assert model.llm.weight_mapper is None + # A language model whose `load_weights` takes no mapper is called without one, and the + # mapper stays bound to the routed model. + model = _model() + mapper = _bound_mapper(model) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, mapper) + assert model.llm.loaded == {"model.layers.0.weight": 2} + assert mapper.model is model + + +# ---------------------------------------------------- the router's check --- + + +class _Decoder: + """A language model target that neither serves media nor refuses it.""" + + +class _TextOnly: + def validate_multimodal_request_data(self, mm_data): + raise ValueError("text only") + + +class _OwnEncoder: + """A target that encodes media in its own forward and refuses at admission what its + tower cannot take.""" + + def validate_request(self, request): + raise ValueError("a request the tower cannot take") + + +class _TokenRangeCheck: + def validate_request(self, request): + """Checks the token range only.""" + + +class _InheritedValidator(_TokenRangeCheck): + """A language model target whose `validate_request` is a base class's: it refuses no media + of its own.""" + + +class _Marked: + supports_multimodal = True + + +class _MarkedFalse: + supports_multimodal = False + + +def test_a_vision_language_architecture_routes_to_a_multimodal_or_text_only_target(monkeypatch): + monkeypatch.setattr(router, "_architecture_takes_media", lambda arch: arch == ARCH) + for target in (_TinyForConditionalGeneration, _TextOnly, _OwnEncoder, _Marked): + router._check_multimodal_contract(target, ARCH) + router._check_multimodal_contract(_Decoder, "TinyForCausalLM") + # The marker counts only when it is True; `validate_request` only in the class's own body. + for target in (_Decoder, _MarkedFalse, _InheritedValidator): + with pytest.raises( + ValueError, + match=f"{target.__name__} is the target for '{ARCH}'.*" + "ModelingV2ForConditionalGeneration.*validate_multimodal", + ) as error: + router._check_multimodal_contract(target, ARCH) + # Not a `NoTarget`, which `auto` turns into a fallback to the built-in model. + assert not isinstance(error.value, NoTarget) + + +@pytest.mark.parametrize("mode", ["auto", "require"]) +def test_a_target_that_breaks_the_contract_fails_at_startup(monkeypatch, mode): + """Not a `NoTarget`: `auto` does not fall back, as the target is wrong, not the + deployment.""" + monkeypatch.setattr(router, "registered_routes", lambda: {ARCH: lambda *args: _Decoder}) + monkeypatch.setattr(router, "_device_compute_capability", lambda: (10, 0)) + monkeypatch.setattr(router, "_architecture_takes_media", lambda arch: True) + config = SimpleNamespace( + pretrained_config=SimpleNamespace(architectures=[ARCH]), moe_tactic=None, spec_config=None + ) + with pytest.raises(ValueError, match=f"_Decoder is the target for '{ARCH}'"): + router.modeling_v2_resolve(config, SimpleNamespace(modeling_v2=mode)) + + +def test_media_architectures_are_known_by_their_input_processor(): + assert router._architecture_takes_media("Qwen4ExpForConditionalGeneration") + assert not router._architecture_takes_media("DeepseekV3ForCausalLM") + assert not router._architecture_takes_media("NoSuchArchitectureForCausalLM") diff --git a/tests/unittest/_torch/pyexecutor/test_model_engine.py b/tests/unittest/_torch/pyexecutor/test_model_engine.py index b827d48e5441..f4ddfdcf1920 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_engine.py +++ b/tests/unittest/_torch/pyexecutor/test_model_engine.py @@ -692,6 +692,29 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( assert engine.llm_args.torch_compile_config is None +@pytest.mark.cpu_only +@pytest.mark.parametrize("takes_rows", [False, True], ids=["fused", "language_model_takes_rows"]) +def test_breakable_gate_for_a_composite_model_with_its_own_tower(takes_rows: bool) -> None: + """A composite model whose language model is a decoder, with a local tower. When the + language model takes `input_ids` and places the rows itself (`language_model_takes_rows`), + as it does when routed alone, the breakable prefill graph is allowed; with the rows fused + into `inputs_embeds` it is refused at startup. The message is not matched.""" + engine = object.__new__(ModelEngine) + engine.model = DummyLegacyMultimodalIndexModel() + engine.model.llm = Mock(spec=DecoderModelForCausalLM) + engine.model.mm_encoder = torch.nn.Module() + engine.model.language_model_takes_rows = takes_rows + engine.input_processor = Mock(spec=BaseMultimodalInputProcessor) + engine.llm_args = SimpleNamespace( + prefill_cuda_graph_backend=PrefillCudaGraphBackend.BREAKABLE, disable_mm_encoder=False + ) + if takes_rows: + engine._validate_breakable_cuda_graph_compatibility() + else: + with pytest.raises(ValueError): + engine._validate_breakable_cuda_graph_compatibility() + + class ModelEngineTestCase(unittest.TestCase): def test_breakable_allows_text_decoder_with_multimodal_processor(self) -> None: engine = object.__new__(ModelEngine) diff --git a/tests/unittest/inputs/test_create_input_processor.py b/tests/unittest/inputs/test_create_input_processor.py index 1bee1497d3c9..ecddfbe59320 100644 --- a/tests/unittest/inputs/test_create_input_processor.py +++ b/tests/unittest/inputs/test_create_input_processor.py @@ -22,10 +22,15 @@ import tensorrt_llm._torch.model_config as model_config_mod import tensorrt_llm._torch.models as models_mod +import tensorrt_llm.inputs.registry as registry_mod from tensorrt_llm.inputs.registry import ( INPUT_PROCESSOR_REGISTRY, DefaultInputProcessor, + InputProcessorRegistry, + MultimodalPlaceholderMetadata, + MultimodalPlaceholderRegistry, create_input_processor, + register_input_processor, ) pytestmark = pytest.mark.cpu_only @@ -107,6 +112,32 @@ def _resolve_to_model_without_processor(monkeypatch: pytest.MonkeyPatch) -> None ) +def test_registry_returns_the_input_processor_registered_for_a_model_class( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`get_input_processor_cls` returns what `register_input_processor` stored, else None.""" + monkeypatch.setattr(registry_mod, "INPUT_PROCESSOR_REGISTRY", InputProcessorRegistry()) + monkeypatch.setattr( + registry_mod, "MULTIMODAL_PLACEHOLDER_REGISTRY", MultimodalPlaceholderRegistry() + ) + + class _Model: + pass + + class _Processor: + pass + + register_input_processor( + _Processor, + model_type="fake_vision_language", + placeholder_metadata=MultimodalPlaceholderMetadata(placeholder_map={"image": ""}), + )(_Model) + + registry = registry_mod.INPUT_PROCESSOR_REGISTRY + assert registry.get_input_processor_cls(_Model) is _Processor + assert registry.get_input_processor_cls(_FakeTextOnlyModel) is None + + @pytest.mark.parametrize("checkpoint_format", ["HF", "MX"]) def test_create_input_processor_uses_registered_processor_for_hf_format_checkpoints( config_loads: list[str], From a6860efadf82230bf1e36d1fed33fad518dd9233 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:24 +0000 Subject: [PATCH 5/6] [TRTLLM-17780][feat] Serve qwen4_exp images through the shared ForConditionalGeneration base Problem The qwen4_exp NVFP4 Modeling V2 target is a DecoderModelForCausalLM that runs its vision tower inside the language model forward. The multimodal runtime does not see it as a multimodal model: no encoder cache, no encoder memory reserve at startup, and the tower runs again on the whole request for every prefill chunk with media. Solution - Route Qwen4ExpForConditionalGeneration to a 29-line subclass of ModelingV2ForConditionalGeneration, ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration. It declares the language model target, the tower class (VisionTower, which the base builds), the model.visual. weight prefix, the encoder hook and the encoder cache. It takes the rows mode: the PLE layer hashes input_ids, so the language model writes the rows at mm_token_indices in its own prefill step. - VisionTower is also a MultimodalEncoderMixin. It builds its own attention arguments per call, so setup_attn_metadata is a no-op. - check_deployment declines an encoder token budget over 32768 while the tower is built: startup profiling fills the budget with dummy images, and one tower call takes at most 32768 patches. auto then uses Modeling V1; require fails and names the setting. The default budget (8192) fits. - vision.encode gives one request's rows, whole and in prompt order. It needs no M-RoPE ids, so profiling can encode its dummy image. The mixin keeps the chunk's rows; vision.multimodal_embeds is removed. - The language model takes multimodal_rows and no longer builds, loads or checks the tower. The tower's admission check runs from validate_media_request. - Breakable prefill runs as on main, with or without a tower: the subclass takes the rows mode, which the engine's gate accepts. The breakable accuracy cases do not change. - The FP8 target's routing test now expects this class for an NVFP4 checkpoint. - The FP8 target does not change: it encodes only the frames of the current prefill chunk in its own forward. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs the one before: 8696 vs 8691 cases, 0 changed outcomes; 11 new tests pass (what the subclass declares, the real tower's construction and weight keys, the budget decline under auto and require, the tower's admission check, the encoder hook, the rows handed to the language model); 6 tests of the removed shell bookkeeping moved there or to the base tests. - The FP8 target's routing tests on CPU, with only its native op library stubbed at import (this image does not ship it): 159 passed, as on main. - One GB200 GPU, all markers, on the series head (the last commit changes no qwen4_exp file): qwen4_exp NVFP4 test_multimodal_model, test_vision, test_vision_warm_up and test_routing: 155 passed; main 42c12879 149 passed, 0 changed outcomes, 5 moved. - Pre-commit hooks pass. - Not run: every qwen4_exp GPU path needs GB300 (sm_103): boot at each layout, MMMU, encoder cache hits, chunked images, MTP with images and the breakable prefill accuracy cases with a tower. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- .../modeling.py | 89 +++-- .../vision.py | 117 ++---- .../weights.py | 8 +- .../modeling_v2/models/qwen4_exp/routing.py | 7 +- .../test_routing_and_layout.py | 5 +- .../test_multimodal_model.py | 363 ++++++++++++++++++ .../test_routing.py | 28 +- .../test_vision.py | 103 +---- 8 files changed, 501 insertions(+), 219 deletions(-) create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 64b890379671..7e7fbb28e6e9 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -418,6 +418,7 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( gdn_prefill_conv_post, ) +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM from tensorrt_llm._torch.modules.fla.cached_replay import CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE from tensorrt_llm._torch.modules.fla.flashinfer_cached_replay import ( @@ -431,7 +432,6 @@ from tensorrt_llm._torch.utils import ActivationType, is_gdn_replay_enabled from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.functional import AllReduceFusionOp -from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -1038,8 +1038,8 @@ class _DraftBound(NamedTuple): class Qwen4ExpDefaultModelingV2(DecoderModel): def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig): """`composite_config` is passed in because the text sub-config adopted as - `model_config.pretrained_config` carries neither the vision half nor the media - token ids.""" + `model_config.pretrained_config` carries no media token ids. The vision tower is + the vision-language model's, not the core's.""" super().__init__(model_config) cfg = model_config.pretrained_config mapping = model_config.mapping @@ -1226,10 +1226,7 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig # device-address view that owns none of those pages. self._ple_table_host = None - # ---- the vision half (the tower declares its own weights) ---------- - vcfg = getattr(composite_config, "vision_config", None) - assert vcfg is not None, "the composite config carries no vision_config" - self.vision_config = vcfg + # ---- the media placeholder ids -------------------------------------- self.image_token_id = int(composite_config.image_token_id) self.video_token_id = int(composite_config.video_token_id) @@ -6972,6 +6969,14 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"{mm_config.encoder_attention_backend}; the target's vision tower runs its own " "attention and does not apply it" ) + # The engine's encoder token budget, which startup profiling fills with dummy images. + encoder_budget = llm_args.encoder_max_num_tokens or llm_args.max_num_tokens + if not model_config.disable_mm_encoder and encoder_budget > _vision.VISION_CALL_MAX_PATCHES: + raise NoTarget( + f"an encoder token budget of {encoder_budget} (encoder_max_num_tokens, or " + "max_num_tokens when it is unset); one call of the target's vision tower takes " + f"at most {_vision.VISION_CALL_MAX_PATCHES} patches" + ) if spec is None: return if not is_gdn_replay_enabled(): @@ -7051,13 +7056,6 @@ def __init__(self, model_config: ModelConfig): # init). self._mm_token_ids_value = (self.model.image_token_id, self.model.video_token_id) self._mm_token_ids_cache: torch.Tensor | None = None - # Replicated on every rank. None with `disable_mm_encoder`: then no vision weight is - # declared or loaded and a media request is refused at admission. - self.vision_tower: _vision.VisionTower | None = ( - None - if model_config.disable_mm_encoder - else _vision.VisionTower(self.model.vision_config, model_config.torch_dtype) - ) @property def multimodal_token_ids(self) -> torch.Tensor: @@ -7068,29 +7066,6 @@ def multimodal_token_ids(self) -> torch.Tensor: self._mm_token_ids_cache = torch.tensor(self._mm_token_ids_value, dtype=torch.int32) return self._mm_token_ids_cache - @property - def multimodal_data_device_paths(self) -> list[str]: - """Tensors under `multimodal_data` the executor moves to the device. Absent, it moves - everything, putting the `(t, h, w)` grids the tower reads on the host there too.""" - return list(_vision.DEVICE_PATHS) - - def validate_request(self, request) -> None: - """Admission check, run by the executor on every new request: a media request the - vision tower would assert on fails here as its own error (`vision.validate_request`); - a text request returns at once. Without a tower `validate_multimodal_request_data` - refuses media instead.""" - if self.vision_tower is not None: - _vision.validate_request(self.vision_tower, request) - - def validate_multimodal_request_data(self, mm_data: dict) -> None: - """Admission check: with `disable_mm_encoder` the target serves text only, so a - request with media or embeddings fails alone. M-RoPE positions alone pass.""" - if self.vision_tower is None and has_media_or_embeddings(mm_data): - raise ValueError( - f"{type(self).__name__} runs with disable_mm_encoder and serves text only; " - "send no images, videos or embeddings" - ) - def forward( self, attn_metadata: AttentionMetadata, @@ -7100,10 +7075,12 @@ def forward( return_context_logits: bool = False, spec_metadata=None, lora_params: dict | None = None, + multimodal_rows: tuple | None = None, **kwargs, ): """One engine step: the core's `begin_step`, then the base forward with every - argument it was handed (`resource_manager` stays in `**kwargs`). + argument it was handed (`resource_manager` stays in `**kwargs`). `multimodal_rows` is + `(rows, mm_token_indices)` from the vision-language model when the step carries media. With a spec worker the shell owns the logits gather (one-model modes gather the rows the sampler verifies, `spec_metadata.gather_ids`) and hands the worker the trunk's @@ -7116,7 +7093,7 @@ def forward( inputs_embeds=inputs_embeds, lora_params=lora_params, spec_metadata=spec_metadata, - vision=_vision.multimodal_embeds(self.vision_tower, kwargs), + vision=multimodal_rows, ) if self.spec_worker is None: assert spec_metadata is None, ( @@ -7186,11 +7163,8 @@ def forward( def load_weights(self, weights, *args, **kwargs): _weights.MODEL_WEIGHTS.load(self, weights) - if self.vision_tower is not None: - self.vision_tower.load_weights(weights) def post_load_weights(self): - # The engine's post-load walk calls the vision tower's own hook. super().post_load_weights() self.model.post_load_weights() if self.one_rank: @@ -7232,3 +7206,34 @@ def _build_heads(self, weight: torch.Tensor) -> None: f"{_FP8_GEMV_HEAD_ROWS} on nvfp4_gemv over lm_head {tuple(nv.data.shape)}", flush=True, ) + + +class ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration( + ModelingV2ForConditionalGeneration +): + """The routed class: the language model target as `llm` and the vision tower as + `mm_encoder`, on the shared base. The language model keeps `input_ids` (its PLE layer + hashes them) and writes the rows at `mm_token_indices` in its own prefill step.""" + + language_model_class = ModelingV2Qwen38FlashNextNvfp4Sm103Default + # Replicated on every rank, at the activation dtype. + mm_encoder_class = _vision.VisionTower + mm_encoder_weight_prefix = "model.visual." + language_model_takes_rows = True + supports_encoder_cache = True + + @property + def multimodal_data_device_paths(self) -> list[str]: + """Tensors under `multimodal_data` the executor moves to the device. Absent, it moves + everything, the `(t, h, w)` grids the tower reads on the host among them.""" + return [*_vision.DEVICE_PATHS, "multimodal_embedding"] + + def validate_media_request(self, request) -> None: + _vision.validate_request(self.mm_encoder, request) + + def encode_multimodal_inputs(self, multimodal_params) -> torch.Tensor: + """Each request's rows, whole and in prompt order, in request order.""" + if self.mm_encoder is None: + raise ValueError("Raw multimodal inputs require a local multimodal encoder.") + rows = [_vision.encode(self.mm_encoder, param) for param in multimodal_params] + return rows[0] if len(rows) == 1 else torch.cat(rows, 0) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py index 1a3f74270a73..53a92365622b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py @@ -10,9 +10,11 @@ the same rows back, since every op is a deterministic function of its operands; under attention DP a rank its own requests'. -The shell calls `multimodal_embeds` on every step, before the core's `begin_step`, and the -executor calls `validate_request` on every new request. Neither runs inside a captured -graph: the tower is eager and reads the request's grid on the host. +The tower is the vision-language model's `mm_encoder`. That model's encoder hook calls +`encode` per request, through the shared multimodal mixin, which caches the rows and keeps +the chunk's; the language model splices them in its own prefill step. The model's +`validate_request` calls `validate_request` here on every new request. Neither runs inside a +captured graph: the tower is eager and reads the request's grid on the host. """ from __future__ import annotations @@ -37,6 +39,7 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.ops.vision.triton_pos_embed_interpolate import ( # noqa: E501 triton_pos_embed_interpolate, ) +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin from tensorrt_llm.inputs.multimodal import get_multimodal_item_metadata from . import weights as _weights @@ -295,12 +298,12 @@ def _rope_tables( return angles.cos().contiguous(), angles.sin().contiguous() -class VisionTower(nn.Module): +class VisionTower(nn.Module, MultimodalEncoderMixin): """One request's images or videos through the 27-block ViT and its patch merger. - Not a target: runs only for a context request carrying media, from the shell, before - the step. Attention is bidirectional per frame (a video's frame group is a frame), - expressed to `thop_attention` as one context "sequence" per frame; running the rows as + Not a target: the vision-language model's encoder, run only for a context request carrying + media, before the step. Attention is bidirectional per frame (a video's frame group is a + frame), expressed to `thop_attention` as one context "sequence" per frame; running the rows as one sequence is far outside tolerance. The position-embedding interpolation and the 2-D rope both emit in spatial-merge order (what the merger's 4-to-1 reshape consumes); a raster-order emitter gives a correctly shaped, silently wrong tower. @@ -341,6 +344,10 @@ def __init__(self, vision_config, dtype: torch.dtype) -> None: # engine's own workspace. self._ws: torch.Tensor | None = None + def setup_attn_metadata(self, max_num_tokens: int, attention_metadata_capacity=None) -> None: + """No engine-built attention metadata: every tower call builds its own encoder-only + attention arguments from its frames (`_run`).""" + def load_weights(self, weights) -> None: """Fill the tower's weights from the checkpoint's `model.visual.` keys.""" _weights.VISION_WEIGHTS.load(self, weights) @@ -481,83 +488,25 @@ def _run(self, pixel_values: torch.Tensor, grid: list[tuple[int, int, int]]) -> return cublas_mm(gelu_erf(cublas_mm(merged, V.m_fc1, V.m_fc1_b)), V.m_fc2, V.m_fc2_b) -# ------------------------------------------------------------ the shell's side --- - - -def multimodal_embeds(tower: VisionTower | None, kwargs: dict): - """Run the vision tower for this batch; returns `(rows, mm_token_indices)` (the tower's - `[R, hidden]` rows in packed-batch order and the `[R]` placeholder rows they replace, - which the prefill target splices into its embedding), or None when the step carries no - media. - - Called by the shell, because the inherited `DecoderModelForCausalLM.forward` drops - `multimodal_params` from `**kwargs` on the no-speculation path. Host-side bookkeeping: - `multimodal_params` is a subset of the context requests, so never zip it positionally; - `multimodal_runtime` (chunked prefill / KV reuse only) gives the cached and in-chunk row - counts over the request's images and videos together, in prompt order; `mm_token_indices` - names every placeholder row. - - The tower runs once per modality a request carries, and `in_prompt_order` merges the - two outputs. Positions are not derived here: every coordinate of a vision span, a - video's temporal one included, is in the payload's `mrope_config.mrope_position_ids`, - which the engine slices into `position_ids`.""" - mm_params = kwargs.get("multimodal_params") or [] - if not mm_params: - return None - rows = [] - for params in mm_params: - data = getattr(params, "multimodal_data", None) or {} - image, video = data.get("image"), data.get("video") - if image is None and video is None: - continue - mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") - assert ( - mrope_ids is not None and mrope_ids.dim() == 3 and mrope_ids.shape[0] == _POSITION_AXES - ), ( - f"a media request needs mrope_config.mrope_position_ids of shape " - f"[{_POSITION_AXES}, 1, L]; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" - ) - runtime = getattr(params, "multimodal_runtime", None) - if runtime is not None and int(runtime.num_mm_tokens_in_chunk) == 0: - # Every placeholder row of this request lies in another chunk. - continue - assert tower is not None, ( - "no vision tower (disable_mm_encoder), so admission refuses media requests" - ) - towers = {} - if image is not None: - towers["image"] = run_tower(tower, image, *MEDIA_KEYS["image"]) - if video is not None: - towers["video"] = run_tower(tower, video, *MEDIA_KEYS["video"]) - embeds = in_prompt_order(tower.merge, params, towers) - if runtime is not None: - # Chunked prefill or reused prefix: splice only this chunk's slice. - start = int(runtime.num_cached_mm_tokens) - count = int(runtime.num_mm_tokens_in_chunk) - total = embeds.shape[0] - assert int(runtime.total_embeds_in_request) == total, ( - f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " - f"this request and the tower produced {total} over its {sorted(towers)}; the " - "two are derived from the same grids and a disagreement means the merger's " - "row count is wrong" - ) - assert 0 <= start <= start + count <= total, (start, count, total) - embeds = embeds[start : start + count] - rows.append(embeds) - if not rows: - return None - vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) - mm_token_indices = kwargs.get("mm_token_indices") - assert mm_token_indices is not None, ( - "the executor shipped multimodal params but no mm_token_indices; that pairing is what " - "the shell's `multimodal_token_ids` buys, and without it there is no sync-free way to " - "find the placeholder rows" - ) - assert mm_token_indices.shape[0] == vision.shape[0], ( - f"the batch has {mm_token_indices.shape[0]} placeholder rows and the tower produced " - f"{vision.shape[0]} embedding rows" - ) - return vision, mm_token_indices +# ------------------------------------------------------------ the encoder hook --- + + +def encode(tower: VisionTower, params) -> torch.Tensor: + """One request's tower rows, one per placeholder of the whole request, in prompt order. + + The tower runs once per modality the request carries, and `in_prompt_order` merges the + outputs. No chunk slice: the multimodal mixin keeps the rows of the current chunk. No + position check either: positions are not derived here (every coordinate of a vision span + is in `mrope_config.mrope_position_ids`, which admission checks), and the engine's memory + profiling hands the encoder a dummy request without them.""" + data = params.multimodal_data + towers = { + modality: run_tower(tower, data[modality], *keys) + for modality, keys in MEDIA_KEYS.items() + if data.get(modality) is not None + } + assert towers, f"the multimodal payload carries {sorted(data)} and no image or video" + return in_prompt_order(tower.merge, params, towers) def run_tower(tower, media: dict, pixel_key: str, grid_key: str): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index b6be1bf1ee56..734f76097d8a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -1550,12 +1550,8 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: fill(model.lm_head.weight, "lm_head.weight", None, cut, pad) # Parameter-side coverage, the other direction: name every registered parameter # outside the target's own ParameterDict instead of assuming lm_head is the only - # one. The vision tower loads and checks its own table. - shell = { - n - for n, _ in model.named_parameters() - if not n.startswith(("model.w.", "vision_tower.")) - } + # one. + shell = {n for n, _ in model.named_parameters() if not n.startswith("model.w.")} assert shell == {"lm_head.weight"}, f"unfed shell parameters: {sorted(shell)}" torch.cuda.synchronize() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py index 2eff81e160ce..c029f1570b9e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py @@ -7,7 +7,9 @@ `qwen3_8_flash_next_fp8__sm_103__unified` at one of `_FP8_LAYOUTS`, each with one-model MTP-Eagle (not at dep2) or no speculative decoding. The target's `check_deployment` then declines the checkpoints and engine choices it does not run, so `auto` falls back to the -built-in implementation instead of failing in the build. +built-in implementation instead of failing in the build. The NVFP4 route returns the target's +vision-language model, which holds its language model and its vision tower; the FP8 target +encodes media in its own forward. """ from __future__ import annotations @@ -106,7 +108,8 @@ def route( return ModelingV2Qwen38FlashNextFp8Sm103Unified from .qwen3_8_flash_next_nvfp4__sm_103__default.modeling import ( ModelingV2Qwen38FlashNextNvfp4Sm103Default, + ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration, ) ModelingV2Qwen38FlashNextNvfp4Sm103Default.check_deployment(model_config, llm_args, member) - return ModelingV2Qwen38FlashNextNvfp4Sm103Default + return ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index d10c638b31ba..2d8d442686c1 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -82,7 +82,7 @@ def _world(member: str) -> int: ) _NVFP4_DEFAULT_TARGET = ( "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." - "ModelingV2Qwen38FlashNextNvfp4Sm103Default" + "ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration" ) # The quantization the engine resolves for the FP8 (128x128 blocks) and the NVFP4 exports. @@ -495,7 +495,8 @@ def test_a_switch_that_would_do_nothing_declines(on_sm103, monkeypatch): @pytest.mark.cpu_only def test_the_quantization_picks_the_checkpoint(on_sm103): - """MIXED_PRECISION routes to the NVFP4 target (its own checks mocked), FP8_BLOCK_SCALES here.""" + """MIXED_PRECISION routes to the NVFP4 target's vision-language model (its own checks + mocked), FP8_BLOCK_SCALES here.""" from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 modeling as nvfp4_modeling, ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py new file mode 100644 index 000000000000..953fe6ca88ca --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The default target's vision-language model on the CPU: what it declares to the shared base +(`ModelingV2ForConditionalGeneration`), its tower, the tower's admission check, its encoder hook +and the rows it hands the language model. The base itself is tested in +`tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py`. + +The language model is never built (its core asserts sm_103): construction swaps in a stub +class, and the other tests call the model's real methods on stubs, with the real multimodal +mixin in between. Towers are stubs over the real `VisionTower.forward`, or real ones on the +meta device. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + vision as V, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + weights as _weights, +) +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin +from tensorrt_llm.inputs.multimodal import ( + MULTIMODAL_ENCODER_ITEM_METADATA_KEY, + MultimodalItemMetadata, + MultimodalParams, + MultimodalRuntimeData, +) + +pytestmark = pytest.mark.cpu_only + +Model = M.ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration + +# Qwen3.8-Flash-Next's vision config. +_VISION = dict( + deepstack_visual_indexes=[], + depth=27, + dtype="bfloat16", + hidden_act="gelu_pytorch_tanh", + hidden_size=1152, + in_channels=3, + intermediate_size=4304, + num_heads=16, + num_position_embeddings=2304, + out_hidden_size=2560, + patch_size=16, + spatial_merge_size=2, + temporal_patch_size=2, +) +HIDDEN = 2560 +MERGE = 2 +PIXEL_WIDTH = 4 +VOCAB = 100 + + +# ------------------------------------------------------------------ stubs --- + + +def _tower(dtype=torch.float32, width=1): + """A `VisionTower` whose per-call body is a stub: merged row `j` of item `i` of modality + `m` holds `100 * m + i` in every column, so a row says where it came from. The real + `forward` (grid checks, frame calls) runs around it.""" + tower = object.__new__(V.VisionTower) + nn.Module.__init__(tower) + tower.dtype, tower.pixel_width, tower.merge = dtype, PIXEL_WIDTH, MERGE + tower.calls = 0 + + def run(pixel_values, grid): + tower.calls += 1 + return pixel_values[:: MERGE * MERGE, :1].expand(-1, width).to(dtype).clone() + + tower._run = run + return tower + + +IMAGE, VIDEO = 1, 2 +IMAGES = [(1, 4, 4), (1, 2, 2)] # 4 and 1 rows +VIDEOS = [(2, 2, 4), (2, 4, 4)] # 4 and 8 rows +MIXED_ORDER = [("video", 0), ("image", 0), ("video", 1), ("image", 1)] +MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] + + +def _media(modality, grids): + """One modality's payload; each item's patch rows hold `100 * modality + item`.""" + pixels = torch.cat( + [ + torch.full((t * h * w, PIXEL_WIDTH), float(100 * modality + item)) + for item, (t, h, w) in enumerate(grids) + ] + ) + if modality == IMAGE: + return {"pixel_values": pixels, "image_grid_thw": torch.tensor(grids)} + return {"pixel_values_videos": pixels, "video_grid_thw": torch.tensor(grids)} + + +def _params(image=None, video=None, order=None, runtime=None, mrope=True): + data = {} + if mrope: + data["mrope_config"] = {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)} + if image is not None: + data["image"] = _media(IMAGE, image) + if video is not None: + data["video"] = _media(VIDEO, video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +def _runtime(total: int, cached: int, chunk_end: int) -> MultimodalRuntimeData: + """A prompt of `total` placeholder tokens, `cached` of them reused, up to `chunk_end`.""" + return MultimodalRuntimeData( + embed_mask_cumsum=torch.arange(1, total + 1, dtype=torch.int64), + past_seen_token_num=cached, + chunk_end_pos=chunk_end, + ) + + +class _Llm(nn.Module): + """Records what the vision-language model hands its language model.""" + + def __init__(self): + super().__init__() + self.model = nn.Linear(2, 2) + self.calls = [] + self.lm_head = SimpleNamespace(num_embeddings=VOCAB) + self.multimodal_token_ids = torch.tensor([90, 91], dtype=torch.int32) + + def forward( + self, + attn_metadata, + input_ids=None, + position_ids=None, + inputs_embeds=None, + return_context_logits=False, + spec_metadata=None, + **kwargs, + ): + args = ( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + ) + self.calls.append((args, kwargs)) + return "logits" + + +def _model(tower=None, llm=None): + """A real instance around stubs, without the constructor.""" + model = Model.__new__(Model) + nn.Module.__init__(model) + model.llm = llm if llm is not None else _Llm() + model.mm_encoder = tower + model.model_config = SimpleNamespace(multimodal_config=None, torch_dtype=torch.bfloat16) + return model + + +# ----------------------------------------------------------- construction --- + + +class _StubLlm(nn.Module): + """Stands in for the language model target: takes its config copy and adopts the text + config on it, as the target does.""" + + def __init__(self, model_config): + super().__init__() + self.got = model_config + model_config.pretrained_config = model_config.pretrained_config.text_config + self.model_config = model_config + + @property + def config(self): + return self.model_config.pretrained_config + + +def _composite(): + return PretrainedConfig( + architectures=["Qwen4ExpForConditionalGeneration"], + text_config=PretrainedConfig(hidden_size=HIDDEN, torch_dtype=torch.bfloat16), + vision_config=Qwen4ExpVisionConfig(**_VISION), + torch_dtype=torch.bfloat16, + ) + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_construction_keeps_the_engine_config_and_builds_the_tower_only_when_asked( + monkeypatch, disable +): + monkeypatch.setattr(Model, "language_model_class", _StubLlm) + composite = _composite() + model_config = ModelConfig(pretrained_config=composite, disable_mm_encoder=disable) + with torch.device("meta"): + model = Model(model_config) + + assert model.model_config is model_config + assert model.config is composite.text_config + assert isinstance(model.llm, _StubLlm) + assert isinstance(model, MultimodalModelMixin) + assert model.dtype == torch.bfloat16 + params = {name.split(".")[0] for name, _ in model.named_parameters()} + if disable: + assert model.mm_encoder is None and not params + else: + assert isinstance(model.mm_encoder, V.VisionTower) + assert isinstance(model.mm_encoder, MultimodalEncoderMixin) + assert model.mm_encoder.dtype == torch.bfloat16 + # Every vision weight is the tower's, and every one is read from under the prefix the + # model hands the tower. + assert params == {"mm_encoder"} + assert all(".w.vis_" in name or ".w.l" in name for name, _ in model.named_parameters()) + sources = _weights.VISION_WEIGHTS.manifest(model.mm_encoder.dims).values() + keys = {key for rows in sources for key, *_ in rows} + assert keys and all(key.startswith(Model.mm_encoder_weight_prefix) for key in keys) + + +# --------------------------------------------------------------- contract --- + + +def test_the_class_declares_what_the_shared_base_needs(): + assert issubclass(Model, ModelingV2ForConditionalGeneration) + assert Model.language_model_class is M.ModelingV2Qwen38FlashNextNvfp4Sm103Default + # The base builds it as `VisionTower(vision_config, dtype)`. + assert Model.mm_encoder_class is V.VisionTower + assert Model.mm_encoder_weight_prefix == "model.visual." + # The PLE layer hashes `input_ids`, so the language model places the rows itself. + assert Model.language_model_takes_rows is True + assert Model.supports_encoder_cache is True + assert Model.supports_mm_encoder_item_scheduling is False + assert Model.supported_encoder_attention_backends == {} + assert Model.get_preferred_kv_cache_manager_version() == "V2" + + +def test_the_tower_is_an_encoder_without_engine_metadata(): + assert issubclass(V.VisionTower, MultimodalEncoderMixin) + assert V.VisionTower.encoder_cuda_graph_modality is None + tower = _tower() + tower.setup_attn_metadata(16, {"attention": 16}) + tower.set_attn_max_seq_len(16) + assert tower.attn_metadata is None + + +# -------------------------------------------------------------- admission --- + +_GEOMETRY = SimpleNamespace(merge=MERGE, patch=16, pixel_width=PIXEL_WIDTH) + + +def _request(tokens=(1, 2, 3), image=None, mrope_len=None, bound=None, **data): + """A request whose prompt is `tokens`, optionally around one image's placeholders.""" + prompt_len = len(tokens) + if image is not None: + rows = sum(t * h * w for t, h, w in image) // (MERGE * MERGE) + prompt_len += rows + data["image"] = _media(IMAGE, image) + data["mrope_config"] = { + "mrope_position_ids": torch.zeros(3, 1, mrope_len or prompt_len, dtype=torch.int32) + } + return SimpleNamespace( + py_multimodal_data=data or None, + prompt_len=prompt_len, + py_prompt_token_bound=bound, + check_token_id_range=lambda vocab: max(tokens) < vocab, + ) + + +def test_admission_runs_the_towers_media_check(): + """The base's `validate_request` checks the token range, then the tower's media check + (`vision.validate_request`); without a tower, media is refused by the base.""" + model = _model(_GEOMETRY) + model.validate_request(_request()) + model.validate_request(_request(image=IMAGES)) + # M-RoPE positions alone (a disaggregated generation request) are not media. + model.validate_request(_request(mrope_config={"mrope_position_deltas": torch.zeros(1, 1)})) + with pytest.raises(ValueError, match="one position per prompt token"): + model.validate_request(_request(image=IMAGES, mrope_len=4)) + with pytest.raises(ValueError, match="Token ID out of range"): + model.validate_request(_request(tokens=(1, VOCAB))) + _model(None).validate_request(_request(image=IMAGES, mrope_len=4)) + + +# ----------------------------------------------------------- encoder hook --- + + +def test_the_encoder_hook_takes_a_profiling_dummy(): + """The engine's memory profiling encodes one dummy image with no position ids and no item + metadata, at the model's dtype: rows of the text hidden width, in that dtype.""" + tower = _tower(dtype=torch.bfloat16, width=HIDDEN) + model = _model(tower) + dummy = MultimodalParams(multimodal_data={"image": _media(IMAGE, [(1, 8, 8)])}) + dummy.multimodal_data["image"]["pixel_values"] = dummy.multimodal_data["image"][ + "pixel_values" + ].to(model.dtype) + rows = model.encode_multimodal_inputs([dummy]) + assert rows.shape == (16, HIDDEN) and rows.dtype == torch.bfloat16 + + +def test_the_encoder_hook_returns_each_request_whole_in_prompt_order(): + tower = _tower() + model = _model(tower) + mixed = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER) + rows = model.encode_multimodal_inputs([_params(image=IMAGES), mixed]) + assert rows[:, 0].tolist() == [100.0] * 4 + [101.0] + MIXED_ROWS + with pytest.raises(ValueError, match="require a local multimodal encoder"): + _model(None).encode_multimodal_inputs([mixed]) + + +# ---------------------------------------------------------------- forward --- + + +def _forward(model, params, num_contexts=1, num_rows=None, input_ids=None): + input_ids = input_ids if input_ids is not None else torch.arange(24, dtype=torch.int32) + kwargs = {"multimodal_params": params, "resource_manager": "rm"} + if num_rows is not None: + kwargs["mm_token_indices"] = torch.arange(1, 1 + num_rows, dtype=torch.int64) + out = model.forward( + SimpleNamespace(num_contexts=num_contexts, num_generations=0), + input_ids, + "position_ids", + None, + False, + "spec_metadata", + **kwargs, + ) + assert out == "logits" + ((args, llm_kwargs),) = model.llm.calls + # `input_ids` reach the language model unchanged, and `inputs_embeds` stays None. + assert args[1] is input_ids + assert args[2:] == ("position_ids", None, False, "spec_metadata") + assert "multimodal_params" not in llm_kwargs and llm_kwargs["resource_manager"] == "rm" + return llm_kwargs["multimodal_rows"], kwargs.get("mm_token_indices") + + +def test_a_media_step_hands_rows_and_placeholder_rows_to_the_language_model(): + model = _model(_tower()) + multimodal_rows, indices = _forward(model, [_params(image=IMAGES)], num_rows=5) + rows, mm_token_indices = multimodal_rows + assert rows[:, 0].tolist() == [100.0] * 4 + [101.0] + assert mm_token_indices is indices + + +def test_a_chunk_hands_over_its_slice_of_the_prompt_order_rows(): + """The mixin keeps the chunk's rows: 3 reused, 6 in this chunk.""" + tower = _tower() + model = _model(tower) + runtime = _runtime(len(MIXED_ROWS), cached=3, chunk_end=9) + params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) + (rows, _), _ = _forward(model, [params], num_rows=6) + assert rows[:, 0].tolist() == MIXED_ROWS[3:9] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py index d320852b53c3..509eb8ec29f7 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py @@ -28,6 +28,7 @@ NGramDecodingConfig, TorchLlmArgs, ) +from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo @@ -38,7 +39,7 @@ _ARCH = "Qwen4ExpForConditionalGeneration" _TARGET = ( "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." - "ModelingV2Qwen38FlashNextNvfp4Sm103Default" + "ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration" ) # The four-rank layouts, as the LLM API's parallel arguments build them. tp4 leaves the @@ -546,3 +547,28 @@ def test_a_misspelled_ple_host_table_mode_fails_at_routing(monkeypatch): modeling_v2_resolve(_model_config(**_LAYOUTS["tep4"]), _llm_args("auto")) monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "Node") _claimed(_model_config(**_LAYOUTS["tep4"])) + + +def test_an_encoder_budget_past_one_tower_call_declines(monkeypatch): + """Startup profiling fills the encoder token budget (`encoder_max_num_tokens`, else + `max_num_tokens`) with dummy images, and one tower call takes at most 32768 patches. A + larger budget falls back under `auto`, with one log line naming the setting and the + limit, and fails under `require`. Without a tower the budget is not used.""" + config = _model_config(**_LAYOUTS["tep4"]) + assert _llm_args().encoder_max_num_tokens is None + assert _llm_args().max_num_tokens == 8192 + _claimed(config) + _claimed(config, encoder_max_num_tokens=32768) + lines = [] + monkeypatch.setattr(logger, "info", lines.append) + _declined( + config, + match=r"budget of 65536 \(encoder_max_num_tokens.* at most 32768 patches", + encoder_max_num_tokens=65536, + ) + routing_lines = [line for line in lines if line.startswith("modeling_v2:")] + assert len(routing_lines) == 1, lines + assert "encoder_max_num_tokens" in routing_lines[0], routing_lines + assert "at most 32768 patches" in routing_lines[0], routing_lines + _declined(config, match="budget of 32769", max_num_tokens=32769) + _claimed(replace(config, disable_mm_encoder=True), encoder_max_num_tokens=65536) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py index c3ab21b0043f..dab307dc4805 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """The default target's vision path on the CPU: the tower's weights, its calls and angle table, -the splice's bookkeeping and the admission check. - -The tower is built over the weight table's own `vis_*` declarations on the meta device, so -what it reads is what `weights.py` declares. Its per-call body is a stub where a test needs one -(the vision ops need a GPU), and so is the whole tower where the shell's bookkeeping is under -test: every row of a stub tower names the item it came from, so the modality dispatch, item -metadata, chunk slice, frame grouping and the order of concatenated calls run with no engine. +the encoder hook's bookkeeping and the admission check. + +The tower is built over its own `vis_*` table on the meta device, so what it reads is what +`weights.py` declares. Its per-call body is a stub where a test needs one (the vision ops need +a GPU), and so is the whole tower where the encoder hook's bookkeeping is under test: every row +of a stub tower names the item it came from, so the modality dispatch, item metadata, frame +grouping and the order of concatenated calls run with no engine. The chunk slice is the +multimodal mixin's (`test_multimodal_model.py`). """ from __future__ import annotations @@ -33,7 +34,7 @@ pytestmark = pytest.mark.cpu_only -Shell = M.ModelingV2Qwen38FlashNextNvfp4Sm103Default +Model = M.ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration # Qwen3.8-Flash-Next's vision config. _VISION = dict( @@ -298,16 +299,10 @@ def _params(image=None, video=None, order=None, runtime=None, mrope=True): def _splice(tower, params, num_rows, padding=0): - """The tower's rows spliced, as the prefill target's break does, over a prompt whose - tokens 1..num_rows are the placeholders and which `padding` rows follow; returns the rows - that landed there, one value per row, or None when the step carries no media.""" - kwargs = { - "multimodal_params": [params], - "mm_token_indices": torch.arange(1, 1 + num_rows, dtype=torch.int64), - } - vision = V.multimodal_embeds(tower, kwargs) - if vision is None: - return None + """The encoder hook's rows spliced, as the prefill target's break does, over a prompt + whose tokens 1..num_rows are the placeholders and which `padding` rows follow; returns the + rows that landed there, one value per row.""" + vision = (V.encode(tower, params), torch.arange(1, 1 + num_rows, dtype=torch.int64)) total = num_rows + 2 h = torch.full((total + padding, HIDDEN), -1.0) target = SimpleNamespace(_step=SimpleNamespace(vision=vision, num_tokens=total)) @@ -328,13 +323,6 @@ def _item(modality, index): MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] -def test_a_text_step_runs_no_tower(): - assert V.multimodal_embeds(None, {}) is None - assert V.multimodal_embeds(None, {"multimodal_params": []}) is None - # Position ids alone (a text prompt through a multimodal processor) are not media. - assert V.multimodal_embeds(None, {"multimodal_params": [_params()]}) is None - - def test_the_splice_fills_the_placeholders_and_zeroes_the_padding(): """An image request's rows land on its placeholders in one tower call and the padding rows after the step are zeroed (`_splice` checks); a text step writes only the padding.""" @@ -365,47 +353,20 @@ def test_a_mixed_request_needs_its_manifest(): _splice(_Tower(), _params(image=IMAGES, video=VIDEOS), len(MIXED_ROWS)) -def test_a_chunk_takes_its_slice_of_the_prompt_order_rows(): - runtime = SimpleNamespace( - num_cached_mm_tokens=3, num_mm_tokens_in_chunk=6, total_embeds_in_request=len(MIXED_ROWS) - ) - params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) - assert _splice(_Tower(), params, 6) == MIXED_ROWS[3:9] - - -def test_a_chunk_without_placeholders_skips_the_tower(): - runtime = SimpleNamespace( - num_cached_mm_tokens=len(MIXED_ROWS), - num_mm_tokens_in_chunk=0, - total_embeds_in_request=len(MIXED_ROWS), - ) - tower = _Tower() - params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) - assert _splice(tower, params, 0) is None - assert tower.calls == 0 - - -def test_a_media_request_needs_the_processors_position_ids(): - with pytest.raises(AssertionError, match="mrope_position_ids"): - _splice(_Tower(), _params(video=VIDEOS, mrope=False), 12) - - -def test_the_placeholder_rows_must_match_the_towers_rows(): - kwargs = { - "multimodal_params": [_params(image=IMAGES)], - "mm_token_indices": torch.arange(4, dtype=torch.int64), - } - with pytest.raises(AssertionError, match="4 placeholder rows and the tower produced 5"): - V.multimodal_embeds(_Tower(), kwargs) +def test_the_encoder_hook_needs_no_position_ids(): + """The engine's memory profiling hands the encoder a dummy image with no position ids; + admission checks them for a real request.""" + assert _splice(_Tower(), _params(image=IMAGES, mrope=False), 5) == [100.0] * 4 + [101.0] def test_the_grids_stay_on_the_host(): """The executor moves only the pixels and the position ids; the grids the tower and the admission check read on the host stay there.""" - paths = Shell.multimodal_data_device_paths.fget(None) + paths = Model.multimodal_data_device_paths.fget(None) assert set(paths) == { "image.pixel_values", "video.pixel_values_videos", + "multimodal_embedding", "mrope_config.mrope_position_ids", "mrope_config.mrope_position_deltas", } @@ -454,8 +415,8 @@ def _request(image=None, video=None, order=None, **overrides): def _admit(request): - """Through the shell's own hook, as the executor calls it.""" - Shell.validate_request(SimpleNamespace(vision_tower=_GEOMETRY), request) + """The tower's admission check, which the model's `validate_request` calls.""" + V.validate_request(_GEOMETRY, request) def test_clean_requests_are_admitted(): @@ -555,25 +516,3 @@ def test_the_placeholder_count_is_the_sum_over_both_modalities(): _admit(request) del request.py_multimodal_data["multimodal_embed_mask_cumsum"] _admit(request) - - -def test_without_a_tower_media_is_refused_and_mrope_positions_pass(): - """With `disable_mm_encoder` the shell builds no tower: a media request fails at - admission as its own error, and an M-RoPE-only payload (a disaggregated generation - request) passes. With a tower the geometry check owns media requests.""" - no_tower = SimpleNamespace(vision_tower=None) - request = _request(image=IMAGES) - # No geometry check without a tower: it would read the tower's geometry. - Shell.validate_request(no_tower, request) - with pytest.raises(ValueError, match="disable_mm_encoder and serves text only"): - Shell.validate_multimodal_request_data(no_tower, request.py_multimodal_data) - with pytest.raises(ValueError, match="serves text only"): - Shell.validate_multimodal_request_data( - no_tower, {"multimodal_embedding": torch.zeros(4, HIDDEN)} - ) - mrope_only = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} - Shell.validate_request(no_tower, SimpleNamespace(py_multimodal_data=mrope_only)) - Shell.validate_multimodal_request_data(no_tower, mrope_only) - Shell.validate_multimodal_request_data( - SimpleNamespace(vision_tower=_GEOMETRY), request.py_multimodal_data - ) From e273bcb6e2165786883422c9a9d085f1bf5e8f62 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:27:25 +0000 Subject: [PATCH 6/6] [TRTLLM-17780][test] Check and document the Modeling V2 multimodal contract Problem The router checks a routed class at startup, which needs the target's GPU. Nothing finds a broken vision-language target on a CPU machine, and the README does not say how to add one or how to name it. Solution - test_modeling_v2_contracts.py reads every class a route returns and checks those routed for an architecture whose input processor takes media (the router's own test, not the class name). A route return that is not a class imported from a target's modeling module fails the check, so no routed class is skipped. Each checked class is either a ModelingV2ForConditionalGeneration subclass that sets language_model_class, mm_encoder_weight_prefix and its tower (mm_encoder_class set to a class, or a build_mm_encoder override) and defines encode_multimodal_inputs, or a text-only target whose validate_multimodal_request_data is run on its own with the real has_media_or_embeddings: it must refuse media, embeddings and unknown keys with the shared message and admit M-RoPE positions alone. A target that encodes media in its own forward is listed by name (today only the qwen4_exp FP8 target) and must define a validate_request that refuses something. An entry fails when it is no longer routed for media, or when its class moved onto the base or serves text only, so the list only shrinks. A forward that refuses media uses the same helper. Fixture families show each broken case is reported. - The same file checks the naming rule of V2 vision-language models: every routed class that is a multimodal model (by its base) subclasses ModelingV2ForConditionalGeneration and its name ends with ForConditionalGeneration, and no class in a target's modeling.py is named ...VLM. The ModelingV2 part is left to review. A stand-in that breaks each part is reported. - Modeling V2 README: a "Multimodal targets" section with the base, a short template, the two row modes, the hook for extra language model inputs, the text-only rule, the listed targets that encode media themselves, disable_mm_encoder, breakable prefill and the encoder side stream with a tower, and the rule: code every V2 vision-language model needs lives in the one shared base, and a class that serves media is ModelingV2ForConditionalGeneration (never ...VLM or ...Model) while its language model target stays ModelingV2. It says which parts the test checks. The code layout, the Modeling V1 code rule, the naming section and the lifecycle name the base. adding_new_model.md states the rule in a short "Multimodal Models" part and points to the section. Validation - CPU, CUDA hidden, main 8ba86b8b test image running the code of main 42c12879, 8 test targets (all Modeling V2 unit tests, multimodal mixin and engine, multimodal scheduler, model engine, executor, inputs), this commit vs the one before: 8724 vs 8696 cases, 0 changed outcomes; the 28 new contract tests pass on today's targets and report each broken stand-in. - This series vs main 42c12879: 8618 common cases, 0 changed outcomes, 106 new tests pass, 5 moved. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/torch/adding_new_model.md | 12 + tensorrt_llm/_torch/modeling_v2/README.md | 99 ++- .../modeling_v2/test_modeling_v2_contracts.py | 587 +++++++++++++++++- 3 files changed, 689 insertions(+), 9 deletions(-) diff --git a/docs/source/torch/adding_new_model.md b/docs/source/torch/adding_new_model.md index 006941981f8e..bc22e8324bf6 100644 --- a/docs/source/torch/adding_new_model.md +++ b/docs/source/torch/adding_new_model.md @@ -128,6 +128,18 @@ Optionally, you may replace the native PyTorch modules with our implementations For a concrete reference, check out `tensorrt_llm/_torch/models/modeling_llama.py`. +#### Multimodal Models + +A Modeling V2 vision-language model has the composite shape of a Modeling V1 +one: it holds the Modeling V2 language model as `llm` and the tower as +`mm_encoder`, and `MultimodalModelMixin` runs it. Name it +`ModelingV2ForConditionalGeneration` (never `...VLM` or `...Model`) and +subclass `ModelingV2ForConditionalGeneration`, the shared base. Declare only +what differs from the base; code that several targets would copy goes into the +base. The language model target keeps its name, `ModelingV2`. See +"Multimodal targets" in `tensorrt_llm/_torch/modeling_v2/README.md` for the +template. + ### Weight Loading The base class `DecoderModelForCausalLM` provides a `load_weights` method that loads the weights from the checkpoint file and assigns them to the corresponding layers in the model. However, if the default method does not work for `MyModelForCausalLM`, you need to implement your own `load_weights`: diff --git a/tensorrt_llm/_torch/modeling_v2/README.md b/tensorrt_llm/_torch/modeling_v2/README.md index 25642bf13647..c8f8e3b37e12 100644 --- a/tensorrt_llm/_torch/modeling_v2/README.md +++ b/tensorrt_llm/_torch/modeling_v2/README.md @@ -81,7 +81,9 @@ double underscores between the three parts: The target class name contains the same three parts in CamelCase, for example `ModelingV2DeepseekR10528Nvfp4Sm10xDefault` in [`r1_0528_nvfp4__sm_10x__default`](models/deepseek_v3/r1_0528_nvfp4__sm_10x__default/). -These names are conventions, not requirements. +These names are conventions, not requirements. The one rule is for a +vision-language model, whose class name ends with `ForConditionalGeneration` +(see [Multimodal targets](#multimodal-targets)). A target can contain several execution paths. The DeepSeek R1 target, for example, has separate `PrefillTarget` and `DecodeTarget` classes and picks one @@ -93,6 +95,7 @@ on each forward step. In this guide, "target" always means the whole directory. modeling_v2/ ├── _router_index.py # Route registration and target resolution ├── _native.py # Loads family native libraries +├── multimodal.py # Base class of vision-language targets ├── catalog/ │ ├── _op.py # OpWrapper: the operation contract │ └── / # Operations used by more than one family @@ -135,8 +138,10 @@ it uses only runtime interfaces, for example: * `DecoderModel`, `DecoderModelForCausalLM`, `ModelConfig` and `Mapping`. * `MultimodalModelMixin` and `MultimodalEncoderMixin`. A vision-language target - may inherit these. It does not subclass or call a Modeling V1 model, vision - tower, layer, attention backend or MoE. + inherits the first through `ModelingV2ForConditionalGeneration` + ([`multimodal.py`](multimodal.py)), and its tower the second. It does not + subclass or call a Modeling V1 model, vision tower, layer, attention backend + or MoE. * Attention and speculative metadata, and the KV cache managers. A target does not import, subclass or call Modeling V1 model code: the model @@ -176,9 +181,10 @@ example that uses no Modeling V1 model code. A target plugs into TensorRT-LLM through the existing `DecoderModel` and `DecoderModelForCausalLM` interfaces. A language model target inherits these -classes directly, not a Modeling V1 model class. A vision-language target is a -composite `MultimodalModelMixin` model that holds such a language model target -as `llm`. Either can override any of these steps: +classes directly, not a Modeling V1 model class. A vision-language target +subclasses `ModelingV2ForConditionalGeneration`, which holds such a language +model target as `llm` (see [Multimodal targets](#multimodal-targets)). Either +can override any of these steps: * **Constructor:** builds the model structure and parameters. * **`load_weights`:** loads checkpoint weights into the model. Put weight @@ -197,6 +203,87 @@ classes are fine when they make the code clearer or faster. The its [weight loader](models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py) are a small example. +## Multimodal targets + +A vision-language architecture routes to one of two kinds of target. The +resolver checks every class a `route` returns: when the architecture's input +processor takes media, the class must be one of these two (or an existing +target that encodes media itself, see below), or startup fails. + +**A vision-language model** runs media through the shared multimodal runtime +and has the composite shape of a Modeling V1 vision-language model: it holds a +language model target as `llm` and the encoder tower as `mm_encoder`, and +`MultimodalModelMixin` runs it. Subclass `ModelingV2ForConditionalGeneration` +from [`multimodal.py`](multimodal.py). This shared base does the common work +once: the config split, admission, encoder caching and the current chunk's rows +(through `MultimodalModelMixin`), the compile scope, the weight split, and what +the engine reads off the routed class and model. A target declares what +differs: + +```python +class ModelingV2MyTargetForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = ModelingV2MyTarget # the language model target + mm_encoder_class = MyTower # MyTower(vision_config, dtype) + mm_encoder_weight_prefix = "model.visual." # the tower's checkpoint keys + + def encode_multimodal_inputs(self, multimodal_params): + return torch.cat([encode(self.mm_encoder, p) for p in multimodal_params]) +``` + +`route` returns this class. The tower is an `nn.Module` and +`MultimodalEncoderMixin` that declares and loads its own weights. The base +builds it as `mm_encoder_class(vision_config, dtype)`, from the composite +config's `vision_config` at the activation dtype; for a tower built another +way, override `build_mm_encoder` instead. By default +the rows reach the language model fused into `inputs_embeds`. With +`language_model_takes_rows = True`, `input_ids` stay as they are and the +language model takes `multimodal_rows=(rows, mm_token_indices)`; the +[qwen4_exp target](models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py) +does this because its PLE layer hashes `input_ids`. Override +`validate_media_request` to refuse, at admission, media the tower cannot +encode. To pass extra inputs to the language model, as V1 Qwen3-VL does for +DeepStack, override `get_language_model_extra_forward_kwargs`. + +**A text-only target** serves the text half of the checkpoint. It defines +`validate_multimodal_request_data` and raises when +`tensorrt_llm.inputs.multimodal.has_media_or_embeddings` is true, so a request +with media fails alone at admission instead of failing its batch in `forward`. +M-RoPE positions alone, as a disaggregated generation request carries, are not +media. + +**A target that encodes media itself** does not use the base. The qwen4_exp +FP8 target encodes, in its own forward, only the frames of the current prefill +chunk. It refuses at admission, in a `validate_request` of its own class (the +resolver does not count an inherited one), what its tower cannot take, and it +gets no shared encoder cache. The contract test lists such targets by name; a +new target is not added to that list, and an entry that is no longer needed +fails the test. + +With `disable_mm_encoder` the model builds no tower, loads no encoder weights +and refuses media like a text-only target. With a tower, the breakable prefill +CUDA graph runs when the language model takes the rows itself +(`language_model_takes_rows`), as for the language model alone; with rows fused +into `inputs_embeds` it is refused at startup. A model with a tower does not +run its encoder on a side stream: the route declines +`encoder_side_stream_max_ahead > 0`. + +**The rule for vision-language models:** + +* One shared base. Code that every V2 vision-language model needs lives in + `ModelingV2ForConditionalGeneration`. A target declares only what differs; + code that several targets would copy goes into the base. +* One name. A V2 class that serves media is + `ModelingV2ForConditionalGeneration` and subclasses the base, never + `...VLM` or `...Model`. Its language model target stays `ModelingV2`. + A text-only target routed for a vision-language checkpoint keeps its name + until it serves media. + +`test_modeling_v2_contracts.py` checks both kinds of target from the source, and +runs each text-only target's admission hook on media, embeddings and M-RoPE +positions. For the name, it checks that a routed multimodal model subclasses the base and that +its name ends with `ForConditionalGeneration`, and that no target class ends +with `VLM`. Review checks the `ModelingV2` part. + ## Routing Each family's `routing.py` defines one `route` function, registered with diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py index a36231c4b081..f83f6140fd56 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py @@ -1,13 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Code contracts of the catalog entries and their cells. +"""Code contracts of Modeling V2 code: the catalog entries and their cells, and the +multimodal contract and naming rule of the targets of vision-language architectures. An entry computes a pure function of its arguments, so its ``CELLS`` cover its -whole input space. These checks keep that true. They read the source and import -nothing, so they run on any CPU machine. +whole input space. A vision-language target runs media through the multimodal +runtime or refuses it at admission, and a vision-language model subclasses the +shared base and its name ends with ``ForConditionalGeneration``. These checks +keep that true. They read the source and import no target, so they run on any +CPU machine; a text-only target's admission hook runs on its own, with the shared +helper. Which architectures take media is the router's own check, from the +built-in input processors. """ from __future__ import annotations +import __future__ import ast import re @@ -197,3 +204,577 @@ def test_target_cells_feed_entries_the_target_calls(test): assert re.search(rf"catalog\.{area}\.{entry} import", used), ( f"{test.name}: declares cells for {target}, which never imports {area}/{entry}" ) + + +# --- the multimodal contract of vision-language targets ---------------------------- + + +def _routed_classes() -> list[tuple[str, Path, str]]: + """`(architecture, modeling.py, class name)` for every class a family's `route` returns. + + Each `return` of `route` names a class that `route` imports from a target's `modeling` + module, as every route does today. Any other return fails here, so no routed class is + skipped unread.""" + found, unread = [], [] + for routing in sorted((_ROOT / "models").glob("*/routing.py")): + for fn in ast.parse(routing.read_text()).body: + if not isinstance(fn, ast.FunctionDef): + continue + archs = [ + arg.value + for dec in fn.decorator_list + if isinstance(dec, ast.Call) and getattr(dec.func, "id", None) == "routes" + for arg in dec.args + if isinstance(arg, ast.Constant) + ] + if not archs: + continue + imported = { + alias.asname or alias.name: Path(routing.parent, *node.module.split(".")) + for node in ast.walk(fn) + if isinstance(node, ast.ImportFrom) + and node.level == 1 + and (node.module or "").endswith(".modeling") + for alias in node.names + } + for node in ast.walk(fn): + if not isinstance(node, ast.Return): + continue + name = getattr(node.value, "id", None) + if name in imported: + found += [(arch, imported[name].with_suffix(".py"), name) for arch in archs] + else: + unread.append(f"{routing.relative_to(_ROOT)}:{node.lineno}") + assert not unread, ( + f"{unread}: return the class that route imports from the target's modeling module " + "directly, so these checks can read it" + ) + return found + + +def _takes_media(architecture: str) -> bool: + """Whether requests for `architecture` can carry media, as the router decides it.""" + from tensorrt_llm._torch.modeling_v2._router_index import _architecture_takes_media + + return _architecture_takes_media(architecture) + + +def _media_routes() -> list[tuple[Path, str]]: + """The classes routed for an architecture whose requests can carry media.""" + return sorted({(path, name) for arch, path, name in _routed_classes() if _takes_media(arch)}) + + +def _calls(fn: ast.AST, name: str) -> bool: + return any( + isinstance(n, ast.Call) and getattr(n.func, "id", getattr(n.func, "attr", None)) == name + for n in ast.walk(fn) + ) + + +def _raises(fn: ast.AST) -> bool: + return any(isinstance(n, ast.Raise) for n in ast.walk(fn)) + + +def _refuses_anything(fn: ast.AST) -> bool: + """Whether the method calls or raises anything: a docstring or `pass` alone refuses nothing.""" + return _raises(fn) or any(isinstance(n, ast.Call) for n in ast.walk(fn)) + + +#: A text-only target's admission error, as the shared helper's users word it. +_TEXT_ONLY_MESSAGE = "{} serves text only; send no images, videos or embeddings" +#: `multimodal_data` a text-only target refuses (the last key is one no helper knows), and the +#: ones it admits: none, or M-RoPE positions alone. +_REFUSED = ( + {"image": {"pixel_values": 0}}, + {"video": {"pixel_values_videos": 0}}, + {"multimodal_embedding": 0}, + {"new_modality": 0}, +) +_ADMITTED = ({}, {"mrope_config": {"mrope_position_deltas": 0}}) + + +def _admission_problems(where: str, name: str, hook: ast.FunctionDef) -> list[str]: + """Run a text-only target's `validate_multimodal_request_data` on its own, as a method of a + class named `name`, with the real `has_media_or_embeddings`: it refuses exactly what the + helper flags, with the shared message. The target module itself is not imported.""" + from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + + namespace = {"has_media_or_embeddings": has_media_or_embeddings} + code = compile( + ast.Module(body=[hook], type_ignores=[]), + where, + "exec", + flags=__future__.annotations.compiler_flag, + dont_inherit=True, + ) + exec(code, namespace) + model = type(name, (), {hook.name: namespace[hook.name]})() + wrong = [] + for data in _ADMITTED: + try: + model.validate_multimodal_request_data(data) + except Exception as error: + wrong.append(f"refuses {sorted(data)}: {error!r}") + for data in _REFUSED: + try: + model.validate_multimodal_request_data(data) + except ValueError as error: + if str(error) != _TEXT_ONLY_MESSAGE.format(name): + wrong.append(f"refuses {sorted(data)} with {str(error)!r}") + except Exception as error: + wrong.append(f"refuses {sorted(data)} with {error!r}, not a ValueError") + else: + wrong.append(f"admits {sorted(data)}") + if not wrong: + return [] + return [ + f"{where}: validate_multimodal_request_data must refuse what has_media_or_embeddings " + f"flags with {_TEXT_ONLY_MESSAGE.format(name)!r} and admit the rest, but {wrong}" + ] + + +#: The shared base of vision-language models, and what a subclass declares: the base does +#: the rest. Its tower is `mm_encoder_class`, or an override of `build_mm_encoder`. +_BASE = "ModelingV2ForConditionalGeneration" +_SUBCLASS_ATTRIBUTES = ("language_model_class", "mm_encoder_weight_prefix") +_SUBCLASS_METHODS = ("encode_multimodal_inputs",) + + +def _class_def(path: Path, name: str) -> ast.ClassDef | None: + """The top-level class `name` of the module at `path`, or None.""" + for node in ast.parse(path.read_text()).body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def _base_names(cls: ast.ClassDef) -> set[str]: + return {getattr(b, "id", getattr(b, "attr", None)) for b in cls.bases} + + +def _assignments(cls: ast.ClassDef) -> dict[str, ast.expr | None]: + """The names the class body assigns, each with its value (None for a bare annotation).""" + return { + t.id: n.value + for n in cls.body + if isinstance(n, (ast.Assign, ast.AnnAssign)) + for t in (n.targets if isinstance(n, ast.Assign) else [n.target]) + if isinstance(t, ast.Name) + } + + +#: Targets that encode media in their own forward instead of on the shared base, as +#: `(target directory under models/, class name)`: the qwen4_exp FP8 target encodes only the +#: frames of the current prefill chunk. Each refuses at admission, in a `validate_request` of +#: its own class, what its tower cannot take; the router accepts it for that alone. Add no +#: entry: a new vision-language target subclasses the base. An entry whose class moves onto +#: the base or serves text only fails until it is deleted, so the set only shrinks. +_OWN_ENCODER_TARGETS = frozenset( + { + ( + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified", + "ModelingV2Qwen38FlashNextFp8Sm103Unified", + ), + } +) + + +def _target_key(path: Path, name: str) -> tuple[str, str]: + return path.parent.relative_to(_ROOT / "models").as_posix(), name + + +def _multimodal_contract_problems(path: Path, name: str) -> list[str]: + """A class routed for media is either a `ModelingV2ForConditionalGeneration` subclass that + declares `_SUBCLASS_ATTRIBUTES` and its tower and defines `_SUBCLASS_METHODS`, or serves + text only and refuses media at admission as `_admission_problems` checks, or is listed in + `_OWN_ENCODER_TARGETS`, still needs to be, and defines a `validate_request` that refuses + something. A `forward` that raises on multimodal input uses `has_media_or_embeddings`.""" + where = f"{path.relative_to(_ROOT)}:{name}" + cls = _class_def(path, name) + if cls is None: + return [f"{where}: no such class"] + defs = {n.name: n for n in cls.body if isinstance(n, ast.FunctionDef)} + assigned = _assignments(cls) + problems = [] + listed = _target_key(path, name) in _OWN_ENCODER_TARGETS + if listed and (_BASE in _base_names(cls) or "validate_multimodal_request_data" in defs): + # The class meets the contract another way now: drop the entry, check it as any other. + problems.append(f"{where}: no longer encodes media itself; delete its entry") + listed = False + if listed: + hook = defs.get("validate_request") + if hook is None: + problems.append(f"{where}: encodes media itself and does not define validate_request") + elif not _refuses_anything(hook): + problems.append( + f"{where}: encodes media itself and its validate_request refuses nothing" + ) + elif _BASE in _base_names(cls): + problems += [ + f"{where}: the subclass does not declare {m}" + for m in _SUBCLASS_ATTRIBUTES + if assigned.get(m) is None + ] + problems += [ + f"{where}: the subclass does not define {m}" for m in _SUBCLASS_METHODS if m not in defs + ] + # A bare annotation or `None` builds no tower. + tower = assigned.get("mm_encoder_class") + if "build_mm_encoder" not in defs and ( + tower is None or (isinstance(tower, ast.Constant) and tower.value is None) + ): + problems.append( + f"{where}: the subclass neither sets mm_encoder_class to a tower class nor " + "defines build_mm_encoder" + ) + else: + hook = defs.get("validate_multimodal_request_data") + if hook is None: + problems.append( + f"{where}: neither a {_BASE} subclass nor a text-only target with " + "validate_multimodal_request_data" + ) + else: + problems += _admission_problems(where, name, hook) + forward = defs.get("forward") + if ( + forward is not None + and _raises(forward) + and "multimodal_params" in ast.unparse(forward) + and not _calls(forward, "has_media_or_embeddings") + ): + problems.append( + f"{where}: forward refuses multimodal input without has_media_or_embeddings" + ) + return problems + + +def test_vision_language_targets_keep_the_multimodal_contract(): + """Every class routed for an architecture whose requests can carry media either runs media + through the multimodal runtime or refuses it at admission, where only that request fails. + Each text-only target's admission hook runs on media, embeddings and M-RoPE positions. The + router checks the contract's shape at startup; this finds a broken target without a GPU.""" + routed = _media_routes() + families = {path.relative_to(_ROOT / "models").parts[0] for path, _ in routed} + assert {"glm5_next", "kimi_k3_vl", "minimax_m3", "qwen4_exp"} <= families, families + text_only = { + path.relative_to(_ROOT / "models").parts[0] + for path, name in routed + if any( + isinstance(node, ast.FunctionDef) and node.name == "validate_multimodal_request_data" + for node in getattr(_class_def(path, name), "body", ()) + ) + } + assert {"glm5_next", "kimi_k3_vl", "minimax_m3"} <= text_only, text_only + stale = _OWN_ENCODER_TARGETS - {_target_key(path, name) for path, name in routed} + assert not stale, f"{sorted(stale)}: no longer routed for media; delete the entry" + problems = [p for path, name in routed for p in _multimodal_contract_problems(path, name)] + assert not problems, problems + + +_FIXTURE_ROUTING = """ +from ..._router_index import routes + + +@routes("{arch}") +def route(model_config, llm_args, compute_capability): + from .target.modeling import {cls} + + return {cls} +""" + +_TEXT_ONLY_OK = """ +class Good(Base): + def validate_multimodal_request_data(self, mm_data: dict) -> None: + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + + def forward(self, *args, **kwargs): + params = kwargs.pop("multimodal_params", None) or [] + if any(has_media_or_embeddings(p.multimodal_data) for p in params): + raise ValueError("text only") +""" + +_SUBCLASS_OK = """ +class Good(ModelingV2ForConditionalGeneration): + language_model_class = Llm + mm_encoder_class = Tower + mm_encoder_weight_prefix = "model.visual." + + def encode_multimodal_inputs(self, multimodal_params): + return None +""" +_BUILT_TOWER = """ + def build_mm_encoder(self, pretrained_config): + return Tower(pretrained_config.audio_config) +""" + + +@pytest.mark.parametrize( + "source, problem", + [ + (_TEXT_ONLY_OK, None), + (_SUBCLASS_OK, None), + ("class Good(Base):\n pass\n", "neither a ModelingV2ForConditionalGeneration"), + ( + _TEXT_ONLY_OK.replace("if has_media_or_embeddings(mm_data)", "if mm_data"), + "but [\"refuses ['mrope_config']", + ), + ( + _TEXT_ONLY_OK.replace("if has_media_or_embeddings", "if not has_media_or_embeddings"), + 'but ["refuses []', + ), + ( + _TEXT_ONLY_OK.replace("raise ValueError(", "ValueError("), + "but [\"admits ['image']\"", + ), + ( + _TEXT_ONLY_OK.replace("serves text only", "is text only"), + "but [\"refuses ['image'] with 'Good is text only", + ), + ( + _TEXT_ONLY_OK.replace("raise ValueError(", "raise RuntimeError("), + "not a ValueError", + ), + ( + _TEXT_ONLY_OK.replace( + "any(has_media_or_embeddings(p.multimodal_data) for p in params)", "params" + ), + "forward refuses multimodal input", + ), + ( + _SUBCLASS_OK.replace(" language_model_class = Llm\n", ""), + "does not declare language_model_class", + ), + ( + _SUBCLASS_OK.replace("def encode_multimodal_inputs", "def other"), + "does not define encode_multimodal_inputs", + ), + (_SUBCLASS_OK.replace(" mm_encoder_class = Tower\n", "") + _BUILT_TOWER, None), + ( + _SUBCLASS_OK.replace(" mm_encoder_class = Tower\n", ""), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ( + _SUBCLASS_OK.replace("mm_encoder_class = Tower", "mm_encoder_class = None"), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ( + _SUBCLASS_OK.replace("mm_encoder_class = Tower", "mm_encoder_class: type"), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ], + ids=[ + "text_only_ok", + "subclass_ok", + "no_hook", + "truthiness_hook", + "inverted_hook", + "hook_never_raises", + "other_message", + "other_error_type", + "truthiness_forward", + "subclass_without_language_model_class", + "subclass_without_encoder_hook", + "subclass_building_its_tower", + "subclass_without_tower", + "subclass_with_a_none_tower", + "subclass_with_a_bare_tower_annotation", + ], +) +def test_the_multimodal_contract_check_reports_a_broken_target( + tmp_path, monkeypatch, source, problem +): + """The architecture's input processor picks the classes to check, not its name: the + media architecture is named like a text one and the text one like a vision-language + one.""" + for arch, family in (("FooForCausalLM", "foo"), ("BarForConditionalGeneration", "bar")): + target = tmp_path / "models" / family / "target" + target.mkdir(parents=True) + (target.parent / "routing.py").write_text(_FIXTURE_ROUTING.format(arch=arch, cls="Good")) + # The text architecture's class is never checked. + (target / "modeling.py").write_text( + source if family == "foo" else "class Good:\n pass\n" + ) + module = sys.modules[__name__] + monkeypatch.setattr(module, "_ROOT", tmp_path) + monkeypatch.setattr(module, "_takes_media", lambda arch: arch == "FooForCausalLM") + routed = _media_routes() + assert routed == [(tmp_path / "models/foo/target/modeling.py", "Good")] + problems = [p for path, name in routed for p in _multimodal_contract_problems(path, name)] + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems + + +_OWN_ENCODER_OK = """ +class Good(Base): + def validate_request(self, request): + \"\"\"Refuse a media request the tower would assert on.\"\"\" + vision.validate_request(self.vision_tower, request) +""" + + +@pytest.mark.parametrize( + "source, problem", + [ + (_OWN_ENCODER_OK, None), + ("class Good(Base):\n pass\n", "does not define validate_request"), + ( + _OWN_ENCODER_OK.replace("vision.validate_request(self.vision_tower, request)", "pass"), + "its validate_request refuses nothing", + ), + (_SUBCLASS_OK, "no longer encodes media itself; delete its entry"), + (_TEXT_ONLY_OK, "no longer encodes media itself; delete its entry"), + ], + ids=[ + "own_encoder_ok", + "own_encoder_without_validate_request", + "own_encoder_refusing_nothing", + "moved_onto_the_base", + "now_text_only", + ], +) +def test_a_listed_own_encoder_target_must_refuse_at_admission( + tmp_path, monkeypatch, source, problem +): + """A listed target needs a `validate_request` of its own that refuses something, and the + entry must still be needed: a class that moved onto the base or serves text only is checked + as such, and its entry is reported for deletion.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + (target / "modeling.py").write_text(source) + module = sys.modules[__name__] + monkeypatch.setattr(module, "_ROOT", tmp_path) + monkeypatch.setattr(module, "_OWN_ENCODER_TARGETS", frozenset({("foo/target", "Good")})) + problems = _multimodal_contract_problems(target / "modeling.py", "Good") + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems + + +def test_a_route_return_the_checks_cannot_read_fails(tmp_path, monkeypatch): + """A route that returns its class through a local name fails instead of being skipped.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + routing = _FIXTURE_ROUTING.format(arch="FooForConditionalGeneration", cls="Good") + (target.parent / "routing.py").write_text( + routing.replace(" return Good", " target_class = Good\n return target_class") + ) + (target / "modeling.py").write_text(_SUBCLASS_OK) + monkeypatch.setattr(sys.modules[__name__], "_ROOT", tmp_path) + with pytest.raises(AssertionError, match=r"models/foo/routing\.py:10\b"): + _routed_classes() + + +# --- the naming rule of vision-language models ---------------------------------------- + +#: A class with one of these bases is a multimodal model. +_MULTIMODAL_BASES = {_BASE, "MultimodalModelMixin"} + + +def _routed_multimodal_models() -> list[tuple[Path, ast.ClassDef]]: + """Every routed class that is a multimodal model, whatever its architecture.""" + found = [] + for path, name in sorted({(path, name) for _, path, name in _routed_classes()}): + cls = _class_def(path, name) + if cls is not None and _base_names(cls) & _MULTIMODAL_BASES: + found.append((path, cls)) + return found + + +def _naming_rule_problems() -> list[str]: + """A routed multimodal model subclasses `ModelingV2ForConditionalGeneration` and its name + ends with `ForConditionalGeneration`; no class of a target's `modeling.py` is named + `...VLM`.""" + problems = [] + for path, cls in _routed_multimodal_models(): + where = f"{path.relative_to(_ROOT)}:{cls.name}" + if _BASE not in _base_names(cls): + problems.append(f"{where}: a multimodal model that does not subclass {_BASE}") + if not cls.name.endswith("ForConditionalGeneration"): + problems.append(f"{where}: a multimodal model not named ...ForConditionalGeneration") + for path in sorted((_ROOT / "models").glob("*/*/modeling.py")): + problems += [ + f"{path.relative_to(_ROOT)}:{node.name}: a target class named ...VLM" + for node in ast.walk(ast.parse(path.read_text())) + if isinstance(node, ast.ClassDef) and node.name.upper().endswith("VLM") + ] + return problems + + +def test_vision_language_models_follow_the_naming_rule(): + """Every routed multimodal model subclasses the shared base and its name ends with + `ForConditionalGeneration`, and no target class is named `...VLM`. The rest of the name, + `ModelingV2`, is left to review. A language model target is not a multimodal + model, so its `ModelingV2` name is not checked here.""" + families = { + path.relative_to(_ROOT / "models").parts[0] for path, _ in _routed_multimodal_models() + } + assert "qwen4_exp" in families, families + problems = _naming_rule_problems() + assert not problems, problems + + +_NAMED_OK = """ +class ModelingV2Foo(Base): + pass + + +class ModelingV2FooForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = ModelingV2Foo +""" + + +@pytest.mark.parametrize( + "routed, source, problem", + [ + ("ModelingV2FooForConditionalGeneration", _NAMED_OK, None), + # A language model target routed on its own is not a multimodal model. + ("ModelingV2Foo", _NAMED_OK, None), + ( + "ModelingV2FooForConditionalGeneration", + _NAMED_OK.replace(f"({_BASE})", "(MultimodalModelMixin, PreTrainedModel)"), + f"does not subclass {_BASE}", + ), + ( + "ModelingV2FooModel", + _NAMED_OK.replace("ModelingV2FooForConditionalGeneration", "ModelingV2FooModel"), + "not named ...ForConditionalGeneration", + ), + ( + "ModelingV2FooForConditionalGeneration", + _NAMED_OK.replace("ModelingV2Foo(", "ModelingV2FooVLM(").replace( + "= ModelingV2Foo\n", "= ModelingV2FooVLM\n" + ), + "ModelingV2FooVLM: a target class named ...VLM", + ), + ], + ids=[ + "model_ok", + "language_model_ok", + "mixin_not_the_base", + "model_suffix", + "vlm_suffix", + ], +) +def test_the_naming_rule_check_reports_each_broken_part( + tmp_path, monkeypatch, routed, source, problem +): + """Each stand-in breaks one part of the rule and gets exactly that problem.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + (target.parent / "routing.py").write_text( + _FIXTURE_ROUTING.format(arch="FooForConditionalGeneration", cls=routed) + ) + (target / "modeling.py").write_text(source) + monkeypatch.setattr(sys.modules[__name__], "_ROOT", tmp_path) + problems = _naming_rule_problems() + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems