diff --git a/docs/source/torch/adding_new_model.md b/docs/source/torch/adding_new_model.md index 006941981f8e..bc22e8324bf6 100644 --- a/docs/source/torch/adding_new_model.md +++ b/docs/source/torch/adding_new_model.md @@ -128,6 +128,18 @@ Optionally, you may replace the native PyTorch modules with our implementations For a concrete reference, check out `tensorrt_llm/_torch/models/modeling_llama.py`. +#### Multimodal Models + +A Modeling V2 vision-language model has the composite shape of a Modeling V1 +one: it holds the Modeling V2 language model as `llm` and the tower as +`mm_encoder`, and `MultimodalModelMixin` runs it. Name it +`ModelingV2ForConditionalGeneration` (never `...VLM` or `...Model`) and +subclass `ModelingV2ForConditionalGeneration`, the shared base. Declare only +what differs from the base; code that several targets would copy goes into the +base. The language model target keeps its name, `ModelingV2`. See +"Multimodal targets" in `tensorrt_llm/_torch/modeling_v2/README.md` for the +template. + ### Weight Loading The base class `DecoderModelForCausalLM` provides a `load_weights` method that loads the weights from the checkpoint file and assigns them to the corresponding layers in the model. However, if the default method does not work for `MyModelForCausalLM`, you need to implement your own `load_weights`: diff --git a/tensorrt_llm/_torch/modeling_v2/README.md b/tensorrt_llm/_torch/modeling_v2/README.md index 3a00db3e0066..c8f8e3b37e12 100644 --- a/tensorrt_llm/_torch/modeling_v2/README.md +++ b/tensorrt_llm/_torch/modeling_v2/README.md @@ -81,7 +81,9 @@ double underscores between the three parts: The target class name contains the same three parts in CamelCase, for example `ModelingV2DeepseekR10528Nvfp4Sm10xDefault` in [`r1_0528_nvfp4__sm_10x__default`](models/deepseek_v3/r1_0528_nvfp4__sm_10x__default/). -These names are conventions, not requirements. +These names are conventions, not requirements. The one rule is for a +vision-language model, whose class name ends with `ForConditionalGeneration` +(see [Multimodal targets](#multimodal-targets)). A target can contain several execution paths. The DeepSeek R1 target, for example, has separate `PrefillTarget` and `DecodeTarget` classes and picks one @@ -93,6 +95,7 @@ on each forward step. In this guide, "target" always means the whole directory. modeling_v2/ ├── _router_index.py # Route registration and target resolution ├── _native.py # Loads family native libraries +├── multimodal.py # Base class of vision-language targets ├── catalog/ │ ├── _op.py # OpWrapper: the operation contract │ └── / # Operations used by more than one family @@ -134,6 +137,11 @@ family's and its own operations and kernels. From the rest of TensorRT-LLM, it uses only runtime interfaces, for example: * `DecoderModel`, `DecoderModelForCausalLM`, `ModelConfig` and `Mapping`. +* `MultimodalModelMixin` and `MultimodalEncoderMixin`. A vision-language target + inherits the first through `ModelingV2ForConditionalGeneration` + ([`multimodal.py`](multimodal.py)), and its tower the second. It does not + subclass or call a Modeling V1 model, vision tower, layer, attention backend + or MoE. * Attention and speculative metadata, and the KV cache managers. A target does not import, subclass or call Modeling V1 model code: the model @@ -172,8 +180,11 @@ example that uses no Modeling V1 model code. ## Model lifecycle A target plugs into TensorRT-LLM through the existing `DecoderModel` and -`DecoderModelForCausalLM` interfaces. It inherits these classes directly, not -a Modeling V1 model class, and can override any of these steps: +`DecoderModelForCausalLM` interfaces. A language model target inherits these +classes directly, not a Modeling V1 model class. A vision-language target +subclasses `ModelingV2ForConditionalGeneration`, which holds such a language +model target as `llm` (see [Multimodal targets](#multimodal-targets)). Either +can override any of these steps: * **Constructor:** builds the model structure and parameters. * **`load_weights`:** loads checkpoint weights into the model. Put weight @@ -192,6 +203,87 @@ classes are fine when they make the code clearer or faster. The its [weight loader](models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py) are a small example. +## Multimodal targets + +A vision-language architecture routes to one of two kinds of target. The +resolver checks every class a `route` returns: when the architecture's input +processor takes media, the class must be one of these two (or an existing +target that encodes media itself, see below), or startup fails. + +**A vision-language model** runs media through the shared multimodal runtime +and has the composite shape of a Modeling V1 vision-language model: it holds a +language model target as `llm` and the encoder tower as `mm_encoder`, and +`MultimodalModelMixin` runs it. Subclass `ModelingV2ForConditionalGeneration` +from [`multimodal.py`](multimodal.py). This shared base does the common work +once: the config split, admission, encoder caching and the current chunk's rows +(through `MultimodalModelMixin`), the compile scope, the weight split, and what +the engine reads off the routed class and model. A target declares what +differs: + +```python +class ModelingV2MyTargetForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = ModelingV2MyTarget # the language model target + mm_encoder_class = MyTower # MyTower(vision_config, dtype) + mm_encoder_weight_prefix = "model.visual." # the tower's checkpoint keys + + def encode_multimodal_inputs(self, multimodal_params): + return torch.cat([encode(self.mm_encoder, p) for p in multimodal_params]) +``` + +`route` returns this class. The tower is an `nn.Module` and +`MultimodalEncoderMixin` that declares and loads its own weights. The base +builds it as `mm_encoder_class(vision_config, dtype)`, from the composite +config's `vision_config` at the activation dtype; for a tower built another +way, override `build_mm_encoder` instead. By default +the rows reach the language model fused into `inputs_embeds`. With +`language_model_takes_rows = True`, `input_ids` stay as they are and the +language model takes `multimodal_rows=(rows, mm_token_indices)`; the +[qwen4_exp target](models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py) +does this because its PLE layer hashes `input_ids`. Override +`validate_media_request` to refuse, at admission, media the tower cannot +encode. To pass extra inputs to the language model, as V1 Qwen3-VL does for +DeepStack, override `get_language_model_extra_forward_kwargs`. + +**A text-only target** serves the text half of the checkpoint. It defines +`validate_multimodal_request_data` and raises when +`tensorrt_llm.inputs.multimodal.has_media_or_embeddings` is true, so a request +with media fails alone at admission instead of failing its batch in `forward`. +M-RoPE positions alone, as a disaggregated generation request carries, are not +media. + +**A target that encodes media itself** does not use the base. The qwen4_exp +FP8 target encodes, in its own forward, only the frames of the current prefill +chunk. It refuses at admission, in a `validate_request` of its own class (the +resolver does not count an inherited one), what its tower cannot take, and it +gets no shared encoder cache. The contract test lists such targets by name; a +new target is not added to that list, and an entry that is no longer needed +fails the test. + +With `disable_mm_encoder` the model builds no tower, loads no encoder weights +and refuses media like a text-only target. With a tower, the breakable prefill +CUDA graph runs when the language model takes the rows itself +(`language_model_takes_rows`), as for the language model alone; with rows fused +into `inputs_embeds` it is refused at startup. A model with a tower does not +run its encoder on a side stream: the route declines +`encoder_side_stream_max_ahead > 0`. + +**The rule for vision-language models:** + +* One shared base. Code that every V2 vision-language model needs lives in + `ModelingV2ForConditionalGeneration`. A target declares only what differs; + code that several targets would copy goes into the base. +* One name. A V2 class that serves media is + `ModelingV2ForConditionalGeneration` and subclasses the base, never + `...VLM` or `...Model`. Its language model target stays `ModelingV2`. + A text-only target routed for a vision-language checkpoint keeps its name + until it serves media. + +`test_modeling_v2_contracts.py` checks both kinds of target from the source, and +runs each text-only target's admission hook on media, embeddings and M-RoPE +positions. For the name, it checks that a routed multimodal model subclasses the base and that +its name ends with `ForConditionalGeneration`, and that no target class ends +with `VLM`. Review checks the `ModelingV2` part. + ## Routing Each family's `routing.py` defines one `route` function, registered with diff --git a/tensorrt_llm/_torch/modeling_v2/_router_index.py b/tensorrt_llm/_torch/modeling_v2/_router_index.py index 56570da39a4e..0d8cf89ebf38 100644 --- a/tensorrt_llm/_torch/modeling_v2/_router_index.py +++ b/tensorrt_llm/_torch/modeling_v2/_router_index.py @@ -306,6 +306,71 @@ def _refuse_unhonored_moe_tactic( ) +def _architecture_takes_media(architecture: str) -> bool: + """Whether requests for ``architecture`` can carry media: the input processor registered + for its built-in class, which the LLM API serves every deployment of it with, is a + multimodal one.""" + from tensorrt_llm._torch.models.modeling_utils import get_registered_model_class + from tensorrt_llm.inputs.registry import INPUT_PROCESSOR_REGISTRY, BaseMultimodalInputProcessor + + model_cls = get_registered_model_class(architecture) + processor_cls = INPUT_PROCESSOR_REGISTRY.get_input_processor_cls(model_cls) + return processor_cls is not None and issubclass(processor_cls, BaseMultimodalInputProcessor) + + +def _check_multimodal_contract(target: type, architecture: str) -> None: + """Raise ``ValueError`` when a target of an architecture whose requests can carry media + neither runs media through the multimodal runtime nor refuses it at admission. + + The first is a ``ModelingV2ForConditionalGeneration`` (``multimodal.py``), or any other + ``MultimodalModelMixin`` model or class that sets the ``supports_multimodal`` marker; the + second defines ``validate_multimodal_request_data`` and raises there on media. A target that + encodes media in its own forward refuses what its tower cannot take in a ``validate_request`` + of its own class; an inherited one, such as a token range check, does not count. A target + that does none of these would run placeholder tokens as text or fail a whole batch + in ``forward``: a bug in the target, so this raises in every mode (a plain ``ValueError``, + not a ``NoTarget``). + """ + from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin + + if ( + issubclass(target, MultimodalModelMixin) + or getattr(target, "supports_multimodal", False) is True + or callable(getattr(target, "validate_multimodal_request_data", None)) + or callable(vars(target).get("validate_request")) + ): + return + if _architecture_takes_media(architecture): + raise ValueError( + f"{target.__qualname__} is the target for {architecture!r}, whose requests can " + "carry images, videos or audio. Make it a ModelingV2ForConditionalGeneration " + "subclass to serve media, or define validate_multimodal_request_data to refuse " + "media at admission and serve text only." + ) + + +def _refuse_encoder_side_stream(config: "ModelConfig", target: type) -> None: + """Raise ``NoTarget`` when a ``ModelingV2ForConditionalGeneration`` target would build its + tower and the engine runs the encoder on a side stream: the base runs its encoder inside + the step only. ``auto`` then uses the built-in implementation. An encoder-only deployment + (``mm_encoder_only``) builds the architecture's encoder model, not this one.""" + from .multimodal import ModelingV2ForConditionalGeneration + + if ( + not issubclass(target, ModelingV2ForConditionalGeneration) + or config.disable_mm_encoder + or config.mm_encoder_only + ): + return + mm_config = config.multimodal_config + if mm_config is not None and mm_config.encoder_side_stream_max_ahead > 0: + raise NoTarget( + "multimodal_config.encoder_side_stream_max_ahead=" + f"{mm_config.encoder_side_stream_max_ahead}; {target.__name__} does not run its " + "encoder on a side stream" + ) + + def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: """Decide which modeling_v2 target, if any, this deployment builds. @@ -314,7 +379,8 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti for the architecture, or when that ``route`` raises ``NoTarget``. In ``require`` every one of those but ``off`` raises ``ValueError`` instead. It also returns None for a ``@disable_v2_by_default`` target when - ``modeling_v2`` was left at its default. + ``modeling_v2`` was left at its default. A target that breaks the multimodal + contract (:func:`_check_multimodal_contract`) raises in every mode. """ mode = ModelingV2Mode.of(llm_args) @@ -334,6 +400,8 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti target = route(config, llm_args, _device_compute_capability()) if target is not None: _refuse_unhonored_moe_tactics(config, target, arch) + _check_multimodal_contract(target, arch) + _refuse_encoder_side_stream(config, target) except NoTarget as why: if mode is ModelingV2Mode.REQUIRE: raise ValueError(f"modeling_v2='require', but no target fits {arch!r}: {why}") from None diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py index f1a5567008af..3e091bbd8328 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/modeling.py @@ -52,6 +52,7 @@ from tensorrt_llm._torch.speculative import get_spec_worker, should_use_separate_draft_kv_cache from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.functional import AllReduceFusionOp, AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping as ParallelMapping @@ -73,7 +74,6 @@ if TYPE_CHECKING: from tensorrt_llm._torch.pyexecutor.guided_decoder import CapturableGuidedDecoder - from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.speculative.interface import SpecMetadata #: The TP all-reduce of more rows than this runs on NCCL; fewer on the one-shot kernel, @@ -746,15 +746,13 @@ def infer_max_seq_len(self) -> int: """The text config's context limit; the model has no RoPE (NoPE MLA, KDA).""" return int(self.text_config.max_position_embeddings) - def validate_request(self, request: LlmRequest) -> None: - """Reject media on admission, as that request's own error: the vision tower is not built, - and the forward would assert on the whole step.""" - data = request.py_multimodal_data or {} - media = [kind for kind in ("image", "video") if data.get(kind)] - if media: + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"this GLM-5.3-Flash deployment serves text only (disable_mm_encoder); the " - f"request carries {' and '.join(media)} input" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) def set_guided_decoder(self, guided_decoder: CapturableGuidedDecoder) -> bool: diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py index f8795c714634..000cde915130 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/modeling.py @@ -68,6 +68,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import unwrap_glm5_next_text_config from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm._utils import async_tensor_h2d +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -84,9 +85,6 @@ # and the certification is per arch. _SM = (10, 0) -#: ``multimodal_data`` entries that carry media for an encoder, or encoder output in place of it. -_MEDIA_KEYS = frozenset({"audio", "image", "video", "multimodal_embedding"}) - #: Stock code the generic path runs outside the catalog, declared so it is not consumed silently: every #: tensorrt_llm import of this module that computes (this target's unit tests check the list both ways, with #: ``_glm5_next_generic_calls.py``). An entry leaves the list when a catalog entry replaces it. @@ -128,6 +126,8 @@ "tensorrt_llm._torch.modules.embedding.Embedding", "tensorrt_llm._torch.distributed.AllReduce", "tensorrt_llm._torch.modules.multi_stream_utils.maybe_execute_in_parallel", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) @@ -2087,14 +2087,10 @@ def load_weights(self, weights: Any, weight_mapper: BaseWeightMapper | None = No _weights.load(self, weights, weight_mapper) def validate_multimodal_request_data(self, mm_data: dict) -> None: - """Refuse, at admission, a request whose multimodal payload carries media: this target runs no encoder. - - Refusing here fails that request alone; the forward would otherwise serve its text without the media. - A payload without media is accepted. - """ - media = sorted(key for key in _MEDIA_KEYS if mm_data.get(key) is not None) - if media: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"{type(self).__name__} serves this checkpoint's language model only and does not accept " - f"multimodal input; the request carries {', '.join(media)}" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py index 6a78ccb11edf..fcf9249fe215 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py @@ -76,6 +76,7 @@ from tensorrt_llm._torch.speculative.spec_tail_kernels import strict_accept from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm._utils import async_tensor_h2d +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -93,9 +94,6 @@ # and the certification is per arch. _SM = (10, 0) -#: ``multimodal_data`` entries that carry media for an encoder, or encoder output in place of it. -_MEDIA_KEYS = frozenset({"audio", "image", "video", "multimodal_embedding"}) - #: Stock code the generic path runs outside the catalog, declared so it is not consumed silently: every #: tensorrt_llm import of this module that computes (this target's unit tests check the list both ways, with #: ``_glm5_next_generic_calls.py``). An entry leaves the list when a catalog entry replaces it. @@ -144,6 +142,8 @@ "tensorrt_llm._torch.distributed.ops.allgather", "tensorrt_llm._torch.speculative.interface.INVALID_PROMPT_LOOKAHEAD_TOKEN", "tensorrt_llm._torch.moe.fused_moe.moe_load_balancer_set_repeated_for_next_layer", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) @@ -2479,14 +2479,10 @@ def load_weights(self, weights: Any, weight_mapper: BaseWeightMapper | None = No _weights.load(self, weights, weight_mapper) def validate_multimodal_request_data(self, mm_data: dict) -> None: - """Refuse, at admission, a request whose multimodal payload carries media: this target runs no encoder. - - Refusing here fails that request alone; the forward would otherwise serve its text without the media. - A payload without media is accepted. - """ - media = sorted(key for key in _MEDIA_KEYS if mm_data.get(key) is not None) - if media: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): raise ValueError( - f"{type(self).__name__} serves this checkpoint's language model only and does not accept " - f"multimodal input; the request carries {', '.join(media)}" + f"{type(self).__name__} serves text only; send no images, videos or embeddings" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py index ed5c5c9205a9..fcf64283d8d8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/common/weight_io.py @@ -18,6 +18,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional from tensorrt_llm._torch.models.checkpoints.base_weight_loader import ConsumableWeightsDict +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger __all__ = [ @@ -89,8 +90,12 @@ def text_model_weights(weights) -> Any: def check_text_only(multimodal_params: Optional[List[Any]]) -> None: - """Refuse a step carrying multimodal input (the engine passes only requests with content).""" - if multimodal_params: + """Refuse a step carrying media or embeddings. M-RoPE positions alone pass, as they pass the + targets' admission check (``has_media_or_embeddings``).""" + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) + for p in multimodal_params or [] + ): raise ValueError( "this Kimi K3 target is text only: it builds no vision tower, and a request carried " "multimodal input" diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py index 4537d92b0e17..d43964df3576 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/modeling.py @@ -140,6 +140,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import is_mla from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -249,6 +250,8 @@ "tensorrt_llm._torch.models.modeling_dspark.draft_is_embedded_in_target", "tensorrt_llm._torch.models.modeling_dflash.DFlashForCausalLM", "tensorrt_llm._torch.pyexecutor.config_utils.is_mla", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) # The K3 decode kernels' bounds: the token-count kernels take one tile of DECODE_MAX_TOKENS rows; the request-aware @@ -3481,6 +3484,15 @@ def _check_step_contract(self, attn_metadata: AttentionMetadata) -> None: ) self._step_checked = True + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -3492,7 +3504,10 @@ def forward( resource_manager=None, **kwargs, ) -> torch.Tensor: - if kwargs.pop("multimodal_params", None): + multimodal_params = kwargs.pop("multimodal_params", None) or [] + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this Kimi K3 target is text only: it loads no vision tower, and a request carried image input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py index c0cbe46118eb..274eeb2da65b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/modeling.py @@ -143,6 +143,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import is_mla from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -252,6 +253,8 @@ "tensorrt_llm._torch.models.modeling_dspark.draft_is_embedded_in_target", "tensorrt_llm._torch.models.modeling_dflash.DFlashForCausalLM", "tensorrt_llm._torch.pyexecutor.config_utils.is_mla", + # The admission check that refuses media (`validate_multimodal_request_data`). + "tensorrt_llm.inputs.multimodal.has_media_or_embeddings", ) # The K3 decode kernels' bounds: the token-count kernels take one tile of DECODE_MAX_TOKENS rows; the request-aware @@ -3595,6 +3598,15 @@ def _check_step_contract(self, attn_metadata: AttentionMetadata) -> None: ) self._step_checked = True + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -3606,7 +3618,10 @@ def forward( resource_manager=None, **kwargs, ) -> torch.Tensor: - if kwargs.pop("multimodal_params", None): + multimodal_params = kwargs.pop("multimodal_params", None) or [] + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this Kimi K3 target is text only: it loads no vision tower, and a request carried image input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py index 876c17dfcf03..379fad49f19a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16/modeling.py @@ -120,7 +120,7 @@ maybe_compiled_copy_, ) from tensorrt_llm._utils import get_sm_version, is_sm_100f -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -5640,6 +5640,15 @@ def set_guided_decoder(self, *args, **kwargs): def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + @torch.inference_mode() def forward( self, diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py index 5f5a478601c9..fc501c343890 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8/modeling.py @@ -130,7 +130,7 @@ ) from tensorrt_llm._utils import get_sm_version, is_sm_100f from tensorrt_llm.functional import AllReduceStrategy -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -7710,5 +7710,14 @@ def runtime_hooks(self) -> RuntimeHooks: """The family's hooks, with ``KimiK3Sm107Helix8CacheManager`` as the KV cache manager.""" return replace(super().runtime_hooks, kv_cache_manager_cls=KimiK3Sm107Helix8CacheManager) + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + __all__ = ["ModelingV2KimiK3Nvfp4Sm107Helix8Moeep8"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py index 42501c8f9179..1377cc89661d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4/modeling.py @@ -130,7 +130,7 @@ ) from tensorrt_llm._utils import get_sm_version, is_sm_100f from tensorrt_llm.functional import AllReduceStrategy -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -7710,5 +7710,14 @@ def runtime_hooks(self) -> RuntimeHooks: """The family's hooks, with ``KimiK3Sm107Helix8CacheManager`` as the KV cache manager.""" return replace(super().runtime_hooks, kv_cache_manager_cls=KimiK3Sm107Helix8CacheManager) + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + __all__ = ["ModelingV2KimiK3Nvfp4Sm107Helix8Moetp2ep4"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py index c94888750224..26fcd2282233 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1/modeling.py @@ -97,7 +97,7 @@ maybe_compiled_copy_, ) from tensorrt_llm._utils import get_sm_version, is_sm_100f -from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.multimodal import MultimodalParams, has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -4701,6 +4701,15 @@ def set_guided_decoder(self, *args, **kwargs): def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + @torch.inference_mode() def forward( self, diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py index 101a3660b623..fee4863d2dc4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py @@ -137,6 +137,7 @@ ) from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -4585,6 +4586,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -4599,7 +4609,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py index da29e313753a..315f035b388f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py @@ -139,6 +139,7 @@ ) from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -4254,6 +4255,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -4268,7 +4278,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py index d8a6afc8b7e6..9b8409b1375d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py @@ -240,6 +240,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen, get_model_extra_attrs from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -6183,6 +6184,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -6197,7 +6207,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft, and on a decode step the spec fold's store of its outputs). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py index a0c944ab0dd7..49ba2ad7dfaa 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py @@ -218,6 +218,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen, is_torch_compiling from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -5647,6 +5648,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -5661,7 +5671,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py index ddb3f5f54c6c..d2f2e4c7517b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py @@ -235,6 +235,7 @@ from tensorrt_llm._torch.utils import ActType_TrtllmGen from tensorrt_llm._utils import mpi_rank, prefer_pinned from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo @@ -5855,6 +5856,15 @@ def post_load_weights(self) -> None: super().post_load_weights() self.model.post_load_weights() + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: this target serves text only, so a request with media or + embeddings fails alone here instead of failing its batch in forward. A payload of + M-RoPE positions only passes.""" + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + def forward( self, attn_metadata: AttentionMetadata, @@ -5869,7 +5879,9 @@ def forward( """One engine step: the target forward, then the logits (or, with speculation, the worker's verify and draft, and on a decode step the spec fold's store of its outputs). Text only.""" multimodal_params = kwargs.pop("multimodal_params", None) or [] - if any(p is not None and getattr(p, "multimodal_data", None) for p in multimodal_params): + if any( + p is not None and has_media_or_embeddings(p.multimodal_data) for p in multimodal_params + ): raise ValueError( "this MiniMax-M3 target is text only; the request carries multimodal input" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 2383908a6d17..7e7fbb28e6e9 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -418,6 +418,7 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( gdn_prefill_conv_post, ) +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM from tensorrt_llm._torch.modules.fla.cached_replay import CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE from tensorrt_llm._torch.modules.fla.flashinfer_cached_replay import ( @@ -1037,8 +1038,8 @@ class _DraftBound(NamedTuple): class Qwen4ExpDefaultModelingV2(DecoderModel): def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig): """`composite_config` is passed in because the text sub-config adopted as - `model_config.pretrained_config` carries neither the vision half nor the media - token ids.""" + `model_config.pretrained_config` carries no media token ids. The vision tower is + the vision-language model's, not the core's.""" super().__init__(model_config) cfg = model_config.pretrained_config mapping = model_config.mapping @@ -1225,10 +1226,7 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig # device-address view that owns none of those pages. self._ple_table_host = None - # ---- the vision half (the weight table declares the tower) --------- - vcfg = getattr(composite_config, "vision_config", None) - assert vcfg is not None, "the composite config carries no vision_config" - self.vision_config = vcfg + # ---- the media placeholder ids -------------------------------------- self.image_token_id = int(composite_config.image_token_id) self.video_token_id = int(composite_config.video_token_id) @@ -6971,6 +6969,14 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"{mm_config.encoder_attention_backend}; the target's vision tower runs its own " "attention and does not apply it" ) + # The engine's encoder token budget, which startup profiling fills with dummy images. + encoder_budget = llm_args.encoder_max_num_tokens or llm_args.max_num_tokens + if not model_config.disable_mm_encoder and encoder_budget > _vision.VISION_CALL_MAX_PATCHES: + raise NoTarget( + f"an encoder token budget of {encoder_budget} (encoder_max_num_tokens, or " + "max_num_tokens when it is unset); one call of the target's vision tower takes " + f"at most {_vision.VISION_CALL_MAX_PATCHES} patches" + ) if spec is None: return if not is_gdn_replay_enabled(): @@ -7050,8 +7056,6 @@ def __init__(self, model_config: ModelConfig): # init). self._mm_token_ids_value = (self.model.image_token_id, self.model.video_token_id) self._mm_token_ids_cache: torch.Tensor | None = None - # Replicated on every rank; built once the weights are real (`post_load_weights`). - self.vision_tower: _vision.VisionTower | None = None @property def multimodal_token_ids(self) -> torch.Tensor: @@ -7062,18 +7066,6 @@ def multimodal_token_ids(self) -> torch.Tensor: self._mm_token_ids_cache = torch.tensor(self._mm_token_ids_value, dtype=torch.int32) return self._mm_token_ids_cache - @property - def multimodal_data_device_paths(self) -> list[str]: - """Tensors under `multimodal_data` the executor moves to the device. Absent, it moves - everything, putting the `(t, h, w)` grids the tower reads on the host there too.""" - return list(_vision.DEVICE_PATHS) - - def validate_request(self, request) -> None: - """Admission check, run by the executor on every new request: a media request the - vision tower would assert on fails here as its own error (`vision.validate_request`); - a text request returns at once.""" - _vision.validate_request(self.vision_tower, request) - def forward( self, attn_metadata: AttentionMetadata, @@ -7083,10 +7075,12 @@ def forward( return_context_logits: bool = False, spec_metadata=None, lora_params: dict | None = None, + multimodal_rows: tuple | None = None, **kwargs, ): """One engine step: the core's `begin_step`, then the base forward with every - argument it was handed (`resource_manager` stays in `**kwargs`). + argument it was handed (`resource_manager` stays in `**kwargs`). `multimodal_rows` is + `(rows, mm_token_indices)` from the vision-language model when the step carries media. With a spec worker the shell owns the logits gather (one-model modes gather the rows the sampler verifies, `spec_metadata.gather_ids`) and hands the worker the trunk's @@ -7099,7 +7093,7 @@ def forward( inputs_embeds=inputs_embeds, lora_params=lora_params, spec_metadata=spec_metadata, - vision=_vision.multimodal_embeds(self.vision_tower, kwargs), + vision=multimodal_rows, ) if self.spec_worker is None: assert spec_metadata is None, ( @@ -7173,8 +7167,6 @@ def load_weights(self, weights, *args, **kwargs): def post_load_weights(self): super().post_load_weights() self.model.post_load_weights() - self.vision_tower = _vision.VisionTower(self.model.w, self.model.vision_config) - self.vision_tower.warm_up() if self.one_rank: self._build_heads(self.lm_head.weight.data) @@ -7214,3 +7206,34 @@ def _build_heads(self, weight: torch.Tensor) -> None: f"{_FP8_GEMV_HEAD_ROWS} on nvfp4_gemv over lm_head {tuple(nv.data.shape)}", flush=True, ) + + +class ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration( + ModelingV2ForConditionalGeneration +): + """The routed class: the language model target as `llm` and the vision tower as + `mm_encoder`, on the shared base. The language model keeps `input_ids` (its PLE layer + hashes them) and writes the rows at `mm_token_indices` in its own prefill step.""" + + language_model_class = ModelingV2Qwen38FlashNextNvfp4Sm103Default + # Replicated on every rank, at the activation dtype. + mm_encoder_class = _vision.VisionTower + mm_encoder_weight_prefix = "model.visual." + language_model_takes_rows = True + supports_encoder_cache = True + + @property + def multimodal_data_device_paths(self) -> list[str]: + """Tensors under `multimodal_data` the executor moves to the device. Absent, it moves + everything, the `(t, h, w)` grids the tower reads on the host among them.""" + return [*_vision.DEVICE_PATHS, "multimodal_embedding"] + + def validate_media_request(self, request) -> None: + _vision.validate_request(self.mm_encoder, request) + + def encode_multimodal_inputs(self, multimodal_params) -> torch.Tensor: + """Each request's rows, whole and in prompt order, in request order.""" + if self.mm_encoder is None: + raise ValueError("Raw multimodal inputs require a local multimodal encoder.") + rows = [_vision.encode(self.mm_encoder, param) for param in multimodal_params] + return rows[0] if len(rows) == 1 else torch.cat(rows, 0) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py index f14708d1110b..53a92365622b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/vision.py @@ -10,9 +10,11 @@ the same rows back, since every op is a deterministic function of its operands; under attention DP a rank its own requests'. -The shell calls `multimodal_embeds` on every step, before the core's `begin_step`, and the -executor calls `validate_request` on every new request. Neither runs inside a captured -graph: the tower is eager and reads the request's grid on the host. +The tower is the vision-language model's `mm_encoder`. That model's encoder hook calls +`encode` per request, through the shared multimodal mixin, which caches the rows and keeps +the chunk's; the language model splices them in its own prefill step. The model's +`validate_request` calls `validate_request` here on every new request. Neither runs inside a +captured graph: the tower is eager and reads the request's grid on the host. """ from __future__ import annotations @@ -21,6 +23,7 @@ from typing import NamedTuple import torch +from torch import nn from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm @@ -36,8 +39,11 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.ops.vision.triton_pos_embed_interpolate import ( # noqa: E501 triton_pos_embed_interpolate, ) +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin from tensorrt_llm.inputs.multimodal import get_multimodal_item_metadata +from . import weights as _weights + #: Tower rope base (`vision_config` has no rope field); unrelated to the text 1e7. VISION_ROPE_THETA = 10000.0 @@ -292,12 +298,12 @@ def _rope_tables( return angles.cos().contiguous(), angles.sin().contiguous() -class VisionTower: +class VisionTower(nn.Module, MultimodalEncoderMixin): """One request's images or videos through the 27-block ViT and its patch merger. - Not a target: runs only for a context request carrying media, from the shell, before - the step. Attention is bidirectional per frame (a video's frame group is a frame), - expressed to `thop_attention` as one context "sequence" per frame; running the rows as + Not a target: the vision-language model's encoder, run only for a context request carrying + media, before the step. Attention is bidirectional per frame (a video's frame group is a + frame), expressed to `thop_attention` as one context "sequence" per frame; running the rows as one sequence is far outside tolerance. The position-embedding interpolation and the 2-D rope both emit in spatial-merge order (what the merger's 4-to-1 reshape consumes); a raster-order emitter gives a correctly shaped, silently wrong tower. @@ -306,11 +312,15 @@ class VisionTower: frames: attention is per frame, the position embedding and rope are per grid row and repeat per frame, the norms and MLPs are row-wise, and a frame's `h * w` is a multiple of `merge**2`, so a cut keeps every merger block whole. One frame past the bound cannot - be served; `validate_request` refuses it at admission.""" + be served; `validate_request` refuses it at admission. + + The tower declares and loads its own weights (`weights.VISION_WEIGHTS`), so a model + built without it (`disable_mm_encoder`) allocates no vision weight.""" - def __init__(self, w: dict, vision_config) -> None: - """`w` is the core's weight table after load; `vision_config` the composite - config's vision half, which `check_deployment` matched against this checkpoint's.""" + def __init__(self, vision_config, dtype: torch.dtype) -> None: + """`vision_config` is the composite config's vision half, which `check_deployment` + matched against this checkpoint's; `dtype` the activation dtype the tower runs at.""" + super().__init__() vcfg = vision_config self.depth = vcfg.depth self.hidden = vcfg.hidden_size @@ -324,12 +334,32 @@ def __init__(self, w: dict, vision_config) -> None: self.merge_dim = self.hidden * self.merge * self.merge # The position table is a square grid of this side. self.grid_side = math.isqrt(vcfg.num_position_embeddings) - self.dtype = w["embed"].dtype - self._w = _vision_views(w, self.depth) + self.dtype = dtype + # Every weight, declared here (meta-init intercepts `torch.empty`) and filled by + # `load_weights`; the views the ops read are built after load. + self.dims = _weights.vision_dims(vcfg, dtype) + self.w = _weights.VISION_WEIGHTS.declare(self.dims) + self._w: _VisionWeights | None = None # Persistent scratch: `thop_attention` grows `workspace` in place; never the # engine's own workspace. self._ws: torch.Tensor | None = None + def setup_attn_metadata(self, max_num_tokens: int, attention_metadata_capacity=None) -> None: + """No engine-built attention metadata: every tower call builds its own encoder-only + attention arguments from its frames (`_run`).""" + + def load_weights(self, weights) -> None: + """Fill the tower's weights from the checkpoint's `model.visual.` keys.""" + _weights.VISION_WEIGHTS.load(self, weights) + + def post_load_weights(self) -> None: + """Build the views the ops read, then warm the kernels up; once, though the + engine's post-load walk may reach this module more than once.""" + if self._w is not None: + return + self._w = _vision_views(self.w, self.depth) + self.warm_up() + def warm_up(self) -> None: """Run the tower once per `WARM_UP_GRIDS` grid on zero pixels, so every JIT kernel it launches is compiled at load rather than on the first media request. The rope and @@ -340,7 +370,7 @@ def warm_up(self) -> None: rows = sum(t * h * w for t, h, w in grid) self._run(torch.zeros(rows, self.pixel_width, dtype=self.dtype, device=device), grid) - def __call__(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: """`pixel_values` is `[sum(t*h*w), in_channels*kt*kh*kw]`, one flat patch per row, channel-slowest. `grid_thw` is one `(frames, height, width)` row per image or video and stays on the host (no synchronization). Returns @@ -458,81 +488,25 @@ def _run(self, pixel_values: torch.Tensor, grid: list[tuple[int, int, int]]) -> return cublas_mm(gelu_erf(cublas_mm(merged, V.m_fc1, V.m_fc1_b)), V.m_fc2, V.m_fc2_b) -# ------------------------------------------------------------ the shell's side --- - - -def multimodal_embeds(tower: VisionTower | None, kwargs: dict): - """Run the vision tower for this batch; returns `(rows, mm_token_indices)` (the tower's - `[R, hidden]` rows in packed-batch order and the `[R]` placeholder rows they replace, - which the prefill target splices into its embedding), or None when the step carries no - media. - - Called by the shell, because the inherited `DecoderModelForCausalLM.forward` drops - `multimodal_params` from `**kwargs` on the no-speculation path. Host-side bookkeeping: - `multimodal_params` is a subset of the context requests, so never zip it positionally; - `multimodal_runtime` (chunked prefill / KV reuse only) gives the cached and in-chunk row - counts over the request's images and videos together, in prompt order; `mm_token_indices` - names every placeholder row. - - The tower runs once per modality a request carries, and `in_prompt_order` merges the - two outputs. Positions are not derived here: every coordinate of a vision span, a - video's temporal one included, is in the payload's `mrope_config.mrope_position_ids`, - which the engine slices into `position_ids`.""" - mm_params = kwargs.get("multimodal_params") or [] - if not mm_params: - return None - rows = [] - for params in mm_params: - data = getattr(params, "multimodal_data", None) or {} - image, video = data.get("image"), data.get("video") - if image is None and video is None: - continue - mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") - assert ( - mrope_ids is not None and mrope_ids.dim() == 3 and mrope_ids.shape[0] == _POSITION_AXES - ), ( - f"a media request needs mrope_config.mrope_position_ids of shape " - f"[{_POSITION_AXES}, 1, L]; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" - ) - runtime = getattr(params, "multimodal_runtime", None) - if runtime is not None and int(runtime.num_mm_tokens_in_chunk) == 0: - # Every placeholder row of this request lies in another chunk. - continue - assert tower is not None, "post_load_weights builds the vision tower before any step" - towers = {} - if image is not None: - towers["image"] = run_tower(tower, image, *MEDIA_KEYS["image"]) - if video is not None: - towers["video"] = run_tower(tower, video, *MEDIA_KEYS["video"]) - embeds = in_prompt_order(tower.merge, params, towers) - if runtime is not None: - # Chunked prefill or reused prefix: splice only this chunk's slice. - start = int(runtime.num_cached_mm_tokens) - count = int(runtime.num_mm_tokens_in_chunk) - total = embeds.shape[0] - assert int(runtime.total_embeds_in_request) == total, ( - f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " - f"this request and the tower produced {total} over its {sorted(towers)}; the " - "two are derived from the same grids and a disagreement means the merger's " - "row count is wrong" - ) - assert 0 <= start <= start + count <= total, (start, count, total) - embeds = embeds[start : start + count] - rows.append(embeds) - if not rows: - return None - vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) - mm_token_indices = kwargs.get("mm_token_indices") - assert mm_token_indices is not None, ( - "the executor shipped multimodal params but no mm_token_indices; that pairing is what " - "the shell's `multimodal_token_ids` buys, and without it there is no sync-free way to " - "find the placeholder rows" - ) - assert mm_token_indices.shape[0] == vision.shape[0], ( - f"the batch has {mm_token_indices.shape[0]} placeholder rows and the tower produced " - f"{vision.shape[0]} embedding rows" - ) - return vision, mm_token_indices +# ------------------------------------------------------------ the encoder hook --- + + +def encode(tower: VisionTower, params) -> torch.Tensor: + """One request's tower rows, one per placeholder of the whole request, in prompt order. + + The tower runs once per modality the request carries, and `in_prompt_order` merges the + outputs. No chunk slice: the multimodal mixin keeps the rows of the current chunk. No + position check either: positions are not derived here (every coordinate of a vision span + is in `mrope_config.mrope_position_ids`, which admission checks), and the engine's memory + profiling hands the encoder a dummy request without them.""" + data = params.multimodal_data + towers = { + modality: run_tower(tower, data[modality], *keys) + for modality, keys in MEDIA_KEYS.items() + if data.get(modality) is not None + } + assert towers, f"the multimodal payload carries {sorted(data)} and no image or video" + return in_prompt_order(tower.merge, params, towers) def run_tower(tower, media: dict, pixel_key: str, grid_key: str): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index a2c2a81932e3..734f76097d8a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -90,7 +90,8 @@ `load(model, weights)` fills every declared parameter once and asserts coverage both ways: every parameter written, and every checkpoint key consumed (a key read through a cut is consumed) or predicted by `expected_unconsumed` -- the experts outside this -rank's windows, and the MTP module when there is no speculation. +rank's windows, the MTP module when there is no speculation, and the vision tower, whose +weights are a second table (`VISION_WEIGHTS`) that the tower module declares and loads. """ import mmap @@ -1271,134 +1272,6 @@ class Qwen4ExpDefaultWeights: src=lambda d, i: [(f"{_ngram_prefix(d.ple_layer)}.weight_scale", None)], layers=None, ), - # --- the vision tower: all bf16, replicated, one-to-one, no transform --- - W( - "vis_patch_w", - shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), - src=_visual("patch_embed.proj.weight"), - layers=None, - ), - W( - "vis_patch_b", - shape=lambda d: (d.v_hidden,), - src=_visual("patch_embed.proj.bias"), - layers=None, - ), - W( - "vis_pos", - shape=lambda d: (d.v_pos_rows, d.v_hidden), - src=_visual("pos_embed.weight"), - layers=None, - ), - # LayerNorm **with a bias**, unlike every norm in the language tower. - W( - "vis_n1_w", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm1.weight"), - layers=_vision_blocks, - ), - W( - "vis_n1_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm1.bias"), - layers=_vision_blocks, - ), - # One fused `[q | k | v]` projection, MHA, with a bias. - W( - "vis_qkv_w", - shape=lambda d: (3 * d.v_hidden, d.v_hidden), - src=_vision_block("attn.qkv.weight"), - layers=_vision_blocks, - ), - W( - "vis_qkv_b", - shape=lambda d: (3 * d.v_hidden,), - src=_vision_block("attn.qkv.bias"), - layers=_vision_blocks, - ), - W( - "vis_proj_w", - shape=lambda d: (d.v_hidden, d.v_hidden), - src=_vision_block("attn.proj.weight"), - layers=_vision_blocks, - ), - W( - "vis_proj_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("attn.proj.bias"), - layers=_vision_blocks, - ), - W( - "vis_n2_w", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm2.weight"), - layers=_vision_blocks, - ), - W( - "vis_n2_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("norm2.bias"), - layers=_vision_blocks, - ), - # A **non-gated** MLP: nothing here is split in half, unlike every SwiGLU in the - # language tower. - W( - "vis_fc1_w", - shape=lambda d: (d.v_inter, d.v_hidden), - src=_vision_block("mlp.linear_fc1.weight"), - layers=_vision_blocks, - ), - W( - "vis_fc1_b", - shape=lambda d: (d.v_inter,), - src=_vision_block("mlp.linear_fc1.bias"), - layers=_vision_blocks, - ), - W( - "vis_fc2_w", - shape=lambda d: (d.v_hidden, d.v_inter), - src=_vision_block("mlp.linear_fc2.weight"), - layers=_vision_blocks, - ), - W( - "vis_fc2_b", - shape=lambda d: (d.v_hidden,), - src=_vision_block("mlp.linear_fc2.bias"), - layers=_vision_blocks, - ), - # The patch merger. Its norm is applied **before** the 4-to-1 shuffle, so it is - # `hidden` wide and not `merge_dim` wide. - W( - "vis_m_n_w", - shape=lambda d: (d.v_hidden,), - src=_visual("merger.norm.weight"), - layers=None, - ), - W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), - W( - "vis_m_fc1_w", - shape=lambda d: (d.v_merge_dim, d.v_merge_dim), - src=_visual("merger.linear_fc1.weight"), - layers=None, - ), - W( - "vis_m_fc1_b", - shape=lambda d: (d.v_merge_dim,), - src=_visual("merger.linear_fc1.bias"), - layers=None, - ), - W( - "vis_m_fc2_w", - shape=lambda d: (d.hidden, d.v_merge_dim), - src=_visual("merger.linear_fc2.weight"), - layers=None, - ), - W( - "vis_m_fc2_b", - shape=lambda d: (d.hidden,), - src=_visual("merger.linear_fc2.bias"), - layers=None, - ), # --- the MTP draft module, at layer index num_hidden_layers, only when live --- # # Structurally one more decoder layer of this architecture with a two-branch @@ -1516,15 +1389,13 @@ class Qwen4ExpDefaultWeights: def dims(self, core) -> SimpleNamespace: """This rank's dimension bundle, from the core's normalized text config, the - composite's vision config (the core holds it as `vision_config` before it - declares anything), the engine's `Mapping` and the core's engine choices.""" + engine's `Mapping` and the core's engine choices.""" quant_config = ( moe_experts_quant_config(core.model_config, f"{_MTP_LAYER}.mlp.experts") or core.model_config.quant_config ) return rank_dims( core.model_config.pretrained_config, - core.vision_config, core.model_config.mapping, mtp_enabled=getattr(core.model_config, "spec_config", None) is not None, trunk_moe_backend=core.trunk_moe_backend, @@ -1538,9 +1409,11 @@ def expected_unconsumed(self, d: SimpleNamespace, keys) -> set: routed expert outside its window (its per-tensor scalars are read on every rank; under the intermediate split the window is every expert, each read at this rank's slice), and, with MTP live, the draft experts outside its draft window; with MTP off, the - whole `mtp.` subtree. The windows are enumerated from `d`, not read off the + whole `mtp.` subtree. The `model.visual.` subtree is always left: the vision tower + loads it with its own table (`VISION_WEIGHTS`), or nothing does with + `disable_mm_encoder`. The windows are enumerated from `d`, not read off the checkpoint, so a key that belongs to a real weight can never land here.""" - out = set() + out = {k for k in keys if k.startswith(f"{_VISUAL}.")} start, count = _trunk_window(d) for i in range(d.num_layers): for e in range(d.num_experts): @@ -1614,6 +1487,18 @@ def read(self, weights, ckpt_key, cut, transform, d, device=None) -> torch.Tenso assert len(src_all) == 1, "a multi-key row needs a transform" return src_all[0] + def fill(self, weights, d, consumed: set, param, ckpt_key, index, cut, transform) -> None: + """Copy one manifest source into `param` (or its row `index`) and record the + checkpoint keys it read in `consumed`.""" + dst = param.data if index is None else param.data[index] + # Materialize where the destination lives: the expert relayouts are row + # gathers and the scale swizzle is a kernel, both on the device. + src = self.read(weights, ckpt_key, cut, transform, d, dst.device) + assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) + assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + def load(self, model, weights) -> None: """Fill every declared parameter. The n-gram table is declared here first, each source is read through its cut, multi-key rows feed their transform several @@ -1640,14 +1525,7 @@ def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int consumed: set = set() def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: - dst = param.data if index is None else param.data[index] - # Materialize where the destination lives: the expert relayouts are row - # gathers and the scale swizzle is a kernel, both on the device. - src = self.read(weights, ckpt_key, cut, transform, d, dst.device) - assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) - assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) - dst.copy_(src, non_blocking=True) - consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + self.fill(weights, d, consumed, param, ckpt_key, index, cut, transform) assert set(manifest.keys()) == set(core.w.keys()), ( "manifest/parameter drift", @@ -1681,7 +1559,7 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: expected = self.expected_unconsumed(d, weights.keys()) assert leftover == expected, ( "checkpoint coverage: leftover keys are not exactly this rank's off-window " - f"experts{'' if d.mtp_enabled else ' and the MTP module'}; unexpected " + f"experts{'' if d.mtp_enabled else ', the MTP module'} and the vision tower; unexpected " f"{sorted(leftover - expected)[:8]}, missed {sorted(expected - leftover)[:8]}" ) @@ -1689,6 +1567,168 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: MODEL_WEIGHTS = Qwen4ExpDefaultWeights() +class Qwen4ExpVisionWeights(Qwen4ExpDefaultWeights): + """The vision tower's weights: all bf16, replicated, one-to-one, no transform. + + The same `declare`, `manifest`, `read` and `fill` passes over a table of its own, which + the tower module (`vision.VisionTower`) declares and loads, so the language model's table + holds no vision weight and `disable_mm_encoder` allocates none. Its dimension bundle is + `vision_dims`. + """ + + WEIGHTS: tuple[W, ...] = ( + W( + "vis_patch_w", + shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), + src=_visual("patch_embed.proj.weight"), + layers=None, + ), + W( + "vis_patch_b", + shape=lambda d: (d.v_hidden,), + src=_visual("patch_embed.proj.bias"), + layers=None, + ), + W( + "vis_pos", + shape=lambda d: (d.v_pos_rows, d.v_hidden), + src=_visual("pos_embed.weight"), + layers=None, + ), + # LayerNorm **with a bias**, unlike every norm in the language tower. + W( + "vis_n1_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.weight"), + layers=_vision_blocks, + ), + W( + "vis_n1_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.bias"), + layers=_vision_blocks, + ), + # One fused `[q | k | v]` projection, MHA, with a bias. + W( + "vis_qkv_w", + shape=lambda d: (3 * d.v_hidden, d.v_hidden), + src=_vision_block("attn.qkv.weight"), + layers=_vision_blocks, + ), + W( + "vis_qkv_b", + shape=lambda d: (3 * d.v_hidden,), + src=_vision_block("attn.qkv.bias"), + layers=_vision_blocks, + ), + W( + "vis_proj_w", + shape=lambda d: (d.v_hidden, d.v_hidden), + src=_vision_block("attn.proj.weight"), + layers=_vision_blocks, + ), + W( + "vis_proj_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("attn.proj.bias"), + layers=_vision_blocks, + ), + W( + "vis_n2_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.weight"), + layers=_vision_blocks, + ), + W( + "vis_n2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.bias"), + layers=_vision_blocks, + ), + # A **non-gated** MLP: nothing here is split in half, unlike every SwiGLU in the + # language tower. + W( + "vis_fc1_w", + shape=lambda d: (d.v_inter, d.v_hidden), + src=_vision_block("mlp.linear_fc1.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc1_b", + shape=lambda d: (d.v_inter,), + src=_vision_block("mlp.linear_fc1.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc2_w", + shape=lambda d: (d.v_hidden, d.v_inter), + src=_vision_block("mlp.linear_fc2.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("mlp.linear_fc2.bias"), + layers=_vision_blocks, + ), + # The patch merger. Its norm is applied **before** the 4-to-1 shuffle, so it is + # `hidden` wide and not `merge_dim` wide. + W( + "vis_m_n_w", + shape=lambda d: (d.v_hidden,), + src=_visual("merger.norm.weight"), + layers=None, + ), + W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), + W( + "vis_m_fc1_w", + shape=lambda d: (d.v_merge_dim, d.v_merge_dim), + src=_visual("merger.linear_fc1.weight"), + layers=None, + ), + W( + "vis_m_fc1_b", + shape=lambda d: (d.v_merge_dim,), + src=_visual("merger.linear_fc1.bias"), + layers=None, + ), + W( + "vis_m_fc2_w", + shape=lambda d: (d.hidden, d.v_merge_dim), + src=_visual("merger.linear_fc2.weight"), + layers=None, + ), + W( + "vis_m_fc2_b", + shape=lambda d: (d.hidden,), + src=_visual("merger.linear_fc2.bias"), + layers=None, + ), + ) + + def load(self, tower, weights) -> None: + """Fill every parameter of `tower.w` and assert coverage both ways: every + parameter written, and every `model.visual.` checkpoint key read.""" + d = tower.dims + manifest = self.manifest(d) + assert set(manifest.keys()) == set(tower.w.keys()), ( + "manifest/parameter drift", + set(manifest.keys()) ^ set(tower.w.keys()), + ) + consumed: set = set() + for param_key, sources in manifest.items(): + for source in sources: + self.fill(weights, d, consumed, tower.w[param_key], *source) + visual = {k for k in weights.keys() if k.startswith(f"{_VISUAL}.")} + assert consumed == visual, ( + "vision checkpoint coverage: unexpected " + f"{sorted(visual - consumed)[:8]}, missed {sorted(consumed - visual)[:8]}" + ) + + +VISION_WEIGHTS = Qwen4ExpVisionWeights() + + def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: """(heads on `rank`, its first global head) when `total` heads are split over `ranks`: an even split, or each head repeated on `ranks // total` neighbouring ranks when there @@ -1700,7 +1740,6 @@ def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: def rank_dims( cfg, - vcfg, mapping, *, mtp_enabled: bool, @@ -1713,7 +1752,6 @@ def rank_dims( Args: cfg: the engine-normalized text config (`Qwen4ExpTextConfig`). - vcfg: the composite's vision config. mapping: this rank's `Mapping`. mtp_enabled: whether the engine has a `speculative_config` (the MTP module is live). trunk_moe_backend: the routed backend the trunk runs, which picks the expert layout. @@ -1762,7 +1800,6 @@ def rank_dims( expert_inter = -(-expert_inter_share // EXPERT_INTER_ALIGN) * EXPERT_INTER_ALIGN ngram_heads = cfg.heads_per_ngram * (cfg.ngram_size - 1) - v_hidden = vcfg.hidden_size return SimpleNamespace( num_layers=cfg.num_hidden_layers, is_attn=[t == "full_attention" for t in cfg.layer_types], @@ -1824,7 +1861,22 @@ def rank_dims( ple_head_dim=cfg.ple_embed_dim // ngram_heads, ple_shards=cfg.split_ngram_parts, ple_table_dtype=PLE_TABLE_DTYPE, - # --- the vision tower --- + # --- engine choices --- + mtp_enabled=mtp_enabled, + trunk_moe_backend=trunk_moe_backend, + moe_prefill_copy=moe_prefill_copy, + dtype=dtype, + ) + + +def vision_dims(vcfg, dtype: torch.dtype) -> SimpleNamespace: + """The vision table's dimension bundle, from the composite's vision config. The tower + is replicated, so every rank declares it whole. `hidden` is the merger's output width, + the text hidden size of the rows it replaces.""" + v_hidden = vcfg.hidden_size + return SimpleNamespace( + hidden=vcfg.out_hidden_size, + dtype=dtype, v_depth=vcfg.depth, v_hidden=v_hidden, v_inter=vcfg.intermediate_size, @@ -1833,11 +1885,6 @@ def rank_dims( v_patch=vcfg.patch_size, v_pos_rows=vcfg.num_position_embeddings, v_merge_dim=v_hidden * vcfg.spatial_merge_size * vcfg.spatial_merge_size, - # --- engine choices --- - mtp_enabled=mtp_enabled, - trunk_moe_backend=trunk_moe_backend, - moe_prefill_copy=moe_prefill_copy, - dtype=dtype, ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py index 2eff81e160ce..c029f1570b9e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py @@ -7,7 +7,9 @@ `qwen3_8_flash_next_fp8__sm_103__unified` at one of `_FP8_LAYOUTS`, each with one-model MTP-Eagle (not at dep2) or no speculative decoding. The target's `check_deployment` then declines the checkpoints and engine choices it does not run, so `auto` falls back to the -built-in implementation instead of failing in the build. +built-in implementation instead of failing in the build. The NVFP4 route returns the target's +vision-language model, which holds its language model and its vision tower; the FP8 target +encodes media in its own forward. """ from __future__ import annotations @@ -106,7 +108,8 @@ def route( return ModelingV2Qwen38FlashNextFp8Sm103Unified from .qwen3_8_flash_next_nvfp4__sm_103__default.modeling import ( ModelingV2Qwen38FlashNextNvfp4Sm103Default, + ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration, ) ModelingV2Qwen38FlashNextNvfp4Sm103Default.check_deployment(model_config, llm_args, member) - return ModelingV2Qwen38FlashNextNvfp4Sm103Default + return ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration diff --git a/tensorrt_llm/_torch/modeling_v2/multimodal.py b/tensorrt_llm/_torch/modeling_v2/multimodal.py new file mode 100644 index 000000000000..b6eafa20ba19 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/multimodal.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The shared base class of Modeling V2 vision-language models. + +A vision-language target is composite, as a Modeling V1 vision-language model is: a +:class:`ModelingV2ForConditionalGeneration` holds a Modeling V2 language model target as +``llm`` and its encoder tower as ``mm_encoder``. The shared multimodal runtime +(``MultimodalModelMixin``) selects the step's media requests, runs or reuses the tower +(encoder cache) and keeps the current chunk's rows, so the executor sees one multimodal model +shape for V1 and V2. + +A target subclasses this base as ``ModelingV2ForConditionalGeneration`` and declares +what differs: + +* ``language_model_class``: the language model target. It is built on a copy of the + ``ModelConfig`` and adopts the text config on that copy, as it does when it is routed alone. +* ``mm_encoder_class``: the tower, an ``nn.Module`` and ``MultimodalEncoderMixin`` that + declares and loads its own weights. The base builds it as + ``mm_encoder_class(vision_config, dtype)``: the composite config's ``vision_config`` and the + activation dtype. A tower built another way overrides ``build_mm_encoder`` instead. No tower + is built with ``disable_mm_encoder``. +* ``mm_encoder_weight_prefix``: the checkpoint prefix of the tower's weights. The tower loads + those keys and the language model every other key. +* ``encode_multimodal_inputs``: the tower's rows for the selected requests, in request order. +* ``language_model_takes_rows``: how the rows reach the language model. False (the default): + the mixin fuses them into ``inputs_embeds``. True: ``input_ids`` stay as they are and the + language model takes ``multimodal_rows=(rows, mm_token_indices)`` and places the rows itself; + the engine then allows the breakable prefill CUDA graph, as for the language model alone. + +Optional: ``validate_media_request`` (admission checks of the tower) and the mixin's feature +flags such as ``supports_encoder_cache``. To pass extra inputs to the language model, as V1 +Qwen3-VL does for DeepStack, a target overrides ``get_language_model_extra_forward_kwargs``. +""" + +from __future__ import annotations + +import inspect +from typing import TYPE_CHECKING, Any, ClassVar + +import torch +from torch import nn +from transformers import PretrainedConfig, PreTrainedModel + +from tensorrt_llm._torch.models.checkpoints.model_hooks import call_load_weights +from tensorrt_llm._torch.models.modeling_multimodal_mixin import ( + MultimodalModelMixin, + PreparedLlmInputs, +) +from tensorrt_llm._torch.models.modeling_utils import DecoderModelForCausalLM +from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + +if TYPE_CHECKING: + from tensorrt_llm._torch.model_config import ModelConfig + +__all__ = ["ModelingV2ForConditionalGeneration"] + +#: `multimodal_data` entries the tower encodes, or rows an earlier step already made. +_ENCODED_KEYS = ("image", "video", "audio", "multimodal_embedding") + +#: Language model flags that are read off the routed class, which is the composite, before any +#: model is built: `moe_impls_honored` and `draft_moe_impls_honored` by the router's MoE tactic +#: check (`_router_index._refuse_unhonored_moe_tactics`), `supports_nvfp4_kv_cache` by the +#: model loader's SM107 KV cache check. Each maps to the value its reader takes when a class has +#: none. A subclass that sets `language_model_class` takes that class's values, or these, unless +#: it sets its own, so it never keeps a parent's; without them the router would refuse a set MoE +#: tactic and the loader would switch an NVFP4 KV cache to FP8. Both read the class, so a +#: property cannot do this. +_LANGUAGE_MODEL_CLASS_ATTRIBUTES = { + "moe_impls_honored": frozenset(), + "draft_moe_impls_honored": frozenset(), + "supports_nvfp4_kv_cache": False, +} + + +class ModelingV2ForConditionalGeneration(MultimodalModelMixin, PreTrainedModel): + """A Modeling V2 language model target as ``llm`` and its encoder tower as ``mm_encoder``, + behind the shared multimodal runtime. See the module docstring for what a target declares. + + Same composite shape as a Modeling V1 vision-language model (for example + ``Qwen3VLModelBase``), holding only the code every V2 target needs. + """ + + language_model_class: ClassVar[type[DecoderModelForCausalLM]] + mm_encoder_class: ClassVar[type[nn.Module] | None] = None + mm_encoder_weight_prefix: ClassVar[str] + language_model_takes_rows: ClassVar[bool] = False + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + llm_class = cls.__dict__.get("language_model_class") + for name, default in _LANGUAGE_MODEL_CLASS_ATTRIBUTES.items(): + if llm_class is not None and name not in cls.__dict__: + setattr(cls, name, getattr(llm_class, name, default)) + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None): + return cls.language_model_class.get_preferred_kv_cache_manager_version(pretrained_config) + + def _check_and_adjust_experts_implementation(self, *args, **kwargs): + """No-op: the model holds no MoE layer of its own for the transformers check.""" + return None + + def __init__(self, model_config: ModelConfig): + composite = model_config.pretrained_config + super().__init__(composite) + # The engine's own `ModelConfig`: the loader, the engine and KV sizing read it here. + self.model_config = model_config + # Built as the engine builds a decoder model (`skip_create_weights_in_init`), on its own + # shallow copy, where it adopts the text config. + self.llm = self.language_model_class( + model_config.copy_with(skip_create_weights_in_init=True) + ) + # Everything that sizes the pools reads the text config. + model_config.pretrained_config = self.llm.config + self.config = self.llm.config + # None with `disable_mm_encoder`: no encoder weight is declared or loaded, and a + # media request is refused at admission. + self.mm_encoder = ( + None if model_config.disable_mm_encoder else self.build_mm_encoder(composite) + ) + + def build_mm_encoder(self, pretrained_config: PretrainedConfig) -> nn.Module | None: + """The encoder tower of the composite ``pretrained_config``, at ``self.dtype``: + ``mm_encoder_class(pretrained_config.vision_config, self.dtype)``. Override it for a + tower built another way. None builds no tower, and the model serves text only.""" + if self.mm_encoder_class is None: + return None + return self.mm_encoder_class(pretrained_config.vision_config, self.dtype) + + @property + def language_model(self) -> nn.Module: + return self.llm + + @property + def dtype(self) -> torch.dtype: + """The activation dtype. Set, not read off the first parameter, which may be a + quantized one.""" + return self.model_config.torch_dtype + + @property + def embedding_dim(self) -> int: + return self.config.hidden_size + + @property + def embedding_dtype(self) -> torch.dtype: + return self.dtype + + @property + def text_embedding_layer(self) -> nn.Module: + """The token embedding the fused path embeds text with.""" + return self.llm.model.embed_tokens + + # What the engine reads off the routed model is the language model's. + + @property + def multimodal_token_ids(self): + return getattr(self.llm, "multimodal_token_ids", None) + + @property + def lm_head(self) -> nn.Module: + return self.llm.lm_head + + @property + def draft_config(self): + return self.llm.draft_config + + @property + def draft_model(self): + return self.llm.draft_model + + @property + def spec_worker(self): + return self.llm.spec_worker + + def apply_llm_torch_compile(self, *, backend, fullgraph: bool) -> None: + """Compile only the language model's core; the tower stays eager.""" + self.llm.model = torch.compile(self.llm.model, backend=backend, fullgraph=fullgraph) + + def validate_request(self, request) -> None: + """Admission check, run by the executor on every new request. The executor checks the + token range of a decoder model only, so it is checked here against the language + model's vocabulary; then, with a tower, `validate_media_request`.""" + num_embeddings = self.llm.lm_head.num_embeddings + token_bound = request.py_prompt_token_bound + if token_bound is not None: + if token_bound >= num_embeddings: + raise ValueError("Token ID out of range") + elif not request.check_token_id_range(num_embeddings): + raise ValueError("Token ID out of range") + if self.mm_encoder is not None: + self.validate_media_request(request) + + def validate_media_request(self, request) -> None: + """Refuse, as its own error, a request whose media the tower cannot encode.""" + + def validate_multimodal_request_data(self, mm_data: dict) -> None: + """Admission check: with no tower the model serves text only and refuses media or + embeddings; with one it encodes its own media and refuses precomputed embeddings. + M-RoPE positions alone pass.""" + if self.mm_encoder is None: + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} builds no multimodal encoder (disable_mm_encoder) " + "and serves text only; send no images, videos or embeddings" + ) + elif mm_data.get("multimodal_embedding") is not None: + raise ValueError( + f"{type(self).__name__} encodes its own media; precomputed multimodal " + "embeddings are not supported" + ) + + def select_multimodal_params(self, multimodal_params, num_context_requests): + """The mixin's selection, then only requests with media or their rows: an M-RoPE-only + param (a disaggregated generation request, or a generation request's position + deltas) has nothing to encode.""" + return [ + param + for param in super().select_multimodal_params(multimodal_params, num_context_requests) + if any(param.multimodal_data.get(key) is not None for key in _ENCODED_KEYS) + ] + + @torch.inference_mode() + def forward( + self, + attn_metadata, + input_ids: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + return_context_logits: bool = False, + spec_metadata=None, + **kwargs, + ): + """The step's multimodal rows from the mixin, then the language model with every + other argument the engine passed. The rows go in fused `inputs_embeds`, or with + `language_model_takes_rows` as `multimodal_rows=(rows, mm_token_indices)` beside + unchanged `input_ids`. + + The language model also takes what `get_language_model_extra_forward_kwargs` returns; + the hook gets the same arguments as in the mixin's own forward. In the rows mode its + `mm_inputs` holds `input_ids` and `inputs_embeds` as the language model gets them and + the chunk's extra rows from `after_active_multimodal_embeddings` (one per multimodal + row, not spread over the tokens), and `multimodal_rows` is among its keyword + arguments.""" + raw_input_ids = input_ids + multimodal_params = kwargs.pop("multimodal_params", []) + prepare_kwargs = dict( + input_ids=input_ids, + positions=position_ids, + multimodal_params=multimodal_params, + num_context_requests=attn_metadata.num_contexts, + attn_metadata=attn_metadata, + **kwargs, + ) + if self.language_model_takes_rows: + rows = self.prepare_multimodal_rows(**prepare_kwargs) + # The extra rows are kept even when the chunk has no multimodal row, as fusion does. + input_ids, active_embeddings, extra_embeds = rows or (input_ids, [], ()) + multimodal_rows = None + if active_embeddings: + embeds = ( + active_embeddings[0] + if len(active_embeddings) == 1 + else torch.cat(active_embeddings) + ) + mm_token_indices = kwargs.get("mm_token_indices") + assert ( + mm_token_indices is not None and mm_token_indices.shape[0] == embeds.shape[0] + ), ( + f"the step has {embeds.shape[0]} multimodal rows and " + f"{None if mm_token_indices is None else mm_token_indices.shape[0]} " + "placeholder rows (mm_token_indices)" + ) + multimodal_rows = (embeds, mm_token_indices) + kwargs["multimodal_rows"] = multimodal_rows + mm_inputs = PreparedLlmInputs(input_ids, inputs_embeds, extra_embeds) + else: + mm_inputs = self.prepare_multimodal_inputs(**prepare_kwargs) + if inputs_embeds is not None: + # As in the mixin's forward: the caller's embeddings replace input_ids. + assert mm_inputs.inputs_embeds is None, ( + "the engine passed inputs_embeds for a media step" + ) + mm_inputs = PreparedLlmInputs(None, inputs_embeds, mm_inputs.extra_embeds) + # As in the mixin's forward: the language model arguments and the target's extra ones + # (for example DeepStack rows). They go over every argument the engine passed. + llm_kwargs = self.get_language_model_forward_kwargs( + attn_metadata=attn_metadata, + input_ids=mm_inputs.input_ids, + raw_input_ids=raw_input_ids, + position_ids=position_ids, + inputs_embeds=mm_inputs.inputs_embeds, + mm_inputs=mm_inputs, + return_context_logits=return_context_logits, + multimodal_params=multimodal_params, + num_generation_requests=attn_metadata.num_generations, + spec_metadata=spec_metadata, + **kwargs, + ) + return self.llm.forward(**{"spec_metadata": spec_metadata, **kwargs, **llm_kwargs}) + + def load_weights(self, weights, weight_mapper=None) -> None: + """The tower loads the keys under `mm_encoder_weight_prefix` and the language model + every other key, with the engine's `weight_mapper` when its `load_weights` takes one; + each checks its own coverage. The base renames no key. Without a tower the encoder keys + are left unread. + + The engine binds its mapper to this model. The mapper is bound to the language model + before it gets it, as in the Modeling V1 composite models, since the language model's + keys name its own modules.""" + encoder_weights, text_weights = {}, {} + for key in weights.keys(): + if not key.startswith(self.mm_encoder_weight_prefix): + text_weights[key] = weights[key] + elif self.mm_encoder is not None: + encoder_weights[key] = weights[key] + if self.mm_encoder is not None: + self.mm_encoder.load_weights(encoder_weights) + if ( + weight_mapper is not None + and "weight_mapper" in inspect.getfullargspec(self.llm.load_weights).args + ): + weight_mapper.init_model_and_config(self.llm, self.llm.model_config) + call_load_weights(self.llm.load_weights, text_weights, weight_mapper) diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index 113ac19c2d83..ef56ca61156b 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -1051,8 +1051,7 @@ def _slice_per_item_sibling_fields( sliced[key] = [value[i] for i in item_indices] return sliced - # A future optional mixin-owned forward can build on the same template method. - def prepare_multimodal_inputs( + def prepare_multimodal_rows( self, *, input_ids: torch.Tensor, @@ -1060,12 +1059,15 @@ def prepare_multimodal_inputs( multimodal_params: Optional[Sequence[MultimodalParams]], num_context_requests: int, **forward_kwargs: Any, - ) -> PreparedLlmInputs: - """Prepare multimodal inputs for a concrete model forward. - - This method owns the common framework sequence around a model-specific - encoder hook: retrieve/cache embeddings, select active chunk rows if - needed, run optional model hooks, and fuse rows into text embeds. + ) -> Optional[tuple[torch.Tensor, list[torch.Tensor], Sequence[torch.Tensor]]]: + """Return this step's multimodal rows, before any fusion. + + Selects the params, retrieves or encodes their embeddings, runs the two + embedding hooks and keeps the rows of this chunk. Returns + ``(input_ids, active_embeddings, extra_embeds)``, or None when the step + has no multimodal params. A model whose language model places the rows + itself (for example at the step's ``mm_token_indices``) calls this + instead of `prepare_multimodal_inputs`. """ context_params = list( self.select_multimodal_params( @@ -1074,7 +1076,7 @@ def prepare_multimodal_inputs( ) ) if not context_params: - return PreparedLlmInputs(input_ids=input_ids, inputs_embeds=None) + return None embeddings = self._get_or_encode_multimodal_embeddings(context_params) @@ -1091,6 +1093,35 @@ def prepare_multimodal_inputs( multimodal_params=context_params, **forward_kwargs, ) + return input_ids, active_embeddings, extra_embeds + + # A future optional mixin-owned forward can build on the same template method. + def prepare_multimodal_inputs( + self, + *, + input_ids: torch.Tensor, + positions: Optional[torch.Tensor], + multimodal_params: Optional[Sequence[MultimodalParams]], + num_context_requests: int, + **forward_kwargs: Any, + ) -> PreparedLlmInputs: + """Prepare multimodal inputs for a concrete model forward. + + This method owns the common framework sequence around a model-specific + encoder hook: retrieve/cache embeddings, select active chunk rows if + needed, run optional model hooks (`prepare_multimodal_rows`), and fuse + rows into text embeds. + """ + rows = self.prepare_multimodal_rows( + input_ids=input_ids, + positions=positions, + multimodal_params=multimodal_params, + num_context_requests=num_context_requests, + **forward_kwargs, + ) + if rows is None: + return PreparedLlmInputs(input_ids=input_ids, inputs_embeds=None) + input_ids, active_embeddings, extra_embeds = rows fused_input_ids, inputs_embeds, fused_extra_embeds = self._fuse_multimodal_embeddings( input_ids=input_ids, diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 6e70f167e39a..0ff9b32ac377 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -905,10 +905,13 @@ def _validate_breakable_cuda_graph_compatibility(self) -> None: if isinstance(self.model, DecoderModelForCausalLM): return decoder_model = getattr(self.model, "llm", None) - if ( - self.llm_args.disable_mm_encoder - and isinstance(decoder_model, DecoderModelForCausalLM) - and getattr(self.model, "mm_encoder", None) is None + if isinstance(decoder_model, DecoderModelForCausalLM) and ( + # The language model takes `input_ids` and places the rows itself, so its graph + # runs as for the decoder alone. + getattr(self.model, "language_model_takes_rows", False) + or ( + self.llm_args.disable_mm_encoder and getattr(self.model, "mm_encoder", None) is None + ) ): return if isinstance(self.model, MultimodalModelMixin) or isinstance( diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index 68b7857cb10b..6bf851b86cd0 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -603,6 +603,19 @@ def strip_mm_encoder_inputs(mm_data: Dict[str, Any]) -> None: mm_data.pop(modality, None) +def has_media_or_embeddings(mm_data: Optional[Dict[str, Any]]) -> bool: + """Whether a request's `multimodal_data` carries more than M-RoPE positions. + + Media, precomputed embeddings and placeholder bookkeeping count. A payload + with only `mrope_config` (a disaggregated generation request, or a + text-only warmup request) does not. Any other non-None entry counts, also + one this function does not know, so a text-only model that refuses on it + fails closed. + """ + return any(value is not None for key, value in (mm_data or {}).items() + if key != "mrope_config") + + @dataclass class MultimodalInput: """Per-logical-unit multimodal metadata for KV-cache hashing (C++ layer). diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 62192567483e..f757865c1e9f 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -1140,6 +1140,11 @@ def __init__(self) -> None: self._input_processors_cls_by_model_type: Dict[ Type[nn.Module], Type[InputProcessor]] = {} + def get_input_processor_cls( + self, model_cls: Type[nn.Module]) -> Optional[Type[InputProcessor]]: + """Return the input processor class registered for ``model_cls``, or None.""" + return self._input_processors_cls_by_model_type.get(model_cls) + INPUT_PROCESSOR_REGISTRY = InputProcessorRegistry() @@ -1294,8 +1299,8 @@ def create_input_processor( if config is not None: try: model_cls, _ = get_model_architecture(config) - input_processor_cls = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type \ - .get(model_cls) + input_processor_cls = INPUT_PROCESSOR_REGISTRY.get_input_processor_cls( + model_cls) except (RuntimeError, ConfigInputError, UnsupportedError): # unregistered model logger.info("Unregistered model, using DefaultInputProcessor") diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 6cc2692f57a4..e1cc77102dc9 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -130,6 +130,7 @@ l0_cpu: - unittest/_torch/modeling_v2/models/beam/test_component_contracts.py - unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py - unittest/_torch/modeling_v2/test_modeling_v2_contracts.py + - unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py - unittest/_torch/modeling_v2/test_modeling_v2_package_location.py - unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py index f724e184f318..eb0f3b64e387 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default/test_glm53_flash_default_text_only.py @@ -24,21 +24,19 @@ def test_the_target_defines_the_admission_hook(): @pytest.mark.parametrize( - "mm_data, carried", + "mm_data", [ - ({"image": {"pixel_values": None, "image_grid_thw": None}}, "image"), - ({"video": {"pixel_values_videos": None, "video_grid_thw": None}}, "video"), - ({"audio": object()}, "audio"), - ({"multimodal_embedding": object()}, "multimodal_embedding"), - ({"image": object(), "video": object()}, "image, video"), + {"image": {"pixel_values": None, "image_grid_thw": None}}, + {"video": {"pixel_values_videos": None, "video_grid_thw": None}}, + {"audio": object()}, + {"multimodal_embedding": object()}, + {"image": object(), "video": object()}, ], ids=["image", "video", "audio", "embedding", "image-and-video"], ) -def test_media_is_refused_at_admission(mm_data, carried): +def test_media_is_refused_at_admission(mm_data): # The hook reads no model state, so it runs without building the model. - with pytest.raises( - ValueError, match=f"does not accept multimodal input; the request carries {carried}$" - ): + with pytest.raises(ValueError, match="serves text only; send no images, videos or embeddings$"): getattr(Target, _HOOK)(types.SimpleNamespace(), mm_data) diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py index 919720b00cda..34d054e5d557 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/test_glm53_flash_default_mtp_fused_text_only.py @@ -24,21 +24,19 @@ def test_the_target_defines_the_admission_hook(): @pytest.mark.parametrize( - "mm_data, carried", + "mm_data", [ - ({"image": {"pixel_values": None, "image_grid_thw": None}}, "image"), - ({"video": {"pixel_values_videos": None, "video_grid_thw": None}}, "video"), - ({"audio": object()}, "audio"), - ({"multimodal_embedding": object()}, "multimodal_embedding"), - ({"image": object(), "video": object()}, "image, video"), + {"image": {"pixel_values": None, "image_grid_thw": None}}, + {"video": {"pixel_values_videos": None, "video_grid_thw": None}}, + {"audio": object()}, + {"multimodal_embedding": object()}, + {"image": object(), "video": object()}, ], ids=["image", "video", "audio", "embedding", "image-and-video"], ) -def test_media_is_refused_at_admission(mm_data, carried): +def test_media_is_refused_at_admission(mm_data): # The hook reads no model state, so it runs without building the model. - with pytest.raises( - ValueError, match=f"does not accept multimodal input; the request carries {carried}$" - ): + with pytest.raises(ValueError, match="serves text only; send no images, videos or embeddings$"): getattr(Target, _HOOK)(types.SimpleNamespace(), mm_data) diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py index da8de5a6dba3..89cdb001bf4b 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py @@ -588,12 +588,13 @@ def test_fp8_constructor_refuses_what_route_declines(): @pytest.mark.parametrize( "multimodal_data, refused", [ - (None, None), - ({}, None), - ({"image": {"pixel_values": torch.zeros(1)}}, "image"), - ({"video": {"pixel_values_videos": torch.zeros(1)}}, "video"), + (None, False), + ({}, False), + ({"mrope_config": {"mrope_position_deltas": None}}, False), + ({"image": {"pixel_values": torch.zeros(1)}}, True), + ({"video": {"pixel_values_videos": torch.zeros(1)}}, True), ], - ids=["text", "empty-payload", "image", "video"], + ids=["text", "empty-payload", "mrope-only", "image", "video"], ) def test_fp8_target_refuses_media_on_admission(multimodal_data, refused): """The text-only target fails a media request on admission, as that request's own error.""" @@ -603,9 +604,9 @@ def test_fp8_target_refuses_media_on_admission(multimodal_data, refused): ModelingV2Glm53FlashFp8Sm10xTp4, ) - request = SimpleNamespace(py_multimodal_data=multimodal_data) - if refused is None: - ModelingV2Glm53FlashFp8Sm10xTp4.validate_request(None, request) + validate = ModelingV2Glm53FlashFp8Sm10xTp4.validate_multimodal_request_data + if not refused: + validate(SimpleNamespace(), multimodal_data) return - with pytest.raises(ValueError, match=f"carries {refused} input"): - ModelingV2Glm53FlashFp8Sm10xTp4.validate_request(None, request) + with pytest.raises(ValueError, match="serves text only"): + validate(SimpleNamespace(), multimodal_data) diff --git a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py index 067d96d789e1..e56b33b49b9b 100644 --- a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py +++ b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/common/test_k3_common.py @@ -13,6 +13,7 @@ import importlib from pathlib import Path +from types import SimpleNamespace import pytest @@ -132,11 +133,14 @@ def test_text_model_weights_skip_the_vision_tower(): def test_check_text_only(): + """The forward refuses what the targets' admission check refuses, and passes the rest.""" fam = _common("weight_io") fam.check_text_only(None) fam.check_text_only([]) - with pytest.raises(ValueError, match="text only"): - fam.check_text_only([object()]) + fam.check_text_only([SimpleNamespace(multimodal_data={"mrope_config": {}})]) + for data in ({"image": {"pixel_values": 1}}, {"multimodal_embedding": 1}): + with pytest.raises(ValueError, match="text only"): + fam.check_text_only([SimpleNamespace(multimodal_data=data)]) def test_common_imports_no_target_code(): diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index d10c638b31ba..2d8d442686c1 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -82,7 +82,7 @@ def _world(member: str) -> int: ) _NVFP4_DEFAULT_TARGET = ( "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." - "ModelingV2Qwen38FlashNextNvfp4Sm103Default" + "ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration" ) # The quantization the engine resolves for the FP8 (128x128 blocks) and the NVFP4 exports. @@ -495,7 +495,8 @@ def test_a_switch_that_would_do_nothing_declines(on_sm103, monkeypatch): @pytest.mark.cpu_only def test_the_quantization_picks_the_checkpoint(on_sm103): - """MIXED_PRECISION routes to the NVFP4 target (its own checks mocked), FP8_BLOCK_SCALES here.""" + """MIXED_PRECISION routes to the NVFP4 target's vision-language model (its own checks + mocked), FP8_BLOCK_SCALES here.""" from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 modeling as nvfp4_modeling, ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py new file mode 100644 index 000000000000..953fe6ca88ca --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_multimodal_model.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The default target's vision-language model on the CPU: what it declares to the shared base +(`ModelingV2ForConditionalGeneration`), its tower, the tower's admission check, its encoder hook +and the rows it hands the language model. The base itself is tested in +`tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py`. + +The language model is never built (its core asserts sm_103): construction swaps in a stub +class, and the other tests call the model's real methods on stubs, with the real multimodal +mixin in between. Towers are stubs over the real `VisionTower.forward`, or real ones on the +meta device. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + vision as V, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + weights as _weights, +) +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration +from tensorrt_llm._torch.models.modeling_multimodal_encoder import MultimodalEncoderMixin +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin +from tensorrt_llm.inputs.multimodal import ( + MULTIMODAL_ENCODER_ITEM_METADATA_KEY, + MultimodalItemMetadata, + MultimodalParams, + MultimodalRuntimeData, +) + +pytestmark = pytest.mark.cpu_only + +Model = M.ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration + +# Qwen3.8-Flash-Next's vision config. +_VISION = dict( + deepstack_visual_indexes=[], + depth=27, + dtype="bfloat16", + hidden_act="gelu_pytorch_tanh", + hidden_size=1152, + in_channels=3, + intermediate_size=4304, + num_heads=16, + num_position_embeddings=2304, + out_hidden_size=2560, + patch_size=16, + spatial_merge_size=2, + temporal_patch_size=2, +) +HIDDEN = 2560 +MERGE = 2 +PIXEL_WIDTH = 4 +VOCAB = 100 + + +# ------------------------------------------------------------------ stubs --- + + +def _tower(dtype=torch.float32, width=1): + """A `VisionTower` whose per-call body is a stub: merged row `j` of item `i` of modality + `m` holds `100 * m + i` in every column, so a row says where it came from. The real + `forward` (grid checks, frame calls) runs around it.""" + tower = object.__new__(V.VisionTower) + nn.Module.__init__(tower) + tower.dtype, tower.pixel_width, tower.merge = dtype, PIXEL_WIDTH, MERGE + tower.calls = 0 + + def run(pixel_values, grid): + tower.calls += 1 + return pixel_values[:: MERGE * MERGE, :1].expand(-1, width).to(dtype).clone() + + tower._run = run + return tower + + +IMAGE, VIDEO = 1, 2 +IMAGES = [(1, 4, 4), (1, 2, 2)] # 4 and 1 rows +VIDEOS = [(2, 2, 4), (2, 4, 4)] # 4 and 8 rows +MIXED_ORDER = [("video", 0), ("image", 0), ("video", 1), ("image", 1)] +MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] + + +def _media(modality, grids): + """One modality's payload; each item's patch rows hold `100 * modality + item`.""" + pixels = torch.cat( + [ + torch.full((t * h * w, PIXEL_WIDTH), float(100 * modality + item)) + for item, (t, h, w) in enumerate(grids) + ] + ) + if modality == IMAGE: + return {"pixel_values": pixels, "image_grid_thw": torch.tensor(grids)} + return {"pixel_values_videos": pixels, "video_grid_thw": torch.tensor(grids)} + + +def _params(image=None, video=None, order=None, runtime=None, mrope=True): + data = {} + if mrope: + data["mrope_config"] = {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)} + if image is not None: + data["image"] = _media(IMAGE, image) + if video is not None: + data["video"] = _media(VIDEO, video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +def _runtime(total: int, cached: int, chunk_end: int) -> MultimodalRuntimeData: + """A prompt of `total` placeholder tokens, `cached` of them reused, up to `chunk_end`.""" + return MultimodalRuntimeData( + embed_mask_cumsum=torch.arange(1, total + 1, dtype=torch.int64), + past_seen_token_num=cached, + chunk_end_pos=chunk_end, + ) + + +class _Llm(nn.Module): + """Records what the vision-language model hands its language model.""" + + def __init__(self): + super().__init__() + self.model = nn.Linear(2, 2) + self.calls = [] + self.lm_head = SimpleNamespace(num_embeddings=VOCAB) + self.multimodal_token_ids = torch.tensor([90, 91], dtype=torch.int32) + + def forward( + self, + attn_metadata, + input_ids=None, + position_ids=None, + inputs_embeds=None, + return_context_logits=False, + spec_metadata=None, + **kwargs, + ): + args = ( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + ) + self.calls.append((args, kwargs)) + return "logits" + + +def _model(tower=None, llm=None): + """A real instance around stubs, without the constructor.""" + model = Model.__new__(Model) + nn.Module.__init__(model) + model.llm = llm if llm is not None else _Llm() + model.mm_encoder = tower + model.model_config = SimpleNamespace(multimodal_config=None, torch_dtype=torch.bfloat16) + return model + + +# ----------------------------------------------------------- construction --- + + +class _StubLlm(nn.Module): + """Stands in for the language model target: takes its config copy and adopts the text + config on it, as the target does.""" + + def __init__(self, model_config): + super().__init__() + self.got = model_config + model_config.pretrained_config = model_config.pretrained_config.text_config + self.model_config = model_config + + @property + def config(self): + return self.model_config.pretrained_config + + +def _composite(): + return PretrainedConfig( + architectures=["Qwen4ExpForConditionalGeneration"], + text_config=PretrainedConfig(hidden_size=HIDDEN, torch_dtype=torch.bfloat16), + vision_config=Qwen4ExpVisionConfig(**_VISION), + torch_dtype=torch.bfloat16, + ) + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_construction_keeps_the_engine_config_and_builds_the_tower_only_when_asked( + monkeypatch, disable +): + monkeypatch.setattr(Model, "language_model_class", _StubLlm) + composite = _composite() + model_config = ModelConfig(pretrained_config=composite, disable_mm_encoder=disable) + with torch.device("meta"): + model = Model(model_config) + + assert model.model_config is model_config + assert model.config is composite.text_config + assert isinstance(model.llm, _StubLlm) + assert isinstance(model, MultimodalModelMixin) + assert model.dtype == torch.bfloat16 + params = {name.split(".")[0] for name, _ in model.named_parameters()} + if disable: + assert model.mm_encoder is None and not params + else: + assert isinstance(model.mm_encoder, V.VisionTower) + assert isinstance(model.mm_encoder, MultimodalEncoderMixin) + assert model.mm_encoder.dtype == torch.bfloat16 + # Every vision weight is the tower's, and every one is read from under the prefix the + # model hands the tower. + assert params == {"mm_encoder"} + assert all(".w.vis_" in name or ".w.l" in name for name, _ in model.named_parameters()) + sources = _weights.VISION_WEIGHTS.manifest(model.mm_encoder.dims).values() + keys = {key for rows in sources for key, *_ in rows} + assert keys and all(key.startswith(Model.mm_encoder_weight_prefix) for key in keys) + + +# --------------------------------------------------------------- contract --- + + +def test_the_class_declares_what_the_shared_base_needs(): + assert issubclass(Model, ModelingV2ForConditionalGeneration) + assert Model.language_model_class is M.ModelingV2Qwen38FlashNextNvfp4Sm103Default + # The base builds it as `VisionTower(vision_config, dtype)`. + assert Model.mm_encoder_class is V.VisionTower + assert Model.mm_encoder_weight_prefix == "model.visual." + # The PLE layer hashes `input_ids`, so the language model places the rows itself. + assert Model.language_model_takes_rows is True + assert Model.supports_encoder_cache is True + assert Model.supports_mm_encoder_item_scheduling is False + assert Model.supported_encoder_attention_backends == {} + assert Model.get_preferred_kv_cache_manager_version() == "V2" + + +def test_the_tower_is_an_encoder_without_engine_metadata(): + assert issubclass(V.VisionTower, MultimodalEncoderMixin) + assert V.VisionTower.encoder_cuda_graph_modality is None + tower = _tower() + tower.setup_attn_metadata(16, {"attention": 16}) + tower.set_attn_max_seq_len(16) + assert tower.attn_metadata is None + + +# -------------------------------------------------------------- admission --- + +_GEOMETRY = SimpleNamespace(merge=MERGE, patch=16, pixel_width=PIXEL_WIDTH) + + +def _request(tokens=(1, 2, 3), image=None, mrope_len=None, bound=None, **data): + """A request whose prompt is `tokens`, optionally around one image's placeholders.""" + prompt_len = len(tokens) + if image is not None: + rows = sum(t * h * w for t, h, w in image) // (MERGE * MERGE) + prompt_len += rows + data["image"] = _media(IMAGE, image) + data["mrope_config"] = { + "mrope_position_ids": torch.zeros(3, 1, mrope_len or prompt_len, dtype=torch.int32) + } + return SimpleNamespace( + py_multimodal_data=data or None, + prompt_len=prompt_len, + py_prompt_token_bound=bound, + check_token_id_range=lambda vocab: max(tokens) < vocab, + ) + + +def test_admission_runs_the_towers_media_check(): + """The base's `validate_request` checks the token range, then the tower's media check + (`vision.validate_request`); without a tower, media is refused by the base.""" + model = _model(_GEOMETRY) + model.validate_request(_request()) + model.validate_request(_request(image=IMAGES)) + # M-RoPE positions alone (a disaggregated generation request) are not media. + model.validate_request(_request(mrope_config={"mrope_position_deltas": torch.zeros(1, 1)})) + with pytest.raises(ValueError, match="one position per prompt token"): + model.validate_request(_request(image=IMAGES, mrope_len=4)) + with pytest.raises(ValueError, match="Token ID out of range"): + model.validate_request(_request(tokens=(1, VOCAB))) + _model(None).validate_request(_request(image=IMAGES, mrope_len=4)) + + +# ----------------------------------------------------------- encoder hook --- + + +def test_the_encoder_hook_takes_a_profiling_dummy(): + """The engine's memory profiling encodes one dummy image with no position ids and no item + metadata, at the model's dtype: rows of the text hidden width, in that dtype.""" + tower = _tower(dtype=torch.bfloat16, width=HIDDEN) + model = _model(tower) + dummy = MultimodalParams(multimodal_data={"image": _media(IMAGE, [(1, 8, 8)])}) + dummy.multimodal_data["image"]["pixel_values"] = dummy.multimodal_data["image"][ + "pixel_values" + ].to(model.dtype) + rows = model.encode_multimodal_inputs([dummy]) + assert rows.shape == (16, HIDDEN) and rows.dtype == torch.bfloat16 + + +def test_the_encoder_hook_returns_each_request_whole_in_prompt_order(): + tower = _tower() + model = _model(tower) + mixed = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER) + rows = model.encode_multimodal_inputs([_params(image=IMAGES), mixed]) + assert rows[:, 0].tolist() == [100.0] * 4 + [101.0] + MIXED_ROWS + with pytest.raises(ValueError, match="require a local multimodal encoder"): + _model(None).encode_multimodal_inputs([mixed]) + + +# ---------------------------------------------------------------- forward --- + + +def _forward(model, params, num_contexts=1, num_rows=None, input_ids=None): + input_ids = input_ids if input_ids is not None else torch.arange(24, dtype=torch.int32) + kwargs = {"multimodal_params": params, "resource_manager": "rm"} + if num_rows is not None: + kwargs["mm_token_indices"] = torch.arange(1, 1 + num_rows, dtype=torch.int64) + out = model.forward( + SimpleNamespace(num_contexts=num_contexts, num_generations=0), + input_ids, + "position_ids", + None, + False, + "spec_metadata", + **kwargs, + ) + assert out == "logits" + ((args, llm_kwargs),) = model.llm.calls + # `input_ids` reach the language model unchanged, and `inputs_embeds` stays None. + assert args[1] is input_ids + assert args[2:] == ("position_ids", None, False, "spec_metadata") + assert "multimodal_params" not in llm_kwargs and llm_kwargs["resource_manager"] == "rm" + return llm_kwargs["multimodal_rows"], kwargs.get("mm_token_indices") + + +def test_a_media_step_hands_rows_and_placeholder_rows_to_the_language_model(): + model = _model(_tower()) + multimodal_rows, indices = _forward(model, [_params(image=IMAGES)], num_rows=5) + rows, mm_token_indices = multimodal_rows + assert rows[:, 0].tolist() == [100.0] * 4 + [101.0] + assert mm_token_indices is indices + + +def test_a_chunk_hands_over_its_slice_of_the_prompt_order_rows(): + """The mixin keeps the chunk's rows: 3 reused, 6 in this chunk.""" + tower = _tower() + model = _model(tower) + runtime = _runtime(len(MIXED_ROWS), cached=3, chunk_end=9) + params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) + (rows, _), _ = _forward(model, [params], num_rows=6) + assert rows[:, 0].tolist() == MIXED_ROWS[3:9] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py index d320852b53c3..509eb8ec29f7 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py @@ -28,6 +28,7 @@ NGramDecodingConfig, TorchLlmArgs, ) +from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo @@ -38,7 +39,7 @@ _ARCH = "Qwen4ExpForConditionalGeneration" _TARGET = ( "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." - "ModelingV2Qwen38FlashNextNvfp4Sm103Default" + "ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration" ) # The four-rank layouts, as the LLM API's parallel arguments build them. tp4 leaves the @@ -546,3 +547,28 @@ def test_a_misspelled_ple_host_table_mode_fails_at_routing(monkeypatch): modeling_v2_resolve(_model_config(**_LAYOUTS["tep4"]), _llm_args("auto")) monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "Node") _claimed(_model_config(**_LAYOUTS["tep4"])) + + +def test_an_encoder_budget_past_one_tower_call_declines(monkeypatch): + """Startup profiling fills the encoder token budget (`encoder_max_num_tokens`, else + `max_num_tokens`) with dummy images, and one tower call takes at most 32768 patches. A + larger budget falls back under `auto`, with one log line naming the setting and the + limit, and fails under `require`. Without a tower the budget is not used.""" + config = _model_config(**_LAYOUTS["tep4"]) + assert _llm_args().encoder_max_num_tokens is None + assert _llm_args().max_num_tokens == 8192 + _claimed(config) + _claimed(config, encoder_max_num_tokens=32768) + lines = [] + monkeypatch.setattr(logger, "info", lines.append) + _declined( + config, + match=r"budget of 65536 \(encoder_max_num_tokens.* at most 32768 patches", + encoder_max_num_tokens=65536, + ) + routing_lines = [line for line in lines if line.startswith("modeling_v2:")] + assert len(routing_lines) == 1, lines + assert "encoder_max_num_tokens" in routing_lines[0], routing_lines + assert "at most 32768 patches" in routing_lines[0], routing_lines + _declined(config, match="budget of 32769", max_num_tokens=32769) + _claimed(replace(config, disable_mm_encoder=True), encoder_max_num_tokens=65536) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py index 29c546ac3e11..dab307dc4805 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """The default target's vision path on the CPU: the tower's weights, its calls and angle table, -the splice's bookkeeping and the admission check. - -The tower is built over the weight table's own `vis_*` declarations on the meta device, so -what it reads is what `weights.py` declares. Its per-call body is a stub where a test needs one -(the vision ops need a GPU), and so is the whole tower where the shell's bookkeeping is under -test: every row of a stub tower names the item it came from, so the modality dispatch, item -metadata, chunk slice, frame grouping and the order of concatenated calls run with no engine. +the encoder hook's bookkeeping and the admission check. + +The tower is built over its own `vis_*` table on the meta device, so what it reads is what +`weights.py` declares. Its per-call body is a stub where a test needs one (the vision ops need +a GPU), and so is the whole tower where the encoder hook's bookkeeping is under test: every row +of a stub tower names the item it came from, so the modality dispatch, item metadata, frame +grouping and the order of concatenated calls run with no engine. The chunk slice is the +multimodal mixin's (`test_multimodal_model.py`). """ from __future__ import annotations @@ -16,6 +17,7 @@ import pytest import torch +from torch import nn from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 @@ -24,9 +26,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 vision as V, ) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 - weights as W, -) from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalItemMetadata, @@ -35,7 +34,7 @@ pytestmark = pytest.mark.cpu_only -Shell = M.ModelingV2Qwen38FlashNextNvfp4Sm103Default +Model = M.ModelingV2Qwen38FlashNextNvfp4Sm103DefaultForConditionalGeneration # Qwen3.8-Flash-Next's vision config. _VISION = dict( @@ -57,28 +56,12 @@ def _tower_over_the_weight_table() -> V.VisionTower: - """The tower over every `vis_*` weight the table declares, at the checkpoint's shapes, - on meta; `embed` stands in for the activation dtype the tower reads off it.""" - vcfg = Qwen4ExpVisionConfig(**_VISION) - d = SimpleNamespace( - hidden=HIDDEN_OUT, - dtype=torch.bfloat16, - v_depth=vcfg.depth, - v_hidden=vcfg.hidden_size, - v_inter=vcfg.intermediate_size, - v_channels=vcfg.in_channels, - v_tpatch=vcfg.temporal_patch_size, - v_patch=vcfg.patch_size, - v_pos_rows=vcfg.num_position_embeddings, - v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, - ) - w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="meta")} - for entry in W.MODEL_WEIGHTS.WEIGHTS: - if not entry.name.startswith("vis_"): - continue - for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): - w[key] = torch.empty(*entry.shape(d), dtype=entry.dtype or d.dtype, device="meta") - return V.VisionTower(w, vcfg) + """The tower at the checkpoint's shapes, on meta: it declares its own `vis_*` table + (`weights.VISION_WEIGHTS`), and the views are what `post_load_weights` builds.""" + with torch.device("meta"): + tower = V.VisionTower(Qwen4ExpVisionConfig(**_VISION), torch.bfloat16) + tower._w = V._vision_views(tower.w, tower.depth) + return tower def test_the_tower_reads_the_weight_tables_vision_rows_at_the_ops_layouts(): @@ -212,6 +195,7 @@ def _stub_tower(monkeypatch, cap): the request.""" monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", cap) tower = object.__new__(V.VisionTower) + nn.Module.__init__(tower) tower.dtype = torch.float32 tower.pixel_width = PIXEL_WIDTH tower.merge = MERGE @@ -315,16 +299,10 @@ def _params(image=None, video=None, order=None, runtime=None, mrope=True): def _splice(tower, params, num_rows, padding=0): - """The tower's rows spliced, as the prefill target's break does, over a prompt whose - tokens 1..num_rows are the placeholders and which `padding` rows follow; returns the rows - that landed there, one value per row, or None when the step carries no media.""" - kwargs = { - "multimodal_params": [params], - "mm_token_indices": torch.arange(1, 1 + num_rows, dtype=torch.int64), - } - vision = V.multimodal_embeds(tower, kwargs) - if vision is None: - return None + """The encoder hook's rows spliced, as the prefill target's break does, over a prompt + whose tokens 1..num_rows are the placeholders and which `padding` rows follow; returns the + rows that landed there, one value per row.""" + vision = (V.encode(tower, params), torch.arange(1, 1 + num_rows, dtype=torch.int64)) total = num_rows + 2 h = torch.full((total + padding, HIDDEN), -1.0) target = SimpleNamespace(_step=SimpleNamespace(vision=vision, num_tokens=total)) @@ -345,13 +323,6 @@ def _item(modality, index): MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] -def test_a_text_step_runs_no_tower(): - assert V.multimodal_embeds(None, {}) is None - assert V.multimodal_embeds(None, {"multimodal_params": []}) is None - # Position ids alone (a text prompt through a multimodal processor) are not media. - assert V.multimodal_embeds(None, {"multimodal_params": [_params()]}) is None - - def test_the_splice_fills_the_placeholders_and_zeroes_the_padding(): """An image request's rows land on its placeholders in one tower call and the padding rows after the step are zeroed (`_splice` checks); a text step writes only the padding.""" @@ -382,47 +353,20 @@ def test_a_mixed_request_needs_its_manifest(): _splice(_Tower(), _params(image=IMAGES, video=VIDEOS), len(MIXED_ROWS)) -def test_a_chunk_takes_its_slice_of_the_prompt_order_rows(): - runtime = SimpleNamespace( - num_cached_mm_tokens=3, num_mm_tokens_in_chunk=6, total_embeds_in_request=len(MIXED_ROWS) - ) - params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) - assert _splice(_Tower(), params, 6) == MIXED_ROWS[3:9] - - -def test_a_chunk_without_placeholders_skips_the_tower(): - runtime = SimpleNamespace( - num_cached_mm_tokens=len(MIXED_ROWS), - num_mm_tokens_in_chunk=0, - total_embeds_in_request=len(MIXED_ROWS), - ) - tower = _Tower() - params = _params(image=IMAGES, video=VIDEOS, order=MIXED_ORDER, runtime=runtime) - assert _splice(tower, params, 0) is None - assert tower.calls == 0 - - -def test_a_media_request_needs_the_processors_position_ids(): - with pytest.raises(AssertionError, match="mrope_position_ids"): - _splice(_Tower(), _params(video=VIDEOS, mrope=False), 12) - - -def test_the_placeholder_rows_must_match_the_towers_rows(): - kwargs = { - "multimodal_params": [_params(image=IMAGES)], - "mm_token_indices": torch.arange(4, dtype=torch.int64), - } - with pytest.raises(AssertionError, match="4 placeholder rows and the tower produced 5"): - V.multimodal_embeds(_Tower(), kwargs) +def test_the_encoder_hook_needs_no_position_ids(): + """The engine's memory profiling hands the encoder a dummy image with no position ids; + admission checks them for a real request.""" + assert _splice(_Tower(), _params(image=IMAGES, mrope=False), 5) == [100.0] * 4 + [101.0] def test_the_grids_stay_on_the_host(): """The executor moves only the pixels and the position ids; the grids the tower and the admission check read on the host stay there.""" - paths = Shell.multimodal_data_device_paths.fget(None) + paths = Model.multimodal_data_device_paths.fget(None) assert set(paths) == { "image.pixel_values", "video.pixel_values_videos", + "multimodal_embedding", "mrope_config.mrope_position_ids", "mrope_config.mrope_position_deltas", } @@ -471,8 +415,8 @@ def _request(image=None, video=None, order=None, **overrides): def _admit(request): - """Through the shell's own hook, as the executor calls it.""" - Shell.validate_request(SimpleNamespace(vision_tower=_GEOMETRY), request) + """The tower's admission check, which the model's `validate_request` calls.""" + V.validate_request(_GEOMETRY, request) def test_clean_requests_are_admitted(): diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py index fddaf5b036af..9abb4eec46fc 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_vision_warm_up.py @@ -4,22 +4,17 @@ `post_load_weights` runs, a media request of any grid compiles no Triton kernel. The tower is one block deep (the block's kernels are every block's) over random weights at the -checkpoint's shapes, built from the weight table's own `vis_*` declarations. +checkpoint's shapes, declared by the tower itself (`weights.VISION_WEIGHTS`). """ from __future__ import annotations -from types import SimpleNamespace - import torch from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 vision as V, ) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 - weights as W, -) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.ops.attention.vision_rope_2d_packed_qk import ( # noqa: E501 _rotary_neox_inplace_kernel, ) @@ -58,27 +53,13 @@ def _tower() -> V.VisionTower: vcfg = Qwen4ExpVisionConfig(**_VISION) - d = SimpleNamespace( - hidden=HIDDEN_OUT, - dtype=torch.bfloat16, - v_depth=vcfg.depth, - v_hidden=vcfg.hidden_size, - v_inter=vcfg.intermediate_size, - v_channels=vcfg.in_channels, - v_tpatch=vcfg.temporal_patch_size, - v_patch=vcfg.patch_size, - v_pos_rows=vcfg.num_position_embeddings, - v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, - ) + with torch.device("cuda"): + tower = V.VisionTower(vcfg, torch.bfloat16) gen = torch.Generator(device="cuda").manual_seed(7) - w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="cuda")} - for entry in W.MODEL_WEIGHTS.WEIGHTS: - if not entry.name.startswith("vis_"): - continue - for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): - shape, dtype = entry.shape(d), entry.dtype or d.dtype - w[key] = (torch.randn(*shape, generator=gen, device="cuda") * 0.02).to(dtype) - return V.VisionTower(w, vcfg) + for param in tower.w.values(): + param.data.copy_(torch.randn(param.shape, generator=gen, device="cuda") * 0.02) + tower._w = V._vision_views(tower.w, tower.depth) + return tower def _variants() -> dict: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py index 70d77773904a..eb28f6fca4a7 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_weight_shards.py @@ -44,6 +44,9 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 pool as P, ) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + vision as V, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 weights as WT, ) @@ -131,10 +134,9 @@ def _mapping(member: str, rank: int) -> Mapping: def _dims(configs, member: str, rank: int, mtp: bool = True, prefill_copy: bool = False): - text, vision = configs + text, _ = configs return WT.rank_dims( text, - vision, _mapping(member, rank), mtp_enabled=mtp, trunk_moe_backend="TRTLLM", @@ -169,6 +171,10 @@ def _rank_manifest(stored, d, member: str) -> dict[str, list[tuple]]: return manifest +def _vision_dims(configs): + return WT.vision_dims(configs[1], torch.bfloat16) + + def _keys_and_cuts(ckpt_key, cut): if isinstance(ckpt_key, tuple): return list(zip(ckpt_key, cut if cut is not None else (None,) * len(ckpt_key))) @@ -253,6 +259,10 @@ def test_every_checkpoint_tensor_is_claimed(stored, configs, member, mtp): f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" ) read_by.update(claimed) + # The tower's own table reads the `model.visual.` keys the language table leaves. + vision_claimed = _claimed(WT.VISION_WEIGHTS.manifest(_vision_dims(configs))) + assert vision_claimed == {k for k in everything if k.startswith("model.visual.")} + read_by.update(vision_claimed) mtp_keys = set() if mtp else {k for k in everything if k.startswith("mtp.")} assert set(read_by) == everything - mtp_keys, "tensors no rank reads: " + str( @@ -278,8 +288,11 @@ def test_rank_operands_have_their_parameter_shapes(stored, configs, member, rank fills its parameter exactly; the rank reads each tensor at its share, and once per layout it is kept in (the routed experts twice with the CuteDSL prefill copy).""" d = _dims(configs, member, rank, mtp=True, prefill_copy=prefill_copy) + vd = _vision_dims(configs) with torch.device("meta"): params = dict(WT.MODEL_WEIGHTS.declare(d)) + # The vision tower's table, which every rank declares whole. + params.update(WT.VISION_WEIGHTS.declare(vd)) # What `load` adds beside the table: the n-gram table and the shell's lm_head. params["ple_table"] = torch.empty( d.ple_shards * _rows_per_shard(stored, d), d.ple_head_dim, dtype=d.ple_table_dtype @@ -289,6 +302,7 @@ def test_rank_operands_have_their_parameter_shapes(stored, configs, member, rank ) entries = {key: _HeaderEntry(*spec) for key, spec in stored.items()} manifest = _rank_manifest(stored, d, member) + manifest.update(WT.VISION_WEIGHTS.manifest(vd)) assert manifest.keys() == params.keys() read = collections.Counter() @@ -507,6 +521,9 @@ def test_load_fills_every_parameter_and_covers_the_checkpoint( model = _stub_model(configs, member, rank, mtp) WT.MODEL_WEIGHTS.load(model, _meta_checkpoint(stored)) assert tuple(model.model.w["ple_table"].shape) == (320_001_536, 160) + with torch.device("meta"): + tower = V.VisionTower(configs[1], torch.bfloat16) + tower.load_weights(_meta_checkpoint(stored)) @pytest.mark.parametrize("expert_override", (None, QuantAlgo.NVFP4)) @@ -1044,6 +1061,7 @@ def test_dep2_ranks_read_the_checkpoint_in_two_256_expert_windows(stored, config f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" ) read_by.update(claimed) + read_by.update(_claimed(WT.VISION_WEIGHTS.manifest(_vision_dims(configs)))) dep4 = _dims(configs, "dep4", rank, mtp=False) non_expert = {k for k in claimed if not _TRUNK_EXPERT_TENSOR.match(k)} @@ -1095,3 +1113,93 @@ def test_dep2_load_fills_every_parameter_and_covers_the_checkpoint(stored, confi model = _stub_model(configs, "dep2", 1, False) WT.MODEL_WEIGHTS.load(model, _meta_checkpoint(stored)) assert model.lm_head.weight.shape[0] == 248320 + + +# ------------------------------------------------------- the vision table -- + +#: Every vision-tower weight, as the language table declared them before the tower took +#: its own table: the split loses none. +_VISION_ENTRIES = { + "vis_patch_w", "vis_patch_b", "vis_pos", + "vis_n1_w", "vis_n1_b", "vis_qkv_w", "vis_qkv_b", "vis_proj_w", "vis_proj_b", + "vis_n2_w", "vis_n2_b", "vis_fc1_w", "vis_fc1_b", "vis_fc2_w", "vis_fc2_b", + "vis_m_n_w", "vis_m_n_b", "vis_m_fc1_w", "vis_m_fc1_b", "vis_m_fc2_w", "vis_m_fc2_b", +} # fmt: skip + +#: A small vision config: two blocks, every width shrunk. +_SMALL_VISION = SimpleNamespace( + depth=2, + hidden_size=8, + num_heads=2, + intermediate_size=12, + in_channels=3, + temporal_patch_size=2, + patch_size=2, + num_position_embeddings=16, + spatial_merge_size=2, + out_hidden_size=6, +) + + +def test_the_vision_weights_are_a_table_of_their_own(): + """The language table declares no vision weight, so `disable_mm_encoder` allocates + none; the tower's table declares exactly the entries the language table had.""" + assert not [e.name for e in WT.MODEL_WEIGHTS.WEIGHTS if e.name.startswith("vis_")] + assert {e.name for e in WT.VISION_WEIGHTS.WEIGHTS} == _VISION_ENTRIES + + +@pytest.mark.parametrize("mtp", [False, True], ids=["no-mtp", "mtp"]) +def test_the_language_table_leaves_every_vision_key(mtp): + d = SimpleNamespace( + num_layers=1, + num_experts=2, + expert_offset=0, + local_experts=2, + mtp_enabled=mtp, + mtp_expert_offset=0, + mtp_local_experts=2, + ) + keys = { + "model.visual.patch_embed.proj.weight", + "model.visual.blocks.0.norm1.weight", + "model.visual.merger.linear_fc2.bias", + "model.language_model.embed_tokens.weight", + "mtp.fc_hidden.weight", + } + unread = WT.MODEL_WEIGHTS.expected_unconsumed(d, keys) + assert {k for k in unread if not k.startswith("mtp.")} == { + k for k in keys if k.startswith("model.visual.") + } + + +def _small_vision_checkpoint(tower) -> dict: + """One distinct tensor per key the tower's table reads, at its declared shape.""" + checkpoint = {} + for value, (key, sources) in enumerate(WT.VISION_WEIGHTS.manifest(tower.dims).items(), 1): + ((ckpt_key, index, cut, transform),) = sources + assert index is None and cut is None and transform is None, ckpt_key + param = tower.w[key] + checkpoint[ckpt_key] = torch.full(param.shape, float(value), dtype=param.dtype) + return checkpoint + + +def test_the_tower_loads_its_table_and_covers_the_vision_keys(): + """The tower module declares its table at a small config, `load_weights` fills every + parameter from `model.visual.` keys and reads them all, and the language model's keys + are not its to read. An unknown or missing vision key fails the load.""" + tower = V.VisionTower(_SMALL_VISION, torch.float32) + # Twelve entries per block, at two blocks. + assert sum(k.startswith(("l0_", "l1_")) for k in tower.w) == 24 + checkpoint = _small_vision_checkpoint(tower) + assert tuple(tower.w["vis_m_fc2_w"].shape) == (6, 32) + tower.load_weights({**checkpoint, "model.language_model.embed_tokens.weight": None}) + manifest = WT.VISION_WEIGHTS.manifest(tower.dims) + for key, param in tower.w.items(): + assert torch.equal(param.data, checkpoint[manifest[key][0][0]]), key + + with pytest.raises(AssertionError, match="vision checkpoint coverage"): + tower.load_weights({**checkpoint, "model.visual.extra.weight": torch.zeros(1)}) + missing = dict(checkpoint) + del missing["model.visual.pos_embed.weight"] + with pytest.raises(AssertionError, match="checkpoint key missing"): + tower.load_weights(missing) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py index a36231c4b081..f83f6140fd56 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_contracts.py @@ -1,13 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Code contracts of the catalog entries and their cells. +"""Code contracts of Modeling V2 code: the catalog entries and their cells, and the +multimodal contract and naming rule of the targets of vision-language architectures. An entry computes a pure function of its arguments, so its ``CELLS`` cover its -whole input space. These checks keep that true. They read the source and import -nothing, so they run on any CPU machine. +whole input space. A vision-language target runs media through the multimodal +runtime or refuses it at admission, and a vision-language model subclasses the +shared base and its name ends with ``ForConditionalGeneration``. These checks +keep that true. They read the source and import no target, so they run on any +CPU machine; a text-only target's admission hook runs on its own, with the shared +helper. Which architectures take media is the router's own check, from the +built-in input processors. """ from __future__ import annotations +import __future__ import ast import re @@ -197,3 +204,577 @@ def test_target_cells_feed_entries_the_target_calls(test): assert re.search(rf"catalog\.{area}\.{entry} import", used), ( f"{test.name}: declares cells for {target}, which never imports {area}/{entry}" ) + + +# --- the multimodal contract of vision-language targets ---------------------------- + + +def _routed_classes() -> list[tuple[str, Path, str]]: + """`(architecture, modeling.py, class name)` for every class a family's `route` returns. + + Each `return` of `route` names a class that `route` imports from a target's `modeling` + module, as every route does today. Any other return fails here, so no routed class is + skipped unread.""" + found, unread = [], [] + for routing in sorted((_ROOT / "models").glob("*/routing.py")): + for fn in ast.parse(routing.read_text()).body: + if not isinstance(fn, ast.FunctionDef): + continue + archs = [ + arg.value + for dec in fn.decorator_list + if isinstance(dec, ast.Call) and getattr(dec.func, "id", None) == "routes" + for arg in dec.args + if isinstance(arg, ast.Constant) + ] + if not archs: + continue + imported = { + alias.asname or alias.name: Path(routing.parent, *node.module.split(".")) + for node in ast.walk(fn) + if isinstance(node, ast.ImportFrom) + and node.level == 1 + and (node.module or "").endswith(".modeling") + for alias in node.names + } + for node in ast.walk(fn): + if not isinstance(node, ast.Return): + continue + name = getattr(node.value, "id", None) + if name in imported: + found += [(arch, imported[name].with_suffix(".py"), name) for arch in archs] + else: + unread.append(f"{routing.relative_to(_ROOT)}:{node.lineno}") + assert not unread, ( + f"{unread}: return the class that route imports from the target's modeling module " + "directly, so these checks can read it" + ) + return found + + +def _takes_media(architecture: str) -> bool: + """Whether requests for `architecture` can carry media, as the router decides it.""" + from tensorrt_llm._torch.modeling_v2._router_index import _architecture_takes_media + + return _architecture_takes_media(architecture) + + +def _media_routes() -> list[tuple[Path, str]]: + """The classes routed for an architecture whose requests can carry media.""" + return sorted({(path, name) for arch, path, name in _routed_classes() if _takes_media(arch)}) + + +def _calls(fn: ast.AST, name: str) -> bool: + return any( + isinstance(n, ast.Call) and getattr(n.func, "id", getattr(n.func, "attr", None)) == name + for n in ast.walk(fn) + ) + + +def _raises(fn: ast.AST) -> bool: + return any(isinstance(n, ast.Raise) for n in ast.walk(fn)) + + +def _refuses_anything(fn: ast.AST) -> bool: + """Whether the method calls or raises anything: a docstring or `pass` alone refuses nothing.""" + return _raises(fn) or any(isinstance(n, ast.Call) for n in ast.walk(fn)) + + +#: A text-only target's admission error, as the shared helper's users word it. +_TEXT_ONLY_MESSAGE = "{} serves text only; send no images, videos or embeddings" +#: `multimodal_data` a text-only target refuses (the last key is one no helper knows), and the +#: ones it admits: none, or M-RoPE positions alone. +_REFUSED = ( + {"image": {"pixel_values": 0}}, + {"video": {"pixel_values_videos": 0}}, + {"multimodal_embedding": 0}, + {"new_modality": 0}, +) +_ADMITTED = ({}, {"mrope_config": {"mrope_position_deltas": 0}}) + + +def _admission_problems(where: str, name: str, hook: ast.FunctionDef) -> list[str]: + """Run a text-only target's `validate_multimodal_request_data` on its own, as a method of a + class named `name`, with the real `has_media_or_embeddings`: it refuses exactly what the + helper flags, with the shared message. The target module itself is not imported.""" + from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + + namespace = {"has_media_or_embeddings": has_media_or_embeddings} + code = compile( + ast.Module(body=[hook], type_ignores=[]), + where, + "exec", + flags=__future__.annotations.compiler_flag, + dont_inherit=True, + ) + exec(code, namespace) + model = type(name, (), {hook.name: namespace[hook.name]})() + wrong = [] + for data in _ADMITTED: + try: + model.validate_multimodal_request_data(data) + except Exception as error: + wrong.append(f"refuses {sorted(data)}: {error!r}") + for data in _REFUSED: + try: + model.validate_multimodal_request_data(data) + except ValueError as error: + if str(error) != _TEXT_ONLY_MESSAGE.format(name): + wrong.append(f"refuses {sorted(data)} with {str(error)!r}") + except Exception as error: + wrong.append(f"refuses {sorted(data)} with {error!r}, not a ValueError") + else: + wrong.append(f"admits {sorted(data)}") + if not wrong: + return [] + return [ + f"{where}: validate_multimodal_request_data must refuse what has_media_or_embeddings " + f"flags with {_TEXT_ONLY_MESSAGE.format(name)!r} and admit the rest, but {wrong}" + ] + + +#: The shared base of vision-language models, and what a subclass declares: the base does +#: the rest. Its tower is `mm_encoder_class`, or an override of `build_mm_encoder`. +_BASE = "ModelingV2ForConditionalGeneration" +_SUBCLASS_ATTRIBUTES = ("language_model_class", "mm_encoder_weight_prefix") +_SUBCLASS_METHODS = ("encode_multimodal_inputs",) + + +def _class_def(path: Path, name: str) -> ast.ClassDef | None: + """The top-level class `name` of the module at `path`, or None.""" + for node in ast.parse(path.read_text()).body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def _base_names(cls: ast.ClassDef) -> set[str]: + return {getattr(b, "id", getattr(b, "attr", None)) for b in cls.bases} + + +def _assignments(cls: ast.ClassDef) -> dict[str, ast.expr | None]: + """The names the class body assigns, each with its value (None for a bare annotation).""" + return { + t.id: n.value + for n in cls.body + if isinstance(n, (ast.Assign, ast.AnnAssign)) + for t in (n.targets if isinstance(n, ast.Assign) else [n.target]) + if isinstance(t, ast.Name) + } + + +#: Targets that encode media in their own forward instead of on the shared base, as +#: `(target directory under models/, class name)`: the qwen4_exp FP8 target encodes only the +#: frames of the current prefill chunk. Each refuses at admission, in a `validate_request` of +#: its own class, what its tower cannot take; the router accepts it for that alone. Add no +#: entry: a new vision-language target subclasses the base. An entry whose class moves onto +#: the base or serves text only fails until it is deleted, so the set only shrinks. +_OWN_ENCODER_TARGETS = frozenset( + { + ( + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified", + "ModelingV2Qwen38FlashNextFp8Sm103Unified", + ), + } +) + + +def _target_key(path: Path, name: str) -> tuple[str, str]: + return path.parent.relative_to(_ROOT / "models").as_posix(), name + + +def _multimodal_contract_problems(path: Path, name: str) -> list[str]: + """A class routed for media is either a `ModelingV2ForConditionalGeneration` subclass that + declares `_SUBCLASS_ATTRIBUTES` and its tower and defines `_SUBCLASS_METHODS`, or serves + text only and refuses media at admission as `_admission_problems` checks, or is listed in + `_OWN_ENCODER_TARGETS`, still needs to be, and defines a `validate_request` that refuses + something. A `forward` that raises on multimodal input uses `has_media_or_embeddings`.""" + where = f"{path.relative_to(_ROOT)}:{name}" + cls = _class_def(path, name) + if cls is None: + return [f"{where}: no such class"] + defs = {n.name: n for n in cls.body if isinstance(n, ast.FunctionDef)} + assigned = _assignments(cls) + problems = [] + listed = _target_key(path, name) in _OWN_ENCODER_TARGETS + if listed and (_BASE in _base_names(cls) or "validate_multimodal_request_data" in defs): + # The class meets the contract another way now: drop the entry, check it as any other. + problems.append(f"{where}: no longer encodes media itself; delete its entry") + listed = False + if listed: + hook = defs.get("validate_request") + if hook is None: + problems.append(f"{where}: encodes media itself and does not define validate_request") + elif not _refuses_anything(hook): + problems.append( + f"{where}: encodes media itself and its validate_request refuses nothing" + ) + elif _BASE in _base_names(cls): + problems += [ + f"{where}: the subclass does not declare {m}" + for m in _SUBCLASS_ATTRIBUTES + if assigned.get(m) is None + ] + problems += [ + f"{where}: the subclass does not define {m}" for m in _SUBCLASS_METHODS if m not in defs + ] + # A bare annotation or `None` builds no tower. + tower = assigned.get("mm_encoder_class") + if "build_mm_encoder" not in defs and ( + tower is None or (isinstance(tower, ast.Constant) and tower.value is None) + ): + problems.append( + f"{where}: the subclass neither sets mm_encoder_class to a tower class nor " + "defines build_mm_encoder" + ) + else: + hook = defs.get("validate_multimodal_request_data") + if hook is None: + problems.append( + f"{where}: neither a {_BASE} subclass nor a text-only target with " + "validate_multimodal_request_data" + ) + else: + problems += _admission_problems(where, name, hook) + forward = defs.get("forward") + if ( + forward is not None + and _raises(forward) + and "multimodal_params" in ast.unparse(forward) + and not _calls(forward, "has_media_or_embeddings") + ): + problems.append( + f"{where}: forward refuses multimodal input without has_media_or_embeddings" + ) + return problems + + +def test_vision_language_targets_keep_the_multimodal_contract(): + """Every class routed for an architecture whose requests can carry media either runs media + through the multimodal runtime or refuses it at admission, where only that request fails. + Each text-only target's admission hook runs on media, embeddings and M-RoPE positions. The + router checks the contract's shape at startup; this finds a broken target without a GPU.""" + routed = _media_routes() + families = {path.relative_to(_ROOT / "models").parts[0] for path, _ in routed} + assert {"glm5_next", "kimi_k3_vl", "minimax_m3", "qwen4_exp"} <= families, families + text_only = { + path.relative_to(_ROOT / "models").parts[0] + for path, name in routed + if any( + isinstance(node, ast.FunctionDef) and node.name == "validate_multimodal_request_data" + for node in getattr(_class_def(path, name), "body", ()) + ) + } + assert {"glm5_next", "kimi_k3_vl", "minimax_m3"} <= text_only, text_only + stale = _OWN_ENCODER_TARGETS - {_target_key(path, name) for path, name in routed} + assert not stale, f"{sorted(stale)}: no longer routed for media; delete the entry" + problems = [p for path, name in routed for p in _multimodal_contract_problems(path, name)] + assert not problems, problems + + +_FIXTURE_ROUTING = """ +from ..._router_index import routes + + +@routes("{arch}") +def route(model_config, llm_args, compute_capability): + from .target.modeling import {cls} + + return {cls} +""" + +_TEXT_ONLY_OK = """ +class Good(Base): + def validate_multimodal_request_data(self, mm_data: dict) -> None: + if has_media_or_embeddings(mm_data): + raise ValueError( + f"{type(self).__name__} serves text only; send no images, videos or embeddings" + ) + + def forward(self, *args, **kwargs): + params = kwargs.pop("multimodal_params", None) or [] + if any(has_media_or_embeddings(p.multimodal_data) for p in params): + raise ValueError("text only") +""" + +_SUBCLASS_OK = """ +class Good(ModelingV2ForConditionalGeneration): + language_model_class = Llm + mm_encoder_class = Tower + mm_encoder_weight_prefix = "model.visual." + + def encode_multimodal_inputs(self, multimodal_params): + return None +""" +_BUILT_TOWER = """ + def build_mm_encoder(self, pretrained_config): + return Tower(pretrained_config.audio_config) +""" + + +@pytest.mark.parametrize( + "source, problem", + [ + (_TEXT_ONLY_OK, None), + (_SUBCLASS_OK, None), + ("class Good(Base):\n pass\n", "neither a ModelingV2ForConditionalGeneration"), + ( + _TEXT_ONLY_OK.replace("if has_media_or_embeddings(mm_data)", "if mm_data"), + "but [\"refuses ['mrope_config']", + ), + ( + _TEXT_ONLY_OK.replace("if has_media_or_embeddings", "if not has_media_or_embeddings"), + 'but ["refuses []', + ), + ( + _TEXT_ONLY_OK.replace("raise ValueError(", "ValueError("), + "but [\"admits ['image']\"", + ), + ( + _TEXT_ONLY_OK.replace("serves text only", "is text only"), + "but [\"refuses ['image'] with 'Good is text only", + ), + ( + _TEXT_ONLY_OK.replace("raise ValueError(", "raise RuntimeError("), + "not a ValueError", + ), + ( + _TEXT_ONLY_OK.replace( + "any(has_media_or_embeddings(p.multimodal_data) for p in params)", "params" + ), + "forward refuses multimodal input", + ), + ( + _SUBCLASS_OK.replace(" language_model_class = Llm\n", ""), + "does not declare language_model_class", + ), + ( + _SUBCLASS_OK.replace("def encode_multimodal_inputs", "def other"), + "does not define encode_multimodal_inputs", + ), + (_SUBCLASS_OK.replace(" mm_encoder_class = Tower\n", "") + _BUILT_TOWER, None), + ( + _SUBCLASS_OK.replace(" mm_encoder_class = Tower\n", ""), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ( + _SUBCLASS_OK.replace("mm_encoder_class = Tower", "mm_encoder_class = None"), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ( + _SUBCLASS_OK.replace("mm_encoder_class = Tower", "mm_encoder_class: type"), + "neither sets mm_encoder_class to a tower class nor defines build_mm_encoder", + ), + ], + ids=[ + "text_only_ok", + "subclass_ok", + "no_hook", + "truthiness_hook", + "inverted_hook", + "hook_never_raises", + "other_message", + "other_error_type", + "truthiness_forward", + "subclass_without_language_model_class", + "subclass_without_encoder_hook", + "subclass_building_its_tower", + "subclass_without_tower", + "subclass_with_a_none_tower", + "subclass_with_a_bare_tower_annotation", + ], +) +def test_the_multimodal_contract_check_reports_a_broken_target( + tmp_path, monkeypatch, source, problem +): + """The architecture's input processor picks the classes to check, not its name: the + media architecture is named like a text one and the text one like a vision-language + one.""" + for arch, family in (("FooForCausalLM", "foo"), ("BarForConditionalGeneration", "bar")): + target = tmp_path / "models" / family / "target" + target.mkdir(parents=True) + (target.parent / "routing.py").write_text(_FIXTURE_ROUTING.format(arch=arch, cls="Good")) + # The text architecture's class is never checked. + (target / "modeling.py").write_text( + source if family == "foo" else "class Good:\n pass\n" + ) + module = sys.modules[__name__] + monkeypatch.setattr(module, "_ROOT", tmp_path) + monkeypatch.setattr(module, "_takes_media", lambda arch: arch == "FooForCausalLM") + routed = _media_routes() + assert routed == [(tmp_path / "models/foo/target/modeling.py", "Good")] + problems = [p for path, name in routed for p in _multimodal_contract_problems(path, name)] + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems + + +_OWN_ENCODER_OK = """ +class Good(Base): + def validate_request(self, request): + \"\"\"Refuse a media request the tower would assert on.\"\"\" + vision.validate_request(self.vision_tower, request) +""" + + +@pytest.mark.parametrize( + "source, problem", + [ + (_OWN_ENCODER_OK, None), + ("class Good(Base):\n pass\n", "does not define validate_request"), + ( + _OWN_ENCODER_OK.replace("vision.validate_request(self.vision_tower, request)", "pass"), + "its validate_request refuses nothing", + ), + (_SUBCLASS_OK, "no longer encodes media itself; delete its entry"), + (_TEXT_ONLY_OK, "no longer encodes media itself; delete its entry"), + ], + ids=[ + "own_encoder_ok", + "own_encoder_without_validate_request", + "own_encoder_refusing_nothing", + "moved_onto_the_base", + "now_text_only", + ], +) +def test_a_listed_own_encoder_target_must_refuse_at_admission( + tmp_path, monkeypatch, source, problem +): + """A listed target needs a `validate_request` of its own that refuses something, and the + entry must still be needed: a class that moved onto the base or serves text only is checked + as such, and its entry is reported for deletion.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + (target / "modeling.py").write_text(source) + module = sys.modules[__name__] + monkeypatch.setattr(module, "_ROOT", tmp_path) + monkeypatch.setattr(module, "_OWN_ENCODER_TARGETS", frozenset({("foo/target", "Good")})) + problems = _multimodal_contract_problems(target / "modeling.py", "Good") + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems + + +def test_a_route_return_the_checks_cannot_read_fails(tmp_path, monkeypatch): + """A route that returns its class through a local name fails instead of being skipped.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + routing = _FIXTURE_ROUTING.format(arch="FooForConditionalGeneration", cls="Good") + (target.parent / "routing.py").write_text( + routing.replace(" return Good", " target_class = Good\n return target_class") + ) + (target / "modeling.py").write_text(_SUBCLASS_OK) + monkeypatch.setattr(sys.modules[__name__], "_ROOT", tmp_path) + with pytest.raises(AssertionError, match=r"models/foo/routing\.py:10\b"): + _routed_classes() + + +# --- the naming rule of vision-language models ---------------------------------------- + +#: A class with one of these bases is a multimodal model. +_MULTIMODAL_BASES = {_BASE, "MultimodalModelMixin"} + + +def _routed_multimodal_models() -> list[tuple[Path, ast.ClassDef]]: + """Every routed class that is a multimodal model, whatever its architecture.""" + found = [] + for path, name in sorted({(path, name) for _, path, name in _routed_classes()}): + cls = _class_def(path, name) + if cls is not None and _base_names(cls) & _MULTIMODAL_BASES: + found.append((path, cls)) + return found + + +def _naming_rule_problems() -> list[str]: + """A routed multimodal model subclasses `ModelingV2ForConditionalGeneration` and its name + ends with `ForConditionalGeneration`; no class of a target's `modeling.py` is named + `...VLM`.""" + problems = [] + for path, cls in _routed_multimodal_models(): + where = f"{path.relative_to(_ROOT)}:{cls.name}" + if _BASE not in _base_names(cls): + problems.append(f"{where}: a multimodal model that does not subclass {_BASE}") + if not cls.name.endswith("ForConditionalGeneration"): + problems.append(f"{where}: a multimodal model not named ...ForConditionalGeneration") + for path in sorted((_ROOT / "models").glob("*/*/modeling.py")): + problems += [ + f"{path.relative_to(_ROOT)}:{node.name}: a target class named ...VLM" + for node in ast.walk(ast.parse(path.read_text())) + if isinstance(node, ast.ClassDef) and node.name.upper().endswith("VLM") + ] + return problems + + +def test_vision_language_models_follow_the_naming_rule(): + """Every routed multimodal model subclasses the shared base and its name ends with + `ForConditionalGeneration`, and no target class is named `...VLM`. The rest of the name, + `ModelingV2`, is left to review. A language model target is not a multimodal + model, so its `ModelingV2` name is not checked here.""" + families = { + path.relative_to(_ROOT / "models").parts[0] for path, _ in _routed_multimodal_models() + } + assert "qwen4_exp" in families, families + problems = _naming_rule_problems() + assert not problems, problems + + +_NAMED_OK = """ +class ModelingV2Foo(Base): + pass + + +class ModelingV2FooForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = ModelingV2Foo +""" + + +@pytest.mark.parametrize( + "routed, source, problem", + [ + ("ModelingV2FooForConditionalGeneration", _NAMED_OK, None), + # A language model target routed on its own is not a multimodal model. + ("ModelingV2Foo", _NAMED_OK, None), + ( + "ModelingV2FooForConditionalGeneration", + _NAMED_OK.replace(f"({_BASE})", "(MultimodalModelMixin, PreTrainedModel)"), + f"does not subclass {_BASE}", + ), + ( + "ModelingV2FooModel", + _NAMED_OK.replace("ModelingV2FooForConditionalGeneration", "ModelingV2FooModel"), + "not named ...ForConditionalGeneration", + ), + ( + "ModelingV2FooForConditionalGeneration", + _NAMED_OK.replace("ModelingV2Foo(", "ModelingV2FooVLM(").replace( + "= ModelingV2Foo\n", "= ModelingV2FooVLM\n" + ), + "ModelingV2FooVLM: a target class named ...VLM", + ), + ], + ids=[ + "model_ok", + "language_model_ok", + "mixin_not_the_base", + "model_suffix", + "vlm_suffix", + ], +) +def test_the_naming_rule_check_reports_each_broken_part( + tmp_path, monkeypatch, routed, source, problem +): + """Each stand-in breaks one part of the rule and gets exactly that problem.""" + target = tmp_path / "models" / "foo" / "target" + target.mkdir(parents=True) + (target.parent / "routing.py").write_text( + _FIXTURE_ROUTING.format(arch="FooForConditionalGeneration", cls=routed) + ) + (target / "modeling.py").write_text(source) + monkeypatch.setattr(sys.modules[__name__], "_ROOT", tmp_path) + problems = _naming_rule_problems() + if problem is None: + assert not problems, problems + else: + assert len(problems) == 1 and problem in problems[0], problems diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py new file mode 100644 index 000000000000..dcd1b75253a9 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_multimodal.py @@ -0,0 +1,755 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The shared base of Modeling V2 vision-language targets, and the router's multimodal check. + +A tiny language model and a tiny tower stand in for a target's: construction, the runtime +contract, admission (also through the executor), the forward's hand-off in both row modes +through the real multimodal mixin, the compile scope and the weight split. Then the check the +router runs on every class a route returns. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2 import NoTarget +from tensorrt_llm._torch.modeling_v2 import _router_index as router +from tensorrt_llm._torch.modeling_v2.multimodal import ModelingV2ForConditionalGeneration +from tensorrt_llm._torch.models.checkpoints.model_hooks import call_load_weights +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin +from tensorrt_llm._torch.pyexecutor.engine import model_setup +from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor +from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData +from tensorrt_llm.llmapi.llm_args import MultimodalConfig + +pytestmark = pytest.mark.cpu_only + +VOCAB, HIDDEN = 32, 4 +IMAGE_TOKEN = VOCAB - 1 # in vocabulary, so the fused path embeds the whole prompt once +MAX_ITEM = 16 # the most rows the tiny tower takes in one item +ARCH = "TinyForConditionalGeneration" + + +# ------------------------------------------------------------------ stubs --- + + +class _TinyLlm(nn.Module): + """A language model target in miniature: it adopts the text config on its own config + copy, as a target does, and records what it is handed.""" + + moe_impls_honored = frozenset({"tiny_moe"}) + + def __init__(self, model_config): + super().__init__() + self.got = model_config + model_config.pretrained_config = model_config.pretrained_config.text_config + self.model_config = model_config + self.model = nn.Module() + self.model.embed_tokens = nn.Embedding(VOCAB, HIDDEN) + self.lm_head = nn.Embedding(VOCAB, HIDDEN) + self.draft_config = None + self.draft_model = object() + self.spec_worker = object() + self.multimodal_token_ids = torch.tensor([IMAGE_TOKEN], dtype=torch.int32) + self.calls = [] + self.loaded = None + + @property + def config(self): + return self.model_config.pretrained_config + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None): + return "V2" + + def forward( + self, + attn_metadata, + input_ids=None, + position_ids=None, + inputs_embeds=None, + return_context_logits=False, + spec_metadata=None, + **kwargs, + ): + """Records the call: the first six arguments in the order a language model takes them + (`DecoderModelForCausalLM.forward`), and the others by name.""" + args = ( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + ) + self.calls.append((args, kwargs)) + return "logits" + + def load_weights(self, weights): + self.loaded = weights + + +class _TinyTower(nn.Module): + """Each row it returns holds its input pixel's value; it refuses an item of more than + `MAX_ITEM` rows, as a tower with a per-call bound does.""" + + def __init__(self, vision_config, dtype): + super().__init__() + self.vision_config = vision_config + self.scale = nn.Parameter(torch.ones((), dtype=dtype)) + self.calls = 0 + self.loaded = None + + def forward(self, pixels): + self.calls += 1 + assert pixels.shape[0] <= MAX_ITEM, f"an item of {pixels.shape[0]} rows" + return pixels.expand(-1, HIDDEN) * self.scale + + def load_weights(self, weights): + self.loaded = weights + + +class _TinyForConditionalGeneration(ModelingV2ForConditionalGeneration): + language_model_class = _TinyLlm + mm_encoder_class = _TinyTower + mm_encoder_weight_prefix = "model.visual." + + def validate_media_request(self, request): + image = (request.py_multimodal_data or {}).get("image") + if image is not None and image["pixel_values"].shape[0] > MAX_ITEM: + raise ValueError("an image larger than the tower takes") + + def encode_multimodal_inputs(self, multimodal_params): + if self.mm_encoder is None: + raise ValueError("no tower") + return torch.cat( + [self.mm_encoder(p.multimodal_data["image"]["pixel_values"]) for p in multimodal_params] + ) + + +class _TinyRowsForConditionalGeneration(_TinyForConditionalGeneration): + language_model_takes_rows = True + + +def _model_config(disable=False, side_stream=0, encoder_only=False): + composite = PretrainedConfig( + architectures=[ARCH], + text_config=PretrainedConfig(hidden_size=HIDDEN, torch_dtype=torch.float32), + vision_config=PretrainedConfig(), + torch_dtype=torch.float32, + ) + return ModelConfig( + pretrained_config=composite, + disable_mm_encoder=disable, + mm_encoder_only=encoder_only, + multimodal_config=MultimodalConfig(encoder_side_stream_max_ahead=side_stream), + ) + + +def _model(cls=_TinyForConditionalGeneration, disable=False): + model = cls(_model_config(disable=disable)) + with torch.no_grad(): + # Token `i` embeds to `i` in every column, so a fused row says where it came from. + model.llm.model.embed_tokens.weight.copy_( + torch.arange(VOCAB, dtype=torch.float32)[:, None].expand(VOCAB, HIDDEN) + ) + return model + + +def _image(*values): + """One image whose rows hold `values`.""" + pixels = torch.tensor(values, dtype=torch.float32)[:, None] + return {"image": {"pixel_values": pixels}} + + +def _params(*values, runtime=None, mrope=False): + data = _image(*values) if values else {} + if mrope: + data["mrope_config"] = {"mrope_position_deltas": torch.zeros(1, 1, dtype=torch.int32)} + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +# ----------------------------------------------------------- construction --- + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_construction_keeps_the_engine_config_and_builds_the_tower_only_when_asked( + monkeypatch, disable +): + model_config = _model_config(disable=disable) + composite = model_config.pretrained_config + # The tower takes the activation dtype: the text config's, not the composite's. + composite.text_config.torch_dtype = torch.bfloat16 + built = [] + + class Tower(_TinyTower): + def __init__(self, vision_config, dtype): + built.append(dtype) + super().__init__(vision_config, dtype) + + monkeypatch.setattr(_TinyForConditionalGeneration, "mm_encoder_class", Tower) + model = _TinyForConditionalGeneration(model_config) + + # The loader's own ModelConfig, now carrying the text config the pools are sized from. + assert model.model_config is model_config + assert model_config.pretrained_config is composite.text_config + assert model.config is composite.text_config + # The language model was built on its own copy, as the engine builds a decoder model. + assert model.llm.got is not model_config + assert model.llm.got.skip_create_weights_in_init is True + assert model.language_model is model.llm + if disable: + # Never built, not built and dropped. + assert built == [] + assert model.mm_encoder is None + assert not any(name.startswith("mm_encoder.") for name, _ in model.named_parameters()) + else: + assert built == [torch.bfloat16] + assert isinstance(model.mm_encoder, Tower) + assert model.mm_encoder.vision_config is composite.vision_config + assert model.mm_encoder.scale.dtype == torch.bfloat16 + + +def test_build_mm_encoder_overrides_mm_encoder_class(): + """A tower not built as `mm_encoder_class(vision_config, dtype)` comes from an override of + `build_mm_encoder`. With neither, the model builds no tower and serves text only.""" + + class Built(_TinyForConditionalGeneration): + def build_mm_encoder(self, pretrained_config): + return nn.Identity() + + class NoTower(_TinyForConditionalGeneration): + mm_encoder_class = None + + assert isinstance(_model(Built).mm_encoder, nn.Identity) + assert _model(NoTower).mm_encoder is None + + +@pytest.mark.parametrize("mode", ["auto", "require"]) +@pytest.mark.parametrize( + "disable, side_stream, encoder_only", + [(False, 2, False), (False, 0, False), (True, 2, False), (False, 2, True)], + ids=["tower", "tower_encoder_inline", "disable_mm_encoder", "mm_encoder_only"], +) +def test_the_route_declines_the_encoder_side_stream_while_a_tower_is_built( + monkeypatch, disable, side_stream, encoder_only, mode +): + """A `NoTarget` from the router, not the constructor: `auto` falls back to the built-in + model and `require` fails. Nothing is declined without a tower, with the encoder in the + step (`encoder_side_stream_max_ahead=0`, the default), or on an encoder-only worker, which + builds the architecture's encoder model and not this one.""" + monkeypatch.setattr( + router, "registered_routes", lambda: {ARCH: lambda *args: _TinyForConditionalGeneration} + ) + monkeypatch.setattr(router, "_device_compute_capability", lambda: (10, 0)) + config = _model_config(disable=disable, side_stream=side_stream, encoder_only=encoder_only) + llm_args = SimpleNamespace(modeling_v2=mode) + if disable or side_stream == 0 or encoder_only: + assert router.modeling_v2_resolve(config, llm_args) is _TinyForConditionalGeneration + elif mode == "auto": + assert router.modeling_v2_resolve(config, llm_args) is None + else: + with pytest.raises(ValueError, match="encoder_side_stream_max_ahead=2"): + router.modeling_v2_resolve(config, llm_args) + + +# --------------------------------------------------------------- contract --- + + +def test_the_model_declares_the_runtime_contract(): + model = _model() + assert isinstance(model, MultimodalModelMixin) + assert model.dtype == torch.float32 + assert (model.embedding_dim, model.embedding_dtype) == (HIDDEN, torch.float32) + assert model.text_embedding_layer is model.llm.model.embed_tokens + assert model.samples_from_vocab_shards is False + # What the engine reads off the routed model is the language model's. + llm = model.llm + assert model.lm_head is llm.lm_head + assert model.draft_config is None + assert (model.draft_model, model.spec_worker) == (llm.draft_model, llm.spec_worker) + assert model.multimodal_token_ids is llm.multimodal_token_ids + # So are the class-level hooks the router and the loader read off the routed class. + assert _TinyForConditionalGeneration.get_preferred_kv_cache_manager_version() == "V2" + assert _TinyForConditionalGeneration.moe_impls_honored == frozenset({"tiny_moe"}) + assert _TinyRowsForConditionalGeneration.moe_impls_honored == frozenset({"tiny_moe"}) + # A flag the language model does not set takes the value its reader takes for none. + assert _TinyForConditionalGeneration.draft_moe_impls_honored == frozenset() + assert _TinyForConditionalGeneration.supports_nvfp4_kv_cache is False + + class Own(_TinyForConditionalGeneration): + language_model_class = _TinyLlm + moe_impls_honored = frozenset({"own"}) + + class FlaglessLlm(nn.Module): + pass + + class Swapped(_TinyForConditionalGeneration): + language_model_class = FlaglessLlm + + assert Own.moe_impls_honored == frozenset({"own"}) + # A subclass that swaps in another language model does not keep its parent's flags. + assert Swapped.moe_impls_honored == frozenset() + + +# -------------------------------------------------------------- admission --- + + +def _request(tokens=(1, 2, 3), bound=None, **data): + return SimpleNamespace( + py_multimodal_data=data or None, + py_prompt_token_bound=bound, + check_token_id_range=lambda vocab: max(tokens) < vocab, + ) + + +def _admit(model, request): + """Both admission hooks, in the executor's order.""" + model.validate_request(request) + if request.py_multimodal_data: + model.validate_multimodal_request_data(request.py_multimodal_data) + + +MROPE_ONLY = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + + +def test_admission_checks_the_token_range_and_the_towers_media(): + model = _model() + _admit(model, _request()) + _admit(model, _request(tokens=(1, IMAGE_TOKEN), **_image(1.0, 2.0))) + _admit(model, _request(bound=VOCAB - 1)) + # M-RoPE positions alone (a disaggregated generation request) are not media. + _admit(model, _request(**MROPE_ONLY)) + with pytest.raises(ValueError, match="Token ID out of range"): + _admit(model, _request(tokens=(1, VOCAB))) + with pytest.raises(ValueError, match="Token ID out of range"): + _admit(model, _request(bound=VOCAB)) + with pytest.raises(ValueError, match="larger than the tower takes"): + _admit(model, _request(**_image(*range(MAX_ITEM + 1)))) + with pytest.raises(ValueError, match="precomputed multimodal embeddings"): + _admit(model, _request(multimodal_embedding=torch.zeros(2, HIDDEN))) + + +def test_without_a_tower_media_is_refused_and_mrope_positions_pass(): + model = _model(disable=True) + _admit(model, _request()) + _admit(model, _request(**MROPE_ONLY)) + # No tower check without a tower; the media check refuses the request instead. + model.validate_request(_request(**_image(*range(MAX_ITEM + 1)))) + for data in (_image(1.0), {"multimodal_embedding": torch.zeros(2, HIDDEN)}): + with pytest.raises(ValueError, match="disable_mm_encoder.*serves text only"): + _admit(model, _request(**data)) + + +def test_the_executor_fails_only_the_request_the_model_refuses(): + """Through the executor's own admission (`_fetch_and_activate_new_requests`): the bad + request fails with its own error and no budget charge; the others are admitted.""" + model = _model() + requests = [ + _request(), + _request(**_image(*range(MAX_ITEM + 1))), + _request(**MROPE_ONLY), + _request(tokens=(1, VOCAB)), + _request(**_image(1.0)), + _request(multimodal_embedding=torch.zeros(2, HIDDEN)), + ] + for request_id, request in enumerate(requests): + request.request_id = request_id + request.py_disaggregated_params = None + request.sampling_config = None + request.kv_cache_retention_config = None + executor = SimpleNamespace( + model_engine=SimpleNamespace(model=model), + _is_kv_manager_v2=False, + kv_cache_manager=None, + kv_cache_transceiver=None, + max_beam_width=1, + sampler=Mock(), + waiting_queue=object(), + active_requests=[], + _mm_encoder_item_scheduling_enabled=False, + _fetch_new_requests=Mock(return_value=requests), + _configure_csa2_remote_tail=Mock(), + _validate_token_id_range=Mock(), + _validate_request_budget=Mock(), + _handle_errors=Mock(), + ) + executor._validate_request = PyExecutor._validate_request.__get__(executor) + + admitted = PyExecutor._fetch_and_activate_new_requests(executor) + + assert admitted == [requests[0], requests[2], requests[4]] + failed = [ + (call.kwargs["requests"], call.kwargs["charge_budget"]) + for call in executor._handle_errors.call_args_list + ] + assert failed == [([requests[1]], False), ([requests[3]], False), ([requests[5]], False)] + + +# ---------------------------------------------------------------- forward --- + + +def _forward(model, params, num_contexts=1, mm_rows=None, inputs_embeds=None): + """One step over a 6-token prompt whose tokens 1..`mm_rows` are placeholders.""" + input_ids = torch.tensor([1, IMAGE_TOKEN, IMAGE_TOKEN, IMAGE_TOKEN, 4, 5], dtype=torch.int32) + kwargs = {"multimodal_params": params, "resource_manager": "rm"} + if mm_rows is not None: + kwargs["mm_token_indices"] = torch.arange(1, 1 + mm_rows, dtype=torch.int64) + kwargs["text_token_indices"] = torch.tensor([0, *range(1 + mm_rows, 6)], dtype=torch.int64) + out = model.forward( + SimpleNamespace(num_contexts=num_contexts, num_generations=0), + input_ids, + "position_ids", + inputs_embeds, + False, + "spec_metadata", + **kwargs, + ) + assert out == "logits" + ((args, llm_kwargs),) = model.llm.calls + assert args[0].num_contexts == num_contexts + assert args[2] == "position_ids" and args[4:] == (False, "spec_metadata") + assert "multimodal_params" not in llm_kwargs and llm_kwargs["resource_manager"] == "rm" + return input_ids, args, llm_kwargs + + +def test_rows_mode_hands_the_rows_and_their_placeholder_rows_to_the_language_model(): + model = _model(_TinyRowsForConditionalGeneration) + input_ids, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)], mm_rows=3) + # `input_ids` reach the language model unchanged, and `inputs_embeds` stays None. + assert args[1] is input_ids and args[3] is None + rows, mm_token_indices = llm_kwargs["multimodal_rows"] + assert rows[:, 0].tolist() == [7.0, 8.0, 9.0] + assert mm_token_indices.tolist() == [1, 2, 3] + + +def test_fused_mode_hands_fused_embeddings_to_the_language_model(): + model = _model() + _, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)], mm_rows=3) + assert args[1] is None + assert args[3][:, 0].tolist() == [1.0, 7.0, 8.0, 9.0, 4.0, 5.0] + assert "multimodal_rows" not in llm_kwargs + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_a_chunk_hands_over_its_slice_of_the_rows(cls): + """The mixin keeps the chunk's rows: 2 of 5 reused, 3 in this chunk.""" + model = _model(cls) + runtime = MultimodalRuntimeData( + embed_mask_cumsum=torch.arange(1, 6, dtype=torch.int64), + past_seen_token_num=2, + chunk_end_pos=5, + ) + _, args, llm_kwargs = _forward( + model, [_params(5.0, 6.0, 7.0, 8.0, 9.0, runtime=runtime)], mm_rows=3 + ) + if cls is _TinyRowsForConditionalGeneration: + assert llm_kwargs["multimodal_rows"][0][:, 0].tolist() == [7.0, 8.0, 9.0] + else: + assert args[3][1:4, 0].tolist() == [7.0, 8.0, 9.0] + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +@pytest.mark.parametrize( + "params, num_contexts", + [ + ([], 1), + ([MultimodalParams(multimodal_data={})], 1), + # A text context mixed with a generation request's position deltas. + ([_params(mrope=True)], 1), + # A generation-only step. + ([_params(7.0)], 0), + # A chunk after the image: its 3 placeholders were all in earlier chunks. + ( + [ + _params( + 7.0, + 8.0, + 9.0, + runtime=MultimodalRuntimeData( + embed_mask_cumsum=torch.tensor([0, 1, 2, 3, 3, 3]), + past_seen_token_num=4, + chunk_end_pos=6, + ), + ) + ], + 1, + ), + ], + ids=["no_params", "empty", "mrope_only", "generation", "chunk_after_the_placeholders"], +) +def test_a_step_without_media_runs_no_tower(params, num_contexts, cls): + model = _model(cls) + input_ids, args, llm_kwargs = _forward(model, params, num_contexts=num_contexts) + assert args[1] is input_ids and args[3] is None + assert llm_kwargs.get("multimodal_rows") is None + assert model.mm_encoder.calls == 0 + + +def test_the_rows_must_match_the_placeholder_rows(): + with pytest.raises(AssertionError, match="3 multimodal rows and 2 placeholder rows"): + _forward(_model(_TinyRowsForConditionalGeneration), [_params(7.0, 8.0, 9.0)], mm_rows=2) + + +class _DeepStack: + """V1 Qwen3-VL's DeepStack pattern in miniature: an extra feature for each of the chunk's + rows (10 times the row), handed to the language model by the mixin's hook.""" + + def after_active_multimodal_embeddings(self, *, active_embeddings, **kwargs): + return active_embeddings, (torch.cat(active_embeddings) * 10,) + + def get_language_model_extra_forward_kwargs(self, *, mm_inputs, spec_metadata, **kwargs): + self.hook_got = dict(kwargs, mm_inputs=mm_inputs) + # A V1 hook also returns `spec_metadata`; the language model still takes it once. + return {"deepstack_embeds": list(mm_inputs.extra_embeds), "spec_metadata": spec_metadata} + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_what_the_extra_forward_kwargs_hook_returns_reaches_the_language_model(cls): + model = _model(type("DeepStackForConditionalGeneration", (_DeepStack, cls), {})) + params = [_params(7.0, 8.0, 9.0)] + input_ids, args, llm_kwargs = _forward(model, params, mm_rows=3) + (deepstack,) = llm_kwargs["deepstack_embeds"] + got = model.hook_got + # The hook gets the same arguments as in the mixin's own forward. + assert got["raw_input_ids"] is input_ids and got["position_ids"] == "position_ids" + assert got["multimodal_params"] is params and got["num_generation_requests"] == 0 + assert got["resource_manager"] == "rm" + if cls is _TinyRowsForConditionalGeneration: + # The rows mode: the unchanged input_ids, no inputs_embeds, one extra row per row. + assert got["mm_inputs"].input_ids is input_ids and got["mm_inputs"].inputs_embeds is None + assert got["multimodal_rows"] is llm_kwargs["multimodal_rows"] + assert deepstack[:, 0].tolist() == [70.0, 80.0, 90.0] + else: + # The fused mode: the fused inputs, the extra rows spread over the tokens. + assert got["mm_inputs"].input_ids is None and got["mm_inputs"].inputs_embeds is args[3] + assert deepstack[:, 0].tolist() == [0.0, 70.0, 80.0, 90.0, 0.0, 0.0] + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_by_default_the_language_model_takes_no_extra_argument(cls): + _, _, llm_kwargs = _forward(_model(cls), [_params(7.0, 8.0, 9.0)], mm_rows=3) + expected = {"resource_manager", "mm_token_indices", "text_token_indices"} + if cls is _TinyRowsForConditionalGeneration: + expected.add("multimodal_rows") + assert set(llm_kwargs) == expected + + +class _ExtraRowsOnly(_DeepStack): + """A hook that keeps no multimodal row of the chunk and returns extra rows.""" + + def after_active_multimodal_embeddings(self, *, active_embeddings, **kwargs): + return [], (torch.cat(active_embeddings) * 10,) + + +@pytest.mark.parametrize( + "cls", [_TinyForConditionalGeneration, _TinyRowsForConditionalGeneration], ids=["fused", "rows"] +) +def test_extra_rows_reach_the_hook_when_the_chunk_has_no_multimodal_row(cls): + model = _model(type("ExtraRowsOnlyForConditionalGeneration", (_ExtraRowsOnly, cls), {})) + input_ids, args, llm_kwargs = _forward(model, [_params(7.0, 8.0, 9.0)]) + # Nothing is fused or handed over as rows, and the extra rows still reach the hook. + assert args[1] is input_ids and args[3] is None + assert llm_kwargs.get("multimodal_rows") is None + (extra,) = llm_kwargs["deepstack_embeds"] + assert extra[:, 0].tolist() == [70.0, 80.0, 90.0] + + +def test_in_the_fused_mode_caller_inputs_embeds_replace_input_ids_as_in_the_mixin(): + model = _model( + type("DeepStackForConditionalGeneration", (_DeepStack, _TinyForConditionalGeneration), {}) + ) + inputs_embeds = torch.ones(6, HIDDEN) + _, args, _ = _forward(model, [], inputs_embeds=inputs_embeds) + assert args[1] is None and args[3] is inputs_embeds + mm_inputs = model.hook_got["mm_inputs"] + assert mm_inputs.input_ids is None and mm_inputs.inputs_embeds is inputs_embeds + + +# ---------------------------------------------------- compile and weights --- + + +def test_torch_compile_reaches_only_the_language_models_core(monkeypatch): + compiled = [] + + def spy(module, **kwargs): + compiled.append(module) + return module + + monkeypatch.setattr(torch, "compile", spy) + model = _model() + core = model.llm.model + returned, prefill_only = model_setup.apply_torch_compile( + model, "inductor", llm_args=SimpleNamespace(), fullgraph=False + ) + assert returned is model and prefill_only is False + assert compiled == [core] + + +class _Checkpoint(Mapping): + """Checkpoint weights that record which keys are read.""" + + def __init__(self, weights): + self.weights, self.read = weights, set() + + def __getitem__(self, key): + self.read.add(key) + return self.weights[key] + + def __iter__(self): + return iter(self.weights) + + def __len__(self): + return len(self.weights) + + +@pytest.mark.parametrize("disable", [False, True], ids=["tower", "disable_mm_encoder"]) +def test_the_tower_and_the_language_model_each_load_their_own_keys(disable): + model = _model(disable=disable) + weights = _Checkpoint( + {"model.visual.patch.weight": 1, "model.layers.0.weight": 2, "lm_head.weight": 3} + ) + model.load_weights(weights) + assert model.llm.loaded == {"model.layers.0.weight": 2, "lm_head.weight": 3} + if not disable: + assert model.mm_encoder.loaded == {"model.visual.patch.weight": 1} + # Without a tower, the tower's keys stay unread. + tower_keys = set() if disable else {"model.visual.patch.weight"} + assert weights.read == {"model.layers.0.weight", "lm_head.weight"} | tower_keys + + +class _MappedLlm(_TinyLlm): + def load_weights(self, weights, weight_mapper=None): + self.loaded = weights + self.weight_mapper = weight_mapper + + +class _MappedForConditionalGeneration(_TinyForConditionalGeneration): + language_model_class = _MappedLlm + + +class _Mapper: + """The model a weight mapper is bound to, as `BaseWeightMapper` binds it.""" + + def init_model_and_config(self, model, config): + self.model, self.config = model, config + + +def _bound_mapper(model): + """A mapper the engine bound to the routed model.""" + mapper = _Mapper() + mapper.init_model_and_config(model, model.model_config) + return mapper + + +def test_the_engines_weight_mapper_reaches_a_language_model_that_takes_one(): + """The engine passes `weight_mapper` only to a `load_weights` that names it + (`call_load_weights`); the base names it and hands it on the same way, bound to the + language model, whose modules the keys name.""" + model = _model(_MappedForConditionalGeneration) + mapper = _bound_mapper(model) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, mapper) + assert model.llm.weight_mapper is mapper + assert (mapper.model, mapper.config) == (model.llm, model.llm.model_config) + # Without a mapper, the language model gets none. + model = _model(_MappedForConditionalGeneration) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, None) + assert model.llm.weight_mapper is None + # A language model whose `load_weights` takes no mapper is called without one, and the + # mapper stays bound to the routed model. + model = _model() + mapper = _bound_mapper(model) + call_load_weights(model.load_weights, {"model.layers.0.weight": 2}, mapper) + assert model.llm.loaded == {"model.layers.0.weight": 2} + assert mapper.model is model + + +# ---------------------------------------------------- the router's check --- + + +class _Decoder: + """A language model target that neither serves media nor refuses it.""" + + +class _TextOnly: + def validate_multimodal_request_data(self, mm_data): + raise ValueError("text only") + + +class _OwnEncoder: + """A target that encodes media in its own forward and refuses at admission what its + tower cannot take.""" + + def validate_request(self, request): + raise ValueError("a request the tower cannot take") + + +class _TokenRangeCheck: + def validate_request(self, request): + """Checks the token range only.""" + + +class _InheritedValidator(_TokenRangeCheck): + """A language model target whose `validate_request` is a base class's: it refuses no media + of its own.""" + + +class _Marked: + supports_multimodal = True + + +class _MarkedFalse: + supports_multimodal = False + + +def test_a_vision_language_architecture_routes_to_a_multimodal_or_text_only_target(monkeypatch): + monkeypatch.setattr(router, "_architecture_takes_media", lambda arch: arch == ARCH) + for target in (_TinyForConditionalGeneration, _TextOnly, _OwnEncoder, _Marked): + router._check_multimodal_contract(target, ARCH) + router._check_multimodal_contract(_Decoder, "TinyForCausalLM") + # The marker counts only when it is True; `validate_request` only in the class's own body. + for target in (_Decoder, _MarkedFalse, _InheritedValidator): + with pytest.raises( + ValueError, + match=f"{target.__name__} is the target for '{ARCH}'.*" + "ModelingV2ForConditionalGeneration.*validate_multimodal", + ) as error: + router._check_multimodal_contract(target, ARCH) + # Not a `NoTarget`, which `auto` turns into a fallback to the built-in model. + assert not isinstance(error.value, NoTarget) + + +@pytest.mark.parametrize("mode", ["auto", "require"]) +def test_a_target_that_breaks_the_contract_fails_at_startup(monkeypatch, mode): + """Not a `NoTarget`: `auto` does not fall back, as the target is wrong, not the + deployment.""" + monkeypatch.setattr(router, "registered_routes", lambda: {ARCH: lambda *args: _Decoder}) + monkeypatch.setattr(router, "_device_compute_capability", lambda: (10, 0)) + monkeypatch.setattr(router, "_architecture_takes_media", lambda arch: True) + config = SimpleNamespace( + pretrained_config=SimpleNamespace(architectures=[ARCH]), moe_tactic=None, spec_config=None + ) + with pytest.raises(ValueError, match=f"_Decoder is the target for '{ARCH}'"): + router.modeling_v2_resolve(config, SimpleNamespace(modeling_v2=mode)) + + +def test_media_architectures_are_known_by_their_input_processor(): + assert router._architecture_takes_media("Qwen4ExpForConditionalGeneration") + assert not router._architecture_takes_media("DeepseekV3ForCausalLM") + assert not router._architecture_takes_media("NoSuchArchitectureForCausalLM") diff --git a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py index 7d00f8daa299..1b0d61da20e8 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py @@ -274,6 +274,66 @@ def test_prepare_multimodal_inputs_accepts_tensor_encoder_output_cpu(): _test_prepare_multimodal_inputs_accepts_tensor_encoder_output("cpu") +@pytest.mark.cpu_only +def test_prepare_multimodal_rows_returns_the_rows_that_inputs_fuses(): + """`prepare_multimodal_inputs` is `prepare_multimodal_rows` plus the fuse: fusing the + rows gives the same inputs, and the rows keep `input_ids` for a model that places them + itself.""" + hidden, mm_token_id = 8, 7 + emb = make_embedding(num_embeddings=40, hidden_size=hidden) + input_ids = torch.tensor([0, mm_token_id, mm_token_id, 1], dtype=torch.long) + text_idx = torch.tensor([0, 3], dtype=torch.long) + mm_idx = torch.tensor([1, 2], dtype=torch.long) + mm_emb = torch.randn(mm_idx.shape[0], hidden) + model = TensorEncoderMultimodalModel(emb, torch.tensor([mm_token_id]), mm_emb) + kwargs = dict( + input_ids=input_ids, + positions=None, + multimodal_params=[make_raw_multimodal_param()], + num_context_requests=1, + text_token_indices=text_idx, + mm_token_indices=mm_idx, + ) + + rows = model.prepare_multimodal_rows(**kwargs) + assert rows is not None + row_input_ids, active_embeddings, extra_embeds = rows + assert row_input_ids is input_ids + assert tuple(extra_embeds) == () + torch.testing.assert_close(torch.cat(active_embeddings), mm_emb) + + fused_ids, fused_embeds, fused_extra = model._fuse_multimodal_embeddings( + input_ids=row_input_ids, + multimodal_embeddings=active_embeddings, + mm_token_ids=model.multimodal_token_ids, + embedding_layer=model.text_embedding_layer, + extra_embeds=extra_embeds, + text_token_indices=text_idx, + mm_token_indices=mm_idx, + ) + prepared = model.prepare_multimodal_inputs(**kwargs) + assert prepared.input_ids is None and fused_ids is None + torch.testing.assert_close(prepared.inputs_embeds, fused_embeds) + assert tuple(prepared.extra_embeds) == tuple(fused_extra) == () + + +@pytest.mark.cpu_only +def test_a_text_step_has_no_multimodal_rows(): + emb = make_embedding(num_embeddings=40, hidden_size=8) + model = DummyMultimodalModel(emb, torch.tensor([7])) + input_ids = torch.tensor([0, 1, 2], dtype=torch.long) + for params in ([], None, [MultimodalParams(multimodal_data={})]): + kwargs = dict( + input_ids=input_ids, + positions=None, + multimodal_params=params, + num_context_requests=1, + ) + assert model.prepare_multimodal_rows(**kwargs) is None + prepared = model.prepare_multimodal_inputs(**kwargs) + assert prepared.input_ids is input_ids and prepared.inputs_embeds is None + + def test_prepare_multimodal_inputs_accepts_tensor_encoder_output_cuda(): _test_prepare_multimodal_inputs_accepts_tensor_encoder_output("cuda") diff --git a/tests/unittest/_torch/pyexecutor/test_model_engine.py b/tests/unittest/_torch/pyexecutor/test_model_engine.py index b827d48e5441..f4ddfdcf1920 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_engine.py +++ b/tests/unittest/_torch/pyexecutor/test_model_engine.py @@ -692,6 +692,29 @@ def test_model_engine_resolves_multimodal_prefill_graphs_before_validation( assert engine.llm_args.torch_compile_config is None +@pytest.mark.cpu_only +@pytest.mark.parametrize("takes_rows", [False, True], ids=["fused", "language_model_takes_rows"]) +def test_breakable_gate_for_a_composite_model_with_its_own_tower(takes_rows: bool) -> None: + """A composite model whose language model is a decoder, with a local tower. When the + language model takes `input_ids` and places the rows itself (`language_model_takes_rows`), + as it does when routed alone, the breakable prefill graph is allowed; with the rows fused + into `inputs_embeds` it is refused at startup. The message is not matched.""" + engine = object.__new__(ModelEngine) + engine.model = DummyLegacyMultimodalIndexModel() + engine.model.llm = Mock(spec=DecoderModelForCausalLM) + engine.model.mm_encoder = torch.nn.Module() + engine.model.language_model_takes_rows = takes_rows + engine.input_processor = Mock(spec=BaseMultimodalInputProcessor) + engine.llm_args = SimpleNamespace( + prefill_cuda_graph_backend=PrefillCudaGraphBackend.BREAKABLE, disable_mm_encoder=False + ) + if takes_rows: + engine._validate_breakable_cuda_graph_compatibility() + else: + with pytest.raises(ValueError): + engine._validate_breakable_cuda_graph_compatibility() + + class ModelEngineTestCase(unittest.TestCase): def test_breakable_allows_text_decoder_with_multimodal_processor(self) -> None: engine = object.__new__(ModelEngine) diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index 18d78cb99c7d..ad81541b05a6 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -791,6 +791,36 @@ def test_encoder_input_above_max_num_tokens_fails_only_that_request() -> None: assert kwargs == {"requests": [requests[1]], "charge_budget": False} +def test_text_only_model_fails_only_the_media_request() -> None: + """A text-only model's admission hook refuses media alone: the image request fails with + its own error and no budget charge, while a text request and an M-RoPE-only generation + request (disaggregated serving) are admitted.""" + from tensorrt_llm.inputs.multimodal import has_media_or_embeddings + + def validate_multimodal_request_data(mm_data) -> None: + if has_media_or_embeddings(mm_data): + raise ValueError("serves text only") + + requests = [_make_admission_request(i) for i in range(3)] + requests[0].py_multimodal_data = None + requests[1].py_multimodal_data = {"image": {"pixel_values": torch.zeros(1)}} + requests[2].py_multimodal_data = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + model = types.SimpleNamespace( + validate_multimodal_request_data=Mock(side_effect=validate_multimodal_request_data) + ) + executor = _make_admission_executor(model, requests) + + admitted = PyExecutor._fetch_and_activate_new_requests(executor) + + assert admitted == [requests[0], requests[2]] + assert executor.active_requests == [requests[0], requests[2]] + executor._handle_errors.assert_called_once_with( + "serves text only", requests=[requests[1]], charge_budget=False + ) + # The hook runs only for a request that carries multimodal data. + assert model.validate_multimodal_request_data.call_count == 2 + + def test_model_validation_runs_after_the_sampler_and_before_the_budget() -> None: order = [] model = types.SimpleNamespace(validate_request=lambda request: order.append("model")) diff --git a/tests/unittest/inputs/test_create_input_processor.py b/tests/unittest/inputs/test_create_input_processor.py index 1bee1497d3c9..ecddfbe59320 100644 --- a/tests/unittest/inputs/test_create_input_processor.py +++ b/tests/unittest/inputs/test_create_input_processor.py @@ -22,10 +22,15 @@ import tensorrt_llm._torch.model_config as model_config_mod import tensorrt_llm._torch.models as models_mod +import tensorrt_llm.inputs.registry as registry_mod from tensorrt_llm.inputs.registry import ( INPUT_PROCESSOR_REGISTRY, DefaultInputProcessor, + InputProcessorRegistry, + MultimodalPlaceholderMetadata, + MultimodalPlaceholderRegistry, create_input_processor, + register_input_processor, ) pytestmark = pytest.mark.cpu_only @@ -107,6 +112,32 @@ def _resolve_to_model_without_processor(monkeypatch: pytest.MonkeyPatch) -> None ) +def test_registry_returns_the_input_processor_registered_for_a_model_class( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`get_input_processor_cls` returns what `register_input_processor` stored, else None.""" + monkeypatch.setattr(registry_mod, "INPUT_PROCESSOR_REGISTRY", InputProcessorRegistry()) + monkeypatch.setattr( + registry_mod, "MULTIMODAL_PLACEHOLDER_REGISTRY", MultimodalPlaceholderRegistry() + ) + + class _Model: + pass + + class _Processor: + pass + + register_input_processor( + _Processor, + model_type="fake_vision_language", + placeholder_metadata=MultimodalPlaceholderMetadata(placeholder_map={"image": ""}), + )(_Model) + + registry = registry_mod.INPUT_PROCESSOR_REGISTRY + assert registry.get_input_processor_cls(_Model) is _Processor + assert registry.get_input_processor_cls(_FakeTextOnlyModel) is None + + @pytest.mark.parametrize("checkpoint_format", ["HF", "MX"]) def test_create_input_processor_uses_registered_processor_for_hf_format_checkpoints( config_loads: list[str], diff --git a/tests/unittest/inputs/test_multimodal.py b/tests/unittest/inputs/test_multimodal.py index 01bcd83e5372..ecf842c5dec4 100644 --- a/tests/unittest/inputs/test_multimodal.py +++ b/tests/unittest/inputs/test_multimodal.py @@ -20,6 +20,7 @@ MultimodalRuntimeData, _find_mm_embedding_lengths_from_masks, find_mm_token_lengths, + has_media_or_embeddings, select_multimodal_fields, ) from tensorrt_llm.inputs.multimodal_data import MEDIA_MODALITIES, AudioData, VideoData @@ -1225,3 +1226,44 @@ def test_params_do_not_infer_item_layout_from_raw_buckets(): params = MultimodalParams(multimodal_data={"audio": {}, "image": {}}) assert params.item_metadata is None assert params.get_item_layout() == ((), ()) + + +_MROPE = {"mrope_config": {"mrope_position_deltas": torch.zeros(1, 1)}} + + +@pytest.mark.parametrize( + "mm_data,expected", + [ + (None, False), + ({}, False), + (_MROPE, False), + ({"mrope_config": None, "image": None}, False), + ({"image": {"pixel_values": torch.zeros(1)}}, True), + ({"video": {"pixel_values_videos": torch.zeros(1)}}, True), + ({"audio": {"input_features": torch.zeros(1)}}, True), + ({"multimodal_embedding": torch.zeros(1, 4)}, True), + ({"multimodal_embed_mask_cumsum": torch.arange(3)}, True), + ({**_MROPE, "image": {"pixel_values": torch.zeros(1)}}, True), + ({"some_future_key": 1}, True), + ], + ids=[ + "none", + "empty", + "mrope_only", + "none_values", + "image", + "video", + "audio", + "embedding", + "cumsum_only", + "mrope_and_image", + "unknown_key", + ], +) +def test_has_media_or_embeddings_ignores_only_mrope_positions(mm_data, expected): + """Only M-RoPE positions are not media. + + Text-only models refuse everything else, including keys they do not know, so an + M-RoPE-only disaggregated generation request still passes. + """ + assert has_media_or_embeddings(mm_data) is expected