From 29ff8d8dac3fa10f23453c85be027060ffef3f24 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:56:12 -0700 Subject: [PATCH 1/3] [None][refactor] define a shared multimodal model protocol Problem No written contract says what a multimodal model must provide. The runtime finds multimodal behaviour through MultimodalModelMixin internals, so a model that does not inherit the mixin, such as a Modeling V2 target, has nothing to follow. Solution - Add SupportsMultimodal in tensorrt_llm/_torch/models/multimodal.py: capability flags, attention backends, language_model, embedding properties, the model-owned encoder cache, item scheduling, encoding and fusion. The module imports no model class. - is_multimodal_model() reads the supports_multimodal class marker. get_multimodal_data_device_paths() reads the optional multimodal_data_device_paths of any model. - MultimodalModelMixin sets the marker and implements the protocol. It exposes its cache as multimodal_encoder_cache and initialize_multimodal_encoder_cache(), and declares support_mm_disagg = False by default. - Align the encoder entry points of six models with the mixin's Sequence input, and document the contract in adding_new_model.md. - The executor does not change here, so a model still subclasses the mixin. The executor's mixin checks move to the protocol in a separate PR. Validation - CPU unit tests of the multimodal mixin pass, including the 4 new protocol tests. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/torch/adding_new_model.md | 31 ++++ .../_torch/models/modeling_gemma4mm.py | 6 +- .../models/modeling_glm5_next_vision.py | 2 +- .../models/modeling_multimodal_mixin.py | 28 ++- .../models/modeling_musespark_open_omni.py | 6 +- .../models/modeling_nemotron_h_multimodal.py | 2 +- .../_torch/models/modeling_qwen2vl.py | 4 +- .../_torch/models/modeling_qwen3vl.py | 2 +- tensorrt_llm/_torch/models/multimodal.py | 168 ++++++++++++++++++ .../multimodal/test_multimodal_mixin.py | 53 ++++++ 10 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 tensorrt_llm/_torch/models/multimodal.py diff --git a/docs/source/torch/adding_new_model.md b/docs/source/torch/adding_new_model.md index 006941981f8e..0e3b0113aadf 100644 --- a/docs/source/torch/adding_new_model.md +++ b/docs/source/torch/adding_new_model.md @@ -11,6 +11,7 @@ Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3. [Step-by-Step Guide](#step-by-step-guide) 1. [Model Configuration](#model-configuration) 2. [Model Definition](#model-definition) + 1. [Multimodal Models](#multimodal-models) 3. [Weight Loading](#weight-loading) 4. [Model Registration](#model-registration) 1. [Core Models](#core-models) @@ -128,6 +129,36 @@ Optionally, you may replace the native PyTorch modules with our implementations For a concrete reference, check out `tensorrt_llm/_torch/models/modeling_llama.py`. +#### Multimodal Models + +`tensorrt_llm._torch.models.multimodal.SupportsMultimodal` defines the model-facing +multimodal contract, and `MultimodalModelMixin` implements it. Reuse the mixin for +its common implementation. The `supports_multimodal = True` class marker declares +the contract; it is not runtime signature validation and does not enable optional +features. + +- Declare encoder caching and item scheduling separately. Item scheduling also + requires processor item metadata and a compatible runtime configuration. +- Report attention backends with the `supported_encoder_attention_backends` + class attribute. Its empty default declares no support; concrete integrations + supply their supported sets. +- Keep the existing `MultimodalParams` input and tensor return contracts. + Selected-item preparation runs before H2D, and item outputs retain selection + order. +- `multimodal_data_device_paths` stays optional for every model. Read it through + `get_multimodal_data_device_paths()`; without it, every tensor in + `multimodal_data` moves to the device. +- Expose the existing model-owned cache through `multimodal_encoder_cache` and + `initialize_multimodal_encoder_cache()`. The latter preserves the mixin's + existing initialization hook. Item scheduling and persistent reuse share this + cache; they do not create separate owners. + +`EncoderGroup` continues to describe modalities that share an encoder call. +`MultimodalEncoderMixin` continues to provide native encoder attention metadata +and graph integration. Processor prompt expansion and disaggregated transport +retain their existing interfaces; implementing this Protocol does not enable a +new model, modality, encoder backend, or E/P path by itself. + ### Weight Loading The base class `DecoderModelForCausalLM` provides a `load_weights` method that loads the weights from the checkpoint file and assigns them to the corresponding layers in the model. However, if the default method does not work for `MyModelForCausalLM`, you need to implement your own `load_weights`: diff --git a/tensorrt_llm/_torch/models/modeling_gemma4mm.py b/tensorrt_llm/_torch/models/modeling_gemma4mm.py index 18134737636d..f913cd830180 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4mm.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4mm.py @@ -23,7 +23,7 @@ import copy import dataclasses import math -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from itertools import accumulate, groupby from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Literal, Optional, Tuple @@ -1105,7 +1105,9 @@ def get_multimodal_item_partitions( return None return _gemma4_input_partitions(param.multimodal_data, metadata).get(modality) - def encode_multimodal_inputs(self, multimodal_params: List[MultimodalParams]) -> torch.Tensor: + def encode_multimodal_inputs( + self, multimodal_params: Sequence[MultimodalParams] + ) -> torch.Tensor: """Encode uncached Gemma4 image, video, and audio payloads.""" modality_inputs = ( ("image", "pixel_values"), diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 699f57660ecc..fdcb02bd8052 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -1154,7 +1154,7 @@ def post_config(self): self.config = self.model_config.pretrained_config def encode_multimodal_inputs( - self, multimodal_params: List[MultimodalParams], **encoder_kwargs: Any + self, multimodal_params: Sequence[MultimodalParams], **encoder_kwargs: Any ) -> torch.Tensor: if self.mm_encoder is None: raise ValueError("Raw multimodal inputs require a local multimodal encoder.") diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index 113ac19c2d83..86ec2516e3ca 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -28,6 +28,7 @@ Iterable, Iterator, List, + Literal, Optional, Sequence, Tuple, @@ -58,6 +59,7 @@ fuse_input_embeds, get_multimodal_embeddings, ) +from .multimodal import SupportsMultimodal, get_multimodal_data_device_paths class MultimodalEncoderContractError(ValueError): @@ -407,8 +409,8 @@ def is_full_miss(self) -> bool: class MultimodalModelMixin: """Template-method mixin for PyTorch multimodal causal LM models. - Concrete model forwards can call `prepare_multimodal_inputs` while keeping their explicit - language-model delegation. + Implements ``SupportsMultimodal``. Concrete model forwards can call + `prepare_multimodal_inputs` while keeping their explicit language-model delegation. Current limitations: @@ -424,6 +426,9 @@ class MultimodalModelMixin: supports_vision_gpu_normalize: ClassVar[bool] = False """Whether the model accepts processor-described raw vision pixels.""" + supports_multimodal: ClassVar[Literal[True]] = True + """Opt in to the shared model contract without requiring mixin inheritance.""" + supports_encoder_cache: ClassVar[bool] = False """Whether the model's production forward path uses the persistent encoder cache.""" @@ -439,10 +444,22 @@ class MultimodalModelMixin: supported_encoder_attention_backends: ClassVar[dict[str, frozenset[str]]] = {} """Public backends by logical encoder; an empty set marks fixed attention.""" + support_mm_disagg: ClassVar[bool] = False + """Set by ``support_multimodal_disaggregated`` after validating the processor hook.""" + model_config: ModelConfig _multimodal_encoder_cache: Optional[TensorLRUCache] = None _encoder_forward_context: Callable[[], contextlib.AbstractContextManager[None]] | None = None + @property + def multimodal_encoder_cache(self) -> Optional[TensorLRUCache]: + """Expose the existing model-owned encoder cache without transferring ownership.""" + return self._multimodal_encoder_cache + + def initialize_multimodal_encoder_cache(self, max_bytes: int) -> Optional[TensorLRUCache]: + """Initialize the encoder output cache through the existing implementation hook.""" + return self._initialize_multimodal_encoder_cache(max_bytes) + @classmethod def _cast_multimodal_encoder_dtype( cls, @@ -1742,7 +1759,7 @@ def _dispatch_cross_iter_prefetch( "multimodal_data", "cuda", pin_memory=prefer_pinned(), - target_keywords=getattr(model, "multimodal_data_device_paths", None), + target_keywords=get_multimodal_data_device_paths(model), ) # `to_device` may replace `multimodal_data` with a new dict; reattach # the (possibly new) dict to each request so the next iteration's @@ -1850,3 +1867,8 @@ def maybe_prefetch_mm_encoder_for_next_iter( return 0 _dispatch_cross_iter_prefetch(model, candidates, aux_stream) return len(candidates) + + +if TYPE_CHECKING: + # Check the shared implementation without imposing Protocol inheritance at runtime. + _model_contract: SupportsMultimodal = MultimodalModelMixin() diff --git a/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py b/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py index ed29a1bda2a7..93252831190f 100644 --- a/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py +++ b/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py @@ -46,7 +46,7 @@ sliding-window schedule exactly like ordinary text. """ -from typing import Any, Dict, List, Literal, Optional, Tuple +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple import numpy as np import torch @@ -1045,7 +1045,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): if audio_args.get("input_mode") == "wav": self.audio_tower = MuseSparkOpenAudioEncoder(config.hidden_size) - def forward(self, multimodal_params: List[Any]) -> torch.Tensor: + def forward(self, multimodal_params: Sequence[Any]) -> torch.Tensor: ordered: List[Tuple[str, Any]] = [] for params in multimodal_params: ordered.extend(_items_in_prompt_order(params)) @@ -1299,7 +1299,7 @@ def multimodal_data_device_paths(self) -> List[str]: "multimodal_embedding", ] - def encode_multimodal_inputs(self, multimodal_params: List[Any]) -> torch.Tensor: + def encode_multimodal_inputs(self, multimodal_params: Sequence[Any]) -> torch.Tensor: """The mixin's uniform encoder entry: one tower pass over the batch. Called by the startup KV-cache profiler, which runs the encoder at the diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index 8b1bde2d68fc..f4504d99611f 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -3496,7 +3496,7 @@ def mm_encoder_groups(self) -> Tuple[EncoderGroup, ...]: ) def encode_multimodal_inputs( - self, multimodal_params: List[MultimodalParams], **_: Any + self, multimodal_params: Sequence[MultimodalParams], **_: Any ) -> torch.Tensor: # Raw encoder hook per the `MultimodalModelMixin` contract. Cache # bookkeeping belongs to callers (`_get_or_encode_multimodal_embeddings`, diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index b2ce944c6474..440e0a987d58 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -6,7 +6,7 @@ import math import re from functools import lru_cache -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union import numpy as np import torch @@ -2462,7 +2462,7 @@ def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() def encode_multimodal_inputs(self, - multimodal_params: List[MultimodalParams], + multimodal_params: Sequence[MultimodalParams], **encoder_kwargs: Any) -> torch.Tensor: """Uniform encoder entry (``MultimodalModelMixin`` contract). diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index c2383c624bda..c0e04867d003 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -1632,7 +1632,7 @@ def get_multimodal_item_partitions( return _qwen_vision_item_partitions(param, modality) def encode_multimodal_inputs( - self, multimodal_params: List[MultimodalParams], **encoder_kwargs: Any + self, multimodal_params: Sequence[MultimodalParams], **encoder_kwargs: Any ) -> torch.Tensor: """Uniform encoder entry (``MultimodalModelMixin`` contract). diff --git a/tensorrt_llm/_torch/models/multimodal.py b/tensorrt_llm/_torch/models/multimodal.py new file mode 100644 index 000000000000..fbf54b252f73 --- /dev/null +++ b/tensorrt_llm/_torch/models/multimodal.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Multimodal model contracts shared by the executor and model implementations. + +This module imports no model implementation, so any model can implement the contract +without depending on the shared mixin. +""" + +from __future__ import annotations + +from collections.abc import Hashable, Sequence +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeGuard + +__all__ = [ + "SupportsMultimodal", + "get_multimodal_data_device_paths", + "is_multimodal_model", +] + +if TYPE_CHECKING: + import torch + + from tensorrt_llm._torch.model_config import ModelConfig + from tensorrt_llm._torch.tensor_lru_cache import TensorLRUCache + from tensorrt_llm.inputs.multimodal import MultimodalParams + + from .modeling_multimodal_mixin import EncoderGroup, PreparedLlmInputs + + +class SupportsMultimodal(Protocol): + """The model-facing multimodal contract, independent of mixin inheritance. + + Models opt in with ``supports_multimodal = True`` and implement this contract. + ``MultimodalModelMixin`` supplies the common implementation. Feature declarations + describe support, while runtime properties/configuration determine what is enabled. + Item scheduling additionally requires the processor's item-metadata contract. + + Optional features keep their declarations false (or return an empty support mapping) + until implemented. Embedding properties and ``language_model`` may raise + ``NotImplementedError`` when the executor can derive or skip their values. + Encoder groups and native encoder metadata/graphs retain their existing roles. + Transport serialization belongs outside the model contract. + + ``multimodal_data_device_paths`` is optional for every model, multimodal or not; + read it through ``get_multimodal_data_device_paths``. + """ + + supports_multimodal: ClassVar[Literal[True]] + supports_encoder_cache: ClassVar[bool] + supports_mm_encoder_item_scheduling: ClassVar[bool] + supported_encoder_attention_backends: ClassVar[dict[str, frozenset[str]]] + supports_vision_gpu_normalize: ClassVar[bool] + support_mm_disagg: ClassVar[bool] + + model_config: ModelConfig[Any] + + @property + def mm_encoder_groups(self) -> Sequence[EncoderGroup]: + """Read the model's existing encoder batching declarations.""" + ... + + @property + def language_model(self) -> torch.nn.Module: + """The inner language model, read for its capabilities and weight-transfer scope.""" + ... + + @property + def samples_from_vocab_shards(self) -> bool: + """Whether the inner model's speculative worker samples target rows from vocabulary + shards.""" + ... + + @property + def embedding_dim(self) -> int: + """Width of one encoder output row, including model-specific extra features.""" + ... + + @property + def embedding_dtype(self) -> torch.dtype: + """Dtype of encoder output rows.""" + ... + + @property + def text_embedding_layer(self) -> torch.nn.Module: + """Text embedding layer used for fusion and output-size inference.""" + ... + + @property + def encoder_cache_active(self) -> bool: + """Whether persistent encoder-output reuse is enabled in this configuration.""" + ... + + @property + def multimodal_encoder_cache(self) -> TensorLRUCache[Hashable] | None: + """The existing model-owned output cache, also used by item scheduling.""" + ... + + def initialize_multimodal_encoder_cache( + self, max_bytes: int + ) -> TensorLRUCache[Hashable] | None: + """Initialize that cache once at startup using the executor's resolved byte budget.""" + ... + + @classmethod + def build_encoder_cache_item_keys( + cls, + multimodal_hashes: Sequence[Sequence[int]] | None, + item_refs: Sequence[tuple[str, int]], + embedding_lengths: Sequence[int], + kwargs_hash: str | None, + ) -> list[Hashable] | None: + """Return prompt-ordered item keys, or None when persistent reuse is unavailable.""" + ... + + def validate_multimodal_request_data(self, mm_data: dict[str, Any]) -> None: + """Reject invalid model inputs before admitting the request to a shared batch.""" + ... + + def encode_multimodal_inputs( + self, multimodal_params: Sequence[MultimodalParams] + ) -> torch.Tensor: + """Return encoder embeddings concatenated in request order.""" + ... + + def prepare_multimodal_encoder_inputs( + self, selected_items: Sequence[tuple[MultimodalParams, int]] + ) -> list[tuple[MultimodalParams, Sequence[int], str]]: + """Prepare selected items before H2D, retaining scheduler order.""" + ... + + def forward_multimodal_encoder_items( + self, encoder_inputs: Sequence[tuple[MultimodalParams, Sequence[int], str]] + ) -> list[torch.Tensor]: + """Return one embedding tensor per selected item, in scheduler order.""" + ... + + def prepare_multimodal_inputs( + self, + *, + input_ids: torch.Tensor, + positions: torch.Tensor | None, + multimodal_params: Sequence[MultimodalParams] | None, + num_context_requests: int, + **forward_kwargs: Any, + ) -> PreparedLlmInputs: + """Fuse model-specific multimodal features into the current LLM inputs.""" + ... + + +def is_multimodal_model(model: object) -> TypeGuard[SupportsMultimodal]: + """Read the explicit class marker without evaluating encoder properties. + + This is a declaration, not runtime signature validation. Implementations are checked + statically against ``SupportsMultimodal``; feature gates still check the relevant + declaration and configuration. Avoid structural introspection on each executor step. + The marker is read from ``model.__class__``, as ``isinstance`` does. + """ + return getattr(model.__class__, "supports_multimodal", False) is True + + +def get_multimodal_data_device_paths(model: object) -> list[str] | None: + """Return the ``multimodal_data`` paths the executor moves to the device for ``model``. + + Any model may declare ``multimodal_data_device_paths``; ``None`` (also when it is + absent) moves every tensor. + """ + return getattr(model, "multimodal_data_device_paths", None) diff --git a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py index 7d00f8daa299..0c99db4c2c8d 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py @@ -25,6 +25,7 @@ MultimodalModelMixin, _assemble_multimodal_encoder_embeddings, ) +from tensorrt_llm._torch.models.multimodal import SupportsMultimodal, is_multimodal_model from tensorrt_llm._torch.modules.embedding import Embedding from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, @@ -36,6 +37,58 @@ from tensorrt_llm.llmapi.llm_args import MultimodalConfig +@pytest.mark.cpu_only +def test_model_contract_detection_does_not_evaluate_encoder_properties() -> None: + model: SupportsMultimodal = MultimodalModelMixin() + # The default embedding properties raise: capability detection must not read them. + assert is_multimodal_model(model) + text_model = torch.nn.Module() + text_model.supports_encoder_cache = True + assert not is_multimodal_model(text_model) + + +@pytest.mark.cpu_only +def test_model_contract_defaults_do_not_enable_optional_features() -> None: + model: SupportsMultimodal = MultimodalModelMixin() + assert not model.supports_encoder_cache + assert not model.supports_mm_encoder_item_scheduling + assert model.supported_encoder_attention_backends == {} + assert not model.support_mm_disagg + assert not model.supports_vision_gpu_normalize + + +@pytest.mark.cpu_only +def test_model_contract_cache_preserves_existing_implementation_hook() -> None: + model = MultimodalModelMixin() + model.model_config = ModelConfig(multimodal_config=MultimodalConfig()) + contract: SupportsMultimodal = model + with patch.object( + model, + "_initialize_multimodal_encoder_cache", + wraps=model._initialize_multimodal_encoder_cache, + ) as initialize: + cache = contract.initialize_multimodal_encoder_cache(1024) + initialize.assert_called_once_with(1024) + assert contract.multimodal_encoder_cache is cache + assert model._multimodal_encoder_cache is cache + assert contract.initialize_multimodal_encoder_cache(1024) is cache + + +@pytest.mark.cpu_only +def test_mixin_defines_every_model_contract_member() -> None: + # CI type checking does not compare the mixin with the Protocol, so compare the names. + members = set(SupportsMultimodal.__annotations__) | { + name for name in vars(SupportsMultimodal) if not name.startswith("_") + } + missing = [ + name + for name in sorted(members) + if not hasattr(MultimodalModelMixin, name) + and name not in MultimodalModelMixin.__annotations__ + ] + assert not missing + + def make_embedding( num_embeddings: int = 100, hidden_size: int = 16, device: str = "cpu" ) -> Embedding: From 867e3e5af13d8aa4b29d97f7f6a88f986bcbbbc5 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:20:10 +0000 Subject: [PATCH 2/3] [None][refactor] add multimodal feature declarations Problem Each multimodal runtime feature has its own flag. A model cannot say why it skips a feature, or limit a feature to some deployments. Solution - MultimodalFeature names the 12 runtime features. - multimodal_opt_outs (per class) and multimodal_feature_limits() (per deployment) list skipped features, each with a reason. - supports_feature() answers per feature: the class marker first, then opt-outs and limits, then today's flags (item scheduling, encoder cache, encoder-to-prefill handoff, encoder parallel modes, the LLM compile hook). Breakable CUDA graphs follow the language model class: its supports_multimodal_breakable_cuda_graph flag, or language_model_takes_rows. Names from open PRs are read with getattr defaults. - The executor does not call it yet, so today's flags stay the source of truth. Item scheduling and the encoder cache stay separate. Validation - New CPU tests cover the marker, each flag, the language model's graph flag, opt-outs, limits and both forms of the encoder parallel mode query. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/torch/adding_new_model.md | 42 +++---- .../models/modeling_multimodal_mixin.py | 15 ++- tensorrt_llm/_torch/models/multimodal.py | 86 ++++++++++++- .../multimodal/test_multimodal_mixin.py | 115 +++++++++++++++++- 4 files changed, 229 insertions(+), 29 deletions(-) diff --git a/docs/source/torch/adding_new_model.md b/docs/source/torch/adding_new_model.md index 0e3b0113aadf..552cb0afc323 100644 --- a/docs/source/torch/adding_new_model.md +++ b/docs/source/torch/adding_new_model.md @@ -131,33 +131,33 @@ For a concrete reference, check out `tensorrt_llm/_torch/models/modeling_llama.p #### Multimodal Models -`tensorrt_llm._torch.models.multimodal.SupportsMultimodal` defines the model-facing -multimodal contract, and `MultimodalModelMixin` implements it. Reuse the mixin for -its common implementation. The `supports_multimodal = True` class marker declares -the contract; it is not runtime signature validation and does not enable optional -features. +`tensorrt_llm._torch.models.multimodal.SupportsMultimodal` lists what the runtime +reads from a multimodal model, and `MultimodalModelMixin` implements it. Subclass +the mixin: the executor still checks for it until it reads the protocol. The +`supports_multimodal = True` class marker declares the contract. It does not check +method signatures and does not turn on any feature. - Declare encoder caching and item scheduling separately. Item scheduling also - requires processor item metadata and a compatible runtime configuration. -- Report attention backends with the `supported_encoder_attention_backends` - class attribute. Its empty default declares no support; concrete integrations - supply their supported sets. -- Keep the existing `MultimodalParams` input and tensor return contracts. - Selected-item preparation runs before H2D, and item outputs retain selection - order. + needs the processor's item metadata and a runtime configuration that allows it. +- List skipped `MultimodalFeature` values in `multimodal_opt_outs`, or in + `multimodal_feature_limits` when the skip depends on the deployment. The + [conformance suite](source:tests/unittest/_torch/multimodal/conformance) checks + them against the required features in the multimodal onboarding skill. +- Declare attention backends in the `supported_encoder_attention_backends` + class attribute. Its empty default declares none. +- Keep the existing `MultimodalParams` inputs and tensor outputs. Selected items + are prepared before they are copied to the GPU, and item outputs keep the + selection order. - `multimodal_data_device_paths` stays optional for every model. Read it through `get_multimodal_data_device_paths()`; without it, every tensor in `multimodal_data` moves to the device. - Expose the existing model-owned cache through `multimodal_encoder_cache` and - `initialize_multimodal_encoder_cache()`. The latter preserves the mixin's - existing initialization hook. Item scheduling and persistent reuse share this - cache; they do not create separate owners. - -`EncoderGroup` continues to describe modalities that share an encoder call. -`MultimodalEncoderMixin` continues to provide native encoder attention metadata -and graph integration. Processor prompt expansion and disaggregated transport -retain their existing interfaces; implementing this Protocol does not enable a -new model, modality, encoder backend, or E/P path by itself. + `initialize_multimodal_encoder_cache()`. Item scheduling and encoder output + reuse share this cache. + +`EncoderGroup` still describes the modalities that share an encoder call, and +`MultimodalEncoderMixin` still provides encoder attention metadata and CUDA +graphs. Prompt expansion and disaggregated serving keep their interfaces. ### Weight Loading diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index 86ec2516e3ca..10352f86e5c8 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -17,6 +17,7 @@ import contextlib import copy import itertools +from collections.abc import Mapping from dataclasses import dataclass from typing import ( TYPE_CHECKING, @@ -59,7 +60,7 @@ fuse_input_embeds, get_multimodal_embeddings, ) -from .multimodal import SupportsMultimodal, get_multimodal_data_device_paths +from .multimodal import MultimodalFeature, SupportsMultimodal, get_multimodal_data_device_paths class MultimodalEncoderContractError(ValueError): @@ -279,6 +280,8 @@ def encode_multimodal_by_groups( if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + from ..pyexecutor.guided_decoder import CapturableGuidedDecoder from ..pyexecutor.llm_request import LlmRequest @@ -429,6 +432,9 @@ class MultimodalModelMixin: supports_multimodal: ClassVar[Literal[True]] = True """Opt in to the shared model contract without requiring mixin inheritance.""" + multimodal_opt_outs: ClassVar[Mapping[MultimodalFeature, str]] = {} + """Features excluded for every deployment, each with a reason and tracking ID.""" + supports_encoder_cache: ClassVar[bool] = False """Whether the model's production forward path uses the persistent encoder cache.""" @@ -451,6 +457,13 @@ class MultimodalModelMixin: _multimodal_encoder_cache: Optional[TensorLRUCache] = None _encoder_forward_context: Callable[[], contextlib.AbstractContextManager[None]] | None = None + @classmethod + def multimodal_feature_limits( + cls, model_config: ModelConfig | None, llm_args: Optional["TorchLlmArgs"] + ) -> Mapping[MultimodalFeature, str]: + """Return additional features excluded for this deployment.""" + return {} + @property def multimodal_encoder_cache(self) -> Optional[TensorLRUCache]: """Expose the existing model-owned encoder cache without transferring ownership.""" diff --git a/tensorrt_llm/_torch/models/multimodal.py b/tensorrt_llm/_torch/models/multimodal.py index fbf54b252f73..cb1a42506005 100644 --- a/tensorrt_llm/_torch/models/multimodal.py +++ b/tensorrt_llm/_torch/models/multimodal.py @@ -9,13 +9,17 @@ from __future__ import annotations -from collections.abc import Hashable, Sequence +from collections.abc import Hashable, Mapping, Sequence from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeGuard +from strenum import StrEnum + __all__ = [ + "MultimodalFeature", "SupportsMultimodal", "get_multimodal_data_device_paths", "is_multimodal_model", + "supports_feature", ] if TYPE_CHECKING: @@ -24,10 +28,26 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.tensor_lru_cache import TensorLRUCache from tensorrt_llm.inputs.multimodal import MultimodalParams + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from .modeling_multimodal_mixin import EncoderGroup, PreparedLlmInputs +class MultimodalFeature(StrEnum): + ITEM_SCHEDULING = "item_scheduling" + ENCODER_CACHE = "encoder_cache" # Independent of item scheduling. + ENCODER_CUDA_GRAPH = "encoder_cuda_graph" + ENCODER_DP = "encoder_dp" + ENCODER_TP = "encoder_tp" + LLM_BREAKABLE_CUDA_GRAPH = "llm_breakable_cuda_graph" + LLM_PIECEWISE_CUDA_GRAPH = "llm_piecewise_cuda_graph" # Includes LLM-scoped torch.compile. + CHUNKED_PREFILL_MEDIA = "chunked_prefill_media" + KV_REUSE_MEDIA = "kv_reuse_media" + DISAGG_E_TO_P = "disagg_e_to_p" + DISAGG_P_TO_D = "disagg_p_to_d" + UINT8_MEDIA = "uint8_media" + + class SupportsMultimodal(Protocol): """The model-facing multimodal contract, independent of mixin inheritance. @@ -47,6 +67,7 @@ class SupportsMultimodal(Protocol): """ supports_multimodal: ClassVar[Literal[True]] + multimodal_opt_outs: ClassVar[Mapping[MultimodalFeature, str]] supports_encoder_cache: ClassVar[bool] supports_mm_encoder_item_scheduling: ClassVar[bool] supported_encoder_attention_backends: ClassVar[dict[str, frozenset[str]]] @@ -55,6 +76,13 @@ class SupportsMultimodal(Protocol): model_config: ModelConfig[Any] + @classmethod + def multimodal_feature_limits( + cls, model_config: ModelConfig[Any] | None, llm_args: TorchLlmArgs | None + ) -> Mapping[MultimodalFeature, str]: + """Return deployment-specific exclusions, each with a reason and tracking ID.""" + ... + @property def mm_encoder_groups(self) -> Sequence[EncoderGroup]: """Read the model's existing encoder batching declarations.""" @@ -126,7 +154,7 @@ def encode_multimodal_inputs( def prepare_multimodal_encoder_inputs( self, selected_items: Sequence[tuple[MultimodalParams, int]] ) -> list[tuple[MultimodalParams, Sequence[int], str]]: - """Prepare selected items before H2D, retaining scheduler order.""" + """Prepare selected items before they are copied to the GPU, in scheduler order.""" ... def forward_multimodal_encoder_items( @@ -151,14 +179,62 @@ def prepare_multimodal_inputs( def is_multimodal_model(model: object) -> TypeGuard[SupportsMultimodal]: """Read the explicit class marker without evaluating encoder properties. - This is a declaration, not runtime signature validation. Implementations are checked - statically against ``SupportsMultimodal``; feature gates still check the relevant - declaration and configuration. Avoid structural introspection on each executor step. + The marker is a declaration; it does not check method signatures. Implementations are + checked statically against ``SupportsMultimodal``; feature gates still check the + relevant declaration and configuration. Avoid structural introspection on each executor step. The marker is read from ``model.__class__``, as ``isinstance`` does. """ return getattr(model.__class__, "supports_multimodal", False) is True +_FEATURE_FLAGS = { + MultimodalFeature.ITEM_SCHEDULING: "supports_mm_encoder_item_scheduling", + MultimodalFeature.ENCODER_CACHE: "supports_encoder_cache", + MultimodalFeature.DISAGG_E_TO_P: "support_mm_disagg", +} + + +def supports_feature( + model_cls: type, + feature: MultimodalFeature, + model_config: ModelConfig[Any] | None = None, + llm_args: TorchLlmArgs | None = None, +) -> bool: + """Return model-class eligibility, subject to any limits for this deployment. + + Breakable CUDA graphs follow the language model: its class declares + ``supports_multimodal_breakable_cuda_graph``, or the model passes the rows to the + language model (``language_model_takes_rows``). A model that builds its language + model at init has no class to read here; the engine checks the instance at setup. + A feature without its own flag is eligible unless the model opts out or limits it. + """ + if getattr(model_cls, "supports_multimodal", False) is not True: + return False + if feature in getattr(model_cls, "multimodal_opt_outs", {}): + return False + limits = getattr(model_cls, "multimodal_feature_limits", None) + if limits is not None and feature in limits(model_config, llm_args): + return False + + if feature in _FEATURE_FLAGS: + return bool(getattr(model_cls, _FEATURE_FLAGS[feature], False)) + if feature in (MultimodalFeature.ENCODER_DP, MultimodalFeature.ENCODER_TP): + get_modes = getattr(model_cls, "get_supported_encoder_parallel_modes", None) + if model_config is not None and callable(get_modes): + modes = get_modes(model_config) + else: + modes = getattr(model_cls, "supported_encoder_parallel_modes", ()) + return ("dp" if feature == MultimodalFeature.ENCODER_DP else "tp") in modes + if feature == MultimodalFeature.LLM_BREAKABLE_CUDA_GRAPH: + language_model = getattr(model_cls, "language_model_class", None) + return getattr(model_cls, "language_model_takes_rows", False) is True or bool( + getattr(language_model, "supports_multimodal_breakable_cuda_graph", False) + ) + if feature == MultimodalFeature.LLM_PIECEWISE_CUDA_GRAPH: + return callable(getattr(model_cls, "apply_llm_torch_compile", None)) + return True + + def get_multimodal_data_device_paths(model: object) -> list[str] | None: """Return the ``multimodal_data`` paths the executor moves to the device for ``model``. diff --git a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py index 0c99db4c2c8d..ef527108e0d1 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py @@ -25,7 +25,12 @@ MultimodalModelMixin, _assemble_multimodal_encoder_embeddings, ) -from tensorrt_llm._torch.models.multimodal import SupportsMultimodal, is_multimodal_model +from tensorrt_llm._torch.models.multimodal import ( + MultimodalFeature, + SupportsMultimodal, + is_multimodal_model, + supports_feature, +) from tensorrt_llm._torch.modules.embedding import Embedding from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, @@ -34,7 +39,7 @@ MultimodalParams, MultimodalRuntimeData, ) -from tensorrt_llm.llmapi.llm_args import MultimodalConfig +from tensorrt_llm.llmapi.llm_args import MultimodalConfig, TorchLlmArgs @pytest.mark.cpu_only @@ -55,6 +60,112 @@ def test_model_contract_defaults_do_not_enable_optional_features() -> None: assert model.supported_encoder_attention_backends == {} assert not model.support_mm_disagg assert not model.supports_vision_gpu_normalize + assert model.multimodal_opt_outs == {} + assert model.multimodal_feature_limits(None, None) == {} + + +@pytest.mark.cpu_only +def test_feature_eligibility_requires_explicit_marker() -> None: + class Unmarked: + supports_encoder_cache = True + + class Truthy(Unmarked): + supports_multimodal = 1 + + for model in (Unmarked, Truthy): + assert not any(supports_feature(model, feature) for feature in MultimodalFeature) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + ("feature", "attribute", "enabled"), + [ + (MultimodalFeature.ITEM_SCHEDULING, "supports_mm_encoder_item_scheduling", True), + (MultimodalFeature.ENCODER_CACHE, "supports_encoder_cache", True), + (MultimodalFeature.DISAGG_E_TO_P, "support_mm_disagg", True), + (MultimodalFeature.LLM_BREAKABLE_CUDA_GRAPH, "language_model_takes_rows", True), + (MultimodalFeature.LLM_PIECEWISE_CUDA_GRAPH, "apply_llm_torch_compile", lambda: None), + ], +) +def test_feature_eligibility_reads_independent_flags( + feature: MultimodalFeature, attribute: str, enabled: object +) -> None: + class Model: + supports_multimodal = True + + defaults = { + MultimodalFeature.ENCODER_CUDA_GRAPH, + MultimodalFeature.CHUNKED_PREFILL_MEDIA, + MultimodalFeature.KV_REUSE_MEDIA, + MultimodalFeature.DISAGG_P_TO_D, + MultimodalFeature.UINT8_MEDIA, + } + assert {f for f in MultimodalFeature if supports_feature(Model, f)} == defaults + setattr(Model, attribute, enabled) + assert {f for f in MultimodalFeature if supports_feature(Model, f)} == defaults | {feature} + setattr(Model, attribute, False) + assert not supports_feature(Model, feature) + + +@pytest.mark.cpu_only +def test_breakable_graph_follows_the_language_model() -> None: + class Decoder: + supports_multimodal_breakable_cuda_graph = True + + class Model: + supports_multimodal = True + # The decoder declares graph support, not the wrapper. + supports_multimodal_breakable_cuda_graph = True + + assert not supports_feature(Model, MultimodalFeature.LLM_BREAKABLE_CUDA_GRAPH) + Model.language_model_class = Decoder + assert supports_feature(Model, MultimodalFeature.LLM_BREAKABLE_CUDA_GRAPH) + + +@pytest.mark.cpu_only +def test_encoder_parallel_features_use_config_when_available() -> None: + config = ModelConfig() + + class StaticModes: + supports_multimodal = True + supported_encoder_parallel_modes = {"tp"} + + class ConfigModes(StaticModes): + @classmethod + def get_supported_encoder_parallel_modes(cls, model_config: ModelConfig) -> set[str]: + assert model_config is config + return {"dp"} + + for model, model_config, dp in ( + (StaticModes, config, False), + (ConfigModes, None, False), + (ConfigModes, config, True), + ): + assert supports_feature(model, MultimodalFeature.ENCODER_DP, model_config) is dp + assert supports_feature(model, MultimodalFeature.ENCODER_TP, model_config) is not dp + + +@pytest.mark.cpu_only +def test_feature_opt_outs_and_deployment_limits_take_precedence() -> None: + config = ModelConfig() + args = MagicMock(spec=TorchLlmArgs) + + class Model: + supports_multimodal = True + supports_encoder_cache = True + multimodal_opt_outs = {MultimodalFeature.ENCODER_CACHE: "No reusable rows (TRTLLM-1)."} + + @classmethod + def multimodal_feature_limits( + cls, model_config: ModelConfig | None, llm_args: TorchLlmArgs | None + ) -> dict[MultimodalFeature, str]: + if model_config is config and llm_args is args: + return {MultimodalFeature.CHUNKED_PREFILL_MEDIA: "Full prompt needed (TRTLLM-2)."} + return {} + + assert not supports_feature(Model, MultimodalFeature.ENCODER_CACHE) + assert supports_feature(Model, MultimodalFeature.CHUNKED_PREFILL_MEDIA) + assert not supports_feature(Model, MultimodalFeature.CHUNKED_PREFILL_MEDIA, config, args) @pytest.mark.cpu_only From 829ff415a5c22dc806351cdbc2f4b7918d8fb7b9 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:24:11 +0000 Subject: [PATCH 3/3] [None][test] add a multimodal conformance suite with feature tiers Problem A new multimodal model can skip a runtime feature and no test fails. Each feature is a separate opt-in, so new models often ship without them and get them months later. Solution - tiers.py lists the P0 and P1 features. A new model must support each P0 feature unless the multimodal owners approve an exception. A P1 opt-out needs a reason and a tracking ID in the model class. - models.py lists every multimodal architecture ID and every V2 target that its routes return. It does not rely on base classes. A text-only V2 target is only checked to refuse media and rows at admission, in validate_multimodal_request_data. - known_gaps.py records today's gaps. Features that no class flag shows are taken from the reviewed support matrix. test_contract.py holds the sizes, so the file can only shrink: the change that adds support deletes the entry, and new gaps go to APPROVED_EXCEPTIONS. - test_contract.py checks gaps, tiers and the class marker. test_processor_contract.py checks the processor hooks that item scheduling needs. An import test keeps the protocol module free of model and router modules. A broken registration fails one test, not the collection. - The multimodal onboarding skill and the Modeling V2 skills point to the same tiers, and a test keeps the skill table equal to tiers.py. - CBTS: a change to that skill runs this test. A Modeling V2 route, shared file or target that imports the multimodal contract runs the conformance suite. - CODEOWNERS lines for known_gaps.py and tiers.py. The owner team is still open. Not in this change, recorded as gaps: tiny model fixtures, the one- and two-GPU runtime feature suites, processor checks with real checkpoints, and the mRoPE, uint8 and media hash checks of open PRs. New models are not yet held to the shared base, fixtures, LLM breakable and piecewise CUDA graphs, or encoder DP/TP, because these are not on main yet. A model that opts out of one of them still needs an exception. Validation - The inventory finds 33 architecture IDs, 2 routed media targets and 14 routed text-only targets (2 refuse media at admission today). - CPU tests with the CPU CI marker on main 8ba86b8b, CBTS tests included: 1501 pass on this branch and 1395 on main. The 1395 shared tests have the same outcome; the 106 new tests pass. - Pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- .../trtllm-model-onboard-multimodal/SKILL.md | 44 +- .github/CODEOWNERS | 2 + .../skills/trtllm-modeling-v2-entry/SKILL.md | 1 + .../trtllm-modeling-v2-features/SKILL.md | 1 + .../references/features.md | 2 + jenkins/scripts/cbts/cbts/rules/README.md | 4 +- jenkins/scripts/cbts/cbts/rules/_helpers.py | 27 ++ .../scripts/cbts/cbts/rules/modeling_v2.py | 74 +++- .../scripts/cbts/cbts/rules/out_of_scope.py | 70 ++- .../multimodal/conformance/known_gaps.py | 283 +++++++++++++ .../_torch/multimodal/conformance/models.py | 243 +++++++++++ .../multimodal/conformance/test_contract.py | 397 ++++++++++++++++++ .../test_multimodal_protocol_imports.py | 84 ++++ .../conformance/test_processor_contract.py | 45 ++ .../conformance/test_skill_tiers.py | 40 ++ .../_torch/multimodal/conformance/tiers.py | 78 ++++ tests/unittest/scripts/cbts/test_docs.py | 33 ++ .../scripts/cbts/test_modeling_v2_rule.py | 18 + 18 files changed, 1420 insertions(+), 26 deletions(-) create mode 100644 tests/unittest/_torch/multimodal/conformance/known_gaps.py create mode 100644 tests/unittest/_torch/multimodal/conformance/models.py create mode 100644 tests/unittest/_torch/multimodal/conformance/test_contract.py create mode 100644 tests/unittest/_torch/multimodal/conformance/test_multimodal_protocol_imports.py create mode 100644 tests/unittest/_torch/multimodal/conformance/test_processor_contract.py create mode 100644 tests/unittest/_torch/multimodal/conformance/test_skill_tiers.py create mode 100644 tests/unittest/_torch/multimodal/conformance/tiers.py diff --git a/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md b/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md index 83f00f710c6b..910d322523af 100644 --- a/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md +++ b/.claude/skills/trtllm-model-onboard-multimodal/SKILL.md @@ -260,6 +260,30 @@ the selected items cross modality-run boundaries. --- +## Required runtime features (P0/P1) + +P0 is the minimum eligibility for a new model. A P0 opt-out needs an entry in `APPROVED_EXCEPTIONS` in the conformance suite's `known_gaps.py`, reviewed by the multimodal owners. For a P1 opt-out, add the `MultimodalFeature` to the model's `multimodal_opt_outs` with one sentence and a TRTLLM issue or nvbug ID. The engine settings determine which eligible features run. A decoder that lacks a graph mode for text needs no multimodal exception for that mode, so do not list it in `multimodal_opt_outs`. Until their runtime gates land, the suite does not hold new models to the features listed in `STRUCTURAL_ONLY` in `tiers.py`, unless the model opts out of one itself. Encoder CUDA graphs count when the tower registered with `@register_vision_encoder` declares `encoder_cuda_graph_modality`. A text-only target of a vision-language architecture has one check: its `validate_multimodal_request_data` raises on media and attached rows. + + +| Feature | Tier | What the model must provide | +| --- | --- | --- | +| `contract` | P0 | Expose the language model and tower. Supply item order, row counts and input partitions. | +| `text_only` | P0 | With `disable_mm_encoder`, omit the tower and refuse raw media and attached rows at admission. | +| `item_scheduling` | P0 | Encode whole items. Supply exact per-item costs, profiling inputs and encoder budgets. | +| `encoder_cache` | P0 | Return whole-item encoder rows with exact row counts and stable media hashes. | +| `chunked_prefill_media` | P0 | Fuse only the media rows in the current prefill chunk. | +| `kv_reuse_media` | P0 | Supply stable media hashes and token counts for prefix reuse. | +| `llm_breakable_cuda_graph` | P0 | Preserve decoder graph support for media. Use a `DecoderModelForCausalLM`, keep encoder attention out of its layer registry and prepare extra graph inputs. | +| `llm_piecewise_cuda_graph` | P0 | Preserve the decoder's piecewise graph support. Scope compilation to the language model. | +| `uint8_media` | P0 | Accept the loader's uint8 media and apply processor normalization. | +| `disagg_p_to_d` | P0 | Preserve media hashes and export and restore mRoPE state when used. | +| `encoder_dp` | P0 | Build the tower from the encoder config so replicas can encode whole items. | +| `encoder_attention_backend` | P0 | Declare supported attention backends per tower in the model class body. | +| `encoder_tp` | P1 | Shard tower weights under the encoder mapping. | +| `encoder_cuda_graph` | P1 | Provide a graph capture adapter for the tower blocks. | +| `disagg_e_to_p` | P1 | Accept attached rows with a complete item layout. Becomes P0 when the generic handoff lands. | + + ## Phases ### Phase 0 — Gather resources @@ -276,6 +300,8 @@ Read `config.json`'s `architectures` and `model_type`. If a `_torch/models/model ### Phase 2 — Model wrapper +Subclass `MultimodalModelMixin` today. When the shared composite base lands, subclass it or document why the target needs to implement the protocol as an adapter. + Create `tensorrt_llm/_torch/models/modeling_{name}.py`. The default pattern below mirrors `modeling_llava_next.py` and `modeling_gemma3vl.py` — a single wrapper class that composes a multimodal encoder + an LLM resolved through `AutoModelForCausalLM.from_config(text_config)`. The `*ModelBase + *Model` Base/non-Base split in `modeling_qwen2vl.py` and `modeling_qwen3vl.py` is an implementation detail for sharing one wrapper between two variants of the same family (Qwen2-VL ↔ Qwen2.5-VL; Qwen3-VL ↔ Qwen3-VL-MoE) — keep the wrapper a single class unless you have the same multi-variant need. ```python @@ -291,8 +317,8 @@ class {Name}VisionModel(nn.Module): class {Name}Model(MultimodalModelMixin, PreTrainedModel): config_class = {Name}Config supports_encoder_cache = True - # Enable after implementing and validating the Phase 3 atomic-item contracts. - supports_mm_encoder_item_scheduling = False + # P0: implement the Phase 3 atomic-item contracts. + supports_mm_encoder_item_scheduling = True def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): config = model_config.pretrained_config @@ -406,11 +432,12 @@ Full-request models can leave unknown costs as `None`; grouped assembly still requires exact output row counts. Do not implement the removed `get_mm_encoder_item_metadata` hook: the registry rejects it at wrapper creation. -**Atomic-item scheduling (optional).** Enable -`supports_mm_encoder_item_scheduling` only when items can be encoded independently. -Supply exact positive `encoder_token_lengths`, `output_embedding_lengths`, and -item selection, and implement `get_max_mm_encoder_output_embeddings` to size -the cache for one legal iteration. Missing scheduling metadata fails admission; +**Atomic-item scheduling (P0).** Set +`supports_mm_encoder_item_scheduling = True`. A model whose items cannot be encoded +independently needs an `APPROVED_EXCEPTIONS` entry. Supply exact positive +`encoder_token_lengths`, `output_embedding_lengths`, and item selection, and +implement `get_max_mm_encoder_output_embeddings` to size the cache for one legal +iteration. Missing scheduling metadata fails admission; it does not trigger a per-request fallback. Keep lengths and partitions on the host: `.cpu().tolist()` can synchronize and does not belong in a forward path. @@ -466,6 +493,8 @@ Inherit from a related mapper for prefix surgery — don't write a one-off trans ### Phase 5 — Tests +Run `pytest tests/unittest/_torch/multimodal/conformance -k ` with the registered architecture or routed target name. + **Per-model unit test (the main one)** at `tests/unittest/_torch/modeling/test_modeling_.py`. Subclass `TestModelingMultimodal` from `tests/unittest/_torch/modeling/test_modeling_multimodal.py` (an abstract `unittest.TestCase`) and implement six abstract methods: `get_model_config`, `get_trtllm_model_class`, `get_hf_model_class`, `get_weight_mapper_class`, `get_model_type`, `get_model_config_class`. The base class drives a `MultimodalScenario`-parameterized run (modality ∈ `image` / `multiple_image` / `video` / `text` / `mixture_text_image` / `audio`, with optional `use_cuda_graph` / `chunked_prefill` / `kv_cache_reuse`) — comparing TRT-LLM logits to HF reference, exercising the KV cache manager, attn metadata, mrope, fusion path, and CUDA graph capture in one harness. Override `get_scenarios` to declare which combinations apply to your model. Reference: `test_modeling_qwen3vl.py`, `test_modeling_qwen2_5vl.py`, `test_modeling_nemotron_h_multimodal.py`. Test data lives under `${LLM_MODELS_ROOT}/multimodals/test_data/`. **Hybrid linear-attention models.** Override `_dummy_request_kwargs` to return `{"use_mrope": True}` if the model uses mRoPE (allocates the 3-D position-id buffer at dummy-request time). The base class's `init_kv_cache_manager` already dispatches on `is_qwen3_hybrid` / `is_nemotron_hybrid` to build `CppMambaHybridCacheManager` — don't override unless you need a different concrete manager. Use `PyKvCacheConfig` from `llmapi.llm_args` (Pydantic), not the C++ bindings `KvCacheConfig` — `CppMambaHybridCacheManager.__init__` reads `mamba_state_cache_interval` which only exists on the Pydantic side. **CUDA-graph capture in the harness doesn't currently address the Mamba SSM state buffer** — keep `use_cuda_graph=False` in `get_scenarios` for hybrid models until that's wired through; production CUDA-graph support is independent and unaffected. @@ -527,6 +556,7 @@ Follow `CONTRIBUTING.md`. Title `[JIRA/NVBUG/None][type] description`, `git comm ## Pre-PR checklist **Architecture & registration** +- [ ] Every P0 opt-out has an owner-reviewed `APPROVED_EXCEPTIONS` entry. Every P1 opt-out has a reason and tracking ID in `multimodal_opt_outs`. - [ ] Decorator stack in correct order: `@support_multimodal_disaggregated` (outermost, optional) → `@register_vision_encoder` → `@register_auto_model` → `@register_input_processor` (innermost). - [ ] Model inherits `MultimodalModelMixin`; an overridden `forward` calls `prepare_multimodal_inputs` and takes no raw MM tensor args. - [ ] `multimodal_data_device_paths` lists every GPU-resident mm field. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 512ab6591f67..d53809fa2525 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -187,6 +187,8 @@ /tests/unittest/_torch/modules/mamba @NVIDIA/trt-llm-models-devs /tests/unittest/_torch/multi_gpu_modeling @NVIDIA/trt-llm-models-devs /tests/unittest/_torch/multimodal @NVIDIA/trt-llm-models-devs +/tests/unittest/_torch/multimodal/conformance/known_gaps.py @NVIDIA/trt-llm-models-devs +/tests/unittest/_torch/multimodal/conformance/tiers.py @NVIDIA/trt-llm-models-devs /tests/unittest/models @NVIDIA/trt-llm-models-devs /tests/unittest/others/test_multimodal_registry.py @NVIDIA/trt-llm-models-devs diff --git a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-entry/SKILL.md b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-entry/SKILL.md index 270e35b2357c..494a0865ee80 100644 --- a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-entry/SKILL.md +++ b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-entry/SKILL.md @@ -76,6 +76,7 @@ file is reported to the planner for ownership; it is not silently omitted. ## Test and certify +- For a vision-language target, also run `pytest tests/unittest/_torch/multimodal/conformance -k `. - Parametrize `op.CELLS`; use `_validating.py`'s `certifying` fixture so the entry's own `__call__` and `reference` both run. Derive the comparison band; bands wider than torch's dtype-aware default need wrong-variant negative diff --git a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/SKILL.md b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/SKILL.md index d0aab15724eb..a591f26e9670 100644 --- a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/SKILL.md +++ b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/SKILL.md @@ -96,6 +96,7 @@ Climb only as far as the claim needs. Each rung says what it cannot see. ## Rules that save hours +- For a media architecture, check each routed target against the [multimodal requirements](references/features.md#6-multimodal-inputs), including text-only admission. - Start from the in-tree test that already runs the feature (each section of the reference names them) rather than writing a launcher, server, transport or harness: the engine moves state and serves, the flow deploys and measures, diff --git a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/references/features.md b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/references/features.md index 684b0b538164..bd97e919e2df 100644 --- a/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/references/features.md +++ b/agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features/references/features.md @@ -370,6 +370,8 @@ extra host sync under overlap costs throughput only, which no gate sees. ## 6. Multimodal inputs +Once the shared V2 composite lands, implement a media-serving target as `ModelingV2ForConditionalGeneration` on that base. It must meet the same P0 requirements as other multimodal models. A text-only target must reject both raw media and attached encoder rows at admission, in `validate_multimodal_request_data`. See the [multimodal onboarding tiers](../../../../../../../../.claude/skills/trtllm-model-onboard-multimodal/SKILL.md#required-runtime-features-p0p1). + **What.** An input processor prepares media, an encoder embeds it, and the embeddings replace placeholder tokens in the decoder's input (`docs/source/features/multi-modality.md`; onboarding hooks in diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index 21e13ae92875..875f6bfc72d8 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -17,11 +17,11 @@ for the overall CBTS architecture. | `test_list.py` | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `visual_gen.py` | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (each excl. docs), and the exact VisualGen serve schema | | `spec_dec.py` | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (each excl. docs) | -| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per target or family for changes under `models/` and `catalog//`, from paths and imports | +| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per target or family for changes under `models/` and `catalog//`, from paths and imports; the multimodal conformance suite for routes, shared files and targets that import the multimodal contract | | `agent_flow.py` | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) → the single `CPU-AgentFlow-UnitTest` stage; not test-db-driven | | `openengine.py` | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) → the `l0_cpu` block containing `unittest/grpc/openengine/` | | `docs.py` | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus documentation-related CPU tests; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | -| `out_of_scope.py` | `OutOfScopeRule` | `noop` / `testsonly` | Reviewed metadata paths, root issue-template and developer-assistant folders, `.gitignore` at any depth, conventional clang filenames under `cpp/`, verified static-check-only `pyproject.toml` edits, QA/dev lists, timing caches, microbenchmarks, and CBTS infrastructure | +| `out_of_scope.py` | `OutOfScopeRule` | `noop` / `testsonly` | Reviewed metadata paths, root issue-template and developer-assistant folders, `.gitignore` at any depth, conventional clang filenames under `cpp/`, verified static-check-only `pyproject.toml` edits, QA/dev lists, timing caches, microbenchmarks, and CBTS infrastructure; a file in `CPU_TEST_INPUTS` selects the CPU test that reads it | ## PythonFormattingRule diff --git a/jenkins/scripts/cbts/cbts/rules/_helpers.py b/jenkins/scripts/cbts/cbts/rules/_helpers.py index 7f33bd892f25..86a7349de0f0 100644 --- a/jenkins/scripts/cbts/cbts/rules/_helpers.py +++ b/jenkins/scripts/cbts/cbts/rules/_helpers.py @@ -48,6 +48,33 @@ def block_filters_for_stem( return filters +def covering_entry_filters( + yaml_index: YAMLIndex, yaml_stem: str, test_path: str +) -> dict[tuple[str, int], dict[str, set[str]]]: + """Keep the longest entry of each `yaml_stem` block that covers `test_path`.""" + filters: dict[tuple[str, int], dict[str, set[str]]] = {} + for block in yaml_index.blocks: + if block.yaml_stem != yaml_stem: + continue + covering = [ + e for e in block.tests if _target_in_filter_subtree(test_path, _entry_target(e)) + ] + if covering: + entry = max(covering, key=lambda e: len(_entry_target(e))) + filters[(block.yaml_stem, block.block_index)] = {_entry_target(entry): {entry}} + return filters + + +def merge_block_filters( + filters: dict[tuple[str, int], dict[str, set[str]]], + more: dict[tuple[str, int], dict[str, set[str]]], +) -> None: + """Add the prefixes and identifiers of `more` to `filters`.""" + for key, prefixes in more.items(): + for prefix, ids in prefixes.items(): + filters.setdefault(key, {}).setdefault(prefix, set()).update(ids) + + def strip_noop_diff_lines(diff: str) -> str: """Normalize a unified diff so blank- and comment-only edits don't count. diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 838dec971d1b..2e5a594edf66 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -85,6 +85,12 @@ CPU guard for shared FP4 test helpers, `moe/test_modeling_v2_fp4_test_helpers.py`, is also kept for every modeling_v2 change. +The multimodal conformance suite (`unittest/_torch/multimodal/conformance`) +reads every route and the targets of vision-language architectures. A shared +file, a `routing.py`, or a file of a target whose files import the multimodal +contract (`tensorrt_llm._torch.models.multimodal` or the subtree's +`multimodal.py`) also selects the CPU stage's entry that covers the suite. + A unit test prefix that no entry of a block sits at or under keeps the longest entry of that block that covers it instead, such as a directory entry for all of `unittest/_torch/modeling_v2`. @@ -127,7 +133,12 @@ from cbts.blocks import Stage, YAMLIndex, _entry_target, _target_in_filter_subtree -from ._helpers import resolve_affected_stages, stages_by_yaml_stem +from ._helpers import ( + covering_entry_filters, + merge_block_filters, + resolve_affected_stages, + stages_by_yaml_stem, +) from .base import PRInputs, Rule, RuleResult from .docs import is_docs_path from .modeling_v2_source import PACKAGE, imported_names, package_names, routed_targets @@ -142,6 +153,8 @@ _IGNORE = re.compile(r"--ignore=(\S+)") _UNIT_ROOT = "unittest/_torch/modeling_v2" +_MULTIMODAL_CONFORMANCE = "unittest/_torch/multimodal/conformance" +_MULTIMODAL_CONTRACT = ("tensorrt_llm._torch.models.multimodal", f"{PACKAGE}.multimodal") _ACCURACY_ROOT = "accuracy/modeling_v2" _PKG_MODULE = PACKAGE _M3_UNIT_ROOT = f"{_UNIT_ROOT}/models/minimax_m3" @@ -298,6 +311,8 @@ def __init__( } # False when a file does not parse: a path through it would be missed. self.complete = True + # Module name -> the absolute names it imports. + self.imports: dict[str, set[str]] = {} self._importers: dict[str, set[str]] = defaultdict(set) # Package -> modules that import a computed name, which may be any module under it. self._computed: dict[str, set[str]] = defaultdict(set) @@ -308,6 +323,7 @@ def __init__( continue package = module if rel.name == "__init__.py" else module.rpartition(".")[0] names, computed = imported_names(tree, package) + self.imports[module] = names if computed: self._computed[package].add(module) # A module depends on its own packages too: importing it runs their __init__. @@ -411,6 +427,8 @@ def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage], repo_root: P # routing.py trees, which both the targets and the import graph read. self._routing_trees: dict[Path, Optional[ast.AST]] = {} self._routed: dict[str, Optional[set[str]]] = {} + # Target directory or family file -> whether its files import the multimodal contract. + self._imports_contract: dict[PurePosixPath, bool] = {} def _parse(self, path: Path) -> Optional[ast.AST]: if path.name != "routing.py" or path.parent.parent != self._src_root / "models": @@ -467,6 +485,54 @@ def _importers(self, rel: PurePosixPath) -> Optional[tuple[set[str], set[str]]]: targets |= self._target_tests(parts[1], parts[2]) return targets, {f"{_UNIT_ROOT}/{test}" for test in tests} + def _reaches_multimodal_contract(self, path: str) -> bool: + """Whether a changed file can change what the multimodal conformance suite reads. + + That is a shared file, a route, or a file of a target whose own files import + the multimodal contract (only the file itself for family code). + """ + if not path.startswith(_MV2_SRC_PREFIXES[0]): + return False + rel = PurePosixPath(path[len(_MV2_SRC_PREFIXES[0]) :]) + if _is_shared(rel) or rel.name == "routing.py": + return True + parts = rel.parts + if parts[0] == "models" and len(parts) > 3 and self._is_target(parts[1], parts[2]): + unit = PurePosixPath(*parts[:3]) + elif rel.suffix == ".py": + unit = rel + else: + return False + if unit not in self._imports_contract: + self._imports_contract[unit] = self._imports_multimodal_contract(unit) + return self._imports_contract[unit] + + def _imports_multimodal_contract(self, unit: PurePosixPath) -> bool: + """True when a file of `unit` imports the multimodal contract, or does not parse. + + Reads the import graph when another change built it, so no file is parsed twice. + """ + if self._graph is not None: + readings = [ + self._graph.imports.get(module) + for module, rel in self._graph.files.items() + if rel == unit or unit in rel.parents + ] + else: + readings = [] + for file in sorted((self._src_root / unit).rglob("*.py")) or [self._src_root / unit]: + rel = PurePosixPath(file.relative_to(self._src_root).as_posix()) + package = _module_name( + rel if rel.name == "__init__.py" else rel.parent / "__init__.py" + ) + tree = self._parse(file) + readings.append(None if tree is None else imported_names(tree, package)[0]) + return not readings or any( + names is None + or any(n == m or n.startswith(f"{m}.") for n in names for m in _MULTIMODAL_CONTRACT) + for names in readings + ) + def _family_unit_tests(self, family: str, subdir: PurePosixPath) -> set[str]: """The mirror unit directory of family code in `subdir`, else every family-level unit test.""" unit = f"{_UNIT_ROOT}/models/{family}" @@ -549,6 +615,12 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: if target: prefix_dict.setdefault(target, set()).add(entry) + if any(self._reaches_multimodal_contract(path) for path in claimed): + merge_block_filters( + block_filters, + covering_entry_filters(self.yaml_index, "l0_cpu", _MULTIMODAL_CONFORMANCE), + ) + if not block_filters: # Defensive: modeling_v2 source changed but no modeling_v2 block # exists in any yaml. Do not fabricate stages -- fall back to diff --git a/jenkins/scripts/cbts/cbts/rules/out_of_scope.py b/jenkins/scripts/cbts/cbts/rules/out_of_scope.py index 9f7b66634df7..c2454595c2ce 100644 --- a/jenkins/scripts/cbts/cbts/rules/out_of_scope.py +++ b/jenkins/scripts/cbts/cbts/rules/out_of_scope.py @@ -14,12 +14,14 @@ """OutOfScopeRule — claims changes outside product-test selection. Emits scope="noop" with no product-test contribution and sanity / perfsanity -off. CBTS infrastructure edits select CPU scripts tests. Noop changes add no -CPU stages. Build and repository checks run upstream. +off. CBTS infrastructure edits select CPU scripts tests, and a file that a CPU +test reads selects that test. Noop changes add no CPU stages. Build and +repository checks run upstream. `is_out_of_scope` is the single source of truth for these path patterns; `TestsDefRule` imports it to skip the same paths, so out-of-scope claims -are not overridden by a same-file narrow contribution. +are not overridden by a same-file narrow contribution. A CPU test input +outside those paths keeps the stages that other rules select. """ from __future__ import annotations @@ -31,7 +33,13 @@ from cbts.blocks import Stage, YAMLIndex -from ._helpers import reconstruct_diff_pre_image, resolve_affected_stages, stages_by_yaml_stem +from ._helpers import ( + covering_entry_filters, + merge_block_filters, + reconstruct_diff_pre_image, + resolve_affected_stages, + stages_by_yaml_stem, +) from .base import PRInputs, Rule, RuleResult # Exact files that affect repository metadata or external gates, but not the @@ -117,6 +125,20 @@ "jenkins/scripts/cbts/", ) +# Files that a CPU test reads. DocsRule also selects the docs stage for the V2 skill. +_V2_FEATURES = ( + "agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/trtllm-modeling-v2-features" +) +_TIER_TEST = "unittest/_torch/multimodal/conformance/test_skill_tiers.py" +CPU_TEST_INPUTS: dict[str, str] = { + ".claude/skills/trtllm-model-onboard-multimodal/SKILL.md": _TIER_TEST, + f"{_V2_FEATURES}/SKILL.md": _TIER_TEST, + f"{_V2_FEATURES}/references/features.md": _TIER_TEST, + "jenkins/scripts/cbts/cbts/rules/modeling_v2_source.py": ( + "unittest/_torch/multimodal/conformance" + ), +} + # Excluded on purpose: # - General `.md` / `.rst` matches: outside the exempt subtrees, DocsRule # routes documentation changes to CPU-Build_Docs. @@ -203,28 +225,44 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: claimed = { path for path in pr.changed_files - if is_out_of_scope(path) or self._is_check_only_pyproject(path, pr) + if path in CPU_TEST_INPUTS + or is_out_of_scope(path) + or self._is_check_only_pyproject(path, pr) } if not claimed: return None - if any(path.startswith("jenkins/scripts/cbts/") for path in claimed): - # The scripts suite includes CBTS tests and the infrastructure - # consumers of its command-line interfaces and selection artifacts. - filters = { - (block.yaml_stem, block.block_index): {"unittest/scripts": {"unittest/scripts"}} - for block in self.yaml_index.blocks - if block.yaml_stem == "l0_cpu" and "unittest/scripts" in block.tests - } - scripts_registered = bool(filters) + cbts = any(path.startswith("jenkins/scripts/cbts/") for path in claimed) + inputs = sorted({CPU_TEST_INPUTS[path] for path in claimed if path in CPU_TEST_INPUTS}) + if cbts or inputs: + filters: dict[tuple[str, int], dict[str, set[str]]] = {} + registered = True + if cbts: + # The scripts suite includes CBTS tests and the infrastructure + # consumers of its command-line interfaces and selection artifacts. + filters = { + (block.yaml_stem, block.block_index): {"unittest/scripts": {"unittest/scripts"}} + for block in self.yaml_index.blocks + if block.yaml_stem == "l0_cpu" and "unittest/scripts" in block.tests + } + registered = bool(filters) + for test in inputs: + found = covering_entry_filters(self.yaml_index, "l0_cpu", test) + registered = registered and bool(found) + merge_block_filters(filters, found) stages = resolve_affected_stages(filters, self.yaml_index, self._stages_by_yaml) + reason = "CBTS infrastructure changes require CPU scripts tests" + if inputs: + reason = "files that CPU tests read require those tests" + if cbts: + reason += " and CPU scripts tests" return RuleResult( handled_files=claimed, affected_stages=stages, - scope="testsonly" if scripts_registered and stages else None, + scope="testsonly" if registered and stages else None, block_filters=filters, sanity_relevant=False, perfsanity_relevant=False, - reason="outofscope: CBTS infrastructure changes require CPU scripts tests", + reason=f"outofscope: {reason}", ) return RuleResult( handled_files=claimed, diff --git a/tests/unittest/_torch/multimodal/conformance/known_gaps.py b/tests/unittest/_torch/multimodal/conformance/known_gaps.py new file mode 100644 index 000000000000..5f7e9eea3250 --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/known_gaps.py @@ -0,0 +1,283 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This snapshot only shrinks. New gaps go in APPROVED_EXCEPTIONS. +The change that adds support deletes its entry, and test_contract.py holds the sizes. +The multimodal owners review changes to this file. +""" + +from _torch.multimodal.conformance.models import Key + +from tensorrt_llm._torch.models.multimodal import MultimodalFeature as F + +_GLM = "Glm5NextForConditionalGeneration" +_KIMI = "KimiK3ForConditionalGeneration" +_MINIMAX = "MiniMaxM3SparseForConditionalGeneration" +_QWEN = "Qwen4ExpForConditionalGeneration" +_QWEN_FP8 = "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified" +_QWEN_NVFP4 = "qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default" + +SNAPSHOT_KEYS: frozenset[Key] = frozenset( + { + ("Cosmos3EdgeForConditionalGeneration", None), + ("Cosmos3ForConditionalGeneration", None), + ("DeepseekV41ForConditionalGeneration", None), + ("Exaone4_5_ForConditionalGeneration", None), + ("Gemma3ForConditionalGeneration", None), + ("Gemma4ForConditionalGeneration", None), + ("Gemma4UnifiedForConditionalGeneration", None), + (_GLM, None), + (_GLM, "glm5_next/glm_5_3_flash_fp8__sm_10x__tp4"), + (_GLM, "glm5_next/glm_5_3_flash_nvfp4__sm_100__default"), + (_GLM, "glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused"), + ("HCXVisionForCausalLM", None), + ("KimiK25ForConditionalGeneration", None), + (_KIMI, None), + (_KIMI, "kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1"), + (_KIMI, "kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1"), + ("Llama4ForConditionalGeneration", None), + ("LlavaLlamaModel", None), + ("LlavaNextForConditionalGeneration", None), + ("MiniCPMV4_6ForConditionalGeneration", None), + (_MINIMAX, None), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_103__tp4"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_103__tp8"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp2"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp4"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp8"), + ("Mistral3ForConditionalGeneration", None), + ("MllamaForConditionalGeneration", None), + ("MuseSparkForConditionalGeneration", None), + ("NemotronH_Nano_Omni_Reasoning_V3", None), + ("NemotronH_Nano_VL_V2", None), + ("NemotronH_Omni_Reasoning_V3", None), + ("Phi4MMForCausalLM", None), + ("PixtralForConditionalGeneration", None), + ("Qwen2VLForConditionalGeneration", None), + ("Qwen2_5_VLForConditionalGeneration", None), + ("Qwen3VLForConditionalGeneration", None), + ("Qwen3VLMoeForConditionalGeneration", None), + ("Qwen3_5ForConditionalGeneration", None), + ("Qwen3_5MoeForConditionalGeneration", None), + (_QWEN, None), + (_QWEN, _QWEN_FP8), + (_QWEN, _QWEN_NVFP4), + ("Step3p7ForConditionalGeneration", None), + ("WhisperForConditionalGeneration", None), + } +) + +_UNMARKED = frozenset( + { + F.CHUNKED_PREFILL_MEDIA, + F.DISAGG_E_TO_P, + F.DISAGG_P_TO_D, + F.ENCODER_CACHE, + F.ENCODER_CUDA_GRAPH, + F.ENCODER_DP, + F.ENCODER_TP, + F.ITEM_SCHEDULING, + F.KV_REUSE_MEDIA, + F.LLM_BREAKABLE_CUDA_GRAPH, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + } +) + +_PARALLEL_AND_BREAKABLE = frozenset( + { + F.ENCODER_DP, + F.ENCODER_TP, + F.LLM_BREAKABLE_CUDA_GRAPH, + } +) + +KNOWN_OPT_OUTS: dict[Key, frozenset[F]] = { + ("Cosmos3EdgeForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE | {F.ENCODER_CUDA_GRAPH}, + ("Cosmos3ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE, + ("DeepseekV41ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | { + F.DISAGG_E_TO_P, + F.ENCODER_CACHE, + F.ENCODER_CUDA_GRAPH, + F.ITEM_SCHEDULING, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + }, + ("Exaone4_5_ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | {F.DISAGG_E_TO_P, F.ENCODER_CACHE, F.ENCODER_CUDA_GRAPH, F.UINT8_MEDIA}, + ("Gemma3ForConditionalGeneration", None): _UNMARKED, + ("Gemma4ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | { + F.DISAGG_E_TO_P, + F.ENCODER_CUDA_GRAPH, + F.KV_REUSE_MEDIA, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + }, + ("Gemma4UnifiedForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | { + F.DISAGG_E_TO_P, + F.ENCODER_CUDA_GRAPH, + F.ITEM_SCHEDULING, + F.KV_REUSE_MEDIA, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + }, + (_GLM, None): _PARALLEL_AND_BREAKABLE + | { + F.DISAGG_E_TO_P, + F.ENCODER_CACHE, + F.ITEM_SCHEDULING, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + }, + ("HCXVisionForCausalLM", None): _UNMARKED, + ("KimiK25ForConditionalGeneration", None): _UNMARKED, + (_KIMI, None): _UNMARKED, + ("Llama4ForConditionalGeneration", None): _UNMARKED, + ("LlavaLlamaModel", None): _UNMARKED, + ("LlavaNextForConditionalGeneration", None): _UNMARKED, + ("MiniCPMV4_6ForConditionalGeneration", None): _UNMARKED, + (_MINIMAX, None): _UNMARKED, + ("Mistral3ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | {F.DISAGG_E_TO_P, F.ENCODER_CUDA_GRAPH, F.LLM_PIECEWISE_CUDA_GRAPH, F.UINT8_MEDIA}, + ("MllamaForConditionalGeneration", None): _UNMARKED, + ("MuseSparkForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | { + F.DISAGG_E_TO_P, + F.ENCODER_CACHE, + F.ENCODER_CUDA_GRAPH, + F.ITEM_SCHEDULING, + F.LLM_PIECEWISE_CUDA_GRAPH, + F.UINT8_MEDIA, + }, + ("NemotronH_Nano_Omni_Reasoning_V3", None): _PARALLEL_AND_BREAKABLE + | {F.ENCODER_CACHE, F.ITEM_SCHEDULING, F.LLM_PIECEWISE_CUDA_GRAPH, F.UINT8_MEDIA}, + ("NemotronH_Nano_VL_V2", None): _PARALLEL_AND_BREAKABLE + | {F.ENCODER_CACHE, F.ITEM_SCHEDULING, F.LLM_PIECEWISE_CUDA_GRAPH, F.UINT8_MEDIA}, + ("NemotronH_Omni_Reasoning_V3", None): _PARALLEL_AND_BREAKABLE + | {F.ENCODER_CACHE, F.ITEM_SCHEDULING, F.LLM_PIECEWISE_CUDA_GRAPH, F.UINT8_MEDIA}, + ("Phi4MMForCausalLM", None): _UNMARKED, + ("PixtralForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | {F.DISAGG_E_TO_P, F.ENCODER_CUDA_GRAPH, F.LLM_PIECEWISE_CUDA_GRAPH, F.UINT8_MEDIA}, + ("Qwen2VLForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | {F.DISAGG_E_TO_P, F.ENCODER_CACHE, F.ENCODER_CUDA_GRAPH, F.UINT8_MEDIA}, + ("Qwen2_5_VLForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE + | {F.ENCODER_CACHE, F.ENCODER_CUDA_GRAPH, F.UINT8_MEDIA}, + ("Qwen3VLForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE, + ("Qwen3VLMoeForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE, + ("Qwen3_5ForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE, + ("Qwen3_5MoeForConditionalGeneration", None): _PARALLEL_AND_BREAKABLE, + (_QWEN, None): _PARALLEL_AND_BREAKABLE | {F.DISAGG_E_TO_P}, + (_QWEN, _QWEN_FP8): _UNMARKED, + (_QWEN, _QWEN_NVFP4): _UNMARKED, + ("Step3p7ForConditionalGeneration", None): _UNMARKED, + ("WhisperForConditionalGeneration", None): _UNMARKED, +} + +# The shared base is not on main yet. Each family moves onto it in its own change. +_WAVE_A = "Moves with the port of the Qwen vision-language base." +_WAVE_B = "Mixin wrapper with its own forward or init; its owner ports it." +_WAVE_C = "Wrapper without the shared mixin; its owner ports it." + +NOT_ON_BASE: dict[Key, str] = { + ("Cosmos3EdgeForConditionalGeneration", None): _WAVE_A, + ("Cosmos3ForConditionalGeneration", None): _WAVE_A, + ("DeepseekV41ForConditionalGeneration", None): _WAVE_B, + ("Exaone4_5_ForConditionalGeneration", None): _WAVE_B, + ("Gemma3ForConditionalGeneration", None): _WAVE_C, + ("Gemma4ForConditionalGeneration", None): _WAVE_B, + ("Gemma4UnifiedForConditionalGeneration", None): _WAVE_B, + (_GLM, None): _WAVE_B, + ("HCXVisionForCausalLM", None): _WAVE_C, + ("KimiK25ForConditionalGeneration", None): _WAVE_C, + (_KIMI, None): _WAVE_C, + ("Llama4ForConditionalGeneration", None): _WAVE_C, + ("LlavaLlamaModel", None): _WAVE_C, + ("LlavaNextForConditionalGeneration", None): _WAVE_C, + ("MiniCPMV4_6ForConditionalGeneration", None): _WAVE_C, + (_MINIMAX, None): _WAVE_C, + ("Mistral3ForConditionalGeneration", None): _WAVE_B, + ("MllamaForConditionalGeneration", None): "Registered media stub.", + ("MuseSparkForConditionalGeneration", None): _WAVE_B, + ("NemotronH_Nano_Omni_Reasoning_V3", None): _WAVE_B, + ("NemotronH_Nano_VL_V2", None): _WAVE_B, + ("NemotronH_Omni_Reasoning_V3", None): _WAVE_B, + ("Phi4MMForCausalLM", None): _WAVE_C, + ("PixtralForConditionalGeneration", None): _WAVE_B, + ("Qwen2VLForConditionalGeneration", None): _WAVE_A, + ("Qwen2_5_VLForConditionalGeneration", None): _WAVE_A, + ("Qwen3VLForConditionalGeneration", None): _WAVE_A, + ("Qwen3VLMoeForConditionalGeneration", None): _WAVE_A, + ("Qwen3_5ForConditionalGeneration", None): _WAVE_A, + ("Qwen3_5MoeForConditionalGeneration", None): _WAVE_A, + (_QWEN, None): _WAVE_A, + (_QWEN, _QWEN_FP8): "Encodes the frames of the current chunk itself (TRTLLM-17780).", + (_QWEN, _QWEN_NVFP4): "Moves onto the shared Modeling V2 base (TRTLLM-17780).", + ("Step3p7ForConditionalGeneration", None): _WAVE_C, + ("WhisperForConditionalGeneration", None): "Encoder-decoder model.", +} + +MISSING_FIXTURE: frozenset[Key] = frozenset( + { + ("Cosmos3EdgeForConditionalGeneration", None), + ("Cosmos3ForConditionalGeneration", None), + ("DeepseekV41ForConditionalGeneration", None), + ("Exaone4_5_ForConditionalGeneration", None), + ("Gemma3ForConditionalGeneration", None), + ("Gemma4ForConditionalGeneration", None), + ("Gemma4UnifiedForConditionalGeneration", None), + (_GLM, None), + ("HCXVisionForCausalLM", None), + ("KimiK25ForConditionalGeneration", None), + (_KIMI, None), + ("Llama4ForConditionalGeneration", None), + ("LlavaLlamaModel", None), + ("LlavaNextForConditionalGeneration", None), + ("MiniCPMV4_6ForConditionalGeneration", None), + (_MINIMAX, None), + ("Mistral3ForConditionalGeneration", None), + ("MllamaForConditionalGeneration", None), + ("MuseSparkForConditionalGeneration", None), + ("NemotronH_Nano_Omni_Reasoning_V3", None), + ("NemotronH_Nano_VL_V2", None), + ("NemotronH_Omni_Reasoning_V3", None), + ("Phi4MMForCausalLM", None), + ("PixtralForConditionalGeneration", None), + ("Qwen2VLForConditionalGeneration", None), + ("Qwen2_5_VLForConditionalGeneration", None), + ("Qwen3VLForConditionalGeneration", None), + ("Qwen3VLMoeForConditionalGeneration", None), + ("Qwen3_5ForConditionalGeneration", None), + ("Qwen3_5MoeForConditionalGeneration", None), + (_QWEN, None), + ("Step3p7ForConditionalGeneration", None), + ("WhisperForConditionalGeneration", None), + } +) + +# Text-only targets that do not refuse media and rows at admission yet. +MISSING_ADMISSION_CHECK: frozenset[Key] = frozenset( + { + (_GLM, "glm5_next/glm_5_3_flash_fp8__sm_10x__tp4"), + (_KIMI, "kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1"), + (_KIMI, "kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__dep16"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moeep8"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__helix8_moetp2ep4"), + (_KIMI, "kimi_k3_vl/kimi_k3_nvfp4__sm_107__tp1"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_103__tp4"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_103__tp8"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp2"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp4"), + (_MINIMAX, "minimax_m3/minimax_m3_nvfp4__sm_107__tp8"), + } +) + +# (Architecture and target, feature or "base", "fixture", "text_only") -> (reason, tracking ID, +# approver). +APPROVED_EXCEPTIONS: dict[tuple[Key, F | str], tuple[str, str, str]] = {} diff --git a/tests/unittest/_torch/multimodal/conformance/models.py b/tests/unittest/_torch/multimodal/conformance/models.py new file mode 100644 index 000000000000..dd792e2a0372 --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/models.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Multimodal registrations and the target classes their routes return.""" + +from __future__ import annotations + +import ast +import inspect +from collections import defaultdict +from dataclasses import dataclass +from functools import cache +from importlib import import_module +from importlib.util import resolve_name +from pathlib import Path + +from tensorrt_llm._torch.modeling_v2._router_index import NoTarget, registered_routes +from tensorrt_llm._torch.models._arch_index import MODEL_ARCH_TO_MODULE, is_builtin_zoo_module +from tensorrt_llm._torch.models.modeling_mistral import Mistral3VLM, MistralNativeInputProcessor +from tensorrt_llm._torch.models.modeling_mllama import MllamaForConditionalGeneration +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin +from tensorrt_llm._torch.models.modeling_utils import ( + MODEL_CLASS_MAPPING, + MODEL_CLASS_VISION_ENCODER_MAPPING, +) +from tensorrt_llm.inputs.registry import INPUT_PROCESSOR_REGISTRY + +__extra_import_path__ = ["~/jenkins/scripts/cbts/cbts/rules"] +from modeling_v2_source import route_returns # noqa: E402 + +Key = tuple[str, str | None] +_ROOT = Path(__file__).resolve().parents[5] +_BASES = {"MultimodalForConditionalGeneration", "ModelingV2ForConditionalGeneration"} +TINY_CONFIGS: dict[Key, str] = {} +_B300_TARGETS = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified", + "qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default", +} + + +@dataclass(frozen=True) +class ModelEntry: + model_cls: type | None + processor_cls: type | None = None + fixture: str | None = None + media: bool = True + class_name: str = "" + # A text-only target defines validate_multimodal_request_data and raises in it. + refuses_media: bool = False + # A registered tower of the architecture declares an encoder graph modality. + graph_tower: bool = False + + @property + def on_base(self) -> bool: + return self.model_cls is not None and any( + cls.__name__ in _BASES for cls in self.model_cls.__mro__ + ) + + @property + def marked(self) -> bool: + return getattr(self.model_cls, "supports_multimodal", False) is True + + @property + def processor_classes(self) -> tuple[type, ...]: + processors = (self.processor_cls,) if self.processor_cls else () + if self.model_cls is Mistral3VLM: + return (*processors, MistralNativeInputProcessor) + return processors + + +def _returns(source: str, where: str) -> dict[str, str]: + """Map each routed directory to its class, without choosing a deployment.""" + fn = ast.parse(source) + imports = defaultdict(set) + for node in ast.walk(fn): + if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module: + for alias in node.names: + imports[alias.asname or alias.name].add((node.module, alias.name)) + targets = {} + for node, directory in route_returns(fn): + origins = imports.get(node.value.id, set()) if isinstance(node.value, ast.Name) else set() + assert directory and len(origins) == 1, f"{where}:{node.lineno}: ambiguous target return" + ((module, name),) = origins + assert module == f"{directory}.modeling", f"{where}: return a target modeling class" + assert directory not in targets or targets[directory] == name, ( + f"{where}: different classes returned from {directory}" + ) + targets[directory] = name + assert targets, f"{where}: media route has no target return" + return targets + + +@cache +def _source(module: str) -> tuple[dict[str, ast.ClassDef], dict[str, tuple[str, str]]]: + path = _ROOT / (module.replace(".", "/") + ".py") + body = ast.parse(path.read_text()).body + imports = { + alias.asname or alias.name: ( + resolve_name("." * node.level + node.module, module.rpartition(".")[0]), + alias.name, + ) + for node in body + if isinstance(node, ast.ImportFrom) and node.module + for alias in node.names + } + for node in body: + if isinstance(node, ast.Import): + for alias in node.names: + module_name = alias.name if alias.asname else alias.name.partition(".")[0] + imports[alias.asname or module_name] = ("", module_name) + return {node.name: node for node in body if isinstance(node, ast.ClassDef)}, imports + + +def _contract_in_source(module: str, name: str) -> bool: + """Follow local and imported base classes to the marker, including mixin adapters.""" + if name in _BASES or name == "MultimodalModelMixin": + return True + if ( + not module.startswith("tensorrt_llm.") + or not (_ROOT / (module.replace(".", "/") + ".py")).is_file() + ): + return False + classes, imports = _source(module) + cls = classes.get(name) + if cls is None: + return False + if any( + isinstance(node, ast.Name) and node.id == "supports_multimodal" + for statement in cls.body + if isinstance(statement, (ast.Assign, ast.AnnAssign)) + for node in ast.walk(statement) + ): + return True + for base in cls.bases: + if isinstance(base, ast.Subscript): + base = base.value + if isinstance(base, ast.Name): + origin, parent = imports.get(base.id, (module, base.id)) + elif isinstance(base, ast.Attribute): + head, _, tail = ast.unparse(base.value).partition(".") + package, imported = imports.get(head, ("", head)) + origin = f"{package}.{imported}.{tail}".strip(".") + parent = base.attr + else: + continue + if _contract_in_source(origin, parent): + return True + return False + + +def _target(module: str, name: str) -> tuple[type | None, bool, bool]: + """The class when it declares the contract, whether it takes media, and whether it + refuses media at admission.""" + if _contract_in_source(module, name): + try: + cls = getattr(import_module(module), name) + except NoTarget as exc: + raise AssertionError( + f"{module}.{name}: expose the feature contract to CPU tests" + ) from exc + media = ( + any(base.__name__ in _BASES for base in cls.__mro__) + or getattr(cls, "supports_multimodal", False) is True + ) + return cls, media, False + # Some targets run the tower and splice its embeddings in their own forward. + methods = { + method.name: method + for method in _source(module)[0][name].body + if isinstance(method, ast.FunctionDef) + } + media = any( + isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "multimodal_embeds" + for node in ast.walk(methods.get("forward", ast.Pass())) + ) + hook = methods.get("validate_multimodal_request_data") + refuses = hook is not None and any(isinstance(node, ast.Raise) for node in ast.walk(hook)) + return None, media, refuses + + +@cache +def inventory() -> dict[Key, ModelEntry]: + # Importing every provider also runs MiniMax's processor factory. + for module in sorted(set(MODEL_ARCH_TO_MODULE.values())): + import_module(f"tensorrt_llm._torch.models.{module}") + processors = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type + keys = { + arch + for arch, cls in MODEL_CLASS_MAPPING.items() + if is_builtin_zoo_module(cls.__module__) + and ( + cls in processors + or issubclass(cls, MultimodalModelMixin) + or cls in (Mistral3VLM, MllamaForConditionalGeneration) + ) + } + entries: dict[Key, ModelEntry] = {} + routes = registered_routes() + jobs = _ROOT / "tests/integration/test_lists/test-db/l0_b300.yml" + listed = { + line.split()[1] for line in jobs.read_text().splitlines() if line.lstrip().startswith("- ") + } + for arch in sorted(keys): + cls = MODEL_CLASS_MAPPING[arch] + processor = processors.get(cls) + towers = MODEL_CLASS_VISION_ENCODER_MAPPING.get(arch) or () + entries[arch, None] = ModelEntry( + cls, + processor, + TINY_CONFIGS.get((arch, None)), + class_name=cls.__name__, + graph_tower=any( + getattr(tower, "encoder_cuda_graph_modality", None) for tower in towers + ), + ) + if not entries[arch, None].processor_classes or arch not in routes: + continue + route = routes[arch] + family = route.__module__.rsplit(".", 2)[-2] + for directory, name in _returns(inspect.getsource(route), route.__module__).items(): + target = f"{family}/{directory}" + module = resolve_name(f".{directory}.modeling", route.__module__.rpartition(".")[0]) + target_cls, media, refuses = _target(module, name) + fixture = TINY_CONFIGS.get((arch, target)) + job = f"unittest/_torch/modeling_v2/models/{target}" + if target in _B300_TARGETS and job in listed: + fixture = f"l0_b300.yml: {job}" + entries[arch, target] = ModelEntry( + target_cls, processor, fixture, media, name, refuses_media=refuses + ) + _source.cache_clear() + return entries + + +def inventory_or_none() -> dict[Key, ModelEntry] | None: + """The inventory for test collection, or None when it cannot be built. + + A test then calls ``inventory()`` and reports the error as one failure, so the error + does not stop the collection of the other multimodal tests. + """ + try: + return inventory() + except Exception: + return None diff --git a/tests/unittest/_torch/multimodal/conformance/test_contract.py b/tests/unittest/_torch/multimodal/conformance/test_contract.py new file mode 100644 index 000000000000..78a82285fe22 --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/test_contract.py @@ -0,0 +1,397 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare every multimodal registration and route to its reviewed gaps.""" + +import ast +import re +import warnings + +import pytest +from _torch.multimodal.conformance import known_gaps as gaps +from _torch.multimodal.conformance import models +from _torch.multimodal.conformance.models import ( + _B300_TARGETS, + Key, + ModelEntry, + _returns, + inventory, + inventory_or_none, +) +from _torch.multimodal.conformance.tiers import FEATURE_TIERS, NOT_DERIVABLE, STRUCTURAL_ONLY + +from tensorrt_llm._torch.models._arch_index import MODEL_ARCH_TO_MODULE +from tensorrt_llm._torch.models.multimodal import MultimodalFeature as Feature +from tensorrt_llm._torch.models.multimodal import supports_feature + +pytestmark = pytest.mark.cpu_only + +_SKILL = ".claude/skills/trtllm-model-onboard-multimodal" +_TRACKING_ID = re.compile(r"\b(?:TRTLLM-\d+|(?:https?://)?nvbugs?[\s:/#-]+\d+)\b", re.I) + + +def _label(key: Key) -> str: + arch, target = key + return f"{arch}/{target}" if target else arch + + +_FIELDS = {"base", "fixture", "text_only", *(feature.value for feature in Feature)} +# The snapshot sizes, as (key, feature) pairs for KNOWN_OPT_OUTS. A change that deletes +# entries lowers these; the snapshot never grows. +_SNAPSHOT_SIZES = { + "SNAPSHOT_KEYS": 49, + "KNOWN_OPT_OUTS": 301, + "NOT_ON_BASE": 35, + "MISSING_FIXTURE": 33, + "MISSING_ADMISSION_CHECK": 12, +} + + +def _contract_problems(key: Key, entry: ModelEntry | None) -> list[str]: + where = _label(key) + known = gaps.KNOWN_OPT_OUTS.get(key, ()) + new = key not in gaps.SNAPSHOT_KEYS + approved = { + field: record for (owner, field), record in gaps.APPROVED_EXCEPTIONS.items() if owner == key + } + problems = [] + for field, (reason, ticket, approver) in approved.items(): + warnings.warn(f"{where}: approved {field}: {reason} ({ticket}, {approver})", stacklevel=2) + if field not in _FIELDS: + problems.append(f"{where}: P0 contract has an unknown exception field '{field}'") + if not (reason.strip() and _TRACKING_ID.fullmatch(ticket) and approver.strip()): + problems.append( + f"{where}: P0 contract exception needs a reason, tracking ID and approver" + ) + + if entry is None: + return [*problems, f"{where}: P0 inventory key is missing: stale entry, delete it"] + snapshots = { + "KNOWN_OPT_OUTS": gaps.KNOWN_OPT_OUTS, + "NOT_ON_BASE": gaps.NOT_ON_BASE, + "MISSING_FIXTURE": gaps.MISSING_FIXTURE, + "MISSING_ADMISSION_CHECK": gaps.MISSING_ADMISSION_CHECK, + } + for name, snapshot in snapshots.items(): + if new and key in snapshot: + problems.append(f"{where}: P0 contract: {name} cannot gain a key (see {_SKILL})") + + if not entry.media: + # A text-only target serves no media, so only its admission check applies. + for name in ("KNOWN_OPT_OUTS", "NOT_ON_BASE", "MISSING_FIXTURE"): + if key in snapshots[name]: + problems.append(f"{where}: text-only target in {name}: stale entry, delete it") + listed = key in gaps.MISSING_ADMISSION_CHECK or "text_only" in approved + if entry.refuses_media and listed: + problems.append(f"{where}: P0 text_only admission is present: stale entry, delete it") + elif not entry.refuses_media and not listed: + problems.append( + f"{where}: P0 text_only: run media through the multimodal runtime, or raise on " + f"media and rows in validate_multimodal_request_data (see {_SKILL})" + ) + return problems + if key in gaps.MISSING_ADMISSION_CHECK: + problems.append(f"{where}: takes media, not text-only: stale entry, delete it") + + if new and not entry.marked: + problems.append(f"{where}: P0 contract requires supports_multimodal = True (see {_SKILL})") + for field, present, snapshot in ( + ("base", entry.on_base, gaps.NOT_ON_BASE), + ("fixture", bool(entry.fixture), gaps.MISSING_FIXTURE), + ): + listed = key in snapshot or field in approved + if present and listed: + problems.append(f"{where}: P0 {field} is present: stale entry, delete it") + elif not present and not listed and not (new and field in STRUCTURAL_ONLY): + problems.append( + f"{where}: missing P0 {field} without APPROVED_EXCEPTIONS (see {_SKILL})" + ) + + opt_outs = getattr(entry.model_cls, "multimodal_opt_outs", {}) + limits = getattr(entry.model_cls, "multimodal_feature_limits", None) + # Limits for an unknown deployment apply to every deployment, like opt-outs. + declared = {**(limits(None, None) if callable(limits) else {}), **opt_outs} + for feature, reason in declared.items(): + tier = FEATURE_TIERS[feature][0] + if not ( + isinstance(reason, str) + and _TRACKING_ID.search(reason) + and re.search(r"\w", _TRACKING_ID.sub("", reason)) + ): + problems.append( + f"{where}: {tier} '{feature}' needs a reason and tracking ID (see {_SKILL})" + ) + + for feature in Feature: + tier = FEATURE_TIERS[feature][0] + label = f"{where}: {tier} feature '{feature}'" + supported = entry.model_cls is not None and supports_feature(entry.model_cls, feature) + if new and feature == Feature.ENCODER_CUDA_GRAPH: + supported = supported and entry.graph_tower + if supported: + if feature in approved or (feature in known and feature not in NOT_DERIVABLE): + problems.append(f"{label} is supported: stale entry, delete it") + elif feature in known or feature in approved: + continue + elif new and feature in STRUCTURAL_ONLY and feature not in declared: + continue + elif tier == "P0": + problems.append( + f"{label} is opted out but has no APPROVED_EXCEPTIONS entry (see {_SKILL})" + ) + elif feature not in declared: + problems.append( + f"{label} needs a multimodal_opt_outs reason and tracking ID (see {_SKILL})" + ) + return problems + + +def _test_id(key: Key, entries: dict[Key, ModelEntry]) -> str: + entry = entries.get(key) + label = _label(key) + return f"{label}-{entry.class_name}" if entry and entry.class_name != key[0] else label + + +def _keys() -> list: + entries = inventory_or_none() + if entries is None: + # The test builds the inventory again and fails with its error. + return [pytest.param(None, id="inventory")] + keys = ( + entries.keys() + | gaps.SNAPSHOT_KEYS + | gaps.KNOWN_OPT_OUTS.keys() + | gaps.NOT_ON_BASE.keys() + | gaps.MISSING_FIXTURE + | gaps.MISSING_ADMISSION_CHECK + | {key for key, _ in gaps.APPROVED_EXCEPTIONS} + ) + return [pytest.param(key, id=_test_id(key, entries)) for key in sorted(keys, key=_label)] + + +@pytest.mark.parametrize("key", _keys()) +def test_multimodal_contract(key: Key) -> None: + assert not (problems := _contract_problems(key, inventory().get(key))), problems + + +def test_snapshot_only_shrinks() -> None: + sizes = { + "SNAPSHOT_KEYS": len(gaps.SNAPSHOT_KEYS), + "KNOWN_OPT_OUTS": sum(map(len, gaps.KNOWN_OPT_OUTS.values())), + "NOT_ON_BASE": len(gaps.NOT_ON_BASE), + "MISSING_FIXTURE": len(gaps.MISSING_FIXTURE), + "MISSING_ADMISSION_CHECK": len(gaps.MISSING_ADMISSION_CHECK), + } + grown = {name: size for name, size in sizes.items() if size > _SNAPSHOT_SIZES[name]} + assert not grown, f"known_gaps.py cannot grow {grown}; add new gaps to APPROVED_EXCEPTIONS" + assert sizes == _SNAPSHOT_SIZES, f"Lower _SNAPSHOT_SIZES to the new sizes: {sizes}" + + +def test_registered_architectures_and_media_targets() -> None: + entries = inventory() + registered = { + key for key in gaps.SNAPSHOT_KEYS if key[1] is None and key[0] in MODEL_ARCH_TO_MODULE + } + assert registered <= entries.keys() + + # These native targets predate the marker. Their media paths have B300 qualification jobs. + for key, entry in entries.items(): + arch, target = key + if key in gaps.SNAPSHOT_KEYS and target is not None and not (entry.marked or entry.on_base): + assert entry.media == (target in _B300_TARGETS), (arch, target) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + (" def helper(): return 0", "no target return"), + ( + " if flag:\n from .one.modeling import A as T\n" + " else:\n from .two.modeling import B as T\n return T", + "ambiguous target return", + ), + ( + " from .one.modeling import A, B\n if flag: return A\n return B", + "different classes returned from one", + ), + ], +) +def test_route_returns_account_for_every_target(body: str, message: str) -> None: + with pytest.raises(AssertionError, match=message): + _returns(f"@routes('Example')\ndef route():\n{body}", "example") + + +class MultimodalForConditionalGeneration: + supports_multimodal = True + supports_mm_encoder_item_scheduling = True + supports_encoder_cache = True + support_mm_disagg = True + + +class _Model(MultimodalForConditionalGeneration): + pass + + +_KEY = ("ExampleVL", None) + + +def _entry(**fields) -> ModelEntry: + return ModelEntry(**{"model_cls": _Model, "fixture": "tiny", "graph_tower": True, **fields}) + + +def test_new_model_requirements(monkeypatch: pytest.MonkeyPatch) -> None: + # New models can leave structural-only features off. + assert _contract_problems(_KEY, _entry()) == [] + monkeypatch.setattr(_Model, "supports_mm_encoder_item_scheduling", False) + monkeypatch.setattr(_Model, "support_mm_disagg", False) + problems = _contract_problems(_KEY, _entry()) + assert len(problems) == 2 + assert any("P0 feature 'item_scheduling'" in p and "APPROVED_EXCEPTIONS" in p for p in problems) + assert any("P1 feature 'disagg_e_to_p'" in p and "multimodal_opt_outs" in p for p in problems) + assert all(_SKILL in p for p in problems) + + feature = Feature.DISAGG_E_TO_P + monkeypatch.setattr( + _Model, "multimodal_opt_outs", {feature: "No layout (nvbug 12345)."}, raising=False + ) + monkeypatch.setitem( + gaps.APPROVED_EXCEPTIONS, + (_KEY, Feature.ITEM_SCHEDULING), + ("Needs item costs.", "TRTLLM-1234", "model owner"), + ) + with pytest.warns(UserWarning, match=r"ExampleVL: approved item_scheduling: .*TRTLLM-1234"): + assert _contract_problems(_KEY, _entry()) == [] + + +@pytest.mark.parametrize("hook", ["multimodal_opt_outs", "multimodal_feature_limits"]) +@pytest.mark.parametrize("feature", sorted(set(STRUCTURAL_ONLY) & set(Feature))) +def test_declared_p0_skip_needs_an_exception_before_its_runtime_lands( + hook: str, feature: Feature, monkeypatch: pytest.MonkeyPatch +) -> None: + skip = {feature: "Not wired yet (TRTLLM-1234)."} + value = skip if hook == "multimodal_opt_outs" else classmethod(lambda cls, config, args: skip) + monkeypatch.setattr(_Model, hook, value, raising=False) + problems = _contract_problems(("FooVL", None), _entry()) + if FEATURE_TIERS[feature][0] == "P1": + assert problems == [] + return + assert problems == [ + f"FooVL: P0 feature '{feature}' is opted out but has no APPROVED_EXCEPTIONS entry " + f"(see {_SKILL})" + ] + + +def test_new_model_needs_a_graph_tower_or_a_reason(monkeypatch: pytest.MonkeyPatch) -> None: + feature = Feature.ENCODER_CUDA_GRAPH + problems = _contract_problems(_KEY, _entry(graph_tower=False)) + assert problems == [ + f"ExampleVL: P1 feature '{feature}' needs a multimodal_opt_outs reason and tracking ID " + f"(see {_SKILL})" + ] + monkeypatch.setattr( + _Model, "multimodal_opt_outs", {feature: "No block adapter (TRTLLM-1234)."}, raising=False + ) + assert _contract_problems(_KEY, _entry(graph_tower=False)) == [] + + +_TEXT_ONLY_TARGETS = """ +class Refuses: + def validate_multimodal_request_data(self, mm_data): + if mm_data: + raise ValueError("text only") + + +class Silent: + def validate_multimodal_request_data(self, mm_data): + pass +""" + + +@pytest.mark.parametrize(("name", "refuses"), [("Refuses", True), ("Silent", False)]) +def test_new_text_only_target_needs_only_admission( + name: str, refuses: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + classes = {node.name: node for node in ast.parse(_TEXT_ONLY_TARGETS).body} + monkeypatch.setattr(models, "_source", lambda module: (classes, {})) + model_cls, media, admission = models._target("example.text_only.modeling", name) + assert (model_cls, media, admission) == (None, False, refuses) + key = ("ExampleVL", "example/text_only") + entry = ModelEntry(model_cls, media=media, class_name=name, refuses_media=admission) + problems = _contract_problems(key, entry) + assert len(problems) == (0 if refuses else 1) + assert all("P0 text_only" in p and _SKILL in p for p in problems) + + +def test_text_only_snapshot_entries(monkeypatch: pytest.MonkeyPatch) -> None: + key = ("ExampleVL", "example/text_only") + monkeypatch.setattr(gaps, "SNAPSHOT_KEYS", gaps.SNAPSHOT_KEYS | {key}) + monkeypatch.setattr(gaps, "MISSING_ADMISSION_CHECK", gaps.MISSING_ADMISSION_CHECK | {key}) + monkeypatch.setitem(gaps.NOT_ON_BASE, key, "Text only.") + entry = ModelEntry(None, media=False, refuses_media=True) + assert _contract_problems(key, entry) == [ + "ExampleVL/example/text_only: text-only target in NOT_ON_BASE: stale entry, delete it", + "ExampleVL/example/text_only: P0 text_only admission is present: stale entry, delete it", + ] + stale = "ExampleVL/example/text_only: takes media, not text-only: stale entry, delete it" + assert stale in _contract_problems(key, _entry()) + + +@pytest.mark.parametrize("reason", ["", "No layout.", "TRTLLM-1234"]) +def test_declared_opt_out_needs_reason_and_id(reason: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + _Model, "multimodal_opt_outs", {Feature.DISAGG_E_TO_P: reason}, raising=False + ) + problems = _contract_problems(_KEY, _entry()) + assert len(problems) == 1 + assert "P1 'disagg_e_to_p' needs a reason and tracking ID" in problems[0] + + +def test_new_key_requires_marker_base_and_fixture(monkeypatch: pytest.MonkeyPatch) -> None: + problems = _contract_problems(_KEY, ModelEntry(model_cls=object)) + assert any("P0 contract requires supports_multimodal" in p and _SKILL in p for p in problems) + assert not any("missing P0 base" in p or "missing P0 fixture" in p for p in problems) + # Once the base and fixtures are available, new keys need them too. + monkeypatch.delitem(STRUCTURAL_ONLY, "base") + monkeypatch.delitem(STRUCTURAL_ONLY, "fixture") + problems = _contract_problems(_KEY, ModelEntry(model_cls=object)) + for rule in ("missing P0 base", "missing P0 fixture"): + assert any(rule in p and _SKILL in p for p in problems) + + +def test_snapshot_rejects_new_and_stale_entries(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(gaps.KNOWN_OPT_OUTS, _KEY, frozenset({Feature.ITEM_SCHEDULING})) + monkeypatch.setitem(gaps.NOT_ON_BASE, _KEY, "Uses its own wrapper.") + monkeypatch.setattr(gaps, "MISSING_FIXTURE", frozenset({_KEY})) + problems = _contract_problems(_KEY, _entry()) + for snapshot in ("KNOWN_OPT_OUTS", "NOT_ON_BASE", "MISSING_FIXTURE"): + assert any(f"P0 contract: {snapshot} cannot gain a key" in p for p in problems) + for rule in ( + "P0 feature 'item_scheduling' is supported", + "P0 base is present", + "P0 fixture is present", + ): + assert any(f"{rule}: stale entry, delete it" in p for p in problems) + assert _contract_problems(_KEY, None) == [ + "ExampleVL: P0 inventory key is missing: stale entry, delete it" + ] + + +def test_snapshot_size_is_fixed(monkeypatch: pytest.MonkeyPatch) -> None: + key = ("Qwen3VLForConditionalGeneration", None) + grown = dict(gaps.KNOWN_OPT_OUTS) + grown[key] |= {Feature.ENCODER_CACHE} + monkeypatch.setattr(gaps, "KNOWN_OPT_OUTS", grown) + with pytest.raises(AssertionError, match="cannot grow"): + test_snapshot_only_shrinks() + del grown[key] + with pytest.raises(AssertionError, match="Lower _SNAPSHOT_SIZES"): + test_snapshot_only_shrinks() + + +def test_exception_requires_a_known_field_and_owner(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(gaps.APPROVED_EXCEPTIONS, (_KEY, "typo"), ("", "", "")) + with pytest.warns(UserWarning, match="ExampleVL: approved typo"): + problems = _contract_problems(_KEY, _entry()) + assert any("P0 contract has an unknown exception field 'typo'" in p for p in problems) + assert any( + "P0 contract exception needs a reason, tracking ID and approver" in p for p in problems + ) diff --git a/tests/unittest/_torch/multimodal/conformance/test_multimodal_protocol_imports.py b/tests/unittest/_torch/multimodal/conformance/test_multimodal_protocol_imports.py new file mode 100644 index 000000000000..cd13fb71b6cf --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/test_multimodal_protocol_imports.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared multimodal interfaces must be usable without a model implementation or router.""" + +import ast +from pathlib import Path + +import pytest + +__extra_import_path__ = ["~/jenkins/scripts/cbts/cbts/rules"] +from modeling_v2_source import imported_names # noqa: E402 + +pytestmark = pytest.mark.cpu_only + +_ROOT = Path(__file__).resolve().parents[5] +_MODELS = "tensorrt_llm._torch.models." +_V2 = "tensorrt_llm._torch.modeling_v2." +# Add the shared composite base when it becomes available. +_GUARDED = (_MODELS + "multimodal",) +_NEUTRAL = { + _MODELS + "modeling_multimodal_mixin", + _MODELS + "modeling_multimodal_encoder", + _MODELS + "modeling_multimodal_utils", + _MODELS + "modeling_utils", + "tensorrt_llm._torch.model_config", + "tensorrt_llm._torch.tensor_lru_cache", + "tensorrt_llm.inputs.multimodal", + "tensorrt_llm.llmapi.llm_args", +} + + +def _forbidden_imports(module: str, root: Path) -> list[str]: + pending = [(module, module)] + visited = set() + violations = [] + while pending: + name, source = pending.pop() + if name in visited or name in _NEUTRAL or name.split(".", 1)[0] != "tensorrt_llm": + continue + visited.add(name) + path = root / Path(*name.split(".")) + package = name.rpartition(".")[0] + if path.is_dir(): + path = path / "__init__.py" + package = name + else: + path = path.with_suffix(".py") + if not path.is_file(): + continue # The imported name is a symbol rather than a source module. + if ( + name.startswith((_MODELS + "modeling_", _V2 + "models.")) + or name == _V2 + "_router_index" + or (name.startswith(_V2) and name.endswith(".routing")) + ): + violations.append(f"{source} imports {name}") + continue + imports, _ = imported_names(ast.parse(path.read_text()), package) + pending.extend((imported, name) for imported in sorted(imports)) + return violations + + +@pytest.mark.parametrize("module", _GUARDED) +def test_multimodal_imports(module: str) -> None: + assert not (violations := _forbidden_imports(module, _ROOT)), violations + + +def test_transitive_type_imports(tmp_path: Path) -> None: + models = tmp_path / "tensorrt_llm/_torch/models" + models.mkdir(parents=True) + (models / "multimodal.py").write_text("from .helper import Config") + (models / "helper.py").write_text( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from . import modeling_example\n" + " from ..modeling_v2._router_index import routes\n" + ) + (models / "modeling_example.py").touch() + v2 = models.parent / "modeling_v2" + v2.mkdir() + (v2 / "_router_index.py").touch() + assert set(_forbidden_imports(_GUARDED[0], tmp_path)) == { + f"{_MODELS}helper imports {_MODELS}modeling_example", + f"{_MODELS}helper imports {_V2}_router_index", + } diff --git a/tests/unittest/_torch/multimodal/conformance/test_processor_contract.py b/tests/unittest/_torch/multimodal/conformance/test_processor_contract.py new file mode 100644 index 000000000000..b689bf392507 --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/test_processor_contract.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Item scheduling needs processor geometry for admission and profiling.""" + +import pytest +from _torch.multimodal.conformance.models import ModelEntry, inventory, inventory_or_none + +from tensorrt_llm.inputs.registry import ( + BaseMultimodalDummyInputsBuilder, + BaseMultimodalInputProcessor, +) + +pytestmark = pytest.mark.cpu_only + + +def _entries() -> list: + entries = inventory_or_none() + if entries is None: + # The test builds the inventory again and fails with its error. + return [pytest.param(None, id="inventory")] + return [ + pytest.param(entry, id=f"{arch}/{target}" if target else arch) + for (arch, target), entry in entries.items() + if getattr(entry.model_cls, "supports_mm_encoder_item_scheduling", False) + ] + + +@pytest.mark.parametrize("entry", _entries()) +def test_item_scheduling_processor(entry: ModelEntry | None) -> None: + if entry is None: + inventory() + pytest.fail("the inventory failed while the tests were collected") + assert entry.processor_classes, "P0 item scheduling requires a registered processor" + for processor in entry.processor_classes: + for default in ( + BaseMultimodalInputProcessor.get_mm_item_metadata, + BaseMultimodalDummyInputsBuilder.get_mm_max_tokens_per_item, + BaseMultimodalDummyInputsBuilder.get_dummy_mm_data, + BaseMultimodalDummyInputsBuilder.get_max_mm_encoder_output_embeddings, + BaseMultimodalDummyInputsBuilder.get_mm_encoder_attention_metadata_capacity, + ): + hook = getattr(processor, default.__name__, None) + assert callable(hook) and hook is not default, ( + f"P0 item scheduling requires {processor.__name__} to implement {default.__name__}" + ) diff --git a/tests/unittest/_torch/multimodal/conformance/test_skill_tiers.py b/tests/unittest/_torch/multimodal/conformance/test_skill_tiers.py new file mode 100644 index 000000000000..cb7f196dd60e --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/test_skill_tiers.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from os.path import relpath +from pathlib import Path + +import pytest +from _torch.multimodal.conformance.tiers import FEATURE_TIERS + +pytestmark = pytest.mark.cpu_only + + +def test_onboarding_skill_matches_feature_tiers(): + root = Path(__file__).resolve().parents[5] + onboarding = root / ".claude/skills/trtllm-model-onboard-multimodal/SKILL.md" + introduction, begin, table = onboarding.read_text().partition( + "" + ) + table, end, _ = table.partition("") + expected = ["| Feature | Tier | What the model must provide |", "| --- | --- | --- |"] + expected.extend( + f"| `{feature}` | {tier} | {requirement} |" + for feature, (tier, requirement) in FEATURE_TIERS.items() + ) + assert begin and end and table.strip().splitlines() == expected, ( + "Update FEATURE_TIERS and the marked table in the multimodal onboarding skill together." + ) + + features = ( + root + / "agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills" + / "trtllm-modeling-v2-features" + ) + reference = features / "references/features.md" + assert "(references/features.md#6-multimodal-inputs)" in (features / "SKILL.md").read_text() + section = ( + reference.read_text().partition("\n## 6. Multimodal inputs\n")[2].partition("\n## ")[0] + ) + assert f"({relpath(onboarding, reference.parent)}#required-runtime-features-p0p1)" in section + assert "\n## Required runtime features (P0/P1)\n" in introduction diff --git a/tests/unittest/_torch/multimodal/conformance/tiers.py b/tests/unittest/_torch/multimodal/conformance/tiers.py new file mode 100644 index 000000000000..b73905616b6f --- /dev/null +++ b/tests/unittest/_torch/multimodal/conformance/tiers.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Multimodal onboarding requirements, in the order used by the onboarding guide.""" + +from tensorrt_llm._torch.models.multimodal import MultimodalFeature as F + +FEATURE_TIERS = { + "contract": ( + "P0", + "Expose the language model and tower. Supply item order, row counts and input partitions.", + ), + "text_only": ( + "P0", + "With `disable_mm_encoder`, omit the tower and refuse raw media and attached rows at admission.", + ), + F.ITEM_SCHEDULING: ( + "P0", + "Encode whole items. Supply exact per-item costs, profiling inputs and encoder budgets.", + ), + F.ENCODER_CACHE: ( + "P0", + "Return whole-item encoder rows with exact row counts and stable media hashes.", + ), + F.CHUNKED_PREFILL_MEDIA: ("P0", "Fuse only the media rows in the current prefill chunk."), + F.KV_REUSE_MEDIA: ("P0", "Supply stable media hashes and token counts for prefix reuse."), + F.LLM_BREAKABLE_CUDA_GRAPH: ( + "P0", + "Preserve decoder graph support for media. Use a `DecoderModelForCausalLM`, keep encoder " + "attention out of its layer registry and prepare extra graph inputs.", + ), + F.LLM_PIECEWISE_CUDA_GRAPH: ( + "P0", + "Preserve the decoder's piecewise graph support. Scope compilation to the language model.", + ), + F.UINT8_MEDIA: ("P0", "Accept the loader's uint8 media and apply processor normalization."), + F.DISAGG_P_TO_D: ("P0", "Preserve media hashes and export and restore mRoPE state when used."), + F.ENCODER_DP: ( + "P0", + "Build the tower from the encoder config so replicas can encode whole items.", + ), + "encoder_attention_backend": ( + "P0", + "Declare supported attention backends per tower in the model class body.", + ), + F.ENCODER_TP: ("P1", "Shard tower weights under the encoder mapping."), + F.ENCODER_CUDA_GRAPH: ("P1", "Provide a graph capture adapter for the tower blocks."), + F.DISAGG_E_TO_P: ( + "P1", + "Accept attached rows with a complete item layout. Becomes P0 when the generic handoff lands.", + ), +} + +UNTIERED = { + "supports_vision_gpu_normalize": "Requires processor statistics and a compatible tensor dtype and layout.", + "openengine_image_routing": "Requires separate qualification for image KV events and routing.", +} + +# New models are not held to these yet, because what they check is not on main. A model +# that opts out of one of them in its class still needs an exception for its tier. +STRUCTURAL_ONLY = { + F.LLM_BREAKABLE_CUDA_GRAPH: "The executor has no multimodal breakable graph path yet.", + F.LLM_PIECEWISE_CUDA_GRAPH: "Only some wrappers scope compilation to the language model.", + F.ENCODER_DP: "The encoder has no separate data parallel mapping yet.", + F.ENCODER_TP: "The encoder has no separate tensor parallel mapping yet.", + "base": "The shared multimodal base class is not available yet.", + "fixture": "The tiny reference model and its fixtures are not available yet.", +} + +# No class flag shows these on main, so a class is eligible unless it opts out. Their +# snapshot entries come from the reviewed support matrix and are never reported as stale; +# the check named here deletes them when it lands. +NOT_DERIVABLE = { + F.ENCODER_CUDA_GRAPH: "The one-GPU encoder graph check.", + F.CHUNKED_PREFILL_MEDIA: "The one-GPU chunked prefill check.", + F.KV_REUSE_MEDIA: "The one-GPU KV reuse check.", + F.DISAGG_P_TO_D: "The decode state export and restore check.", + F.UINT8_MEDIA: "The uint8 versus float input check, with the uint8 loader default.", +} diff --git a/tests/unittest/scripts/cbts/test_docs.py b/tests/unittest/scripts/cbts/test_docs.py index d7c6467937f3..1918827298e7 100644 --- a/tests/unittest/scripts/cbts/test_docs.py +++ b/tests/unittest/scripts/cbts/test_docs.py @@ -979,6 +979,39 @@ def test_cbts_changes_require_registered_cpu_scripts_tests( assert narrowed["l0_cpu"][0]["tests"] == ["unittest/scripts"] +@pytest.mark.parametrize("registered", (True, False)) +def test_cpu_test_inputs_select_the_tests_that_read_them(registered: bool) -> None: + entry = "unittest/_torch/multimodal" if registered else "unittest/_torch/other" + tier_test = f"{entry}/conformance/test_skill_tiers.py" + yaml_index = YAMLIndex() + yaml_index.blocks = [ + Block(yaml_stem="l0_cpu", block_index=0, condition={}, tests=["unittest/tools", entry]), + Block(yaml_stem="l0_cpu", block_index=1, condition={}, tests=[tier_test]), + Block(yaml_stem="l0_h100", block_index=0, condition={}, tests=[entry]), + ] + stages = {"CPU-Generic-x86-1": Stage("CPU-Generic-x86-1", "l0_cpu", "x86_64", 1, 1)} + rules = [DocsRule(yaml_index, stages), OutOfScopeRule(yaml_index, stages)] + features = ( + "agent-flow/agent_flow/workflows/trtllm_modeling_v2/agents/skills/" + "trtllm-modeling-v2-features" + ) + for path, docs_stages in ( + (".claude/skills/trtllm-model-onboard-multimodal/SKILL.md", set()), + (f"{features}/SKILL.md", {DOCS_STAGE}), + (f"{features}/references/features.md", {DOCS_STAGE}), + ): + result = Selector(stages).run(PRInputs(changed_files=[path], diffs={}), rules) + if not registered: + assert result.scope is None, path + continue + assert result.scope == "testsonly", path + assert result.affected_stages == {"CPU-Generic-x86-1"} | docs_stages, path + assert result.block_filters == { + ("l0_cpu", 0): {entry: {entry}}, + ("l0_cpu", 1): {tier_test: {tier_test}}, + }, path + + def test_reviewed_noop_does_not_suppress_mixed_docs_validation(tmp_path: Path) -> None: input_path = tmp_path / "input.json" input_path.write_text( diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 93180aad367f..da749769a31b 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -578,3 +578,21 @@ def test_stage_coverage_requires_an_affected_stage( assert _test_stages(result, index, stages, path) result.affected_stages.clear() assert not _test_stages(result, index, stages, path) + + +def test_multimodal_contract_users_run_the_conformance_suite(rule): + """Routes, shared files and contract importers also run the multimodal conformance suite.""" + cpu = next(block for block in rule.yaml_index.blocks if block.yaml_stem == "l0_cpu") + cpu.tests.append("unittest/_torch/multimodal") + target = rule._src_root / "models/kimi_k3_vl/target" + (target / "modeling.py").write_text( + "from tensorrt_llm._torch.models.multimodal import SupportsMultimodal\n" + ) + (target / "vision.py").write_text("X = 1\n") + for path, runs in ( + (f"{_PY}/models/kimi_k3_vl/target/vision.py", True), + (f"{_PY}/models/gpt_oss/{_GPT_OSS_TARGET}/modeling.py", False), + (f"{_PY}/models/gpt_oss/routing.py", True), + (f"{_PY}/multimodal.py", True), + ): + assert ("unittest/_torch/multimodal" in _selected(_apply(rule, path))) is runs, path