From dbd16deda554fa5889bcccc73ca4964b9d570e70 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:44:18 +0000 Subject: [PATCH 1/6] [None][feat] configure attention backends per multimodal encoder Problem: Encoder backend choices mixed public attention backends with HF names and repeated validation and config-copy logic across model families. Solution: Accept VANILLA, TRTLLM and FLASHINFER. Validate model declarations before weight allocation, reject fixed or disabled encoders, and apply overrides with shared config helpers. Preserve omitted defaults and decoder choices. Update docs and schemas; reuse existing CPU test collection. Validation: Fresh SM100 core build; 230 CPU tests and applicable hooks passed. BF16/TP1 Qwen3-VL and Gemma 4 smoke completed 16 forwards with all public backends, including head64/head72 cases. Fixed elementwise checks met atol/rtol 0.05 in 9/12 comparisons. Gemma FP differences are recorded; maximum relative L2 was 0.005398392211645842. No tolerances were retuned. Pretrained-model, multi-GPU and serving accuracy remain unverified. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/features/multi-modality.md | 36 +++ scripts/generate_trtllm_serve_schemas.py | 8 + tensorrt_llm/_torch/model_config.py | 26 ++ .../models/modeling_deepseekv41_multimodal.py | 2 + .../_torch/models/modeling_gemma4_unified.py | 2 + .../_torch/models/modeling_gemma4_vision.py | 5 +- .../_torch/models/modeling_gemma4mm.py | 11 +- .../models/modeling_glm5_next_vision.py | 3 + .../_torch/models/modeling_kimi_k25.py | 7 + .../_torch/models/modeling_kimi_k3_vl.py | 3 + .../_torch/models/modeling_minicpmv4_6.py | 10 +- .../_torch/models/modeling_minimaxm3.py | 2 + .../_torch/models/modeling_mistral.py | 7 +- .../models/modeling_multimodal_mixin.py | 6 + .../models/modeling_nemotron_h_multimodal.py | 6 + .../_torch/models/modeling_qwen2vl.py | 8 + .../_torch/models/modeling_qwen3vl.py | 6 +- .../_torch/models/modeling_step3p7vl.py | 8 +- .../_torch/pyexecutor/model_loader.py | 44 +++- tensorrt_llm/llmapi/llm_args.py | 25 ++ .../schemas/trtllm-serve-config.schema.json | 21 ++ .../trtllm-serve-disagg-config.schema.json | 21 ++ .../_torch/modeling/test_modeling_qwen3vl.py | 57 ++++ .../test_encoder_attention_backend.py | 244 ++++++++++++++++++ tests/unittest/llmapi/test_llm_args.py | 2 + .../test_trtllm_serve_config_schemas.py | 24 ++ 26 files changed, 585 insertions(+), 9 deletions(-) create mode 100644 tests/unittest/_torch/multimodal/test_encoder_attention_backend.py diff --git a/docs/source/features/multi-modality.md b/docs/source/features/multi-modality.md index dc272451c7a8..0fb295bf3af8 100644 --- a/docs/source/features/multi-modality.md +++ b/docs/source/features/multi-modality.md @@ -27,6 +27,42 @@ TensorRT LLM incorporates some key optimizations to enhance the performance of m Further optimizations are under development and will be updated as they become available. +## Encoder attention backends + +`multimodal_config.encoder_attention_backend` selects a TensorRT-LLM attention backend for each logical encoder. The public values are `VANILLA` (PyTorch scaled dot product attention), `TRTLLM`, and `FLASHINFER`; values are case-insensitive. Image and video inputs share the `vision` encoder. + +```yaml +multimodal_config: + encoder_attention_backend: + vision: FLASHINFER +``` + +The same setting is available in the Python API: + +```python +from tensorrt_llm import LLM + +llm = LLM( + model="Qwen/Qwen3-VL-8B-Instruct", + multimodal_config={"encoder_attention_backend": {"vision": "VANILLA"}}, +) +``` + +Omitted entries preserve the model's existing encoder defaults, including inheritance from `attn_backend` where applicable. An encoder override leaves the language model's backend unchanged. Backend availability and supported tensor shapes still depend on the installed kernels and hardware; floating-point differences between backends are expected. + +| Encoder | Supported values | Default when omitted | +| --- | --- | --- | +| Qwen2.5-VL, Qwen3-VL / Qwen3.5-VL vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | Existing `attn_backend` | +| Gemma 4 vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `TRTLLM` | +| Kimi K2.5 / K3, MiniCPM-V 4.6, Step 3.7 vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | Existing `attn_backend` | +| Mistral3 / Pixtral vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `TRTLLM` | +| Nemotron multimodal vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `FLASHINFER` | +| GLM5-Next vision | `TRTLLM` | Existing `attn_backend` | + +For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. + +Only the declared encoders using TensorRT-LLM's `Attention` support this setting. Qwen2-VL's Hugging Face vision encoder, VILA's Hugging Face CLIP/SigLIP towers, Phi-4-multimodal's Hugging Face encoder, Parakeet audio, Gemma 4 audio, and MiniMax M3 / DeepSeek V4.1 vision keep their fixed attention implementations. Overrides for these encoders fail during initialization, even if the requested value is `VANILLA`. Gemma 4 Unified has no configurable attention encoder. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. + ## Model Support Matrix Please refer to the latest multimodal [support matrix](../models/supported-models.md#multimodal-feature-support-matrix-pytorch-backend). diff --git a/scripts/generate_trtllm_serve_schemas.py b/scripts/generate_trtllm_serve_schemas.py index b2e92e0a6f3e..a7bb8a6f6ea5 100644 --- a/scripts/generate_trtllm_serve_schemas.py +++ b/scripts/generate_trtllm_serve_schemas.py @@ -350,6 +350,14 @@ def mirror_moe_tactic_validators( mirror_moe_tactic_validators( definition, "moe_backend", "moe_tactic", {"enum": ["AUTO", None]} ) + # MultimodalConfig normalizes encoder backend names before Literal validation. + encoder_backend = schema["$defs"]["MultimodalConfig"]["properties"][ + "encoder_attention_backend" + ]["anyOf"][0]["additionalProperties"] + alternatives = [ + "".join(f"[{char}{char.lower()}]" for char in name) for name in encoder_backend.pop("enum") + ] + encoder_backend["pattern"] = "^(?:" + "|".join(alternatives) + r")$(?![\s\S])" # These object/Any annotations are Python API hooks, not arbitrary YAML mappings. # Update these restrictions when hooks are added or gain YAML construction support. for name in ("batched_logits_processor", "checkpoint_loader", "_mpi_session"): diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 4bc12f4607ba..56d9c31132e1 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -492,6 +492,32 @@ class ModelConfig(Generic[TConfig]): # implementation. Set by the model loader, read by `_resolve_class`. modeling_v2_target: Optional[type] = None + def get_encoder_attention_backend(self, encoder: str, default: str) -> str: + """Select a logical encoder's backend, preserving its existing default.""" + overrides = (self.multimodal_config.encoder_attention_backend + if self.multimodal_config is not None else None) + if not overrides or encoder not in overrides: + return default + backend = overrides[encoder] + if backend not in ("VANILLA", "TRTLLM", "FLASHINFER"): + raise ValueError( + f"Invalid attention backend {backend!r} for the {encoder} encoder; " + "expected VANILLA, TRTLLM, or FLASHINFER") + return backend + + def with_encoder_attention_backend(self, + encoder: str, + default: str | None = None + ) -> "ModelConfig": + """Apply an encoder selection to a shallow copy without changing the decoder.""" + backend = self.get_encoder_attention_backend( + encoder, self.attn_backend if default is None else default) + if backend == self.attn_backend: + return self + config = copy.copy(self) + object.__setattr__(config, "attn_backend", backend) + return config + @property def kv_cache_layer_specs(self) -> Optional[Tuple[KVCacheLayerSpec, ...]]: """Return model-provided per-layer KV-cache geometry, if any.""" diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py b/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py index 1d3a1bc7838d..2b38d09d10ed 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py @@ -41,6 +41,8 @@ class DeepseekV41ForConditionalGeneration(MultimodalModelMixin, PreTrainedModel) Attention, Engram, and DSpark remain in the text model. """ + supported_encoder_attention_backends = {"vision": frozenset()} + def __init__(self, model_config: ModelConfig) -> None: config = model_config.pretrained_config super().__init__(config) diff --git a/tensorrt_llm/_torch/models/modeling_gemma4_unified.py b/tensorrt_llm/_torch/models/modeling_gemma4_unified.py index 20981c1bbf3e..dc9e5e322ebb 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4_unified.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4_unified.py @@ -252,6 +252,8 @@ def get_mm_item_metadata( class Gemma4UnifiedForConditionalGeneration(Gemma4MultimodalModelBase): """Gemma 4 12B Unified model with encoder-free multimodal projections.""" + supported_encoder_attention_backends = {} + def __init__(self, model_config: ModelConfig): config = model_config.pretrained_config if config.image_token_id is None: diff --git a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py index 8a0ae3745b84..6f8ed33247c7 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py @@ -279,8 +279,9 @@ class Gemma4VisionAttention(Attention): """Bidirectional attention with q/k/v RMSNorms and 2D RoPE. Inherits ``Attention`` to participate in the TRT-LLM backend dispatch. - Vision sub_config is forced to ``attn_backend="TRTLLM"`` in - ``modeling_gemma4mm.get_sub_model_config`` (same pattern Qwen2.5-VL uses). + Vision defaults to ``attn_backend="TRTLLM"``; an explicit + ``multimodal_config.encoder_attention_backend['vision']`` selects another + supported backend in ``modeling_gemma4mm.get_sub_model_config``. Vision is context-only and runs through ``forward_impl`` with ``PredefinedAttentionMask.FULL`` and a tower-local ``attn_metadata`` (``kv_cache_manager=None``). diff --git a/tensorrt_llm/_torch/models/modeling_gemma4mm.py b/tensorrt_llm/_torch/models/modeling_gemma4mm.py index 92c666a0f4e7..e74ebce25e25 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4mm.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4mm.py @@ -985,6 +985,11 @@ def _normalize_vision_patches( class Gemma4MultimodalModelBase(MultimodalModelMixin, PreTrainedModel): """Shared multimodal encoder flow for Gemma4 conditional generation models.""" + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}), + "audio": frozenset(), + } + supports_encoder_cache = True supports_vision_gpu_normalize: ClassVar[bool] = True lora_owner = DecoderLoraOwner(module_path="language_model") @@ -1246,7 +1251,11 @@ def get_sub_model_config( ) pretrained_config = getattr(model_config.pretrained_config, name) quant_config = model_config.quant_config if name == "text_config" else None - attn_backend = model_config.attn_backend if name == "text_config" else "TRTLLM" + attn_backend = ( + model_config.attn_backend + if name == "text_config" + else model_config.get_encoder_attention_backend(name.removesuffix("_config"), "TRTLLM") + ) sub_config: ModelConfig = dataclasses.replace( model_config, pretrained_config=pretrained_config, diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 60a1aea3e113..699f57660ecc 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -647,6 +647,7 @@ def __init__( vlm_base_model: Optional[type[nn.Module]] = None, ) -> None: super().__init__() + model_config = model_config.with_encoder_attention_backend("vision") self.model_config = model_config self.model_dtype = _text_dtype(model_config) # The tower is excluded from checkpoint quantization (BF16 in @@ -1028,6 +1029,8 @@ class Glm5NextVLM(MultimodalModelMixin, PreTrainedModel): requests bypass the vision tower, which runs full attention without KV cache. """ + supported_encoder_attention_backends = {"vision": frozenset({"TRTLLM"})} + _supports_flash_attn = True _supports_sdpa = True diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index cbe0bd1c630d..b1a1c9f078ee 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -798,6 +798,9 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: self.model_config = copy.copy(model_config) self.model_config.extra_attrs = copy.copy(model_config.extra_attrs) self.model_config._frozen = False + self.model_config.attn_backend = model_config.get_encoder_attention_backend( + "vision", model_config.attn_backend + ) self.model_config.quant_config = QuantConfig( kv_cache_quant_algo=model_config.quant_config.kv_cache_quant_algo ) @@ -1575,6 +1578,10 @@ class KimiK25ForConditionalGeneration(PreTrainedModel): llm: DeepseekV3ForCausalLM (MLA + MoE, 61 layers) """ + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + _LANG_PREFIX = "language_model." # Vision tower class. Subclasses (Kimi K3) override this so the shared # __init__/load_weights MetaInitMode deferral-and-recreation logic diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py b/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py index d4fd119240a8..e72a726b29d0 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py @@ -291,6 +291,9 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: self.model_config = copy.copy(model_config) self.model_config.extra_attrs = copy.copy(model_config.extra_attrs) self.model_config._frozen = False + self.model_config.attn_backend = model_config.get_encoder_attention_backend( + "vision", model_config.attn_backend + ) # Vision tower is not quantized (checkpoint quant ignore list covers # vision_tower.* / mm_projector.*), and its encoder layers keep no KV # cache — so do not carry the text model's kv_cache_quant_algo over: diff --git a/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py b/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py index 8c5b562c6415..605a172b211b 100644 --- a/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py +++ b/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py @@ -526,7 +526,9 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): # projection would then all-reduce data that was never sharded across # ranks and produce wrong output. vision_model_config = dataclasses.replace( - model_config, quant_config=QuantConfig(), mapping=Mapping() + model_config.with_encoder_attention_backend("vision"), + quant_config=QuantConfig(), + mapping=Mapping(), ) self.model_config = vision_model_config self.config = self.vision_config @@ -541,7 +543,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(vision_model_config) self.merger = MiniCPMV4_6Merger(vision_model_config) - self.metadata_cls = get_attention_backend(model_config.attn_backend).Metadata + self.metadata_cls = get_attention_backend(vision_model_config.attn_backend).Metadata # Per-segment capacity required by the shared encoder metadata helper. # Like the Qwen-VL and Cosmos3 vision towers, default to the model's # token budget rather than the live grid length. The engine replaces it @@ -901,6 +903,10 @@ def call_with_text_prompt( ), ) class MiniCPMV4_6Model(PreTrainedModel): + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): config = model_config.pretrained_config super().__init__(config) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3.py b/tensorrt_llm/_torch/models/modeling_minimaxm3.py index 00267c14f162..53831fc16004 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3.py @@ -3196,6 +3196,8 @@ class MiniMaxM3VLForConditionalGeneration(MiniMaxM3ForCausalLM): blobs under that namespace. """ + supported_encoder_attention_backends = {"vision": frozenset()} + lora_owner = DecoderLoraOwner(module_path="model") def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index c17d4f04b653..eb11968a2f78 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -853,6 +853,10 @@ class Mistral3VLM(MultimodalModelMixin, PreTrainedModel): `tensorrt_llm/inputs/utils.py`). """ + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + supports_encoder_cache = True supports_mm_encoder_item_scheduling = True @@ -909,7 +913,8 @@ def __init__( vision_model_config = self._get_sub_model_config( model_config_cp, "vision_config", - attn_backend="TRTLLM", + attn_backend=model_config.get_encoder_attention_backend( + "vision", "TRTLLM"), quant_config=None) self._vision_tower = modeling_pixtral.PixtralVisionModel( diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index a79cb7b0d7c0..cb243e1a483d 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -432,6 +432,12 @@ class MultimodalModelMixin: `BaseMultimodalInputProcessor.get_mm_item_metadata`. This is the executor-layer gate for item scheduling.""" + # TODO: Move capability declarations into the planned single + # SupportsMultimodal Protocol; keep shared execution in this mixin + # and encoder attention-metadata setup in MultimodalEncoderMixin. + supported_encoder_attention_backends: ClassVar[dict[str, frozenset[str]]] = {} + """Public backends by logical encoder; an empty set marks fixed attention.""" + model_config: ModelConfig _multimodal_encoder_cache: Optional[TensorLRUCache] = None diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index 9d5dd7ff74b3..e8e4876e33cc 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -667,6 +667,7 @@ def __init__(self, model_config: ModelConfig[transformers.PretrainedConfig]): self.vision_model = RADIOVisionModel( vision_model_config, disable_quantization=True, + vision_attn_backend=model_config.get_encoder_attention_backend("vision", "FLASHINFER"), encoder_cuda_graph_config=encoder_cuda_graph_config, ) @@ -2902,6 +2903,11 @@ def _resample_audios( placeholder_metadata=_NEMOTRON_H_MM_PLACEHOLDER_METADATA, ) class NemotronHMultimodalModel(MultimodalModelMixin, transformers.PreTrainedModel): + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}), + "audio": frozenset(), + } + _supports_flash_attn = True lora_owner = DecoderLoraOwner(module_path="language_model") diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index d677e2c775aa..7d62cdd21209 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -1461,6 +1461,8 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig], model_class(model_config.pretrained_config.vision_config), self.model_dtype).eval() elif model_class == Qwen2_5_VisionModel: + self.model_config = self.model_config.with_encoder_attention_backend( + "vision") self.visual = MultimodalModelMixin._cast_multimodal_encoder_dtype( model_class(self.model_config), self.model_dtype) else: @@ -2317,6 +2319,10 @@ def _qwen_vision_input_partitions( class Qwen2VLModelBase(PreTrainedModel, MultimodalModelMixin): + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + supports_mm_encoder_item_scheduling = True def get_multimodal_item_partitions( @@ -2600,6 +2606,8 @@ def _get_requests_with_mm_data(self, multimodal_params): )) class Qwen2VLModel(Qwen2VLModelBase): + supported_encoder_attention_backends = {"vision": frozenset()} + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): # NOTE: Since Qwen2-VL is outdated model, we leave it as HF implementation. diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index 62b687071343..f862f0e5a920 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -1516,7 +1516,7 @@ def __init__( model_class: Union[type[PreTrainedModel], type[torch.nn.Module]], ): super().__init__() - self.model_config = model_config + self.model_config = model_config.with_encoder_attention_backend("vision") self.model_dtype = self.model_config.pretrained_config.text_config.dtype # NOTE: Re-setting QuantConfig to exclude vision encoder from quantization, @@ -1617,6 +1617,10 @@ def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tenso class Qwen3VLModelBase(MultimodalModelMixin, PreTrainedModel): + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + supports_mm_encoder_item_scheduling = True supports_vision_gpu_normalize = True diff --git a/tensorrt_llm/_torch/models/modeling_step3p7vl.py b/tensorrt_llm/_torch/models/modeling_step3p7vl.py index 945081bb6210..50577f78e9dd 100644 --- a/tensorrt_llm/_torch/models/modeling_step3p7vl.py +++ b/tensorrt_llm/_torch/models/modeling_step3p7vl.py @@ -730,7 +730,9 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): vision_model_config = ModelConfig( pretrained_config=vision_config, mapping=Mapping(world_size=1, tp_size=1, rank=0), - attn_backend=getattr(model_config, "attn_backend", "TRTLLM"), + attn_backend=model_config.get_encoder_attention_backend( + "vision", model_config.attn_backend + ), skip_create_weights_in_init=False, ) self.vision_model = Step3p7VisionEncoder( @@ -1218,6 +1220,10 @@ class Step3p7VLForConditionalGeneration(nn.Module): benchmarks continue to use the original Step3p7 text path. """ + supported_encoder_attention_backends = { + "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + } + lora_owner = DecoderLoraOwner(module_path="language_model") def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 26639051a017..4c8ab2cb8710 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -38,7 +38,7 @@ from tensorrt_llm.checkpoint_format import CheckpointFormat from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, ExecutorMemoryType, - ModelExpressConfig, + ModelExpressConfig, MultimodalConfig, PrefillCudaGraphBackend, SparseAttentionConfig, TorchLlmArgs) from tensorrt_llm.llmapi.llm_utils import (_resolve_kv_cache_manager_v2_auto, @@ -205,6 +205,42 @@ def _with_fp8_block_scale_weights( {90})) +def _validate_encoder_attention_backend_support( + model_cls: type | None, + mm_config: MultimodalConfig | None, + *, + pretrained_config: object | None = None, + disable_mm_encoder: bool = False) -> None: + """Reject unsupported encoder selections before allocating model weights.""" + if mm_config is None or not mm_config.encoder_attention_backend: + return + supported = getattr(model_cls, "supported_encoder_attention_backends", {}) + model_name = model_cls.__name__ if model_cls is not None else "The model" + for encoder, backend in mm_config.encoder_attention_backend.items(): + if not supported or encoder in supported and not supported[encoder]: + raise ValueError( + f"The {encoder} encoder's attention backend cannot be changed " + f"for {model_name}") + if encoder not in supported: + raise ValueError(f"Unknown encoder {encoder!r} for {model_name}; " + f"available encoders: {sorted(supported)}") + if backend not in supported[encoder]: + raise ValueError( + f"The {encoder} encoder does not support attention backend {backend!r}; " + f"supported backends: {sorted(supported[encoder])}") + if disable_mm_encoder or (pretrained_config is not None and getattr( + pretrained_config, f"{encoder}_config", None) is None): + raise ValueError( + f"The {encoder} encoder is not enabled for {model_name}") + if backend == "FLASHINFER": + from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE + + if not IS_FLASHINFER_AVAILABLE: + raise ValueError( + f"The {encoder} encoder requested FLASHINFER, but FlashInfer is unavailable" + ) + + def _validate_and_adjust_mamba_snapshot_config(config: ModelConfig, llm_args: TorchLlmArgs) -> None: """Validate snapshot reuse after the model and V2 setting are resolved.""" @@ -1179,6 +1215,12 @@ def load_config_and_apply_defaults( preference_cls = get_registered_model_class( architectures[0]) or model_cls + _validate_encoder_attention_backend_support( + preference_cls, + llm_args.multimodal_config, + pretrained_config=config.pretrained_config, + disable_mm_encoder=llm_args.disable_mm_encoder) + # model_cls is None when the architecture is unknown/unsupported. model_defaults = {} if model_cls and hasattr(model_cls, 'get_model_defaults'): diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 286831c8bdd4..1a227323ab58 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -830,6 +830,21 @@ class MultimodalConfig(StrictBaseModel): status="prototype", ) + encoder_attention_backend: dict[str, Literal[ + "VANILLA", "TRTLLM", "FLASHINFER"]] | None = Field( + default=None, + description= + ("TensorRT-LLM attention backend for each logical multimodal encoder, " + "for example {'vision': 'FLASHINFER'}. Public values are VANILLA, TRTLLM, " + "and FLASHINFER. Image and video share the vision encoder. Only encoders " + "using TensorRT-LLM Attention support this option; encoders using fixed " + "attention reject overrides at initialization. Omitted entries retain " + "their model-specific defaults and the language model's attn_backend " + "is unchanged. Unsupported encoders or backends raise an error."), + status="prototype", + telemetry=False, + ) + encoder_cuda_graph: dict[str, MultimodalEncoderCudaGraphConfig] | None = Field( default=None, description= @@ -899,6 +914,16 @@ class MultimodalConfig(StrictBaseModel): def parse_encoder_cache_max_bytes(cls, value): return _parse_binary_byte_string(value) + @field_validator('encoder_attention_backend', mode='before') + @classmethod + def normalize_encoder_attention_backend(cls, value: object) -> object: + if isinstance(value, dict): + return { + name: backend.upper() if isinstance(backend, str) else backend + for name, backend in value.items() + } + return value + @model_validator(mode='after') def validate_encoder_optimization_compatibility(self) -> 'MultimodalConfig': if (self.encoder_cuda_graph is not None diff --git a/tensorrt_llm/schemas/trtllm-serve-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-config.schema.json index 9c5613a420dc..89da95023fe5 100644 --- a/tensorrt_llm/schemas/trtllm-serve-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-config.schema.json @@ -5191,6 +5191,27 @@ "additionalProperties": false, "description": "Multimodal model configuration.", "properties": { + "encoder_attention_backend": { + "anyOf": [ + { + "additionalProperties": { + "pattern": "^(?:[Vv][Aa][Nn][Ii][Ll][Ll][Aa]|[Tt][Rr][Tt][Ll][Ll][Mm]|[Ff][Ll][Aa][Ss][Hh][Ii][Nn][Ff][Ee][Rr])$(?![\\s\\S])", + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "TensorRT-LLM attention backend for each logical multimodal encoder, for example {'vision': 'FLASHINFER'}. Public values are VANILLA, TRTLLM, and FLASHINFER. Image and video share the vision encoder. Only encoders using TensorRT-LLM Attention support this option; encoders using fixed attention reject overrides at initialization. Omitted entries retain their model-specific defaults and the language model's attn_backend is unchanged. Unsupported encoders or backends raise an error.", + "status": "prototype", + "telemetry": { + "exclude": true + }, + "title": "Encoder Attention Backend" + }, "encoder_cache_max_bytes": { "default": 134217728, "description": "Target capacity in bytes for reusable multimodal encoder embeddings; 0 disables cross-request reuse. The capacity applies per rank; ranks do not share a cache. With item scheduling, encoder outputs still use the model-owned cache and its capacity is at least one legal encoder iteration; this value can make that cache larger. String values such as '512MB' and '1GiB' use binary units. Both inline encoding and item scheduling cache individual items, including those of mixed-modality requests. Compatible with side-stream prefetch, which requires inline encoding (item scheduling rejects it).", diff --git a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json index fef922dbc04a..c6f68784c1de 100644 --- a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json @@ -7148,6 +7148,27 @@ "additionalProperties": false, "description": "Multimodal model configuration.", "properties": { + "encoder_attention_backend": { + "anyOf": [ + { + "additionalProperties": { + "pattern": "^(?:[Vv][Aa][Nn][Ii][Ll][Ll][Aa]|[Tt][Rr][Tt][Ll][Ll][Mm]|[Ff][Ll][Aa][Ss][Hh][Ii][Nn][Ff][Ee][Rr])$(?![\\s\\S])", + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "TensorRT-LLM attention backend for each logical multimodal encoder, for example {'vision': 'FLASHINFER'}. Public values are VANILLA, TRTLLM, and FLASHINFER. Image and video share the vision encoder. Only encoders using TensorRT-LLM Attention support this option; encoders using fixed attention reject overrides at initialization. Omitted entries retain their model-specific defaults and the language model's attn_backend is unchanged. Unsupported encoders or backends raise an error.", + "status": "prototype", + "telemetry": { + "exclude": true + }, + "title": "Encoder Attention Backend" + }, "encoder_cache_max_bytes": { "default": 134217728, "description": "Target capacity in bytes for reusable multimodal encoder embeddings; 0 disables cross-request reuse. The capacity applies per rank; ranks do not share a cache. With item scheduling, encoder outputs still use the model-owned cache and its capacity is at least one legal encoder iteration; this value can make that cache larger. String values such as '512MB' and '1GiB' use binary units. Both inline encoding and item scheduling cache individual items, including those of mixed-modality requests. Compatible with side-stream prefetch, which requires inline encoding (item scheduling rejects it).", diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py index 5bea1f78846f..c4f257c9cfee 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py @@ -2218,3 +2218,60 @@ def encode(residuals): encoder.assert_not_called() assert torch.equal(attached[0], params.multimodal_data["multimodal_embedding"]) assert params.multimodal_data["image"]["vision_normalization"] is spec + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + ("decoder_backend", "override"), + [("TRTLLM", None), ("TRTLLM", "VANILLA"), ("FLASHINFER", "VANILLA")], +) +def test_qwen_encoder_backend_preserves_decoder(decoder_backend: str, override: str | None) -> None: + from transformers import PretrainedConfig + + from tensorrt_llm._torch.models.modeling_qwen3vl import Qwen3VisionModelBase + from tensorrt_llm.llmapi.llm_args import MultimodalConfig + + class RecordingVisionEncoder(torch.nn.Module): + def __init__(self, model_config: ModelConfig) -> None: + super().__init__() + self.model_config = model_config + self.weight = torch.nn.Parameter(torch.ones(1)) + + config = ModelConfig( + pretrained_config=PretrainedConfig( + text_config=PretrainedConfig(dtype=torch.float32), + vision_config=PretrainedConfig(), + ), + attn_backend=decoder_backend, + multimodal_config=MultimodalConfig( + encoder_attention_backend={"vision": override} if override else None + ), + ) + config._frozen = True + encoder = Qwen3VisionModelBase(config, RecordingVisionEncoder) + assert encoder.visual.model_config.attn_backend == (override or decoder_backend) + assert encoder.model_config.extra_attrs is config.extra_attrs + assert encoder.model_config._frozen + if override is None: + assert encoder.model_config is config + assert config.attn_backend == decoder_backend + + +@pytest.mark.cpu_only +def test_qwen_encoder_backend_declarations_preserve_fixed_hf_encoder() -> None: + from tensorrt_llm._torch.models.modeling_qwen2vl import Qwen2_5_VLModel, Qwen2VLModel + from tensorrt_llm._torch.models.modeling_qwen3vl_moe import Qwen3MoeVLModel + from tensorrt_llm._torch.pyexecutor.model_loader import ( + _validate_encoder_attention_backend_support, + ) + from tensorrt_llm.llmapi.llm_args import MultimodalConfig + + public_backends = frozenset({"TRTLLM", "FLASHINFER", "VANILLA"}) + for model_cls in (Qwen2_5_VLModel, Qwen3VLModel, Qwen3MoeVLModel): + assert model_cls.supported_encoder_attention_backends == {"vision": public_backends} + assert Qwen2VLModel.supported_encoder_attention_backends == {"vision": frozenset()} + with pytest.raises(ValueError, match="vision.*cannot be changed"): + _validate_encoder_attention_backend_support( + Qwen2VLModel, + MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + ) diff --git a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py new file mode 100644 index 000000000000..e000e8981dac --- /dev/null +++ b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from importlib import import_module +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.pyexecutor.model_loader import _validate_encoder_attention_backend_support +from tensorrt_llm.llmapi.llm_args import MultimodalConfig + +pytestmark = pytest.mark.cpu_only + +_PUBLIC_BACKENDS = frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) + + +@pytest.mark.parametrize("backend", sorted(_PUBLIC_BACKENDS)) +def test_public_backend_names_parse_case_insensitively(backend: str) -> None: + config = MultimodalConfig(encoder_attention_backend={"vision": backend.lower()}) + assert config.encoder_attention_backend == {"vision": backend} + assert MultimodalConfig.model_validate_json(config.model_dump_json()) == config + + +@pytest.mark.parametrize( + "backend", + ["SDPA", "FLASH_ATTENTION_2", "eager", "unknown", "VANILLA\n", "", None, 1, True, [], {}], +) +def test_rejects_non_public_backend_names_and_non_strings(backend: object) -> None: + with pytest.raises(ValidationError, match="encoder_attention_backend.vision"): + MultimodalConfig(encoder_attention_backend={"vision": backend}) + + +@pytest.mark.parametrize("value", ["VANILLA", ["VANILLA"], 1]) +def test_rejects_backend_settings_without_encoder_names(value: object) -> None: + with pytest.raises(ValidationError, match="encoder_attention_backend"): + MultimodalConfig(encoder_attention_backend=value) + + +@pytest.mark.parametrize("overrides", [None, {}]) +def test_omitted_encoder_backends_preserve_defaults(overrides: dict | None) -> None: + config = ModelConfig( + attn_backend="FLASHINFER", + multimodal_config=MultimodalConfig(encoder_attention_backend=overrides), + ) + assert config.get_encoder_attention_backend("vision", "TRTLLM") == "TRTLLM" + assert config.get_encoder_attention_backend("audio", "VANILLA") == "VANILLA" + assert config.with_encoder_attention_backend("vision") is config + assert config.with_encoder_attention_backend("vision", "TRTLLM").attn_backend == "TRTLLM" + assert config.with_encoder_attention_backend("audio", "VANILLA").attn_backend == "VANILLA" + assert config.attn_backend == "FLASHINFER" + + +def test_encoder_override_preserves_other_encoders_decoder_and_shared_state() -> None: + config = ModelConfig( + attn_backend="TRTLLM", + multimodal_config=MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + ) + config.extra_attrs["shared"] = object() + config._frozen = True + vision = config.with_encoder_attention_backend("vision") + assert vision is not config + assert vision.attn_backend == "VANILLA" + assert config.get_encoder_attention_backend("audio", "FLASHINFER") == "FLASHINFER" + assert config.with_encoder_attention_backend("audio", "TRTLLM") is config + assert vision.extra_attrs is config.extra_attrs + assert vision.multimodal_config is config.multimodal_config + assert vision._frozen + assert config.attn_backend == "TRTLLM" + + +def test_explicit_default_backend_does_not_copy_config() -> None: + config = ModelConfig( + attn_backend="TRTLLM", + multimodal_config=MultimodalConfig(encoder_attention_backend={"vision": "TRTLLM"}), + ) + assert config.with_encoder_attention_backend("vision", "VANILLA") is config + + +class _EncoderModel: + supported_encoder_attention_backends = {"vision": _PUBLIC_BACKENDS, "audio": frozenset()} + + +@pytest.mark.parametrize("backend", sorted(_PUBLIC_BACKENDS)) +def test_loader_accepts_declared_public_backends( + backend: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("tensorrt_llm._torch.flashinfer_utils.IS_FLASHINFER_AVAILABLE", True) + _validate_encoder_attention_backend_support( + _EncoderModel, MultimodalConfig(encoder_attention_backend={"vision": backend}) + ) + + +@pytest.mark.parametrize( + ("model_cls", "encoder", "message"), + [ + (_EncoderModel, "video", "video"), + (_EncoderModel, "audio", "audio.*cannot be changed"), + (None, "vision", "vision"), + ], +) +def test_loader_rejects_unknown_and_fixed_encoders( + model_cls: type | None, encoder: str, message: str +) -> None: + with pytest.raises(ValueError, match=message): + _validate_encoder_attention_backend_support( + model_cls, MultimodalConfig(encoder_attention_backend={encoder: "VANILLA"}) + ) + + +def test_loader_rejects_backend_outside_encoder_support() -> None: + class VanillaEncoderModel: + supported_encoder_attention_backends = {"vision": frozenset({"VANILLA"})} + + with pytest.raises(ValueError, match="vision.*TRTLLM"): + _validate_encoder_attention_backend_support( + VanillaEncoderModel, + MultimodalConfig(encoder_attention_backend={"vision": "TRTLLM"}), + ) + + +def test_explicit_flashinfer_does_not_silently_fall_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("tensorrt_llm._torch.flashinfer_utils.IS_FLASHINFER_AVAILABLE", False) + with pytest.raises(ValueError, match="FlashInfer.*unavailable"): + _validate_encoder_attention_backend_support( + _EncoderModel, + MultimodalConfig(encoder_attention_backend={"vision": "FLASHINFER"}), + ) + + +@pytest.mark.parametrize("model_cls", [None, _EncoderModel]) +@pytest.mark.parametrize("overrides", [None, {}]) +def test_loader_keeps_omitted_settings_unchanged( + model_cls: type | None, overrides: dict | None +) -> None: + _validate_encoder_attention_backend_support(model_cls, None) + _validate_encoder_attention_backend_support( + model_cls, MultimodalConfig(encoder_attention_backend=overrides) + ) + + +@pytest.mark.parametrize( + ("module", "model", "expected"), + [ + ( + "gemma4mm", + "Gemma4ForConditionalGeneration", + {"vision": _PUBLIC_BACKENDS, "audio": frozenset()}, + ), + ("gemma4_unified", "Gemma4UnifiedForConditionalGeneration", {}), + ( + "nemotron_h_multimodal", + "NemotronHMultimodalModel", + {"vision": _PUBLIC_BACKENDS, "audio": frozenset()}, + ), + ("glm5_next_vision", "Glm5NextVLM", {"vision": frozenset({"TRTLLM"})}), + ("kimi_k25", "KimiK25ForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), + ("kimi_k3_vl", "KimiK3ForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), + ("step3p7vl", "Step3p7VLForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), + ("minicpmv4_6", "MiniCPMV4_6Model", {"vision": _PUBLIC_BACKENDS}), + ("mistral", "Mistral3VLM", {"vision": _PUBLIC_BACKENDS}), + ("deepseekv41_multimodal", "DeepseekV41ForConditionalGeneration", {"vision": frozenset()}), + ("minimaxm3", "MiniMaxM3VLForConditionalGeneration", {"vision": frozenset()}), + ], +) +def test_model_encoder_backend_declarations(module: str, model: str, expected: dict) -> None: + model_cls = getattr(import_module(f"tensorrt_llm._torch.models.modeling_{module}"), model) + assert model_cls.supported_encoder_attention_backends == expected + for encoder, backends in expected.items(): + assert backends <= _PUBLIC_BACKENDS + if not backends: + with pytest.raises(ValueError, match=f"{encoder}.*cannot be changed"): + _validate_encoder_attention_backend_support( + model_cls, MultimodalConfig(encoder_attention_backend={encoder: "VANILLA"}) + ) + + +@pytest.mark.parametrize( + ("module", "model", "encoder"), + [ + ("vila", "VilaModel", "vision"), + ("phi4mm", "Phi4MMForCausalLM", "vision"), + ("phi4mm", "Phi4MMForCausalLM", "audio"), + ], +) +def test_hf_encoders_reject_backend_overrides(module: str, model: str, encoder: str) -> None: + model_cls = getattr(import_module(f"tensorrt_llm._torch.models.modeling_{module}"), model) + with pytest.raises(ValueError, match=f"{encoder}.*cannot be changed"): + _validate_encoder_attention_backend_support( + model_cls, MultimodalConfig(encoder_attention_backend={encoder: "VANILLA"}) + ) + + +@pytest.mark.parametrize("disabled", [False, True]) +def test_loader_rejects_absent_or_disabled_encoder(disabled: bool) -> None: + config = SimpleNamespace(vision_config=object() if disabled else None) + with pytest.raises(ValueError, match="vision.*not enabled"): + _validate_encoder_attention_backend_support( + _EncoderModel, + MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + pretrained_config=config, + disable_mm_encoder=disabled, + ) + + +def test_loader_accepts_enabled_encoder() -> None: + _validate_encoder_attention_backend_support( + _EncoderModel, + MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + pretrained_config=SimpleNamespace(vision_config=object()), + ) + + +def test_model_config_rejects_invalid_backend_even_without_pydantic_validation() -> None: + config = ModelConfig( + multimodal_config=MultimodalConfig.model_construct( + encoder_attention_backend={"vision": "SDPA"} + ) + ) + with pytest.raises(ValueError, match="Invalid attention backend.*SDPA"): + config.get_encoder_attention_backend("vision", "TRTLLM") + + +@pytest.mark.parametrize("override", [None, "VANILLA", "TRTLLM", "FLASHINFER"]) +def test_gemma4_preserves_default_towers_and_changes_only_vision(override: str | None) -> None: + from transformers import Gemma4Config + + from tensorrt_llm._torch.models.modeling_gemma4mm import Gemma4ForConditionalGeneration + + config = ModelConfig( + pretrained_config=Gemma4Config(text_config={}, vision_config={}, audio_config={}), + attn_backend="FLASHINFER", + multimodal_config=MultimodalConfig( + encoder_attention_backend={"vision": override} if override else None + ), + ) + text = Gemma4ForConditionalGeneration.get_sub_model_config(config, "text_config") + vision = Gemma4ForConditionalGeneration.get_sub_model_config(config, "vision_config") + audio = Gemma4ForConditionalGeneration.get_sub_model_config(config, "audio_config") + assert text.attn_backend == "FLASHINFER" + assert vision.attn_backend == (override or "TRTLLM") + assert audio.attn_backend == "TRTLLM" + assert text.extra_attrs is vision.extra_attrs is audio.extra_attrs is config.extra_attrs diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index cfe5a8350431..56d21f33a31e 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -2536,6 +2536,7 @@ class TestMultimodalConfig: def test_default_encoder_cuda_graph_is_none(self): assert MultimodalConfig().vision_gpu_normalize is False + assert MultimodalConfig().encoder_attention_backend is None assert MultimodalConfig().encoder_cuda_graph is None assert MultimodalConfig().encoder_side_stream_max_ahead == 0 assert MultimodalConfig().encoder_cache_max_bytes == 128 * 1024**2 @@ -2546,6 +2547,7 @@ def test_torch_llm_args_default_multimodal_config(self): args = TorchLlmArgs(model=llama_model_path) assert isinstance(args.multimodal_config, MultimodalConfig) assert args.multimodal_config.vision_gpu_normalize is False + assert args.multimodal_config.encoder_attention_backend is None assert args.multimodal_config.encoder_cuda_graph is None assert args.multimodal_config.encoder_side_stream_max_ahead == 0 assert args.multimodal_config.encoder_cache_max_bytes == 128 * 1024**2 diff --git a/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py b/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py index 37570a63866a..1ba60b4a8c69 100644 --- a/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py +++ b/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py @@ -860,6 +860,30 @@ def test_cross_field_rules_reject_at_config_time( TorchLlmArgs(model=tmp_path, gpus_per_node=1, **config) +@pytest.mark.parametrize( + ("backend", "valid"), + [ + ("VANILLA", True), + ("TRTLLM", True), + ("FLASHINFER", True), + ("vanilla", True), + ("fLaShInFeR", True), + ("VANILLA\n", False), + ("SDPA", False), + ("FLASH_ATTENTION_2", False), + ("unknown", False), + (None, False), + (1, False), + ], +) +def test_encoder_attention_backend_schema( + schemas: dict[str, dict], backend: object, valid: bool +) -> None: + config = {"multimodal_config": {"encoder_attention_backend": {"vision": backend}}} + errors = list(Draft202012Validator(schemas[generator.SERVE_SCHEMA]).iter_errors(config)) + assert (not errors) == valid, [error.message for error in errors] + + def _assert_serving_fields_in_schema(source: str, function_name: str, properties: dict) -> None: """Check literal top-level config accesses, without tracing aliases or dynamic keys.""" function = next( From 26e1d568d33c581d6da82c4fc645ee32d5d0d024 Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:19:32 +0000 Subject: [PATCH 2/6] [None][feat] declare encoder backend support for every multimodal model Declare fixed encoder paths and reuse existing configurable mappings. Reject Cosmos3 Edge overrides that its custom tower previously ignored. Check registry completeness and the full docs table in one CPU test. Validation: 231 CPU tests; applicable hooks; schema and manifest checks. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/features/multi-modality.md | 48 ++++++++++--- .../_torch/models/modeling_cosmos3.py | 2 + .../_torch/models/modeling_cosmos3_edge.py | 3 + .../models/modeling_deepseekv41_multimodal.py | 1 + .../_torch/models/modeling_exaone4_5.py | 2 + .../_torch/models/modeling_gemma3vl.py | 3 + .../_torch/models/modeling_gemma4mm.py | 6 +- .../_torch/models/modeling_hyperclovax.py | 6 ++ .../_torch/models/modeling_kimi_k3_vl.py | 4 ++ tensorrt_llm/_torch/models/modeling_llama.py | 3 + .../_torch/models/modeling_llava_next.py | 4 ++ .../_torch/models/modeling_minimaxm3.py | 1 + tensorrt_llm/_torch/models/modeling_mllama.py | 5 +- .../models/modeling_musespark_open_omni.py | 4 ++ .../models/modeling_nemotron_h_multimodal.py | 2 +- tensorrt_llm/_torch/models/modeling_phi4mm.py | 3 + .../_torch/models/modeling_qwen2vl.py | 2 + .../_torch/models/modeling_qwen3_5.py | 6 ++ .../_torch/models/modeling_qwen3vl.py | 2 + .../_torch/models/modeling_qwen3vl_moe.py | 5 ++ .../_torch/models/modeling_qwen4_exp.py | 2 + tensorrt_llm/_torch/models/modeling_vila.py | 4 ++ .../_torch/models/modeling_whisper.py | 3 + .../test_encoder_attention_backend.py | 70 +++++++++++++++++++ 24 files changed, 180 insertions(+), 11 deletions(-) diff --git a/docs/source/features/multi-modality.md b/docs/source/features/multi-modality.md index 0fb295bf3af8..544768b2d4fd 100644 --- a/docs/source/features/multi-modality.md +++ b/docs/source/features/multi-modality.md @@ -50,18 +50,50 @@ llm = LLM( Omitted entries preserve the model's existing encoder defaults, including inheritance from `attn_backend` where applicable. An encoder override leaves the language model's backend unchanged. Backend availability and supported tensor shapes still depend on the installed kernels and hardware; floating-point differences between backends are expected. -| Encoder | Supported values | Default when omitted | +The table covers every registered multimodal model implementation; model aliases share the same declaration. Image/video share `vision`, and `audio` names a separate encoder. "Not supported" means this option cannot change that encoder's attention backend; it does not mean the model or modality is unsupported. Gemma 4 Unified uses encoder-free projections, and Mllama's vision input is not implemented. The CPU registry test checks this table against each model's explicit `supported_encoder_attention_backends` declaration. + + +| Model | Logical encoder | Supported backends | | --- | --- | --- | -| Qwen2.5-VL, Qwen3-VL / Qwen3.5-VL vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | Existing `attn_backend` | -| Gemma 4 vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `TRTLLM` | -| Kimi K2.5 / K3, MiniCPM-V 4.6, Step 3.7 vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | Existing `attn_backend` | -| Mistral3 / Pixtral vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `TRTLLM` | -| Nemotron multimodal vision | `VANILLA`, `TRTLLM`, `FLASHINFER` | `FLASHINFER` | -| GLM5-Next vision | `TRTLLM` | Existing `attn_backend` | +| `Cosmos3EdgeModel` | `vision` | Not supported | +| `Cosmos3Model` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `DeepseekV41ForConditionalGeneration` | `vision` | Not supported | +| `Exaone4_5_ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Gemma3VLM` | `vision` | Not supported | +| `Gemma4ForConditionalGeneration` | `audio` | Not supported | +| `Gemma4ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Gemma4UnifiedForConditionalGeneration` | None (encoder-free) | Not supported | +| `Glm5NextVLM` | `vision` | `TRTLLM` | +| `HCXVisionForCausalLM` | `vision` | Not supported | +| `KimiK25ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `KimiK3ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Llama4ForConditionalGeneration` | `vision` | Not supported | +| `LlavaNextModel` | `vision` | Not supported | +| `MiniCPMV4_6Model` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `MiniMaxM3VLForConditionalGeneration` | `vision` | Not supported | +| `Mistral3VLM` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `MllamaForConditionalGeneration` | `vision` | Not supported | +| `MuseSparkForConditionalGeneration` | `audio` | Not supported | +| `MuseSparkForConditionalGeneration` | `vision` | Not supported | +| `NemotronHMultimodalModel` | `audio` | Not supported | +| `NemotronHMultimodalModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Phi4MMForCausalLM` | `audio` | Not supported | +| `Phi4MMForCausalLM` | `vision` | Not supported | +| `Qwen2VLModel` | `vision` | Not supported | +| `Qwen2_5_VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Qwen3MoeVLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Qwen3VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Qwen3_5MoeVLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Qwen3_5VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Qwen4ExpForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `Step3p7VLForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` | +| `VilaModel` | `vision` | Not supported | +| `WhisperForConditionalGeneration` | `audio` | Not supported | + For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. -Only the declared encoders using TensorRT-LLM's `Attention` support this setting. Qwen2-VL's Hugging Face vision encoder, VILA's Hugging Face CLIP/SigLIP towers, Phi-4-multimodal's Hugging Face encoder, Parakeet audio, Gemma 4 audio, and MiniMax M3 / DeepSeek V4.1 vision keep their fixed attention implementations. Overrides for these encoders fail during initialization, even if the requested value is `VANILLA`. Gemma 4 Unified has no configurable attention encoder. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. +Supported entries use TensorRT-LLM's `Attention` and apply encoder overrides explicitly. Other encoders keep their fixed implementation, including Hugging Face attention, direct SDPA, and custom attention paths. Overrides for entries marked "Not supported" fail during initialization, even if the requested value is `VANILLA`. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. ## Model Support Matrix diff --git a/tensorrt_llm/_torch/models/modeling_cosmos3.py b/tensorrt_llm/_torch/models/modeling_cosmos3.py index 2960b54c17ec..d295c2e7c835 100644 --- a/tensorrt_llm/_torch/models/modeling_cosmos3.py +++ b/tensorrt_llm/_torch/models/modeling_cosmos3.py @@ -76,6 +76,8 @@ def get_preferred_media_io_kwargs(self) -> Dict[str, Dict[str, Any]]: Cosmos3InputProcessor, model_type="cosmos3_omni", placeholder_metadata=PLACEHOLDER_METADATA ) class Cosmos3Model(Qwen3VLModel): + supported_encoder_attention_backends = Qwen3VLModel.supported_encoder_attention_backends + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): omni_config = model_config.pretrained_config if omni_config is None: diff --git a/tensorrt_llm/_torch/models/modeling_cosmos3_edge.py b/tensorrt_llm/_torch/models/modeling_cosmos3_edge.py index 0b2d41241946..c41ec28cf788 100644 --- a/tensorrt_llm/_torch/models/modeling_cosmos3_edge.py +++ b/tensorrt_llm/_torch/models/modeling_cosmos3_edge.py @@ -1012,6 +1012,9 @@ def _edge_positional_embedding_params(config) -> PositionalEmbeddingParams: placeholder_metadata=PLACEHOLDER_METADATA, ) class Cosmos3EdgeModel(Qwen3VLModelBase): + # The custom SigLIP path does not apply encoder overrides. + supported_encoder_attention_backends = {"vision": frozenset()} + supports_encoder_cache = True def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py b/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py index 2b38d09d10ed..be51636402e6 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv41_multimodal.py @@ -41,6 +41,7 @@ class DeepseekV41ForConditionalGeneration(MultimodalModelMixin, PreTrainedModel) Attention, Engram, and DSpark remain in the text model. """ + # The vision encoder calls SDPA directly. supported_encoder_attention_backends = {"vision": frozenset()} def __init__(self, model_config: ModelConfig) -> None: diff --git a/tensorrt_llm/_torch/models/modeling_exaone4_5.py b/tensorrt_llm/_torch/models/modeling_exaone4_5.py index f06dd8eed359..ea7efdbffe3c 100644 --- a/tensorrt_llm/_torch/models/modeling_exaone4_5.py +++ b/tensorrt_llm/_torch/models/modeling_exaone4_5.py @@ -192,6 +192,8 @@ class Exaone4_5_VisionModel(Qwen2VisionModelBase): ), ) class Exaone4_5_ForConditionalGeneration(Qwen2VLModelBase): + supported_encoder_attention_backends = Qwen2VLModelBase.supported_encoder_attention_backends + def __init__( self, model_config: ModelConfig[PretrainedConfig], diff --git a/tensorrt_llm/_torch/models/modeling_gemma3vl.py b/tensorrt_llm/_torch/models/modeling_gemma3vl.py index 8fe7eabbab35..0d925b4813d9 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma3vl.py +++ b/tensorrt_llm/_torch/models/modeling_gemma3vl.py @@ -181,6 +181,9 @@ def forward(self, vision_outputs: torch.Tensor): )) class Gemma3VLM(PreTrainedModel): + # The native vision path does not apply encoder overrides. + supported_encoder_attention_backends = {"vision": frozenset()} + lora_owner = DecoderLoraOwner(module_path="language_model") @classmethod diff --git a/tensorrt_llm/_torch/models/modeling_gemma4mm.py b/tensorrt_llm/_torch/models/modeling_gemma4mm.py index e74ebce25e25..18134737636d 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4mm.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4mm.py @@ -987,7 +987,7 @@ class Gemma4MultimodalModelBase(MultimodalModelMixin, PreTrainedModel): supported_encoder_attention_backends = { "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}), - "audio": frozenset(), + "audio": frozenset(), # Chunked local attention with relative-position bias. } supports_encoder_cache = True @@ -1398,6 +1398,10 @@ class Gemma4ForConditionalGeneration(Gemma4MultimodalModelBase): - mm_token_type_ids-based bidirectional masking """ + # fmt: off + supported_encoder_attention_backends = Gemma4MultimodalModelBase.supported_encoder_attention_backends + # fmt: on + build_mtp_draft_model_from_config = True supports_mm_encoder_item_scheduling = True diff --git a/tensorrt_llm/_torch/models/modeling_hyperclovax.py b/tensorrt_llm/_torch/models/modeling_hyperclovax.py index 2164a39aabff..4275ed2f81fe 100644 --- a/tensorrt_llm/_torch/models/modeling_hyperclovax.py +++ b/tensorrt_llm/_torch/models/modeling_hyperclovax.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import copy import math import os @@ -1031,6 +1034,9 @@ def forward(self, multimodal_params: List[MultimodalParams]): )) class HCXVisionForCausalLM(PreTrainedModel): + # The native vision path does not apply encoder overrides. + supported_encoder_attention_backends = {"vision": frozenset()} + def __init__(self, model_config: ModelConfig): config = model_config.pretrained_config super().__init__(config) diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py b/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py index e72a726b29d0..073f86ad39af 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k3_vl.py @@ -461,6 +461,10 @@ class KimiK3ForConditionalGeneration(KimiK25ForConditionalGeneration): backbone classes). """ + # fmt: off + supported_encoder_attention_backends = KimiK25ForConditionalGeneration.supported_encoder_attention_backends + # fmt: on + _VISION_MODEL_CLS = KimiK3VisionModel mamba_metadata_cls = KimiLinearForCausalLM.mamba_metadata_cls diff --git a/tensorrt_llm/_torch/models/modeling_llama.py b/tensorrt_llm/_torch/models/modeling_llama.py index 0f65fa73a5e2..104d715172ec 100644 --- a/tensorrt_llm/_torch/models/modeling_llama.py +++ b/tensorrt_llm/_torch/models/modeling_llama.py @@ -1531,6 +1531,9 @@ def call_with_text_prompt( class Llama4ForConditionalGeneration(SpecDecOneEngineForCausalLM[Llama4Model, Llama4Config]): + # The vision encoder uses the Hugging Face implementation. + supported_encoder_attention_backends = {"vision": frozenset()} + @classmethod def get_preferred_kv_cache_manager_version( cls, diff --git a/tensorrt_llm/_torch/models/modeling_llava_next.py b/tensorrt_llm/_torch/models/modeling_llava_next.py index d6eac6adf806..7a300a25bbb5 100644 --- a/tensorrt_llm/_torch/models/modeling_llava_next.py +++ b/tensorrt_llm/_torch/models/modeling_llava_next.py @@ -642,6 +642,10 @@ def forward(self, multimodal_params: List[MultimodalParams]): content_format=ContentFormat.OPENAI, )) class LlavaNextModel(PreTrainedModel): + + # The native vision path does not apply encoder overrides. + supported_encoder_attention_backends = {"vision": frozenset()} + config_class = LlavaNextConfig def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3.py b/tensorrt_llm/_torch/models/modeling_minimaxm3.py index 53831fc16004..d868f1085bf8 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3.py @@ -3196,6 +3196,7 @@ class MiniMaxM3VLForConditionalGeneration(MiniMaxM3ForCausalLM): blobs under that namespace. """ + # The vision encoder calls SDPA directly. supported_encoder_attention_backends = {"vision": frozenset()} lora_owner = DecoderLoraOwner(module_path="model") diff --git a/tensorrt_llm/_torch/models/modeling_mllama.py b/tensorrt_llm/_torch/models/modeling_mllama.py index 23b95e1c6f92..b33dcc680106 100644 --- a/tensorrt_llm/_torch/models/modeling_mllama.py +++ b/tensorrt_llm/_torch/models/modeling_mllama.py @@ -1,6 +1,6 @@ # Adapted from https://github.com/huggingface/transformers/blob/a7f5479b45a8040392af80bf1107a2bdd796931c/src/transformers/models/mllama/modeling_mllama.py # -# Copyright 2024 the NVIDIA Inc. team. All rights reserved. +# Copyright 2024-2026 the NVIDIA Inc. team. All rights reserved. # Copyright 2024 the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -261,6 +261,9 @@ def infer_max_seq_len(self) -> int: @register_auto_model("MllamaForConditionalGeneration") class MllamaForConditionalGeneration(nn.Module): + # Vision input is not implemented. + supported_encoder_attention_backends = {"vision": frozenset()} + def __init__( self, config: ModelConfig[config_mllama.MllamaConfig], diff --git a/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py b/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py index a15b80e8ddef..ed29a1bda2a7 100644 --- a/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py +++ b/tensorrt_llm/_torch/models/modeling_musespark_open_omni.py @@ -1160,6 +1160,10 @@ class MuseSparkForConditionalGeneration(MultimodalModelMixin, PreTrainedModel): `get_multimodal_embeddings` applies as for any mixin model. """ + # The vision tower derives its backend from the decoder's, and the audio + # tower is plain PyTorch; neither applies encoder overrides. + supported_encoder_attention_backends = {"vision": frozenset(), "audio": frozenset()} + config_class = type(None) # resolved from model_config, not from a HF class # -- class-level hooks ------------------------------------------------- diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index e8e4876e33cc..1ecd2d012922 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -2905,7 +2905,7 @@ def _resample_audios( class NemotronHMultimodalModel(MultimodalModelMixin, transformers.PreTrainedModel): supported_encoder_attention_backends = { "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}), - "audio": frozenset(), + "audio": frozenset(), # Parakeet uses fixed relative-position SDPA. } _supports_flash_attn = True diff --git a/tensorrt_llm/_torch/models/modeling_phi4mm.py b/tensorrt_llm/_torch/models/modeling_phi4mm.py index 3f9b65da2653..50d78289d14b 100644 --- a/tensorrt_llm/_torch/models/modeling_phi4mm.py +++ b/tensorrt_llm/_torch/models/modeling_phi4mm.py @@ -955,6 +955,9 @@ def call_with_text_prompt( )) class Phi4MMForCausalLM(transformers.PreTrainedModel): + # Both encoders use Hugging Face implementations. + supported_encoder_attention_backends = {"vision": frozenset(), "audio": frozenset()} # yapf: disable + _supports_flash_attn = True lora_owner = DecoderLoraOwner(module_path="language_model") diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index 7d62cdd21209..f3b2d248b93e 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -2744,6 +2744,8 @@ def build_disagg_prefill_multimodal_inputs( )) class Qwen2_5_VLModel(Qwen2VLModelBase): + supported_encoder_attention_backends = Qwen2VLModelBase.supported_encoder_attention_backends + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): kwargs['vision_model_class'] = Qwen2_5_VisionModel diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 171bc04be5a7..6cc566fdb772 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -763,6 +763,8 @@ class _Qwen3_5VLModel(Qwen3VLModelBase): decorators (outer arch string + input-processor `model_type`). """ + supported_encoder_attention_backends = Qwen3VLModelBase.supported_encoder_attention_backends + supports_encoder_cache = True # The inner Qwen3.5 decoder implements them (see Qwen3NextForCausalLM). model_specific_llm_args = Qwen3NextForCausalLM.model_specific_llm_args @@ -859,6 +861,8 @@ def load_weights( class Qwen3_5MoeVLModel(_Qwen3_5VLModel): """VLM wrapper composing Qwen3 vision encoder with Qwen3.5 MoE text decoder.""" + supported_encoder_attention_backends = _Qwen3_5VLModel.supported_encoder_attention_backends + @classmethod def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: return _get_qwen35_moe_model_defaults(llm_args) @@ -881,3 +885,5 @@ class Qwen3_5VLModel(_Qwen3_5VLModel): instead of `SparseMoeBlock` (the dense text config has a native `intermediate_size` and no `num_experts`). """ + + supported_encoder_attention_backends = _Qwen3_5VLModel.supported_encoder_attention_backends diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index f862f0e5a920..c2383c624bda 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -2075,6 +2075,8 @@ def _get_requests_with_mm_data(self, multimodal_params): ), ) class Qwen3VLModel(Qwen3VLModelBase): + supported_encoder_attention_backends = Qwen3VLModelBase.supported_encoder_attention_backends + supports_encoder_cache = True def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py b/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py index 41912d28a0f9..a923c1bbd56e 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + from typing import Dict, List import torch @@ -45,6 +48,8 @@ ), ) class Qwen3MoeVLModel(Qwen3VLModelBase): + supported_encoder_attention_backends = Qwen3VLModelBase.supported_encoder_attention_backends + supports_encoder_cache = True def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): diff --git a/tensorrt_llm/_torch/models/modeling_qwen4_exp.py b/tensorrt_llm/_torch/models/modeling_qwen4_exp.py index 1688a183cae8..803b94f2122c 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen4_exp.py +++ b/tensorrt_llm/_torch/models/modeling_qwen4_exp.py @@ -1256,6 +1256,8 @@ def load_weights( class Qwen4ExpForConditionalGeneration(Qwen3VLModelBase): """Composite model using the production Qwen3-VL vision stack.""" + supported_encoder_attention_backends = Qwen3VLModelBase.supported_encoder_attention_backends + supports_encoder_cache = True @classmethod diff --git a/tensorrt_llm/_torch/models/modeling_vila.py b/tensorrt_llm/_torch/models/modeling_vila.py index 67ce5d7e0031..a68f59023c42 100644 --- a/tensorrt_llm/_torch/models/modeling_vila.py +++ b/tensorrt_llm/_torch/models/modeling_vila.py @@ -1206,6 +1206,10 @@ def call_with_text_prompt( content_format=ContentFormat.PASSTHROUGH, )) class VilaModel(PreTrainedModel): + + # The vision encoder uses Hugging Face CLIP or SigLIP. + supported_encoder_attention_backends = {"vision": frozenset()} + config_class = VilaConfig def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, diff --git a/tensorrt_llm/_torch/models/modeling_whisper.py b/tensorrt_llm/_torch/models/modeling_whisper.py index cb9cb798daef..0170870ed52e 100644 --- a/tensorrt_llm/_torch/models/modeling_whisper.py +++ b/tensorrt_llm/_torch/models/modeling_whisper.py @@ -869,6 +869,9 @@ def forward( class WhisperForConditionalGeneration(nn.Module, metaclass=PostInitCaller): """Whisper encoder-decoder model with LM head.""" + # The native audio path does not apply encoder overrides. + supported_encoder_attention_backends = {"audio": frozenset()} + def __init__(self, model_config: ModelConfig[WhisperConfig]): super().__init__() self.model_config = model_config diff --git a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py index e000e8981dac..30ba0edf388f 100644 --- a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py +++ b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from importlib import import_module +from pathlib import Path from types import SimpleNamespace import pytest @@ -16,6 +17,75 @@ _PUBLIC_BACKENDS = frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}) +def test_all_multimodal_encoder_backends_are_declared_and_documented() -> None: + from tensorrt_llm._torch.models._arch_index import ( + MODEL_ARCH_TO_MODULE, + MULTIMODAL_MODEL_TYPE_TO_MODULE, + is_builtin_zoo_module, + ) + from tensorrt_llm._torch.models.modeling_gemma4_unified import ( + Gemma4UnifiedForConditionalGeneration, + ) + from tensorrt_llm._torch.models.modeling_mllama import MllamaForConditionalGeneration + from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin + from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING + from tensorrt_llm.inputs.registry import ( + INPUT_PROCESSOR_REGISTRY, + MULTIMODAL_PLACEHOLDER_REGISTRY, + ) + + # Import the lazy zoo before enumerating registries, including mixin models + # that may not register an input processor yet. + for module in sorted(set(MODEL_ARCH_TO_MODULE.values())): + import_module(f"tensorrt_llm._torch.models.{module}") + encoders_by_module = {} + for model_type in MULTIMODAL_PLACEHOLDER_REGISTRY.get_registered_model_types(): + module = MULTIMODAL_MODEL_TYPE_TO_MODULE[model_type] + modalities = MULTIMODAL_PLACEHOLDER_REGISTRY.get_registered_modalities(model_type) + # Aliases and variants in a provider share logical encoders. Union their + # modalities so adding an encoder to any alias cannot escape this check. + encoders_by_module.setdefault(module, set()).update( + "vision" if modality in ("image", "video") else modality for modality in modalities + ) + models = set(INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type) + models.update( + model for model in MODEL_CLASS_MAPPING.values() if issubclass(model, MultimodalModelMixin) + ) + models.add(MllamaForConditionalGeneration) + rows = ["| Model | Logical encoder | Supported backends |", "| --- | --- | --- |"] + for model in sorted(models, key=lambda model: model.__name__): + if not is_builtin_zoo_module(model.__module__): + continue + assert "supported_encoder_attention_backends" in vars(model), model.__name__ + declared = model.supported_encoder_attention_backends + if model is Gemma4UnifiedForConditionalGeneration: + expected = set() # Encoder-free projections have no attention backend. + elif model is MllamaForConditionalGeneration: + expected = {"vision"} # Registered stub; vision input is not implemented. + else: + expected = encoders_by_module[model.__module__.rsplit(".", 1)[1]] + assert set(declared) == expected, model.__name__ + for encoder in sorted(declared) or [None]: + backends = declared[encoder] if encoder else frozenset() + assert isinstance(backends, frozenset), (model.__name__, encoder) + assert backends <= _PUBLIC_BACKENDS, (model.__name__, encoder) + status = ( + ", ".join( + f"`{backend}`" + for backend in ("VANILLA", "TRTLLM", "FLASHINFER") + if backend in backends + ) + or "Not supported" + ) + label = f"`{encoder}`" if encoder else "None (encoder-free)" + rows.append(f"| `{model.__name__}` | {label} | {status} |") + + doc = Path(__file__).resolve().parents[4] / "docs/source/features/multi-modality.md" + table = doc.read_text().split("\n", 1)[1] + table = table.split("\n", 1)[0] + assert table == "\n".join(rows), "Update the encoder backend support table from declarations" + + @pytest.mark.parametrize("backend", sorted(_PUBLIC_BACKENDS)) def test_public_backend_names_parse_case_insensitively(backend: str) -> None: config = MultimodalConfig(encoder_attention_backend={"vision": backend.lower()}) From c030096db7ec9068c663e358040f297d1a18e11e Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:00:45 +0000 Subject: [PATCH 3/6] [None][fix] harden encoder backend overrides from review Problem: An encoder override kept the decoder's FP8 KV-cache setting on Qwen2.5-VL and Kimi K2.5; FlashInfer rejects it without a KV cache. VANILLA derives segment boundaries on the host, so encoder CUDA graph replay would reuse the captured ones. The loader checked encoder presence before model_kwargs were applied. Tests did not check the backend passed to the Pixtral, RADIO and Step3 towers. Solution: Drop KV-cache quantization when an override changes the encoder backend; defaults are unchanged. Reject VANILLA for an encoder that also enables encoder_cuda_graph. Check encoder presence on the config loaded with model_kwargs, before model construction. Add CPU tests for these cases, for the backend each tower receives, and for loader class resolution. Validation: CPU suites of the PR and touched files match upstream main apart from the new passing cases; applicable pre-commit hooks pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/features/multi-modality.md | 2 +- .../_torch/models/modeling_kimi_k25.py | 9 +- .../_torch/models/modeling_qwen2vl.py | 7 +- .../_torch/pyexecutor/model_loader.py | 37 +-- tensorrt_llm/llmapi/llm_args.py | 16 ++ .../_torch/modeling/test_modeling_qwen3vl.py | 44 +++- .../test_encoder_attention_backend.py | 237 +++++++++++++++++- 7 files changed, 318 insertions(+), 34 deletions(-) diff --git a/docs/source/features/multi-modality.md b/docs/source/features/multi-modality.md index 544768b2d4fd..bf66f8c40c6d 100644 --- a/docs/source/features/multi-modality.md +++ b/docs/source/features/multi-modality.md @@ -48,7 +48,7 @@ llm = LLM( ) ``` -Omitted entries preserve the model's existing encoder defaults, including inheritance from `attn_backend` where applicable. An encoder override leaves the language model's backend unchanged. Backend availability and supported tensor shapes still depend on the installed kernels and hardware; floating-point differences between backends are expected. +Omitted entries preserve the model's existing encoder defaults, including inheritance from `attn_backend` where applicable. An encoder override leaves the language model's backend unchanged. Backend availability and supported tensor shapes still depend on the installed kernels and hardware; floating-point differences between backends are expected. `VANILLA` does not support `multimodal_config.encoder_cuda_graph`, so selecting it for an encoder with CUDA graphs enabled fails during configuration. The table covers every registered multimodal model implementation; model aliases share the same declaration. Image/video share `vision`, and `audio` names a separate encoder. "Not supported" means this option cannot change that encoder's attention backend; it does not mean the model or modality is unsupported. Gemma 4 Unified uses encoder-free projections, and Mllama's vision input is not implemented. The CPU registry test checks this table against each model's explicit `supported_encoder_attention_backends` declaration. diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index b1a1c9f078ee..2ddad3dbc98c 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -801,9 +801,12 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: self.model_config.attn_backend = model_config.get_encoder_attention_backend( "vision", model_config.attn_backend ) - self.model_config.quant_config = QuantConfig( - kv_cache_quant_algo=model_config.quant_config.kv_cache_quant_algo - ) + kv_cache_quant_algo = model_config.quant_config.kv_cache_quant_algo + if self.model_config.attn_backend != model_config.attn_backend: + # The encoder keeps no KV cache, and FlashInfer rejects FP8 + # KV-cache quantization without one. + kv_cache_quant_algo = None + self.model_config.quant_config = QuantConfig(kv_cache_quant_algo=kv_cache_quant_algo) self.model_config.pretrained_config = copy.copy(model_config.pretrained_config) # Extract vision config dict (num_heads is resolved here, with this diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index f3b2d248b93e..b2ce944c6474 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -1461,8 +1461,13 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig], model_class(model_config.pretrained_config.vision_config), self.model_dtype).eval() elif model_class == Qwen2_5_VisionModel: - self.model_config = self.model_config.with_encoder_attention_backend( + vision_config = self.model_config.with_encoder_attention_backend( "vision") + if vision_config is not self.model_config: + # The encoder keeps no KV cache, and FlashInfer rejects FP8 + # KV-cache quantization without one. + vision_config.quant_config = QuantConfig() + self.model_config = vision_config self.visual = MultimodalModelMixin._cast_multimodal_encoder_dtype( model_class(self.model_config), self.model_dtype) else: diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 4c8ab2cb8710..dc38c33760cf 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -206,11 +206,7 @@ def _with_fp8_block_scale_weights( def _validate_encoder_attention_backend_support( - model_cls: type | None, - mm_config: MultimodalConfig | None, - *, - pretrained_config: object | None = None, - disable_mm_encoder: bool = False) -> None: + model_cls: type | None, mm_config: MultimodalConfig | None) -> None: """Reject unsupported encoder selections before allocating model weights.""" if mm_config is None or not mm_config.encoder_attention_backend: return @@ -228,10 +224,6 @@ def _validate_encoder_attention_backend_support( raise ValueError( f"The {encoder} encoder does not support attention backend {backend!r}; " f"supported backends: {sorted(supported[encoder])}") - if disable_mm_encoder or (pretrained_config is not None and getattr( - pretrained_config, f"{encoder}_config", None) is None): - raise ValueError( - f"The {encoder} encoder is not enabled for {model_name}") if backend == "FLASHINFER": from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE @@ -241,6 +233,21 @@ def _validate_encoder_attention_backend_support( ) +def _validate_encoder_attention_backend_enabled( + mm_config: MultimodalConfig | None, + pretrained_config: object, + *, + disable_mm_encoder: bool = False) -> None: + """Reject selections for encoders that the effective model config omits.""" + if mm_config is None or not mm_config.encoder_attention_backend: + return + for encoder in mm_config.encoder_attention_backend: + if disable_mm_encoder or getattr(pretrained_config, f"{encoder}_config", + None) is None: + raise ValueError( + f"The {encoder} encoder is not enabled in the model config") + + def _validate_and_adjust_mamba_snapshot_config(config: ModelConfig, llm_args: TorchLlmArgs) -> None: """Validate snapshot reuse after the model and V2 setting are resolved.""" @@ -1215,11 +1222,8 @@ def load_config_and_apply_defaults( preference_cls = get_registered_model_class( architectures[0]) or model_cls - _validate_encoder_attention_backend_support( - preference_cls, - llm_args.multimodal_config, - pretrained_config=config.pretrained_config, - disable_mm_encoder=llm_args.disable_mm_encoder) + _validate_encoder_attention_backend_support(preference_cls, + llm_args.multimodal_config) # model_cls is None when the architecture is unknown/unsupported. model_defaults = {} @@ -2445,6 +2449,11 @@ def _load_and_validate_config( load_config_kwargs['model_kwargs'] = self.llm_args.model_kwargs config = checkpoint_loader.load_config(**load_config_kwargs) + # Check after model_kwargs, which can remove an encoder. + _validate_encoder_attention_backend_enabled( + self.llm_args.multimodal_config, + config.pretrained_config, + disable_mm_encoder=self.llm_args.disable_mm_encoder) # Decided here, after `load_config_and_apply_defaults` resolved the # built-in class and applied its model defaults to `llm_args`: a diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 1a227323ab58..96f2e2f3733f 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -933,6 +933,22 @@ def validate_encoder_optimization_compatibility(self) -> 'MultimodalConfig': "multimodal_config.encoder_side_stream_max_ahead > 0 are " "mutually exclusive. Disable side-stream MM prefetch or " "disable MM encoder CUDA graphs.") + # VANILLA derives segment boundaries on the host, so graph replay + # would reuse the captured ones. Image and video share `vision`. + graph_encoders = { + "vision" if name in ("image", "video") else name + for name in self.encoder_cuda_graph or () + } + vanilla_graph_encoders = sorted( + encoder for encoder, backend in ( + self.encoder_attention_backend or {}).items() + if backend == "VANILLA" and encoder in graph_encoders) + if vanilla_graph_encoders: + raise ValueError( + "multimodal_config.encoder_attention_backend selects VANILLA " + f"for {vanilla_graph_encoders}, which does not support " + "multimodal_config.encoder_cuda_graph. Select TRTLLM or " + "FLASHINFER, or disable CUDA graphs for that encoder.") return self diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py index c4f257c9cfee..69bd17827eb0 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py @@ -2220,6 +2220,13 @@ def encode(residuals): assert params.multimodal_data["image"]["vision_normalization"] is spec +class _RecordingVisionEncoder(torch.nn.Module): + def __init__(self, model_config: ModelConfig) -> None: + super().__init__() + self.model_config = model_config + self.weight = torch.nn.Parameter(torch.ones(1)) + + @pytest.mark.cpu_only @pytest.mark.parametrize( ("decoder_backend", "override"), @@ -2229,13 +2236,6 @@ def test_qwen_encoder_backend_preserves_decoder(decoder_backend: str, override: from transformers import PretrainedConfig from tensorrt_llm._torch.models.modeling_qwen3vl import Qwen3VisionModelBase - from tensorrt_llm.llmapi.llm_args import MultimodalConfig - - class RecordingVisionEncoder(torch.nn.Module): - def __init__(self, model_config: ModelConfig) -> None: - super().__init__() - self.model_config = model_config - self.weight = torch.nn.Parameter(torch.ones(1)) config = ModelConfig( pretrained_config=PretrainedConfig( @@ -2248,7 +2248,7 @@ def __init__(self, model_config: ModelConfig) -> None: ), ) config._frozen = True - encoder = Qwen3VisionModelBase(config, RecordingVisionEncoder) + encoder = Qwen3VisionModelBase(config, _RecordingVisionEncoder) assert encoder.visual.model_config.attn_backend == (override or decoder_backend) assert encoder.model_config.extra_attrs is config.extra_attrs assert encoder.model_config._frozen @@ -2257,6 +2257,34 @@ def __init__(self, model_config: ModelConfig) -> None: assert config.attn_backend == decoder_backend +@pytest.mark.cpu_only +@pytest.mark.parametrize("override", [None, "VANILLA", "FLASHINFER"]) +def test_qwen2_5_vl_encoder_override_drops_decoder_kv_cache_quant( + override: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + from transformers import PretrainedConfig + + monkeypatch.setattr(modeling_qwen2vl, "Qwen2_5_VisionModel", _RecordingVisionEncoder) + pretrained_config = PretrainedConfig(vision_config=PretrainedConfig(num_heads=1)) + pretrained_config.torch_dtype = torch.float32 + config = ModelConfig( + pretrained_config=pretrained_config, + quant_config=QuantConfig(kv_cache_quant_algo=QuantAlgo.FP8), + multimodal_config=MultimodalConfig( + encoder_attention_backend={"vision": override} if override else None + ), + ) + + encoder = modeling_qwen2vl.Qwen2VisionModelBase(config, _RecordingVisionEncoder) + + vision = encoder.visual.model_config + assert vision.attn_backend == (override or "TRTLLM") + # The default keeps today's config. An override drops the decoder's FP8 + # KV-cache setting, which FlashInfer rejects without a KV cache. + assert vision.quant_config.kv_cache_quant_algo == (None if override else QuantAlgo.FP8) + assert config.quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + + @pytest.mark.cpu_only def test_qwen_encoder_backend_declarations_preserve_fixed_hf_encoder() -> None: from tensorrt_llm._torch.models.modeling_qwen2vl import Qwen2_5_VLModel, Qwen2VLModel diff --git a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py index 30ba0edf388f..cf3ae66325b1 100644 --- a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py +++ b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py @@ -6,11 +6,17 @@ from types import SimpleNamespace import pytest +import torch from pydantic import ValidationError from tensorrt_llm._torch.model_config import ModelConfig -from tensorrt_llm._torch.pyexecutor.model_loader import _validate_encoder_attention_backend_support +from tensorrt_llm._torch.pyexecutor.model_loader import ( + _validate_encoder_attention_backend_enabled, + _validate_encoder_attention_backend_support, +) from tensorrt_llm.llmapi.llm_args import MultimodalConfig +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo pytestmark = pytest.mark.cpu_only @@ -266,20 +272,237 @@ def test_hf_encoders_reject_backend_overrides(module: str, model: str, encoder: def test_loader_rejects_absent_or_disabled_encoder(disabled: bool) -> None: config = SimpleNamespace(vision_config=object() if disabled else None) with pytest.raises(ValueError, match="vision.*not enabled"): - _validate_encoder_attention_backend_support( - _EncoderModel, + _validate_encoder_attention_backend_enabled( MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), - pretrained_config=config, + config, disable_mm_encoder=disabled, ) def test_loader_accepts_enabled_encoder() -> None: - _validate_encoder_attention_backend_support( - _EncoderModel, + _validate_encoder_attention_backend_enabled( MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), - pretrained_config=SimpleNamespace(vision_config=object()), + SimpleNamespace(vision_config=object()), + ) + + +def test_loader_checks_encoder_after_model_kwargs() -> None: + from unittest.mock import MagicMock + + from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + + def load_config(**kwargs): + # The checkpoint has a vision encoder; model_kwargs removes it. + pretrained_config = SimpleNamespace(vision_config=object()) + for name, value in kwargs["model_kwargs"].items(): + setattr(pretrained_config, name, value) + return SimpleNamespace(pretrained_config=pretrained_config) + + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + model_kwargs={"vision_config": None}, + multimodal_config=MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + ) + loader = ModelLoader( + llm_args=llm_args, + mapping=MagicMock(), + spec_config=None, + sparse_attention_config=None, + max_num_tokens=128, + max_seq_len=128, + ) + with pytest.raises(ValueError, match="vision.*not enabled"): + loader._load_and_validate_config( + "/tmp/dummy_model", SimpleNamespace(load_config=load_config) + ) + + +def test_loader_validates_registered_model_class(monkeypatch: pytest.MonkeyPatch) -> None: + from tensorrt_llm._torch.pyexecutor import model_loader + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + + monkeypatch.setattr( + model_loader.AutoModelForCausalLM, "_resolve_class", staticmethod(lambda config: None) + ) + checkpoint_loader = SimpleNamespace( + load_config=lambda *args, **kwargs: SimpleNamespace( + pretrained_config=SimpleNamespace(architectures=["Phi4MMForCausalLM"]) + ) + ) + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + multimodal_config=MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), ) + with pytest.raises(ValueError, match="vision.*cannot be changed for Phi4MMForCausalLM"): + model_loader.ModelLoader.load_config_and_apply_defaults( + "/tmp/dummy_model", llm_args, checkpoint_loader + ) + + +@pytest.mark.parametrize("graph_key", ["vision", "image"]) +def test_vanilla_encoder_rejects_encoder_cuda_graph(graph_key: str) -> None: + with pytest.raises(ValidationError, match=r"VANILLA for \['vision'\].*encoder_cuda_graph"): + MultimodalConfig( + encoder_attention_backend={"vision": "VANILLA"}, + encoder_cuda_graph={graph_key: {"max_batch_size": 1}}, + ) + + +@pytest.mark.parametrize( + ("backend", "graph_key"), + [("TRTLLM", "vision"), ("FLASHINFER", "vision"), ("VANILLA", "audio")], +) +def test_encoder_cuda_graph_accepts_graph_safe_backends(backend: str, graph_key: str) -> None: + config = MultimodalConfig( + encoder_attention_backend={"vision": backend}, + encoder_cuda_graph={graph_key: {"max_batch_size": 1}}, + ) + assert config.encoder_attention_backend == {"vision": backend} + + +def _recorder(built: dict, role: str) -> type: + """Stand in for a tower and record the config it receives.""" + + class Recorder(torch.nn.Module): + def __init__(self, model_config: ModelConfig, *args, **kwargs) -> None: + super().__init__() + built[role] = (model_config, kwargs) + self.config = model_config.pretrained_config + + return Recorder + + +def _multimodal_config(override: str | None) -> MultimodalConfig: + return MultimodalConfig(encoder_attention_backend={"vision": override} if override else None) + + +@pytest.mark.parametrize("override", [None, "VANILLA"]) +def test_pixtral_encoder_backend_keeps_decoder( + override: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + from transformers import Mistral3Config + + from tensorrt_llm._torch.models import modeling_mistral + + built = {} + monkeypatch.delenv("TLLM_MULTIMODAL_DISAGGREGATED", raising=False) + monkeypatch.setattr(modeling_mistral, "MistralForCausalLM", _recorder(built, "decoder")) + monkeypatch.setattr( + modeling_mistral.modeling_pixtral, "PixtralVisionModel", _recorder(built, "vision") + ) + monkeypatch.setattr( + modeling_mistral, "Mistral3MultiModalProjector", _recorder(built, "projector") + ) + config = ModelConfig( + pretrained_config=Mistral3Config(architectures=["Mistral3ForConditionalGeneration"]), + attn_backend="FLASHINFER", + multimodal_config=_multimodal_config(override), + ) + + modeling_mistral.Mistral3VLM(config) + + assert built["vision"][0].attn_backend == (override or "TRTLLM") + assert built["decoder"][0].attn_backend == "FLASHINFER" + assert config.attn_backend == "FLASHINFER" + + +@pytest.mark.parametrize("override", [None, "VANILLA"]) +def test_radio_encoder_backend_keeps_decoder( + override: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + from transformers import PretrainedConfig + + from tensorrt_llm._torch.models import modeling_nemotron_h_multimodal + + built = {} + monkeypatch.setattr( + modeling_nemotron_h_multimodal, "RADIOVisionModel", _recorder(built, "vision") + ) + pretrained_config = PretrainedConfig( + force_image_size=32, + patch_size=16, + downsample_ratio=0.5, + ps_version="v2", + vit_hidden_size=4, + projector_hidden_size=8, + llm_config=PretrainedConfig(hidden_size=8, rms_norm_eps=1e-6), + vision_config=PretrainedConfig(), + ) + pretrained_config.torch_dtype = torch.float32 + config = ModelConfig( + pretrained_config=pretrained_config, + attn_backend="TRTLLM", + multimodal_config=_multimodal_config(override), + ) + + modeling_nemotron_h_multimodal.NemotronHVisionEncoder(config) + + assert built["vision"][1]["vision_attn_backend"] == (override or "FLASHINFER") + assert config.attn_backend == "TRTLLM" + + +@pytest.mark.parametrize( + ("decoder_backend", "override"), [("FLASHINFER", None), ("TRTLLM", "VANILLA")] +) +def test_step3_encoder_backend_keeps_decoder( + decoder_backend: str, override: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + from transformers import PretrainedConfig + + from tensorrt_llm._torch.models import modeling_step3p7vl + + built = {} + monkeypatch.setattr(modeling_step3p7vl, "Step3p7VisionEncoder", _recorder(built, "vision")) + pretrained_config = PretrainedConfig( + vision_config=PretrainedConfig(width=4), text_config=PretrainedConfig(hidden_size=8) + ) + pretrained_config.torch_dtype = torch.float32 + config = ModelConfig( + pretrained_config=pretrained_config, + attn_backend=decoder_backend, + multimodal_config=_multimodal_config(override), + ) + + modeling_step3p7vl.Step3p7VisionTower(config) + + assert built["vision"][0].attn_backend == (override or decoder_backend) + assert config.attn_backend == decoder_backend + + +@pytest.mark.parametrize("override", [None, "FLASHINFER"]) +def test_kimi_k25_encoder_override_drops_decoder_kv_cache_quant(override: str | None) -> None: + from transformers import PretrainedConfig + + from tensorrt_llm._torch.models.modeling_kimi_k25 import KimiK25VisionModel + + pretrained_config = PretrainedConfig( + vision_config={ + "vt_hidden_size": 16, + "vt_num_hidden_layers": 1, + "vt_num_attention_heads": 2, + "vt_intermediate_size": 32, + "text_hidden_size": 16, + "init_pos_emb_height": 4, + "init_pos_emb_width": 4, + } + ) + pretrained_config.torch_dtype = torch.float32 + # VANILLA keeps construction on CPU; the vision default follows the decoder. + config = ModelConfig( + pretrained_config=pretrained_config, + attn_backend="VANILLA", + quant_config=QuantConfig(kv_cache_quant_algo=QuantAlgo.FP8), + multimodal_config=_multimodal_config(override), + ) + + vision = KimiK25VisionModel(config).model_config + + assert vision.attn_backend == (override or "VANILLA") + # The default keeps today's config. An override drops the decoder's FP8 + # KV-cache setting, which FlashInfer rejects without a KV cache. + assert vision.quant_config.kv_cache_quant_algo == (None if override else QuantAlgo.FP8) + assert config.quant_config.kv_cache_quant_algo == QuantAlgo.FP8 def test_model_config_rejects_invalid_backend_even_without_pydantic_validation() -> None: From 900ae3bb47994fd219d6d4e2882efa0bedfea80d Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:07:47 +0000 Subject: [PATCH 4/6] [None][fix] Address encoder backend review comments Problem: trtllm-bench sets disable_mm_encoder for text-only runs, so a serving config with encoder_attention_backend failed at startup. Under modeling_v2 auto or require, the Qwen4-Exp target built its own vision tower and ignored the selection. The Nemotron-H RADIO vision default was FLASHINFER, whose plan build for a new image shape can stall during CUDA graph replay. Backend names were repeated outside the config Literal, and this PR also carried a MiniCPM-V 4.6 fix owned by another change. Solution: Skip the encoder-presence check with one info log when the encoder is disabled; model support checks still run. Decline the Qwen4-Exp target when an encoder backend is selected, so auto uses the built-in model and require names the setting. Default RADIO vision to TRTLLM; VANILLA and FLASHINFER stay selectable. Read backend names from a named Literal and document the ModelConfig helpers. Drop the MiniCPM-V 4.6 max_seq_len change and a test that restated model declarations. Validation: CPU suites of the PR and touched files match the previous head apart from the changed tests, which pass; applicable pre-commit hooks pass and the serve schemas are unchanged. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/features/multi-modality.md | 4 +- .../modeling.py | 7 ++ tensorrt_llm/_torch/model_config.py | 38 +++++++- .../models/modeling_nemotron_h_multimodal.py | 4 +- .../_torch/pyexecutor/model_loader.py | 11 ++- tensorrt_llm/llmapi/llm_args.py | 31 +++--- .../modeling_v2/test_modeling_v2_routing.py | 10 ++ .../test_encoder_attention_backend.py | 96 ++++++++++--------- 8 files changed, 132 insertions(+), 69 deletions(-) diff --git a/docs/source/features/multi-modality.md b/docs/source/features/multi-modality.md index bf66f8c40c6d..1d2b72fac3da 100644 --- a/docs/source/features/multi-modality.md +++ b/docs/source/features/multi-modality.md @@ -91,9 +91,9 @@ The table covers every registered multimodal model implementation; model aliases | `WhisperForConditionalGeneration` | `audio` | Not supported | -For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. +For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. The Nemotron-H RADIO vision encoder defaults to `TRTLLM`, and `VANILLA` and `FLASHINFER` remain selectable. -Supported entries use TensorRT-LLM's `Attention` and apply encoder overrides explicitly. Other encoders keep their fixed implementation, including Hugging Face attention, direct SDPA, and custom attention paths. Overrides for entries marked "Not supported" fail during initialization, even if the requested value is `VANILLA`. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. +Supported entries use TensorRT-LLM's `Attention` and apply encoder overrides explicitly. Other encoders keep their fixed implementation, including Hugging Face attention, direct SDPA, and custom attention paths. Overrides for entries marked "Not supported" fail during initialization, even if the requested value is `VANILLA`. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. With `disable_mm_encoder=True`, encoders that the model skips do not use the setting, so text-only runs can reuse a serving configuration. ## Model Support Matrix diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index ebc31a356d05..75f9049084a7 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -6923,6 +6923,13 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: "which the target does not implement; 'breakable' runs the prefill step as a " "breakable CUDA graph" ) + mm_config = llm_args.multimodal_config + if mm_config is not None and mm_config.encoder_attention_backend: + raise NoTarget( + "multimodal_config.encoder_attention_backend=" + f"{mm_config.encoder_attention_backend}; the target's vision tower runs its own " + "attention and does not apply it" + ) if spec is None: return if not is_gdn_replay_enabled(): diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 56d9c31132e1..41963d6ed1e6 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import (TYPE_CHECKING, Any, Callable, Dict, Generic, Iterator, List, - Optional, Sequence, Set, Tuple, TypeVar) + Optional, Sequence, Set, Tuple, TypeVar, get_args) import filelock import torch @@ -41,14 +41,17 @@ torch_dtype_to_binding) from tensorrt_llm.bindings import LayerType as LayerTypeCpp from tensorrt_llm.functional import AllReduceStrategy +# yapf: disable # Callers import MoeLoadBalancerConfig from this module. from tensorrt_llm.llmapi.llm_args import MoeLoadBalancerConfig # noqa: F401 from tensorrt_llm.llmapi.llm_args import (CSA2SparseAttentionConfig, DeepSeekSparseAttentionConfig, DeepSeekV4SparseAttentionConfig, + EncoderAttentionBackend, KvCacheConfig, MoeLoadBalancerConfigType, MultimodalConfig) +# yapf: enable from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig @@ -493,23 +496,48 @@ class ModelConfig(Generic[TConfig]): modeling_v2_target: Optional[type] = None def get_encoder_attention_backend(self, encoder: str, default: str) -> str: - """Select a logical encoder's backend, preserving its existing default.""" + """Return the attention backend selected for a logical encoder. + + Args: + encoder: Logical encoder name, such as ``"vision"`` or ``"audio"``. + default: Backend the model uses for this encoder when + ``multimodal_config.encoder_attention_backend`` omits it. + + Returns: + The backend from ``multimodal_config.encoder_attention_backend``, + or ``default`` when the encoder has no entry. + + Raises: + ValueError: If the selected backend is not a public encoder backend. + """ overrides = (self.multimodal_config.encoder_attention_backend if self.multimodal_config is not None else None) if not overrides or encoder not in overrides: return default backend = overrides[encoder] - if backend not in ("VANILLA", "TRTLLM", "FLASHINFER"): + backends = get_args(EncoderAttentionBackend) + if backend not in backends: raise ValueError( f"Invalid attention backend {backend!r} for the {encoder} encoder; " - "expected VANILLA, TRTLLM, or FLASHINFER") + f"expected one of {', '.join(backends)}") return backend def with_encoder_attention_backend(self, encoder: str, default: str | None = None ) -> "ModelConfig": - """Apply an encoder selection to a shallow copy without changing the decoder.""" + """Return a config whose ``attn_backend`` is the encoder's backend. + + Args: + encoder: Logical encoder name, such as ``"vision"`` or ``"audio"``. + default: Backend the model uses for this encoder when no override + is set. ``None`` means the decoder's ``attn_backend``. + + Returns: + ``self`` when the selected backend equals ``attn_backend``; + otherwise a shallow copy with only ``attn_backend`` changed, so the + decoder config is never modified. + """ backend = self.get_encoder_attention_backend( encoder, self.attn_backend if default is None else default) if backend == self.attn_backend: diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index 1ecd2d012922..2c7664b47e49 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -667,7 +667,9 @@ def __init__(self, model_config: ModelConfig[transformers.PretrainedConfig]): self.vision_model = RADIOVisionModel( vision_model_config, disable_quantization=True, - vision_attn_backend=model_config.get_encoder_attention_backend("vision", "FLASHINFER"), + # FlashInfer's cuDNN ragged-prefill plan build for a new image shape can stall + # indefinitely while the language model replays CUDA graphs, freezing the server. + vision_attn_backend=model_config.get_encoder_attention_backend("vision", "TRTLLM"), encoder_cuda_graph_config=encoder_cuda_graph_config, ) diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index dc38c33760cf..f433c4eb86a7 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -241,9 +241,16 @@ def _validate_encoder_attention_backend_enabled( """Reject selections for encoders that the effective model config omits.""" if mm_config is None or not mm_config.encoder_attention_backend: return + if disable_mm_encoder: + # Text-only runs, such as trtllm-bench without --modality, reuse + # serving configs. A skipped encoder never applies the selection, and + # models that ignore disable_mm_encoder still build it with the setting. + logger.info( + "multimodal_config.encoder_attention_backend is not applied " + "to encoders skipped by disable_mm_encoder=True") + return for encoder in mm_config.encoder_attention_backend: - if disable_mm_encoder or getattr(pretrained_config, f"{encoder}_config", - None) is None: + if getattr(pretrained_config, f"{encoder}_config", None) is None: raise ValueError( f"The {encoder} encoder is not enabled in the model config") diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 96f2e2f3733f..2d64869aa16c 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -816,6 +816,10 @@ class MultimodalEncoderSchedulingPolicy(StrEnum): LLM capacity filtering (``MultimodalEagerEncoderScheduler``).""" +# Public attention backends a multimodal encoder can select. +EncoderAttentionBackend = Literal["VANILLA", "TRTLLM", "FLASHINFER"] + + class MultimodalConfig(StrictBaseModel): """Multimodal model configuration.""" @@ -830,20 +834,19 @@ class MultimodalConfig(StrictBaseModel): status="prototype", ) - encoder_attention_backend: dict[str, Literal[ - "VANILLA", "TRTLLM", "FLASHINFER"]] | None = Field( - default=None, - description= - ("TensorRT-LLM attention backend for each logical multimodal encoder, " - "for example {'vision': 'FLASHINFER'}. Public values are VANILLA, TRTLLM, " - "and FLASHINFER. Image and video share the vision encoder. Only encoders " - "using TensorRT-LLM Attention support this option; encoders using fixed " - "attention reject overrides at initialization. Omitted entries retain " - "their model-specific defaults and the language model's attn_backend " - "is unchanged. Unsupported encoders or backends raise an error."), - status="prototype", - telemetry=False, - ) + encoder_attention_backend: dict[str, EncoderAttentionBackend] | None = Field( + default=None, + description= + ("TensorRT-LLM attention backend for each logical multimodal encoder, " + "for example {'vision': 'FLASHINFER'}. Public values are VANILLA, TRTLLM, " + "and FLASHINFER. Image and video share the vision encoder. Only encoders " + "using TensorRT-LLM Attention support this option; encoders using fixed " + "attention reject overrides at initialization. Omitted entries retain " + "their model-specific defaults and the language model's attn_backend " + "is unchanged. Unsupported encoders or backends raise an error."), + status="prototype", + telemetry=False, + ) encoder_cuda_graph: dict[str, MultimodalEncoderCudaGraphConfig] | None = Field( default=None, diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py index 26fd063dfb92..2e530b4ab76d 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py @@ -57,6 +57,7 @@ CacheTransceiverConfig, KvCacheConfig, MTPDecodingConfig, + MultimodalConfig, NGramDecodingConfig, TorchLlmArgs, ) @@ -1595,6 +1596,15 @@ def test_qwen4_exp_piecewise_prefill_cuda_graph_falls_back(): _declined(config, dict(prefill_cuda_graph_backend="piecewise"), match="piecewise") +@pytest.mark.usefixtures("_qwen4_exp_sm") +def test_qwen4_exp_encoder_attention_backend_falls_back(): + """The target's own vision tower would ignore it; the built-in model applies it.""" + config = _model_config(_qwen4_exp_config()) + selected = MultimodalConfig(encoder_attention_backend={"vision": "TRTLLM"}) + _declined(config, dict(multimodal_config=selected), match="encoder_attention_backend") + _claimed(config, dict(multimodal_config=MultimodalConfig(encoder_attention_backend={}))) + + @pytest.mark.usefixtures("_qwen4_exp_sm") @pytest.mark.parametrize("backend", ["disabled", "breakable"]) def test_qwen4_exp_matches_without_or_with_the_breakable_prefill_cuda_graph(backend): diff --git a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py index cf3ae66325b1..775003854f5c 100644 --- a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py +++ b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py @@ -216,42 +216,6 @@ def test_loader_keeps_omitted_settings_unchanged( ) -@pytest.mark.parametrize( - ("module", "model", "expected"), - [ - ( - "gemma4mm", - "Gemma4ForConditionalGeneration", - {"vision": _PUBLIC_BACKENDS, "audio": frozenset()}, - ), - ("gemma4_unified", "Gemma4UnifiedForConditionalGeneration", {}), - ( - "nemotron_h_multimodal", - "NemotronHMultimodalModel", - {"vision": _PUBLIC_BACKENDS, "audio": frozenset()}, - ), - ("glm5_next_vision", "Glm5NextVLM", {"vision": frozenset({"TRTLLM"})}), - ("kimi_k25", "KimiK25ForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), - ("kimi_k3_vl", "KimiK3ForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), - ("step3p7vl", "Step3p7VLForConditionalGeneration", {"vision": _PUBLIC_BACKENDS}), - ("minicpmv4_6", "MiniCPMV4_6Model", {"vision": _PUBLIC_BACKENDS}), - ("mistral", "Mistral3VLM", {"vision": _PUBLIC_BACKENDS}), - ("deepseekv41_multimodal", "DeepseekV41ForConditionalGeneration", {"vision": frozenset()}), - ("minimaxm3", "MiniMaxM3VLForConditionalGeneration", {"vision": frozenset()}), - ], -) -def test_model_encoder_backend_declarations(module: str, model: str, expected: dict) -> None: - model_cls = getattr(import_module(f"tensorrt_llm._torch.models.modeling_{module}"), model) - assert model_cls.supported_encoder_attention_backends == expected - for encoder, backends in expected.items(): - assert backends <= _PUBLIC_BACKENDS - if not backends: - with pytest.raises(ValueError, match=f"{encoder}.*cannot be changed"): - _validate_encoder_attention_backend_support( - model_cls, MultimodalConfig(encoder_attention_backend={encoder: "VANILLA"}) - ) - - @pytest.mark.parametrize( ("module", "model", "encoder"), [ @@ -268,14 +232,11 @@ def test_hf_encoders_reject_backend_overrides(module: str, model: str, encoder: ) -@pytest.mark.parametrize("disabled", [False, True]) -def test_loader_rejects_absent_or_disabled_encoder(disabled: bool) -> None: - config = SimpleNamespace(vision_config=object() if disabled else None) +def test_loader_rejects_absent_encoder() -> None: with pytest.raises(ValueError, match="vision.*not enabled"): _validate_encoder_attention_backend_enabled( MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), - config, - disable_mm_encoder=disabled, + SimpleNamespace(vision_config=None), ) @@ -318,6 +279,50 @@ def load_config(**kwargs): ) +def test_loader_skips_encoder_check_when_encoder_is_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A text-only run, such as trtllm-bench without --modality, reuses a serving config.""" + from unittest.mock import MagicMock + + from tensorrt_llm._torch.pyexecutor import model_loader + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + + class ReachedTargetResolution(Exception): + pass + + def resolve_target(*args, **kwargs): + raise ReachedTargetResolution + + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + disable_mm_encoder=True, + multimodal_config=MultimodalConfig(encoder_attention_backend={"vision": "VANILLA"}), + ) + messages = [] + monkeypatch.setattr(model_loader.logger, "info", messages.append) + monkeypatch.setattr(model_loader, "modeling_v2_resolve_on_all_ranks", resolve_target) + loader = model_loader.ModelLoader( + llm_args=llm_args, + mapping=MagicMock(), + spec_config=None, + sparse_attention_config=None, + max_num_tokens=128, + max_seq_len=128, + ) + checkpoint_loader = SimpleNamespace( + load_config=lambda **kwargs: SimpleNamespace(pretrained_config=SimpleNamespace()) + ) + + # The check passes without an error and loading continues to target resolution. + with pytest.raises(ReachedTargetResolution): + loader._load_and_validate_config("/tmp/dummy_model", checkpoint_loader) + assert [message for message in messages if "encoder_attention_backend" in message] == [ + "multimodal_config.encoder_attention_backend is not applied " + "to encoders skipped by disable_mm_encoder=True" + ] + + def test_loader_validates_registered_model_class(monkeypatch: pytest.MonkeyPatch) -> None: from tensorrt_llm._torch.pyexecutor import model_loader from tensorrt_llm.llmapi.llm_args import TorchLlmArgs @@ -407,7 +412,7 @@ def test_pixtral_encoder_backend_keeps_decoder( assert config.attn_backend == "FLASHINFER" -@pytest.mark.parametrize("override", [None, "VANILLA"]) +@pytest.mark.parametrize("override", [None, "VANILLA", "FLASHINFER"]) def test_radio_encoder_backend_keeps_decoder( override: str | None, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -432,14 +437,15 @@ def test_radio_encoder_backend_keeps_decoder( pretrained_config.torch_dtype = torch.float32 config = ModelConfig( pretrained_config=pretrained_config, - attn_backend="TRTLLM", + attn_backend="FLASHINFER", multimodal_config=_multimodal_config(override), ) modeling_nemotron_h_multimodal.NemotronHVisionEncoder(config) - assert built["vision"][1]["vision_attn_backend"] == (override or "FLASHINFER") - assert config.attn_backend == "TRTLLM" + # The RADIO default is TRTLLM regardless of the decoder backend. + assert built["vision"][1]["vision_attn_backend"] == (override or "TRTLLM") + assert config.attn_backend == "FLASHINFER" @pytest.mark.parametrize( From d36b959a484ba35d5da09ccfa1d22b78d16912cf Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:24:18 +0000 Subject: [PATCH 5/6] [None][fix] Explain encoder backend copy and hoist test imports Problem: Review asked why with_encoder_attention_backend sets attn_backend with object.__setattr__, and asked to move the function-local imports in the new Qwen-VL encoder backend tests to the module imports. Solution: Comment that the shallow copy keeps _frozen, so setattr raises on a config from from_pretrained, and that object.__setattr__ changes only the copy. Move the tests' PretrainedConfig, Qwen3VisionModelBase, Qwen2_5_VLModel, Qwen2VLModel, Qwen3MoeVLModel and _validate_encoder_attention_backend_support imports to the module imports and drop a duplicate MultimodalConfig import. Validation: The repo's Python pre-commit hooks (isort, yapf, autoflake, ruff, ruff format, codespell) pass on both files. On CPU, test_encoder_attention_backend.py passes 61/61 and test_modeling_qwen3vl.py matches the previous head case for case; the three changed tests pass. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 2 ++ .../_torch/modeling/test_modeling_qwen3vl.py | 19 +++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 41963d6ed1e6..42d1c2715062 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -543,6 +543,8 @@ def with_encoder_attention_backend(self, if backend == self.attn_backend: return self config = copy.copy(self) + # The copy keeps ``_frozen``, so ``setattr`` would raise on a config + # from ``from_pretrained``. ``object.__setattr__`` changes only the copy. object.__setattr__(config, "attn_backend", backend) return config diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py index 69bd17827eb0..c4babc343044 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py @@ -16,7 +16,7 @@ from PIL import Image from test_modeling_multimodal import MultimodalScenario, TestModelingMultimodal from test_modeling_qwen2vl import _make_vision_attention -from transformers import Qwen3VLConfig +from transformers import PretrainedConfig, Qwen3VLConfig from transformers import Qwen3VLForConditionalGeneration as HFQwen3VLForConditionalLM from utils.llm_data import llm_models_root @@ -28,13 +28,17 @@ normalize_vision_pixels, vision_normalization_from_processor, ) +from tensorrt_llm._torch.models.modeling_qwen2vl import Qwen2_5_VLModel, Qwen2VLModel from tensorrt_llm._torch.models.modeling_qwen3vl import ( Qwen3VisionModel, + Qwen3VisionModelBase, Qwen3VLInputProcessorBase, Qwen3VLModel, _triton_pos_embed_interpolate, ) +from tensorrt_llm._torch.models.modeling_qwen3vl_moe import Qwen3MoeVLModel from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import KeyType +from tensorrt_llm._torch.pyexecutor.model_loader import _validate_encoder_attention_backend_support from tensorrt_llm._utils import get_sm_version from tensorrt_llm.inputs.multimodal import MultimodalParams from tensorrt_llm.llmapi.llm_args import MultimodalConfig, MultimodalEncoderCudaGraphConfig @@ -2233,10 +2237,6 @@ def __init__(self, model_config: ModelConfig) -> None: [("TRTLLM", None), ("TRTLLM", "VANILLA"), ("FLASHINFER", "VANILLA")], ) def test_qwen_encoder_backend_preserves_decoder(decoder_backend: str, override: str | None) -> None: - from transformers import PretrainedConfig - - from tensorrt_llm._torch.models.modeling_qwen3vl import Qwen3VisionModelBase - config = ModelConfig( pretrained_config=PretrainedConfig( text_config=PretrainedConfig(dtype=torch.float32), @@ -2262,8 +2262,6 @@ def test_qwen_encoder_backend_preserves_decoder(decoder_backend: str, override: def test_qwen2_5_vl_encoder_override_drops_decoder_kv_cache_quant( override: str | None, monkeypatch: pytest.MonkeyPatch ) -> None: - from transformers import PretrainedConfig - monkeypatch.setattr(modeling_qwen2vl, "Qwen2_5_VisionModel", _RecordingVisionEncoder) pretrained_config = PretrainedConfig(vision_config=PretrainedConfig(num_heads=1)) pretrained_config.torch_dtype = torch.float32 @@ -2287,13 +2285,6 @@ def test_qwen2_5_vl_encoder_override_drops_decoder_kv_cache_quant( @pytest.mark.cpu_only def test_qwen_encoder_backend_declarations_preserve_fixed_hf_encoder() -> None: - from tensorrt_llm._torch.models.modeling_qwen2vl import Qwen2_5_VLModel, Qwen2VLModel - from tensorrt_llm._torch.models.modeling_qwen3vl_moe import Qwen3MoeVLModel - from tensorrt_llm._torch.pyexecutor.model_loader import ( - _validate_encoder_attention_backend_support, - ) - from tensorrt_llm.llmapi.llm_args import MultimodalConfig - public_backends = frozenset({"TRTLLM", "FLASHINFER", "VANILLA"}) for model_cls in (Qwen2_5_VLModel, Qwen3VLModel, Qwen3MoeVLModel): assert model_cls.supported_encoder_attention_backends == {"vision": public_backends} From 3040dc8ca108d30566f5d7297ed8b26340f827bc Mon Sep 17 00:00:00 2001 From: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:26:07 +0000 Subject: [PATCH 6/6] [None][fix] Keep FLASHINFER as the RADIO vision default Problem: This PR had changed the Nemotron-H RADIO vision default from FLASHINFER to TRTLLM to match open PR #21458, with its comment that FlashInfer's cuDNN prefill plan build can stall during CUDA graph replay. That hang matches the GIL deadlock reported in #20011 (one- model speculative decoding plus guided decoding), not a FlashInfer bug, and FlashInfer is faster for large images. Changing the default for every GPU is outside the scope of this PR. Solution: Keep main's FLASHINFER default for the RADIO vision encoder and drop the copied comment. VANILLA and TRTLLM stay selectable through encoder_attention_backend. Update the sentence under the docs table and test_radio_encoder_backend_keeps_decoder (TRTLLM decoder, FLASHINFER default, VANILLA and TRTLLM overrides). Validation: The repo's Python pre-commit hooks pass on the changed files. On CPU, test_encoder_attention_backend.py passes 61/61, and test_modeling_nemotron_h_multimodal.py matches the previous head and main case for case. With no override, the encoder is built with the same backend as on main. Signed-off-by: Yechan Kim <161688079+yechank-nvidia@users.noreply.github.com> --- docs/source/features/multi-modality.md | 2 +- .../_torch/models/modeling_nemotron_h_multimodal.py | 4 +--- .../multimodal/test_encoder_attention_backend.py | 10 +++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/source/features/multi-modality.md b/docs/source/features/multi-modality.md index 1d2b72fac3da..3ad9112e9bba 100644 --- a/docs/source/features/multi-modality.md +++ b/docs/source/features/multi-modality.md @@ -91,7 +91,7 @@ The table covers every registered multimodal model implementation; model aliases | `WhisperForConditionalGeneration` | `audio` | Not supported | -For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. The Nemotron-H RADIO vision encoder defaults to `TRTLLM`, and `VANILLA` and `FLASHINFER` remain selectable. +For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. The Nemotron-H RADIO vision encoder keeps its `FLASHINFER` default, and `VANILLA` and `TRTLLM` remain selectable. Supported entries use TensorRT-LLM's `Attention` and apply encoder overrides explicitly. Other encoders keep their fixed implementation, including Hugging Face attention, direct SDPA, and custom attention paths. Overrides for entries marked "Not supported" fail during initialization, even if the requested value is `VANILLA`. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. With `disable_mm_encoder=True`, encoders that the model skips do not use the setting, so text-only runs can reuse a serving configuration. diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index 2c7664b47e49..1ecd2d012922 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -667,9 +667,7 @@ def __init__(self, model_config: ModelConfig[transformers.PretrainedConfig]): self.vision_model = RADIOVisionModel( vision_model_config, disable_quantization=True, - # FlashInfer's cuDNN ragged-prefill plan build for a new image shape can stall - # indefinitely while the language model replays CUDA graphs, freezing the server. - vision_attn_backend=model_config.get_encoder_attention_backend("vision", "TRTLLM"), + vision_attn_backend=model_config.get_encoder_attention_backend("vision", "FLASHINFER"), encoder_cuda_graph_config=encoder_cuda_graph_config, ) diff --git a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py index 775003854f5c..d0e452928578 100644 --- a/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py +++ b/tests/unittest/_torch/multimodal/test_encoder_attention_backend.py @@ -412,7 +412,7 @@ def test_pixtral_encoder_backend_keeps_decoder( assert config.attn_backend == "FLASHINFER" -@pytest.mark.parametrize("override", [None, "VANILLA", "FLASHINFER"]) +@pytest.mark.parametrize("override", [None, "VANILLA", "TRTLLM"]) def test_radio_encoder_backend_keeps_decoder( override: str | None, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -437,15 +437,15 @@ def test_radio_encoder_backend_keeps_decoder( pretrained_config.torch_dtype = torch.float32 config = ModelConfig( pretrained_config=pretrained_config, - attn_backend="FLASHINFER", + attn_backend="TRTLLM", multimodal_config=_multimodal_config(override), ) modeling_nemotron_h_multimodal.NemotronHVisionEncoder(config) - # The RADIO default is TRTLLM regardless of the decoder backend. - assert built["vision"][1]["vision_attn_backend"] == (override or "TRTLLM") - assert config.attn_backend == "FLASHINFER" + # The RADIO default is FLASHINFER regardless of the decoder backend. + assert built["vision"][1]["vision_attn_backend"] == (override or "FLASHINFER") + assert config.attn_backend == "TRTLLM" @pytest.mark.parametrize(