diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 62192567483e..f255fd55040d 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -1003,6 +1003,9 @@ def _ensure_provider_imported(self, model_type: str) -> None: MULTIMODAL_MODEL_TYPE_TO_MODULE module_name = MULTIMODAL_MODEL_TYPE_TO_MODULE.get(model_type) if module_name is None: + # Not a model type of the zoo: a modeling_v2 family may register + # it (`register_architecture_input_processor`). + _import_modeling_v2_families() return full_name = f"tensorrt_llm._torch.models.{module_name}" try: @@ -1027,6 +1030,7 @@ def _ensure_all_providers_imported(self) -> None: MULTIMODAL_MODEL_TYPE_TO_MODULE for model_type in MULTIMODAL_MODEL_TYPE_TO_MODULE: self._ensure_provider_imported(model_type) + _import_modeling_v2_families() def remove_placeholder_metadata(self, model_type: str): self._ensure_provider_imported(model_type) @@ -1139,6 +1143,10 @@ class InputProcessorRegistry: def __init__(self) -> None: self._input_processors_cls_by_model_type: Dict[ Type[nn.Module], Type[InputProcessor]] = {} + # Processors of architectures no model class of the zoo implements, + # keyed by `config.architectures[0]`. + self._input_processors_cls_by_architecture: Dict[ + str, Type[InputProcessor]] = {} INPUT_PROCESSOR_REGISTRY = InputProcessorRegistry() @@ -1216,6 +1224,57 @@ def wrapper(model_cls: N) -> N: return wrapper +def register_architecture_input_processor( + architecture: str, model_type: str, + placeholder_metadata: MultimodalPlaceholderMetadata): + """Register the decorated input processor for an architecture by name. + + This is for an architecture that no model class of the zoo implements, + such as one only a modeling_v2 target serves. `architecture` is the + checkpoint's `config.architectures[0]`; `create_input_processor` falls + back to these registrations when `get_model_architecture` finds no model + class for it. `model_type` and `placeholder_metadata` register the prompt + placeholders, as `register_input_processor` does. + + A modeling_v2 family registers its processor when its routing module is + imported, which `registered_routes` does; the lookups here import every + family first. + """ + if placeholder_metadata is None: + raise ValueError( + f"A valid placeholder_metadata must be provided but got {placeholder_metadata}" + ) + + def wrapper(processor_cls: Type[InputProcessor]) -> Type[InputProcessor]: + INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_architecture[ + architecture] = processor_cls + MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata( + model_type, + placeholder_metadata, + registrant_module=processor_cls.__module__) + processor_cls._registered_model_type = model_type + return processor_cls + + return wrapper + + +def _import_modeling_v2_families() -> None: + """Import every modeling_v2 family, which registers their processors.""" + # Function-local: the families' processors import this module. + from tensorrt_llm._torch.modeling_v2._router_index import registered_routes + registered_routes() + + +def _architecture_input_processor( + architecture: Optional[str]) -> Optional[Type[InputProcessor]]: + """The processor registered for `architecture` by name, or None.""" + if architecture is None: + return None + _import_modeling_v2_families() + return INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_architecture.get( + architecture) + + def _warn_prompt_token_cache_ignored(processor_name: str) -> None: logger.warning_once( f"prompt_token_cache_config is ignored: {processor_name} does not " @@ -1297,9 +1356,13 @@ def create_input_processor( input_processor_cls = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type \ .get(model_cls) except (RuntimeError, ConfigInputError, - UnsupportedError): # unregistered model - logger.info("Unregistered model, using DefaultInputProcessor") - input_processor_cls = None + UnsupportedError): # no model class of the zoo + model_cls = None + architectures = getattr(config, "architectures", None) + input_processor_cls = _architecture_input_processor( + architectures[0] if architectures else None) + if input_processor_cls is None: + logger.info("Unregistered model, using DefaultInputProcessor") if input_processor_cls is not None: if "vision_gpu_normalize" in kwargs: kwargs["vision_gpu_normalize"] = kwargs[ diff --git a/tests/unittest/inputs/test_architecture_input_processor.py b/tests/unittest/inputs/test_architecture_input_processor.py new file mode 100644 index 000000000000..cbd92b664c79 --- /dev/null +++ b/tests/unittest/inputs/test_architecture_input_processor.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests of input processors registered by architecture name. + +An architecture that no model class of the zoo implements, such as one only a modeling_v2 +target serves, registers its processor and prompt placeholders with +`register_architecture_input_processor`. `create_input_processor` and the placeholder lookups +find them after importing the modeling_v2 families, which register theirs on import. +""" + +import subprocess +import sys +import textwrap +from contextlib import nullcontext +from types import SimpleNamespace + +import pytest + +import tensorrt_llm._torch.model_config as model_config_mod +import tensorrt_llm._torch.models as models_mod +import tensorrt_llm.inputs.registry as registry_mod +from tensorrt_llm._errors import UnsupportedError +from tensorrt_llm.inputs.registry import ( + INPUT_PROCESSOR_REGISTRY, + MULTIMODAL_PLACEHOLDER_REGISTRY, + DefaultInputProcessor, + MultimodalPlaceholderMetadata, + create_input_processor, + register_architecture_input_processor, +) + +pytestmark = pytest.mark.cpu_only + +_ARCHITECTURE = "FakeForConditionalGeneration" +_MODEL_TYPE = "fake_architecture_mm" +_METADATA = MultimodalPlaceholderMetadata(placeholder_map={"image": ""}) + + +class _ModelClass: + """A model class of the zoo, as `get_model_architecture` resolves one.""" + + +class _RecordingInputProcessor: + """An input processor that records its constructor arguments.""" + + def __init__(self, model_path, config, tokenizer, **kwargs) -> None: + self.model_path = model_path + self.config = config + self.tokenizer = tokenizer + self.kwargs = kwargs + + +class _ArchitectureInputProcessor(_RecordingInputProcessor): + """The processor registered for `_ARCHITECTURE` by name.""" + + +@pytest.fixture +def registries(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Empty registration tables, a checkpoint config of `_ARCHITECTURE`, and family imports. + + Each import of the modeling_v2 families is recorded and imports nothing here. + """ + monkeypatch.setattr(INPUT_PROCESSOR_REGISTRY, "_input_processors_cls_by_architecture", {}) + monkeypatch.setattr( + MULTIMODAL_PLACEHOLDER_REGISTRY, "_multimodal_placeholder_by_model_type", {} + ) + imports: list[str] = [] + monkeypatch.setattr( + registry_mod, "_import_modeling_v2_families", lambda: imports.append("families") + ) + config = SimpleNamespace(architectures=[_ARCHITECTURE]) + monkeypatch.setattr( + model_config_mod.ModelConfig, + "from_pretrained", + classmethod(lambda _cls, *_args, **_kwargs: SimpleNamespace(pretrained_config=config)), + ) + monkeypatch.setattr(model_config_mod, "hf_remote_code_lock", nullcontext) + return imports + + +def _no_model_class(monkeypatch: pytest.MonkeyPatch) -> None: + def get_model_architecture(config): + raise UnsupportedError(f"Unknown model architecture: {config.architectures[0]}") + + monkeypatch.setattr(models_mod, "get_model_architecture", get_model_architecture) + + +def _register() -> None: + register_architecture_input_processor( + _ARCHITECTURE, model_type=_MODEL_TYPE, placeholder_metadata=_METADATA + )(_ArchitectureInputProcessor) + + +def test_the_registration_records_the_processor_and_its_placeholders(registries): + _register() + table = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_architecture + assert table == {_ARCHITECTURE: _ArchitectureInputProcessor} + assert MULTIMODAL_PLACEHOLDER_REGISTRY.get_placeholder_metadata(_MODEL_TYPE) is _METADATA + assert _ArchitectureInputProcessor._registered_model_type == _MODEL_TYPE + + +def test_the_registration_requires_placeholder_metadata(): + with pytest.raises(ValueError, match="placeholder_metadata"): + register_architecture_input_processor( + _ARCHITECTURE, model_type=_MODEL_TYPE, placeholder_metadata=None + ) + + +def test_an_architecture_without_a_model_class_gets_its_registered_processor( + registries, monkeypatch +): + _no_model_class(monkeypatch) + _register() + tokenizer = object() + + processor = create_input_processor("/ckpt", tokenizer) + + assert type(processor) is _ArchitectureInputProcessor + assert processor.model_path == "/ckpt" + assert processor.tokenizer is tokenizer + assert registries == ["families"] + + +def test_the_lookup_imports_the_families_that_register_the_processor(registries, monkeypatch): + _no_model_class(monkeypatch) + monkeypatch.setattr(registry_mod, "_import_modeling_v2_families", _register) + + assert type(create_input_processor("/ckpt", object())) is _ArchitectureInputProcessor + + +def test_an_architecture_registered_by_no_one_gets_the_default_processor(registries, monkeypatch): + _no_model_class(monkeypatch) + tokenizer = object() + + processor = create_input_processor("/ckpt", tokenizer) + + assert type(processor) is DefaultInputProcessor + assert processor.tokenizer is tokenizer + + +def test_a_model_class_processor_takes_precedence(registries, monkeypatch): + """With a model class of the zoo, its registration decides. + + The modeling_v2 families are not imported then. + """ + _register() + monkeypatch.setattr( + models_mod, "get_model_architecture", lambda config: (_ModelClass, _ARCHITECTURE) + ) + monkeypatch.setitem( + INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type, + _ModelClass, + _RecordingInputProcessor, + ) + + assert type(create_input_processor("/ckpt", object())) is _RecordingInputProcessor + assert registries == [] + + +def test_a_placeholder_lookup_imports_the_families_for_a_model_type_the_zoo_lacks( + registries, monkeypatch +): + monkeypatch.setattr(registry_mod, "_import_modeling_v2_families", _register) + + assert MULTIMODAL_PLACEHOLDER_REGISTRY.is_valid(_MODEL_TYPE, "image") + assert MULTIMODAL_PLACEHOLDER_REGISTRY.get_placeholder(_MODEL_TYPE, "image") == "" + + +def test_listing_the_model_types_imports_the_families(registries, monkeypatch): + monkeypatch.setattr(registry_mod, "_import_modeling_v2_families", _register) + monkeypatch.setattr( + MULTIMODAL_PLACEHOLDER_REGISTRY, "_ensure_provider_imported", lambda _: None + ) + + assert _MODEL_TYPE in MULTIMODAL_PLACEHOLDER_REGISTRY.get_registered_model_types() + + +def test_importing_the_families_imports_no_model_of_the_zoo(): + """Importing the modeling_v2 families loads no model of the zoo. + + The placeholder lookups import the families for any model type the zoo lacks, text-only ones + included. The zoo's own index and base modules may load: every placeholder lookup reads the + index. + """ + script = textwrap.dedent("""\ + import sys + from tensorrt_llm.inputs import registry + from tensorrt_llm._torch.models import _arch_index as index + + registry._import_modeling_v2_families() + models = { + f"tensorrt_llm._torch.models.{name}" + for table in (index.MODEL_ARCH_TO_MODULE, index.MULTIMODAL_MODEL_TYPE_TO_MODULE) + for name in table.values() + } + loaded = sorted(models & set(sys.modules)) + assert not loaded, f"importing the modeling_v2 families loaded {loaded}" + print("FAMILIES-OK") + """) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=300 + ) + assert result.returncode == 0 and "FAMILIES-OK" in result.stdout, result.stderr