Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions tensorrt_llm/_torch/pyexecutor/_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@
PeftCacheManager, ResourceManager,
ResourceManagerType)
from .runtime_hooks import runtime_hooks_of
from .sampler import EarlyStopWithMMResult, TorchSampler
from .sampler import TorchSampler
from .sampler.penalties import PenaltyStore
from .scheduler import (BindCapacityScheduler, BindMicroBatchScheduler,
KVCacheV2Scheduler, MultimodalEagerEncoderScheduler,
Expand Down Expand Up @@ -6012,7 +6012,6 @@ def instantiate_sampler(
*,
max_batch_size: int,
max_beam_width: int,
mm_encoder_only: bool,
speculative_config: SpeculativeConfig,
max_num_sequences: Optional[int] = None,
):
Expand Down Expand Up @@ -6052,16 +6051,6 @@ def instantiate_sampler(
)
return sampler

if mm_encoder_only:
# NOTE: handle model outputs specially for mm encoder executor/engine.
# Only export results on ranks whose responses reach the frontend
# (see PyExecutor._enqueue_responses): rank 0, plus the attention-DP
# ranks gathered into it. A CUDA IPC handle exported on any other rank
# is never opened, so PyTorch would never release its allocation.
returns_responses = mapping.rank == 0 or (mapping.enable_attention_dp
and 0 in mapping.tp_group)
return EarlyStopWithMMResult(return_mm_results=returns_responses,
max_num_tokens=engine.max_num_tokens)
return TorchSampler(sampler_args)


Expand Down
199 changes: 165 additions & 34 deletions tensorrt_llm/_torch/pyexecutor/engine/model_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,19 @@
from tensorrt_llm._torch.nccl_window_tensor_scope import install_eager_nccl_window_tensor_scopes
from tensorrt_llm._torch.pyexecutor.engine.cuda_graph import plan_cuda_graphs
from tensorrt_llm._torch.pyexecutor.engine.model_call import ModelCaller
from tensorrt_llm._torch.pyexecutor.engine.multimodal import (
setup_mm_encoder_attn_metadata,
setup_mm_encoder_cuda_graphs,
)
from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.runner import PrefillCompiledModel
from tensorrt_llm._torch.pyexecutor.engine.runners.encoder import EncoderRunner
from tensorrt_llm._torch.pyexecutor.engine.runners.interface import ScheduledModelRunner
from tensorrt_llm._torch.pyexecutor.engine.runners.mm_encoder import MultimodalEncoderRunner
from tensorrt_llm._torch.pyexecutor.layerwise_nvtx_marker import LayerwiseNvtxMarker
from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader
from tensorrt_llm._torch.utils import set_torch_compiling
from tensorrt_llm._utils import mpi_disabled
from tensorrt_llm.inputs.registry import InputProcessor, create_input_processor
from tensorrt_llm.llmapi.llm_args import (
DecodingBaseConfig,
PrefillCudaGraphBackend,
Expand Down Expand Up @@ -455,6 +462,23 @@ def create_model_loader(
)


def create_engine_input_processor(model_path: str, llm_args: TorchLlmArgs) -> InputProcessor:
"""Create the checkpoint's input processor with the multimodal options the engine applies."""
input_processor_kwargs = {}
video_pruning_rate = llm_args.multimodal_config.video_pruning_rate
if video_pruning_rate is not None:
input_processor_kwargs["video_pruning_rate"] = video_pruning_rate
if llm_args.multimodal_config.vision_gpu_normalize:
input_processor_kwargs["vision_gpu_normalize"] = True
return create_input_processor(
model_path,
tokenizer=None,
checkpoint_format=llm_args.checkpoint_format,
trust_remote_code=llm_args.trust_remote_code,
**input_processor_kwargs,
)


def prepare_loaded_model(model: torch.nn.Module, llm_args: TorchLlmArgs) -> None:
"""Give the model its extra attributes and the requested NVTX markers."""
# AutoModelForCausalLM sets extra_attrs; a model passed in through
Expand Down Expand Up @@ -492,38 +516,35 @@ def resolve_attention_backend(
class EncoderBuild:
"""An encoder runner built from a checkpoint, with the loader that owns its weights."""

runner: EncoderRunner
runner: ScheduledModelRunner
model_loader: ModelLoader
model_config: ModelConfig
max_batch_size: int
max_num_tokens: int
max_seq_len: int


@dataclass(frozen=True)
class _LoadedEncoder:
model: torch.nn.Module
model_loader: ModelLoader
max_beam_width: int
max_batch_size: int
max_num_tokens: int
max_seq_len: int
prefill_capture_num_tokens: list[int]


@contextmanager
def build_encoder_runner(
def _load_encoder(
llm_args: TorchLlmArgs,
*,
model_path: str,
checkpoint_loader: BaseCheckpointLoader,
mapping: Mapping,
dist: Distributed,
) -> Iterator[EncoderBuild]:
"""Load and compile the checkpoint's model and build an encoder runner for it.

The loader is cleaned up when building or the ``with`` body raises, so the executor
constructed in the body takes the loader over only once it exists.

Args:
llm_args: The resolved arguments.
model_path: The checkpoint directory.
checkpoint_loader: The loader for the checkpoint's format.
mapping: The parallel mapping of this rank.
dist: The communicator across the model's ranks.

Yields:
The runner with its loader, model config and resolved capacity.
"""
) -> Iterator[_LoadedEncoder]:
"""Load the checkpoint's model and resolve its capacity; the loader is cleaned up on a raise."""
init_process_setup(mapping, dist)
configure_autotuner_nvmmh(llm_args)
max_beam_width, max_num_tokens, max_seq_len, max_batch_size = llm_args.get_runtime_sizes()
Expand All @@ -536,7 +557,7 @@ def build_encoder_runner(
)
try:
# Encoder executors run without a MoE load balancer: encode_only has one rank,
# and scheduled pooling rejects the setting.
# and scheduled encoders reject the setting.
model, _ = model_loader.load(checkpoint_dir=model_path, checkpoint_loader=checkpoint_loader)
prepare_loaded_model(model, llm_args)
max_seq_len = resolve_max_seq_len(model, max_seq_len)
Expand All @@ -551,35 +572,145 @@ def build_encoder_runner(
max_seq_len=max_seq_len,
decode_tokens_per_request=1,
)
yield _LoadedEncoder(
model=model,
model_loader=model_loader,
max_beam_width=max_beam_width,
max_batch_size=max_batch_size,
max_num_tokens=max_num_tokens,
max_seq_len=max_seq_len,
prefill_capture_num_tokens=plan.prefill_num_tokens,
)
except BaseException:
model_loader.cleanup()
raise


@contextmanager
def build_encoder_runner(
llm_args: TorchLlmArgs,
*,
model_path: str,
checkpoint_loader: BaseCheckpointLoader,
mapping: Mapping,
dist: Distributed,
) -> Iterator[EncoderBuild]:
"""Load and compile the checkpoint's model and build an encoder runner for it.

The loader is cleaned up when building or the ``with`` body raises, so the executor
constructed in the body takes the loader over only once it exists.

Args:
llm_args: The resolved arguments.
model_path: The checkpoint directory.
checkpoint_loader: The loader for the checkpoint's format.
mapping: The parallel mapping of this rank.
dist: The communicator across the model's ranks.

Yields:
The runner with its loader, model config and resolved capacity.
"""
with _load_encoder(
llm_args,
model_path=model_path,
checkpoint_loader=checkpoint_loader,
mapping=mapping,
dist=dist,
) as loaded:
compiled = compile_model(
model,
loaded.model,
llm_args=llm_args,
mapping=mapping,
dist=dist,
max_num_tokens=max_num_tokens,
prefill_capture_num_tokens=plan.prefill_num_tokens,
max_num_tokens=loaded.max_num_tokens,
prefill_capture_num_tokens=loaded.prefill_capture_num_tokens,
)
runner = EncoderRunner.create(
compiled.model,
llm_args=llm_args,
mapping=mapping,
dist=dist,
max_batch_size=max_batch_size,
max_num_tokens=max_num_tokens,
max_seq_len=max_seq_len,
max_beam_width=max_beam_width,
max_batch_size=loaded.max_batch_size,
max_num_tokens=loaded.max_num_tokens,
max_seq_len=loaded.max_seq_len,
max_beam_width=loaded.max_beam_width,
model_caller=compiled.caller,
attention_backend=resolve_attention_backend(compiled.model, llm_args),
attention_runtime_features=AttentionRuntimeFeatures(),
)
yield EncoderBuild(
runner=runner,
model_loader=model_loader,
model_config=model.model_config,
max_batch_size=max_batch_size,
max_num_tokens=max_num_tokens,
max_seq_len=max_seq_len,
model_loader=loaded.model_loader,
model_config=loaded.model.model_config,
max_batch_size=loaded.max_batch_size,
max_num_tokens=loaded.max_num_tokens,
max_seq_len=loaded.max_seq_len,
)


@contextmanager
def build_mm_encoder_runner(
llm_args: TorchLlmArgs,
*,
model_path: str,
checkpoint_loader: BaseCheckpointLoader,
mapping: Mapping,
dist: Distributed,
) -> Iterator[EncoderBuild]:
"""Load the checkpoint's vision encoder, set up its towers and build a multimodal encoder runner.

The towers size their attention metadata, then capture their block CUDA graphs after
compilation. The loader is cleaned up as in ``build_encoder_runner``.

Args:
llm_args: The resolved arguments, with ``mm_encoder_only`` set.
model_path: The checkpoint directory.
checkpoint_loader: The loader for the checkpoint's format.
mapping: The parallel mapping of this rank.
dist: The communicator across the model's ranks.

Yields:
The runner with its loader, model config and resolved capacity.
"""
with _load_encoder(
llm_args,
model_path=model_path,
checkpoint_loader=checkpoint_loader,
mapping=mapping,
dist=dist,
) as loaded:
# The towers count patches, which a merged item needs more of than prompt tokens,
# so they keep the configured budget rather than the clamped one.
encoder_max_num_tokens = next(
budget
for budget in (
llm_args.encoder_max_num_tokens,
llm_args.max_num_tokens,
loaded.max_num_tokens,
)
if budget is not None
)
input_processor = create_engine_input_processor(model_path, llm_args)
setup_mm_encoder_attn_metadata(loaded.model, input_processor, encoder_max_num_tokens, None)
compiled = compile_model(
loaded.model,
llm_args=llm_args,
mapping=mapping,
dist=dist,
max_num_tokens=loaded.max_num_tokens,
prefill_capture_num_tokens=loaded.prefill_capture_num_tokens,
)
mm_config = llm_args.multimodal_config
setup_mm_encoder_cuda_graphs(
compiled.model,
mm_config.encoder_cuda_graph if mm_config is not None else None,
max_num_tokens=encoder_max_num_tokens,
)
yield EncoderBuild(
runner=MultimodalEncoderRunner.create(compiled.model),
model_loader=loaded.model_loader,
model_config=loaded.model.model_config,
max_batch_size=loaded.max_batch_size,
max_num_tokens=loaded.max_num_tokens,
max_seq_len=loaded.max_seq_len,
)
except BaseException:
model_loader.cleanup()
raise
11 changes: 1 addition & 10 deletions tensorrt_llm/_torch/pyexecutor/engine/runners/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,9 @@

from torch import nn

from tensorrt_llm.llmapi.llm_args import TorchLlmArgs

from .decoder import DecoderRunner
from .encoder import EncoderRunner
from .encoder_decoder import EncoderDecoderRunner
from .mm_encoder import MultimodalEncoderRunner

if TYPE_CHECKING:
from .interface import ModelRunner
Expand All @@ -27,14 +24,8 @@
]


def resolve_runner_type(
model: nn.Module,
llm_args: TorchLlmArgs,
) -> type[ModelRunner]:
def resolve_runner_type(model: nn.Module) -> type[ModelRunner]:
"""Resolve the startup runner class without constructing its dependencies."""
if llm_args.mm_encoder_only:
return MultimodalEncoderRunner

if model.model_config.is_encoder_decoder:
return EncoderDecoderRunner

Expand Down
Loading
Loading