Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tensorrt_llm/_torch/compilation/piecewise_optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def _piecewise_boundary_ops():
"gdn_custom_op_inplace",
"mamba2_custom_op_inplace",
"minimax_m3_attn_custom_op_inplace",
"modeling_v2_m3_prefill_attention",
]
return [
op for op in (get_optional_trtllm_op(op_name) for op_name in op_names)
Expand Down
3 changes: 3 additions & 0 deletions tensorrt_llm/_torch/compilation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,9 @@ def inplace_info():
"minimax_m3_attn_custom_op_inplace": {
1: "output"
},
"modeling_v2_m3_prefill_attention": {
1: "output"
},
# The ordinary outputs are compact Q/index-Q; the next
# three outputs of auto_functionalized are the mutated paged caches.
"minimax_m3_fused_sparse_qkv_producer": {
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import torch
from torch import nn

from tensorrt_llm._torch.custom_ops import inplace_slice_copy

if TYPE_CHECKING:
from tensorrt_llm._torch.model_config import ModelConfig

Expand Down Expand Up @@ -115,8 +117,11 @@ def _pinned_mxfp8_quantize(state: PinnedModelingSupport, /, x: Any) -> Any:


def _split_chunk(state: ModelingSupport, /, rows: int, num_chunks: int) -> List[int]:
"""The MoE scheduler's even split: the first ``rows % num_chunks`` chunks take one more row."""
base, extra = divmod(rows, num_chunks)
"""The MoE scheduler's even split: the first ``rows % num_chunks`` chunks take one more row.

Floor division and modulo (rather than ``divmod``) keep the split traceable on a symbolic row count.
"""
base, extra = rows // num_chunks, rows % num_chunks
return [base + 1] * extra + [base] * (num_chunks - extra)


Expand Down Expand Up @@ -194,6 +199,39 @@ def _fp8_index_qk_norm_rope(
).flatten(1)


def _prepare_prefill_eagle3_capture(spec_metadata) -> Callable[..., None]:
"""Check live rows before the core traces a fixed-size capture copy."""
if spec_metadata.layers_to_capture:
destination = spec_metadata.hidden_states
assert destination is not None
live_rows = spec_metadata.num_capture_tokens
if live_rows > destination.shape[0]:
raise RuntimeError(
f"EAGLE3 hidden-state capture needs {live_rows} rows, got "
f"buffer capacity {destination.shape[0]}"
)
return _capture_prefill_eagle3_hidden_states


def _capture_prefill_eagle3_hidden_states(spec_metadata, layer_id, hidden_states, residual):
if not torch.compiler.is_compiling():
spec_metadata.maybe_capture_hidden_states(layer_id, hidden_states, residual)
return

destination = spec_metadata.hidden_states
assert destination is not None
for index, capture_layer in enumerate(spec_metadata.layers_to_capture):
if capture_layer == layer_id:
# The draft reads only the live prefix. Copy the bucket up to buffer capacity.
rows = destination.shape[0]
source = hidden_states[:rows]
if residual is not None:
source = source + residual[:rows]
width = spec_metadata.hidden_size
inplace_slice_copy(destination, source, index * width, (index + 1) * width)
return


def _eagle3_capture(
state: ModelingSupport,
/,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
flashinfer_mxfp8_autotune_active,
flashinfer_mxfp8_decode_graph_capture_active,
)
from tensorrt_llm._torch.utils import is_torch_compiling


class _Mxfp8GemmState(nn.Module):
Expand All @@ -27,6 +28,17 @@ class _Mxfp8GemmState(nn.Module):
def __init__(self) -> None:
super().__init__()
self.quant_method = MXFP8LinearMethod()
method = self.quant_method
# Compiled prefill bakes its dispatch into the graph, so it reads only state fixed here: never the warmup's
# tuning flags, whose flips would invalidate the traced graph. The autotuned native GEMM keeps profiling in
# the engine's tuning pass and uses the registered tactics after it.
self.compiled_mode = MxMode(
decode_graph=False,
autotune=False,
tuned_quantize_gemm=False,
flashinfer=method.backend == "flashinfer",
native_tuning=method.use_cutlass,
)


@dataclass(frozen=True)
Expand All @@ -39,12 +51,12 @@ class MxMode:
bool # per-bucket tuned quantize + GEMM (decode graphs, tune_mxfp8_backends)
)
flashinfer: bool # FlashInfer's MXFP8 GEMM on the quantized activation
native_tuning: (
bool # the native GEMM through the autotuner (warmup, before its tactics are registered)
)
native_tuning: bool # the native GEMM through the autotuner (warmup, and compiled prefill)


def _mx_mode(state: _Mxfp8GemmState) -> MxMode:
if is_torch_compiling():
return state.compiled_mode
method = state.quant_method
decode_graph = flashinfer_mxfp8_decode_graph_capture_active()
autotune = flashinfer_mxfp8_autotune_active()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Keep request-dependent M3 attention outside captured prefill segments."""

from collections.abc import Callable
from dataclasses import dataclass
from functools import partial
from weakref import ReferenceType, ref

import torch
from torch import nn

from tensorrt_llm._torch.utils import get_model_extra_attrs


@dataclass
class _PrefillCore:
core: ReferenceType[nn.Module]
producer: Callable[..., tuple[torch.Tensor, torch.Tensor] | None]
attention: Callable[..., None]
indexer: Callable[..., torch.Tensor]


_CORES: dict[str, _PrefillCore] = {}


def _forget_core(key: str, reference: ReferenceType[nn.Module]) -> None:
_CORES.pop(key, None)


def register_prefill_core(
core: nn.Module,
producer: Callable[..., tuple[torch.Tensor, torch.Tensor] | None],
attention: Callable[..., None],
indexer: Callable[..., torch.Tensor],
) -> str:
"""Register ``core``'s live-row attention callbacks and return the key the boundary takes.

The registry holds ``core`` weakly, so releasing the model drops its entry.
"""
key = f"m3_prefill_{id(core)}"
_CORES[key] = _PrefillCore(ref(core, partial(_forget_core, key)), producer, attention, indexer)
return key


def run_prefill_attention(
packed: torch.Tensor | None,
position_ids: torch.Tensor | None,
q: torch.Tensor | None,
k: torch.Tensor | None,
v: torch.Tensor | None,
idx_qk: torch.Tensor | None,
output: torch.Tensor,
core_key: str,
layer: int,
) -> None:
"""Run one prefill layer's attention over the live rows of the current attention metadata.

A sparse layer with the five-way projection passes ``packed``: the horizontal producer writes K, V and index-K
to the paged caches and returns Q and index-Q. An unfused sparse layer passes ``idx_qk``, whose index-K is
written before attention. A dense layer passes only ``q``, ``k`` and ``v``. Pad rows of ``output`` are left as
allocated.

Breakable prefill graphs call this function directly at every replay. Compiled prefill calls it through
``prefill_attention``, a piecewise boundary op.
"""
entry = _CORES[core_key]
core = entry.core()
if core is None:
raise RuntimeError("MiniMax-M3 prefill core is no longer available")
metadata = get_model_extra_attrs()["attention_metadata"]()
live = int(metadata.num_tokens)
if position_ids is not None:
position_ids = position_ids[..., :live]
idx_q = None
if packed is not None:
horizontal = entry.producer(
core,
layer,
packed[:live],
position_ids,
metadata.kv_cache_manager,
metadata.msa_idx_k_cache,
metadata.msa_out_cache_loc,
)
if horizontal is None:
raise RuntimeError(
f"MiniMax-M3 sparse layer {layer}: no compatible horizontal producer"
)
q, idx_q = horizontal
k = v = None
elif idx_qk is not None:
# Insert index-K where attention reads it, using this step's live cache slots.
idx_q = entry.indexer(
core,
layer,
idx_qk[:live],
position_ids,
metadata.msa_idx_k_cache,
metadata.msa_out_cache_loc,
)
entry.attention(core, layer, q, k, v, idx_q, metadata, output)


@torch.library.custom_op("trtllm::modeling_v2_m3_prefill_attention", mutates_args=("output",))
def prefill_attention(
packed: torch.Tensor | None,
position_ids: torch.Tensor | None,
q: torch.Tensor | None,
k: torch.Tensor | None,
v: torch.Tensor | None,
idx_qk: torch.Tensor | None,
output: torch.Tensor,
core_key: str,
layer: int,
) -> None:
"""``run_prefill_attention`` as an op the piecewise optimizer splits on (it is not captured)."""
run_prefill_attention(packed, position_ids, q, k, v, idx_qk, output, core_key, layer)


@prefill_attention.register_fake
def _prefill_attention_fake(
packed: torch.Tensor | None,
position_ids: torch.Tensor | None,
q: torch.Tensor | None,
k: torch.Tensor | None,
v: torch.Tensor | None,
idx_qk: torch.Tensor | None,
output: torch.Tensor,
core_key: str,
layer: int,
) -> None:
pass
24 changes: 20 additions & 4 deletions tensorrt_llm/_torch/modeling_v2/models/minimax_m3/reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
from contextlib import AbstractContextManager
from typing import Callable, Optional, Protocol

import torch
Comment thread
2ez4bz marked this conversation as resolved.
Resolved


class _Logger(Protocol):
info: Callable[[str], None]
Expand Down Expand Up @@ -38,6 +40,9 @@ def require_mode(state: ReportingTarget, /) -> bool:
def _log_once(
state: ReportingTarget, /, level: Callable[[str], None], path: str, reason: str, message: str
) -> None:
"""Log eager diagnostics once; compiled regions do not lock or log."""
if torch.compiler.is_compiling():
Comment thread
2ez4bz marked this conversation as resolved.
Resolved
return
key = (path, reason)
with state._lock:
if key in state._logged:
Expand All @@ -47,7 +52,9 @@ def _log_once(


def missed(state: ReportingTarget, /, path: str, rows: Optional[int], reason: str) -> None:
"""``path`` covers this step but cannot run: raise under require, else log once; the caller falls back."""
"""Raise under require, including during tracing; otherwise report only on eager steps."""
if not state._require[0] and torch.compiler.is_compiling():
return
at = "" if rows is None else f" at {rows} rows"
if state._require[0]:
raise state.FastPathUnavailable(
Expand All @@ -67,14 +74,23 @@ def capture_missed(state: ReportingTarget, /, path: str, rows: Optional[int], wh


def taken(state: ReportingTarget, /, path: str, rows: Optional[int], what: str = "") -> None:
"""``path`` ran on this rank at ``rows``: one rank-tagged info line per (path, rows, what), the engagement line
every GEN rank prints for each fast path it takes (``what``: an optional detail, e.g. the projection)."""
"""Report eager engagement once per (path, rows, what) on each rank.

Compiled regions do not report. ``what`` is an optional detail such as the projection. Hot paths call this on
every step, so a line already logged returns before the rank query, the lock and the message formatting.
"""
if torch.compiler.is_compiling():
return
reason = f"taken: {rows}: {what}" if what else f"taken: {rows}"
# Lock-free read: keys are only added under the lock, and a stale miss falls through to _log_once's locked check.
if (path, reason) in state._logged:
return
at = "" if rows is None else f" at {rows} rows"
detail = f" ({what})" if what else ""
state._log_once(
state.logger.info,
path,
f"taken: {rows}" + (f": {what}" if what else ""),
reason,
f"[M3 dispatch] rank {state.mpi_rank()}: {path} taken{at}{detail}",
)

Expand Down
2 changes: 2 additions & 0 deletions tests/integration/test_lists/test-db/l0_cpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ l0_cpu:
backend: generic
orchestrator: mpi
tests:
- unittest/_torch/modeling_v2/models/minimax_m3/test_reporting.py
- unittest/_torch/modeling_v2/models/minimax_m3/test_prefill_fullgraph.py
- unittest/_torch/modeling_v2/models/minimax_m3/test_shared_kernel_contracts.py
- unittest/_torch/modeling_v2/models/minimax_m3/test_collective_launcher_cleanup.py
- unittest/_torch/modeling_v2/models/minimax_m3/test_modeling_v2_eagle3_worker_api.py
Expand Down
68 changes: 68 additions & 0 deletions tests/unittest/_torch/modeling_v2/models/minimax_m3/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import pytest
import torch
from transformers import PretrainedConfig

from tensorrt_llm._torch.model_config import ModelConfig
from tensorrt_llm.llmapi.llm_args import MiniMaxM3SparseAttentionConfig
from tensorrt_llm.mapping import Mapping
from tensorrt_llm.models.modeling_utils import QuantConfig
from tensorrt_llm.quantization.mode import QuantAlgo


@pytest.fixture
def make_m3_config():
def make_config(
tp: int,
*,
moe_layers: tuple[int, ...] = (0, 1),
sparse_layers: tuple[int, ...] = (0, 1),
head_dim: int = 128,
fused_projection: bool = False,
) -> ModelConfig[PretrainedConfig]:
return ModelConfig(
pretrained_config=PretrainedConfig(
dtype=torch.bfloat16,
vocab_size=128,
hidden_size=128,
num_hidden_layers=len(moe_layers),
num_attention_heads=tp,
num_key_value_heads=tp,
head_dim=head_dim,
max_position_embeddings=32,
dense_intermediate_size=256,
shared_intermediate_size=256,
intermediate_size=256,
num_local_experts=8,
num_experts_per_tok=2,
n_shared_experts=1,
rms_norm_eps=1e-6,
moe_layer_freq=moe_layers,
sparse_attention_config={
"sparse_attention_freq": sparse_layers,
"sparse_num_index_heads": tp,
"sparse_index_dim": 128,
},
),
mapping=Mapping(world_size=tp, tp_size=tp),
quant_config=QuantConfig(kv_cache_quant_algo=QuantAlgo.FP8),
sparse_attention_config=MiniMaxM3SparseAttentionConfig(
implementation="msa",
indexer_kv_dtype="fp8",
sparse_num_index_heads=tp,
fuse_qkv_index_projection=fused_projection,
),
allreduce_strategy="NCCL",
max_num_tokens=32,
)

return make_config


@pytest.fixture
def make_cuda_m3_config(make_m3_config):
if not torch.cuda.is_available():
pytest.skip("MiniMax-M3 constructors allocate CUDA resources")
return make_m3_config
Loading
Loading