From 92db50471a2d443a94fbf52b8bb42b22fb3a627d Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:39:45 -0800 Subject: [PATCH 01/51] [None][feat] modeling_v2 qwen4_exp: family operations at the FP8 checkpoint's shapes Family operations of qwen4_exp gain what the Qwen3.8-Flash-Next-FP8 target needs, without changing what the NVFP4 target launches: - hc_lowrank_act, renorm_moe_routing_op, fp8_quantize_1x128: certified cells under CUDA-graph capture at the FP8 target's decode batches. - qsa_paged_index_scores: opt-in prefill query-tile path and wide split (prefill_query_tile=, wide_split=; both default off, so the NVFP4 target's launches are unchanged), one-row-tile variant, no batch specialization of the group kernel, memory-clobbering PDL wait. - qsa_topk_expand_decode: opt-in ordered wide launch (ordered_wide=, default off). - ple_gate_conv_inject_norm, qwen4_exp_hc_up_gate_mean, qwen4_exp_hc_inject_norm, spec_rollback_commit: new parameters, defaulted to the current behavior. - jit_variants: warms the Triton variants a family op launches, so no variant compiles in a served step. - fused_sigmoid_mul_inplace, rms_norm_gated_token_major: direct_launch= (default off), the op's own launch without the custom-op dispatch, same results. - The family operations the FP8 target calls as they are (causal_conv1d_update, gdn_decode_rows_gated_norm, gdn_prefill_conv_post, bf16_to_fp32) list the FP8 target's shapes in their tests. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../activation/fused_sigmoid_mul_inplace.py | 79 ++++- .../ops/attention/qsa_paged_index_scores.py | 183 +++++++++++- .../attention/qsa_paged_sparse_gqa_nvfp4.py | 8 +- .../ops/attention/qsa_topk_expand_decode.py | 80 ++++- .../ops/attention/spec_rollback_commit.py | 69 +++-- .../ops/hc/qwen4_exp_hc_inject_norm.py | 37 ++- .../ops/hc/qwen4_exp_hc_lowrank_act.py | 26 +- .../ops/hc/qwen4_exp_hc_up_gate_mean.py | 126 ++++++-- .../models/qwen4_exp/ops/jit_variants.py | 53 ++++ .../ops/moe/renorm_moe_routing_op.py | 28 +- .../ops/norm/rms_norm_gated_token_major.py | 53 +++- .../ops/ple/ple_gate_conv_inject_norm.py | 274 +++++++++++++++--- .../qwen4_exp/ops/ple/ple_ngram_embed.py | 4 +- .../ops/quantization/fp8_quantize_1x128.py | 25 +- ...t_modeling_v2_fused_sigmoid_mul_inplace.py | 48 ++- ...test_modeling_v2_qsa_paged_index_scores.py | 102 +++++++ ...test_modeling_v2_qsa_topk_expand_decode.py | 23 ++ ...st_modeling_v2_qwen4_exp_hc_lowrank_act.py | 41 ++- ...t_modeling_v2_qwen4_exp_hc_up_gate_mean.py | 21 +- .../test_modeling_v2_renorm_moe_routing_op.py | 71 ++++- ...v2_qwen4_exp_rms_norm_gated_token_major.py | 52 +++- ...t_modeling_v2_ple_gate_conv_inject_norm.py | 84 ++++++ .../ple/test_modeling_v2_ple_ngram_embed.py | 24 ++ .../test_modeling_v2_bf16_to_fp32.py | 20 +- .../test_modeling_v2_fp8_quantize_1x128.py | 69 ++++- .../test_modeling_v2_causal_conv1d_update.py | 28 +- ..._modeling_v2_gdn_decode_rows_gated_norm.py | 48 ++- .../test_modeling_v2_gdn_prefill_conv_post.py | 41 ++- 28 files changed, 1530 insertions(+), 187 deletions(-) create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py index 641965755582..816cf9c801f6 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py @@ -4,21 +4,62 @@ The gated attention's output gate, between the attention core and o_proj, in one Triton kernel (`torch.ops.trtllm.fused_sigmoid_mul_inplace`). Called by the -qwen4_exp target (`qwen3_8_flash_next_nvfp4__sm_103__default`) at every layout -- its -trunk, gate off the fused input-projection row, and its MTP layer, gate as the strided -half of the q projection. +qwen4_exp targets at every layout -- the trunk, gate off the fused input-projection row, +and the MTP layer, gate as the strided half of the q projection. """ from __future__ import annotations import torch +import triton # Registered by the module that defines it: after `import # tensorrt_llm._torch.custom_ops` alone, torch.ops.trtllm has no # `fused_sigmoid_mul_inplace` (measured). -import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate # noqa: F401 +import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate as _gate_mod from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +def _launch(attention_output: torch.Tensor, gate: torch.Tensor) -> bool: + """The registered op's launch, made here when its preconditions hold: the same kernel, + grid, block, warps and PDL policy, without the custom-op dispatch around it. False + (nothing launched) when they do not, for the op to refuse loudly.""" + if attention_output.dim() != 2 or attention_output.stride(-1) != 1 or gate.stride(-1) != 1: + return False + num_tokens, hidden = attention_output.shape + if gate.dim() == 3: + if gate.shape[0] != num_tokens or gate.shape[1] * gate.shape[2] != hidden: + return False + head_dim, gate_stride_head = gate.shape[2], gate.stride(1) + elif gate.dim() == 2 and gate.shape == attention_output.shape: + head_dim, gate_stride_head = hidden, hidden + else: + return False + if num_tokens == 0: + return True + block = min(triton.next_power_of_2(hidden), 1024 if num_tokens < 1024 else 2048) + pdl = ( + dict(LAUNCH_WITH_PDL=True, launch_pdl=True) + if _gate_mod._sigmoid_mul_pdl(attention_output.device.index) + else {} + ) + _gate_mod._fused_sigmoid_mul_kernel[(num_tokens, triton.cdiv(hidden, block))]( + attention_output, + attention_output, + gate, + attention_output.stride(0), + attention_output.stride(0), + gate.stride(0), + gate_stride_head, + hidden_size=hidden, + head_dim=head_dim, + block_size=block, + num_warps=4, + **pdl, + ) + return True + + #: The checkpoint's gated attention: 24 q heads of 256, q projection [T, 24, 512]. _CKPT = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") @@ -135,7 +176,9 @@ class FusedSigmoidMulInplace(OpWrapper): the running device at the first call of each (hidden, head_dim, block) specialization, no autotuner: grid `(T, cdiv(hidden, block))`, `block = min(next_pow2(hidden), 1024 if T < 1024 else 2048)`, 4 warps. The - cells ran on sm_103. + cells ran on sm_103. With `direct_launch=True` (the FP8 target), `_launch` makes + that same launch itself where the preconditions below hold and skips only the + custom-op dispatch; otherwise the op is called and refuses the call. Call sites, each the attention output gate between the attention core and o_proj: @@ -192,7 +235,10 @@ class FusedSigmoidMulInplace(OpWrapper): gate) at 24 heads (`[T, 24, 512]`, hidden 6144) at 1 token, and at 6 heads (a compact `[T, 6, 512]`, hidden 1536) at 1, 128, 512, 1024 and 8192 rows; CUDA-graph capture and replay of the 1536-wide fused-row form at 8 rows and - of the 6-head 3-D gate at 1, 128 and 1024 rows. + of the 6-head 3-D gate at 1, 128 and 1024 rows. With `direct_launch` (the FP8 + target's cells in the test): the 2-D window at hidden 3072 (row stride 7424, + gate column 3584) at 1 and 8192 rows, the 24-head 3-D gate at 128 and 8192 + rows, and graph capture of the 1536-wide fused-row form. Hard limits T * row stride token offset computed in int64 boundary 2^63 not probed @@ -202,19 +248,24 @@ class FusedSigmoidMulInplace(OpWrapper): Not certified: 0 tokens; rows past 8192; the 3-D gate at a token stride other than its own q_proj's (heads x 2 x head_dim, e.g. a gate inside a - fused q|k|v row); the 24-head strided q_proj half above 1 token (a dep4 - rank's MTP layer: drafter 1..32 rows and its verify); a strided - `attention_output`; a gate dtype other than `attention_output.dtype`; - fp16/fp32 operands; hidden sizes other than 1536 and 6144; CUDA-graph - capture of the 6144-wide forms; torch.compile. + fused q|k|v row); the 24-head strided q_proj half above 1 token without + `direct_launch`; a strided `attention_output`; a gate dtype other than + `attention_output.dtype`; fp16/fp32 operands; hidden sizes other than 1536, + 3072 and 6144; CUDA-graph capture of the 6144-wide forms; torch.compile. """ - def __call__(self, attention_output: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: - torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) + def __call__( + self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False + ) -> torch.Tensor: + if not (direct_launch and _launch(attention_output, gate)): + torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) return attention_output - def reference(self, attention_output: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + def reference( + self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False + ) -> torch.Tensor: """What `attention_output` holds after the call; neither input is mutated. + `direct_launch` does not change it. `gate.reshape(T, hidden)` is the per-head `(t, i // D, i % D)` read, for the 2-D and 3-D forms alike. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py index 4084ca92ddd9..5ecd5f289901 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py @@ -8,13 +8,19 @@ import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + warm_variants, +) + # Approved exception: QSA's selection path is Triton called as Python, with no # torch.ops boundary. The engine's own sparse hook (`select_qsa_paged_tokens` # in `_torch/attention/backends/sparse/qsa/module.py`) calls this same callable. +from tensorrt_llm._torch.attention.backends.sparse.qsa import kernels as _qsa_kernels from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( triton_qsa_paged_index_scores as _triton_qsa_paged_index_scores, ) -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber _INTEGER_DTYPES = (torch.int32, torch.int64) @@ -31,6 +37,10 @@ #: 1024 tiles of a 262143-position window; wider batches stream several requests at once. _SPLIT_PROGRAMS_PER_SM = 16 _SPLIT_PROGRAMS_PER_SM_ONE_TILE = 32 +#: With `wide_split`, from this many rows four times as many programs, so the longest windows' +#: chains of column tiles do not set the launch's time. The scores do not depend on the split. +_SPLIT_WIDE_MIN_ROWS = 32 +_SPLIT_PROGRAMS_PER_SM_WIDE = 64 _SPLIT_NUM_WARPS = 2 _SM_COUNT: dict = {} @@ -81,7 +91,8 @@ def _qsa_index_scores_visible_split_kernel( row of their request in the row tile, and different requests run side by side. """ if USE_PDL: - tl.extra.cuda.gdc_wait() + # The memory clobber keeps loads of the predecessor's outputs below the wait. + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() row_base = tl.program_id(0) * ROWS_PER_TILE group_slot = tl.program_id(1) @@ -210,6 +221,7 @@ def _qsa_index_scores_visible_split( programs_per_sm: int | None = None, num_warps: int = _SPLIT_NUM_WARPS, num_stages: int = 2, + wide: bool = False, ) -> torch.Tensor: """One launch of the visible-split kernel after the callable's own checks.""" if index_cache.ndim != 4 or index_cache.shape[2] != 1: @@ -242,7 +254,11 @@ def _qsa_index_scores_visible_split( row_tiles = triton.cdiv(rows, rows_per_tile) if programs_per_sm is None: programs_per_sm = ( - _SPLIT_PROGRAMS_PER_SM_ONE_TILE if row_tiles == 1 else _SPLIT_PROGRAMS_PER_SM + _SPLIT_PROGRAMS_PER_SM_ONE_TILE + if row_tiles == 1 + else _SPLIT_PROGRAMS_PER_SM_WIDE + if wide and rows >= _SPLIT_WIDE_MIN_ROWS + else _SPLIT_PROGRAMS_PER_SM ) # Every (row tile, group slot) pair gets `split` programs; slots past a tile's request # count exit at once. @@ -276,7 +292,9 @@ def _qsa_index_scores_visible_split( TOKENS_PER_BLOCK=tokens_per_block, COMPRESS_RATIO=compress_ratio, MAX_COMPRESSED_BLOCKS=max_compressed_blocks, - ROWS_PER_TILE=rows_per_tile, + # One row tile masks its rows by `num_rows` alone: compile it at the full tile so every + # row count up to `query_tile` shares one variant. + ROWS_PER_TILE=query_tile if row_tiles == 1 else rows_per_tile, QUERY_TILE=query_tile, BLOCK_H=dot_width, BLOCK_N=block_n, @@ -288,7 +306,11 @@ def _qsa_index_scores_visible_split( return output -@triton.jit +# The row count follows the batch; the metadata views begin at any row of a mixed step. +@triton.jit( + do_not_specialize=["rows"], + do_not_specialize_on_alignment=["query_positions", "request_indices"], +) def _qsa_index_scores_group_kernel( q, index_cache, @@ -327,7 +349,8 @@ def _qsa_index_scores_group_kernel( the row-wise kernel's, bit for bit.""" BLOCK_H: tl.constexpr = BLOCK_G * NUM_INDEX_HEADS if USE_PDL: - tl.extra.cuda.gdc_wait() + # The memory clobber keeps loads of the predecessor's outputs below the wait. + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() group = tl.program_id(0) slots = tl.arange(0, BLOCK_G) @@ -453,6 +476,96 @@ def _takes_request_groups( ) +#: With `prefill_query_tile`, a prefill-sized, visible-only context-row call at 4 index heads of +#: 128 runs the engine's query-tiled kernel at a deeper launch than the engine's (32 rows a dot +#: tile, 8 column tiles a program): each gathered key tile serves twice the rows. Bitwise the +#: engine launch's scores (a row's dot and head sum do not depend on the tile shape). +_PREFILL_QUERY_TILE = 64 +_PREFILL_TILES_PER_PROGRAM = 16 +_PREFILL_BLOCK_N = 64 +_PREFILL_HEADS = 4 +_PREFILL_HEAD_DIM = 128 + + +def _takes_prefill_query_tile( + q: torch.Tensor, only_visible_blocks: bool, context_rows: bool, prefetch: bool +) -> bool: + """A context-row call the deeper query-tiled launch serves (`_PREFILL_QUERY_TILE`).""" + return ( + context_rows + and only_visible_blocks + and not prefetch + and q.ndim == 3 + and q.shape[0] > _DECODE_ROW_LIMIT + and q.shape[1] == _PREFILL_HEADS + and q.shape[2] == _PREFILL_HEAD_DIM + and q.dtype == torch.bfloat16 + ) + + +def _qsa_index_scores_prefill_query_tile( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, +) -> torch.Tensor: + """The engine's query-tiled kernel at the deeper launch. Same contract as the engine's + launcher with `context_rows` and `only_visible_blocks`: columns at and past a row's + visible bound are unspecified.""" + rows = q.shape[0] + if index_cache.ndim != 4 or index_cache.shape[2] != 1 or index_cache.shape[3] != q.shape[2]: + raise ValueError("QSA index cache must have shape [pages, tokens, 1, dim]") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA index scoring requires a two-dimensional int32 block table") + if q.dtype != index_cache.dtype: + raise ValueError("QSA index query and cache dtypes must match") + if query_positions.shape != (rows,) or request_indices.shape != (rows,): + raise ValueError("QSA index-scoring metadata must have one entry per row") + max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + output = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + grid = ( + triton.cdiv(rows, _PREFILL_QUERY_TILE), + triton.cdiv(max_blocks, _PREFILL_BLOCK_N * _PREFILL_TILES_PER_PROGRAM), + ) + _qsa_kernels._qsa_paged_index_scores_qtile_kernel[grid]( + q, + index_cache, + block_table, + query_positions, + request_indices, + output, + q.shape[2] ** -0.5, + rows, + q.stride(0), + q.stride(1), + q.stride(2), + index_cache.stride(0), + index_cache.stride(1), + index_cache.stride(2), + index_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + output.stride(0), + output.stride(1), + NUM_INDEX_HEADS=q.shape[1], + INDEX_HEAD_DIM=q.shape[2], + BLOCK_D=q.shape[2], + TOKENS_PER_BLOCK=tokens_per_block, + COMPRESS_RATIO=compress_ratio, + MAX_COMPRESSED_BLOCKS=max_blocks, + QUERY_TILE=_PREFILL_QUERY_TILE, + BLOCK_H=_PREFILL_QUERY_TILE * q.shape[1], + BLOCK_N=_PREFILL_BLOCK_N, + TILES_PER_PROGRAM=_PREFILL_TILES_PER_PROGRAM, + ONLY_VISIBLE_BLOCKS=True, + num_warps=4, + ) + return output + + def _qsa_index_scores_request_groups( q: torch.Tensor, index_cache: torch.Tensor, @@ -494,7 +607,7 @@ def _qsa_index_scores_request_groups( groups = triton.cdiv(rows, rows_per_request) tiles = 1 if groups <= 1 else 2 if groups <= 4 else 4 grid = (groups, triton.cdiv(max_compressed_blocks, _GROUP_BLOCK_N * tiles)) - _qsa_index_scores_group_kernel[grid]( + args = ( q, index_cache, block_table, @@ -513,6 +626,8 @@ def _qsa_index_scores_request_groups( block_table.stride(1), output.stride(0), output.stride(1), + ) + constexprs = dict( NUM_INDEX_HEADS=num_index_heads, INDEX_HEAD_DIM=index_head_dim, BLOCK_D=max(16, triton.next_power_of_2(index_head_dim)), @@ -528,6 +643,28 @@ def _qsa_index_scores_request_groups( num_warps=4, launch_pdl=use_pdl, ) + # `TILES_PER_PROGRAM` follows the request count: compile every depth at the first call. + warm_variants( + _qsa_index_scores_group_kernel, + ( + args[8:18], # the strides + q.dtype, + q.data_ptr() % 16 == 0, + query_positions.dtype, + request_indices.dtype, + num_index_heads, + index_head_dim, + tokens_per_block, + compress_ratio, + max_compressed_blocks, + rows_per_request, + use_pdl, + ), + args, + [{"TILES_PER_PROGRAM": t} for t in (1, 2, 4)], + **constexprs, + ) + _qsa_index_scores_group_kernel[grid](*args, **constexprs) return output @@ -645,9 +782,13 @@ class QsaPagedIndexScores(OpWrapper): from row 0 (a verify step's `predicted_tokens_per_seq`): with `only_visible_blocks`, no `prefetch` and 2, 4 or 8 index heads, the entry's request-group kernel shares one key gather across each run (a run whose rows name different requests is gathered - row by row). Each cell names the kernel it was - measured on. The request-group kernel is bitwise the row-wise one; the - visible-split and query-tiled kernels are not. + row by row). With `prefill_query_tile`, a visible-only context-row call of more + than 256 rows at 4 bf16 index heads of 128 runs the query-tiled kernel at a deeper + launch, bitwise the engine launch's scores. With `wide_split`, a visible-split call + of `_SPLIT_WIDE_MIN_ROWS` rows or more past one row tile takes more programs an SM, + bitwise the narrow split. Both default off. Each cell names the kernel it was + measured on. The request-group kernel is bitwise the row-wise one; the visible-split + and query-tiled kernels are not. Fusion boundary: the table walk, the gather, the dots, relu, head sum, scale and causal bound are inside. Producing `q` (norm, RoPE), writing and @@ -1071,6 +1212,8 @@ def __call__( use_pdl: bool = False, prefetch: bool = False, rows_per_request: int = 1, + prefill_query_tile: bool = False, + wide_split: bool = False, ) -> torch.Tensor: if rows_per_request < 1: raise ValueError("QSA index scoring requires rows_per_request >= 1") @@ -1084,6 +1227,7 @@ def __call__( tokens_per_block, compress_ratio, use_pdl, + wide=wide_split, ) if _takes_request_groups(q, only_visible_blocks, context_rows, prefetch, rows_per_request): return _qsa_index_scores_request_groups( @@ -1097,6 +1241,18 @@ def __call__( use_pdl, rows_per_request, ) + if prefill_query_tile and _takes_prefill_query_tile( + q, only_visible_blocks, context_rows, prefetch + ): + return _qsa_index_scores_prefill_query_tile( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) return _triton_qsa_paged_index_scores( q=q, index_cache=index_cache, @@ -1125,10 +1281,13 @@ def reference( use_pdl: bool = False, prefetch: bool = False, rows_per_request: int = 1, + prefill_query_tile: bool = False, + wide_split: bool = False, ) -> torch.Tensor: """fp32 scores, `-inf` past each row's bound -- or NaN there, meaning - "unspecified", when `only_visible_blocks` is True. `context_rows` and - `rows_per_request` are ignored: they pick a kernel, not a result. + "unspecified", when `only_visible_blocks` is True. `context_rows`, + `rows_per_request`, `prefill_query_tile` and `wide_split` are ignored: they pick a + kernel or a launch, not a result. Walked in row chunks only to bound the gathered keys, which reach 8.7 GiB in fp32 at 8191 rows x 2080 columns. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py index 19aebdf34d6d..5dd8b2f41c3e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py @@ -10,11 +10,13 @@ import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + # Approved exception (as for `qsa_paged_sparse_gqa`): QSA's sparse path has no torch.ops # surface; its row-count PDL policy is reached as Python. The split-K buckets are this # entry's own: an NVFP4 tile needs 16 tokens where the bf16 one takes 64. from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import _qsa_pdl_enabled -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Base-2 online softmax: scores are scaled by log2(e) once. _LOG2_E = tl.constexpr(1.4426950408889634) @@ -499,7 +501,7 @@ def _nvfp4_sparse_gqa_splitk_kernel( kv_head = tl.program_id(1) split = tl.program_id(2) if USE_PDL: - tl.extra.cuda.gdc_wait() + gdc_wait_with_memory_clobber() request = tl.load(request_indices + row).to(tl.int64) valid_request = (request >= 0) & (request < num_requests) safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) @@ -618,7 +620,7 @@ def _nvfp4_merge_splitk_kernel( row = tl.program_id(0).to(tl.int64) head = tl.program_id(1) if USE_PDL: - tl.extra.cuda.gdc_wait() + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() split_offsets = tl.arange(0, BLOCK_SPLITS) dim_offsets = tl.arange(0, HEAD_DIM) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py index 9f3364ac5eff..b663fbe2d10e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py @@ -427,14 +427,14 @@ def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): mSel[row, col] = val -def _cache_slabs(width: int, threads: int, ctas: int) -> int: +def _cache_slabs(width: int, threads: int, ctas: int, extra_words: int = 0) -> int: """Slabs of the wide kernel's window cache: as many of the table's as the shared memory left after the fixed state and the candidate list hold with `ctas` CTAs an SM (11 of 4096 columns at one CTA of 1024 threads: a window of up to 45056 blocks, 180224 tokens, is read from global once).""" slab = KPT * threads per_cta = min(_SMEM_LIMIT, _SMEM_SM // ctas - _SMEM_RESERVED) - room = per_cta - (_WIDE_FIXED_WORDS + 2 * slab) * 4 - _WIDE_PAD + room = per_cta - (_WIDE_FIXED_WORDS + 2 * slab + extra_words) * 4 - _WIDE_PAD return max(0, min(-(-width // slab), room // (slab * 4))) @@ -471,6 +471,7 @@ def __init__( threads=_WIDE_THREADS, ctas=1, prefilter=True, + order=False, ): self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( width, @@ -486,12 +487,20 @@ def __init__( self.warps = threads // 32 self.slab = KPT * threads self.ctas = ctas - self.cache_slabs = _cache_slabs(width, threads, ctas) + # `order`: CTA b takes the row of the b-th longest window (ties to the lower row), so the + # CTAs launched last get the shortest windows. Every row's selection is the same. + self.order = bool(order) + self.cache_slabs = _cache_slabs( + width, threads, ctas, (_ORDER_MAX_ROWS + 32) if self.order else 0 + ) self.per_warp = -(-block_topk // self.warps) self.prefilter = bool(prefilter) and self.per_warp <= 32 def smem_bytes(self) -> int: - return (self.cache_slabs * self.slab + 2 * self.slab + _WIDE_FIXED_WORDS) * 4 + _WIDE_PAD + order_words = (_ORDER_MAX_ROWS + 32) if self.order else 0 + return ( + self.cache_slabs * self.slab + 2 * self.slab + _WIDE_FIXED_WORDS + order_words + ) * 4 + _WIDE_PAD @cute.jit def __call__( @@ -888,6 +897,37 @@ def kernel( k: cutlass.Constexpr = self.k cr: cutlass.Constexpr = self.cr smem = cutlass.utils.SmemAllocator() + if const_expr(self.order): + # 0a. This CTA's row: the one whose window ranks `block_idx` by length, from the + # host-filled row ends (read before the wait). Four threads rank a row, each over + # every fourth row; the ranks are a permutation, so every row has one CTA. + sEnd = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((_ORDER_MAX_ROWS,), stride=(1,)), byte_alignment=128 + ) + sRow = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + nrows = cute.size(mEnd) + for t0 in cutlass.range(0, nrows, nt): + if t0 + tidx < nrows: + sEnd[t0 + tidx] = mEnd[t0 + tidx] + cute.arch.sync_threads() + q4 = tidx % 4 + for r0 in cutlass.range(0, nrows, nt // 4): + tt = r0 + tidx // 4 + cnt = cutlass.Int32(0) + if tt < nrows: + e = sEnd[tt] + for u in cutlass.range(q4, nrows, 4): + eu = sEnd[u] + if (eu > e) | ((eu == e) & (u < tt)): + cnt = cnt + 1 + cnt = cnt + cute.arch.shuffle_sync_bfly(cnt, offset=1) + cnt = cnt + cute.arch.shuffle_sync_bfly(cnt, offset=2) + if (tt < nrows) & (q4 == 0) & (cnt == row): + sRow[0] = tt + cute.arch.sync_threads() + row = sRow[0] sK = None if const_expr(self.cache_slabs > 0): # `KPT` consecutive keys (mode 0) of each thread-slab column (mode 1). @@ -1128,14 +1168,22 @@ def kernel( _SMS: dict = {} -def _wide_config(rows: int, device) -> tuple: - """The wide kernel's `(threads, CTAs an SM, prefilter)` for a call of `rows` rows: one - 1024-thread CTA an SM (the largest window cache) while the rows fit one wave, two once - they do not, so one CTA's walks overlap the other's loads.""" +#: Rows the ordered launch ranks (its shared row-end table). +_ORDER_MAX_ROWS = 1024 + + +def _wide_config(rows: int, device, ordered: bool = False) -> tuple: + """The wide kernel's `(threads, CTAs an SM, prefilter, order)` for a call of `rows` rows: + one 1024-thread CTA an SM (the largest window cache) while the rows fit one wave, two once + they do not, so one CTA's walks overlap the other's loads. With `ordered`, past one wave and + up to `_ORDER_MAX_ROWS` rows: one CTA an SM with the rows ranked by window length, so the + second wave takes the shortest windows. The same selection either way.""" sms = _SMS.get(device.index) if sms is None: sms = _SMS[device.index] = torch.cuda.get_device_properties(device).multi_processor_count - return (_WIDE_THREADS, 1 if rows <= sms else 2, True) + if ordered and sms < rows <= _ORDER_MAX_ROWS: + return (_WIDE_THREADS, 1, True, True) + return (_WIDE_THREADS, 1 if rows <= sms else 2, True, False) _INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} @@ -1151,6 +1199,7 @@ def launch( token_topk, pdl=True, out=None, + ordered_wide=False, ): """The launch behind `QsaTopkExpandDecode.__call__`: writes `topk_indices` and returns the token selection `[rows, token_topk + compress_ratio - 1]` int32 (`out` when given).""" @@ -1174,7 +1223,7 @@ def launch( query_positions.dtype, sequence_lengths.dtype, pdl, - None if width <= MAX_WIDTH else _wide_config(rows, scores.device), + None if width <= MAX_WIDTH else _wide_config(rows, scores.device, ordered_wide), ) ptrs = ( row_ends.data_ptr(), @@ -1197,9 +1246,9 @@ def launch( if width <= MAX_WIDTH: kernel = _TopkExpandKernel(**geometry) else: - threads, ctas, prefilter = key[-1] + threads, ctas, prefilter, order = key[-1] kernel = _TopkExpandWideKernel( - **geometry, threads=threads, ctas=ctas, prefilter=prefilter + **geometry, threads=threads, ctas=ctas, prefilter=prefilter, order=order ) fn = _COMPILED[key] = cute.compile( kernel, @@ -1469,6 +1518,7 @@ def __call__( compress_ratio, token_topk, out=None, + ordered_wide=False, ): return launch( scores, @@ -1479,6 +1529,7 @@ def __call__( compress_ratio, token_topk, out=out, + ordered_wide=ordered_wide, ) def reference( @@ -1491,9 +1542,11 @@ def reference( compress_ratio, token_topk, out=None, + ordered_wide=False, ): """`(topk_indices_after, selection)`: a stable descending sort per row (ties to the - higher block), the k first in block order, then the expansion entry's reference.""" + higher block), the k first in block order, then the expansion entry's reference. + `ordered_wide` picks a launch, not a result.""" rows, _ = scores.shape k = topk_indices.shape[1] out = torch.full_like(topk_indices, -1) @@ -1525,6 +1578,7 @@ def is_valid( compress_ratio, token_topk, out=None, + ordered_wide=False, ): assert scores.dtype == torch.float32 and scores.stride(1) == 1, ( "fp32 scores, unit column stride" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py index 5a69fd373f72..9161717308b3 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py @@ -63,17 +63,21 @@ def _stage_kernel( STATE: tl.constexpr, STATE_BLOCK: tl.constexpr, BLOCK_C: tl.constexpr, + STAGE_CONV: tl.constexpr = True, ): i = tl.program_id(0) cb = tl.program_id(1) slot = tl.load(slots + i).to(tl.int64) - c = cb * BLOCK_C + tl.arange(0, BLOCK_C) - s = tl.arange(0, STATE_BLOCK) - m = (c < channels)[:, None] & (s < STATE)[None, :] - vals = tl.load(conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, mask=m) - tl.store( - conv_prev + (i.to(tl.int64) * channels + c[:, None]) * STATE + s[None, :], vals, mask=m - ) + if STAGE_CONV: + c = cb * BLOCK_C + tl.arange(0, BLOCK_C) + s = tl.arange(0, STATE_BLOCK) + m = (c < channels)[:, None] & (s < STATE)[None, :] + vals = tl.load( + conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, mask=m + ) + tl.store( + conv_prev + (i.to(tl.int64) * channels + c[:, None]) * STATE + s[None, :], vals, mask=m + ) if cb == 0: w = tl.arange(0, W_BLOCK) hist = tl.load(ngram_pool + slot * ngram_stride + w, mask=w < CTX) @@ -118,6 +122,7 @@ def _commit_kernel( STATE_BLOCK: tl.constexpr, BLOCK_C: tl.constexpr, ROW: tl.constexpr, + PREV_IN_POOL: tl.constexpr = False, ): i = tl.program_id(0) cb = tl.program_id(1) @@ -131,13 +136,22 @@ def _commit_kernel( j = a + s m = (c < channels)[:, None] & (s < STATE)[None, :] from_prev = (j < STATE)[None, :] - prev = tl.load( - conv_prev + (i64 * channels + c[:, None]) * STATE + j[None, :], mask=m & from_prev - ) + if PREV_IN_POOL: + # The pool row still holds the pre-step state: every read lands before any store. + prev = tl.load( + conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + j[None, :] * conv_s2, + mask=m & from_prev, + ) + else: + prev = tl.load( + conv_prev + (i64 * channels + c[:, None]) * STATE + j[None, :], mask=m & from_prev + ) xin = tl.load( conv_x + i64 * x_s0 + (j - STATE).to(tl.int64)[None, :] * x_s1 + c[:, None] * x_s2, mask=m & ~from_prev, ) + if PREV_IN_POOL: + tl.debug_barrier() tl.store( conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, tl.where(from_prev, prev, xin), @@ -219,16 +233,21 @@ class SpecRollbackStage(OpWrapper): int32 as the call site hands them, slots 0 and 1873 among those read. """ - def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): + def __call__(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True): + """`conv=False` stages the n-gram stream only and returns `(stream, None)`, for a caller + whose PLE launch left the conv pool rows unpublished (the commit then reads them in + place).""" n = slots.shape[0] ctx = ngram_pool.shape[1] channels, state = conv_pool.shape[1], conv_pool.shape[2] stream = torch.empty([n, ctx + p], dtype=ngram_pool.dtype, device=ngram_pool.device) - conv_prev = torch.empty( - [n, channels, state], dtype=conv_pool.dtype, device=conv_pool.device + conv_prev = ( + torch.empty([n, channels, state], dtype=conv_pool.dtype, device=conv_pool.device) + if conv + else None ) if n: - _stage_kernel[(n, triton.cdiv(channels, _BLOCK_C))]( + _stage_kernel[(n, triton.cdiv(channels, _BLOCK_C) if conv else 1)]( ngram_pool, ngram_pool.stride(0), conv_pool, @@ -238,7 +257,7 @@ def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): slots, ids, stream, - conv_prev, + conv_prev if conv else stream, channels, P=p, CTX=ctx, @@ -246,17 +265,18 @@ def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): STATE=state, STATE_BLOCK=_pow2(state), BLOCK_C=_BLOCK_C, + STAGE_CONV=bool(conv), num_warps=4, ) return stream, conv_prev - def reference(self, ngram_pool, conv_pool, slots, ids, p: int): + def reference(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True): n = slots.shape[0] slots = slots.to(torch.long) rows = torch.reshape(ids[: n * p], [n, p]).to(ngram_pool.dtype) - return torch.cat([ngram_pool[slots], rows], 1), conv_pool[slots] + return torch.cat([ngram_pool[slots], rows], 1), (conv_pool[slots] if conv else None) - def is_valid(self, ngram_pool, conv_pool, slots, ids, p: int) -> None: + def is_valid(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True) -> None: assert ngram_pool.dim() == 2 and ngram_pool.stride(1) == 1 assert ngram_pool.dtype == torch.int64 assert conv_pool.dim() == 3 @@ -335,6 +355,10 @@ def __call__( if not n: return blocks = max(triton.cdiv(channels, _BLOCK_C), index_old.shape[0]) + # `conv_prev` None: the conv pool rows still hold the pre-step state (read in place). + prev_in_pool = conv_prev is None + if prev_in_pool: + conv_prev = conv_pool _commit_kernel[(n, blocks)]( accepted, slots, @@ -371,6 +395,7 @@ def __call__( STATE_BLOCK=_pow2(state), BLOCK_C=_BLOCK_C, ROW=ROW, + PREV_IN_POOL=prev_in_pool, num_warps=4, ) @@ -404,6 +429,8 @@ def reference( """ n, p, _ = conv_x.shape context, state_len = ngram_pool.shape[1], conv_pool.shape[2] + if conv_prev is None: + conv_prev = conv_pool[slots.to(torch.long)] a = accepted.to(torch.long) req = torch.arange(n, device=a.device) # Window k of the unfold is the id history after k rows. @@ -454,11 +481,13 @@ def is_valid( f"stream must be the contiguous [n, context + p] = [{n}, {ngram_pool.shape[1] + p}] " f"spec_rollback_stage wrote; got {tuple(stream.shape)}" ) - assert conv_pool.shape[1] == channels and conv_prev.is_contiguous(), ( + assert conv_pool.shape[1] == channels and ( + conv_prev is None or conv_prev.is_contiguous() + ), ( "conv_prev must be contiguous (read as packed [n, C, state_len] rows), and conv_pool " "as wide as conv_x" ) - assert conv_prev.shape == (n, channels, conv_pool.shape[2]), ( + assert conv_prev is None or conv_prev.shape == (n, channels, conv_pool.shape[2]), ( f"conv_prev is {tuple(conv_prev.shape)}, expected {(n, channels, conv_pool.shape[2])}" ) assert position.dim() == 3 and position.shape[2] == 3, ( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py index 679479d54a4c..e7e1a88fae30 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py @@ -31,9 +31,13 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + warm_variants, +) from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Trigger dependents at kernel start (`griddepcontrol.launch_dependents`); a no-op #: unless the next kernel is launched with programmatic dependent launch. @@ -74,12 +78,16 @@ def _hc_inject_norm_kernel( TOPK: tl.constexpr, STREAMS_PER_PROGRAM: tl.constexpr, PDL_TRIGGER: tl.constexpr, + PDL_WAIT: tl.constexpr = False, ): if PDL_TRIGGER: # Release a programmatic-dependent-launch consumer (the decode mixer's # first projection) at once: it streams its static weights while this # kernel runs and waits for the grid to finish before reading `normed`. tl.extra.cuda.gdc_launch_dependents() + if PDL_WAIT: + # Every input may be the previous kernel's output: nothing is read before its grid ends. + gdc_wait_with_memory_clobber() # One program per `STREAMS_PER_PROGRAM` streams of a row; a stream's `HIDDEN` # columns are one RMS group. A row's programs are adjacent in launch order, so # the row's block output (and the combine's operands) come from HBM once and @@ -1000,7 +1008,7 @@ def __call__( # The fold gathers TOPK rows to form a row's block output: two streams a # program form it half as often (when hc_count is even). spp = 2 if top_k and hc_count % 2 == 0 else 1 - _hc_inject_norm_kernel[(rows * hc_count // spp,)]( + args = ( hyper_input, block_out if has_inject else hyper_input, inject if has_inject else hyper_input, @@ -1021,6 +1029,8 @@ def __call__( hc_out.stride(0), normed.stride(0), eps, + ) + constexprs = dict( HC_COUNT=hc_count, HIDDEN=hidden, BLOCK=block, @@ -1029,8 +1039,29 @@ def __call__( TOPK=top_k, STREAMS_PER_PROGRAM=spp, PDL_TRIGGER=_PDL_TRIGGER, - num_warps=(2 if rows >= _FEW_WARPS_MIN_ROWS else 8) if block >= 2048 else 4, ) + if block >= 2048: + # Both warp counts compile at a geometry's first call. + warps = 2 if rows >= _FEW_WARPS_MIN_ROWS else 8 + warm_variants( + _hc_inject_norm_kernel, + ( + args[10:19], + hyper_input.dtype, + hc_count, + hidden, + has_inject, + has_shared, + top_k, + spp, + ), + args, + [{"num_warps": 8}, {"num_warps": 2}], + **constexprs, + ) + else: + warps = 4 + _hc_inject_norm_kernel[(rows * hc_count // spp,)](*args, **constexprs, num_warps=warps) return hc_out, normed def reference( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py index 2e36fc757808..c058e449497f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py @@ -65,7 +65,8 @@ class Qwen4ExpHcLowrankAct(OpWrapper): ARCHS = frozenset({Arch.SM_103}) - # spec: rows, inject (True), hc_count (4), rank (320), logits_width (384), dtype (bf16). + # spec: rows, inject (True), hc_count (4), rank (320), logits_width (384), dtype (bf16), + # capture (False: eager; True: replayed from a captured CUDA graph over two steps' logits). # The default target's HC mixers run this form on the fresh fp32 `cublas_mm` output # over the first `hc_first_cols` = ceil((320 + 4) / 32) * 32 = 352 columns of the # first projection: hence the 352-wide cells. @@ -93,6 +94,23 @@ class Qwen4ExpHcLowrankAct(OpWrapper): why="default stack-final mixer (no inject): one decode token", spec=dict(rows=1, inject=False, logits_width=352), ), + # The FP8 unified target's HC mixers in its captured decode graph (batches 1..128). + Cell( + why="FP8 unified captured HC mixer, batch 2: 352-wide logits", + spec=dict(rows=2, logits_width=352, capture=True), + ), + Cell( + why="FP8 unified captured HC mixer, batch 72: 352-wide logits", + spec=dict(rows=72, logits_width=352, capture=True), + ), + Cell( + why="FP8 unified captured HC mixer, batch 120: 352-wide logits", + spec=dict(rows=120, logits_width=352, capture=True), + ), + Cell( + why="FP8 unified captured HC mixer, batch 128 (the largest): 352-wide", + spec=dict(rows=128, logits_width=352, capture=True), + ), ) note = """ @@ -107,6 +125,9 @@ class Qwen4ExpHcLowrankAct(OpWrapper): Preconditions: `logits` fp32, unit column stride, at least `rank (+ hc_count)` columns; `hc_count` a power of two. Graph-capturable. + CUDA-graph capture (the `capture` cells): the replay is bitwise the eager + call, and a second step's logits replay bitwise their own eager call. + Source: a Triton kernel JIT-compiled at first call for the running device; the cells ran on sm_103. One program per row with an int64 row offset; 2^31 elements is 6.1e6 rows of 352, far past the envelope. @@ -120,7 +141,8 @@ class Qwen4ExpHcLowrankAct(OpWrapper): Not certified: zero rows; rows past 8192; hc_count other than 4; rank other than 320; float16 outputs. The kernel has no row-count switch, so row counts between the cells take the same path, and the logits width enters - only as the row stride (128 rows is driven at 384 wide only). + only as the row stride. Not driven under capture: the form without the + injection, and the other captured batches. """ def __call__( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py index 5cd56ddc0fe8..bb3a664b6ee9 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py @@ -195,11 +195,22 @@ class HcUpGateMeanKernel: """ def __init__( - self, block_h: int, hc: int, hidden: int, rank: int, num_n_stage: int = 2, fp4: bool = False + self, + block_h: int, + hc: int, + hidden: int, + rank: int, + num_n_stage: int = 2, + fp4: bool = False, + wide: bool = False, ): assert block_h in (32, 64) and hidden % block_h == 0 and rank % 64 == 0 + assert not (wide and fp4), "the NVFP4 epilogue stores 32-column rows" self.bh, self.hc, self.hidden, self.rank = block_h, hc, hidden, rank self.fp4 = fp4 + # `wide`: eight epilogue warps; warps w and w + 4 read the same TMEM lane quarter and + # take the two 16-column halves of each stream's `block_h` columns. + self.wide = wide self.acc_dtype = cutlass.Float32 self.io_dtype = cutlass.BFloat16 self.mma_tiler = (128, hc * block_h, 1) @@ -211,6 +222,12 @@ def __init__( self.tma_warp_id = 5 self.nrm_warp_id = 6 self.threads_per_cta = 32 * 7 + if wide: + assert block_h == 32 + self.epi_cols = 16 + self.epilogue_warp_id = tuple(range(8)) + self.mma_warp_id, self.tma_warp_id, self.nrm_warp_id = 8, 9, 10 + self.threads_per_cta = 32 * 11 self.num_n_stage = num_n_stage self.epilog_sync_bar_id = 1 self.tmem_alloc_sync_bar_id = 2 @@ -235,13 +252,15 @@ def _setup(self, a_major, b_major): smem_capacity = utils.get_smem_capacity_in_bytes() # One normed stage is the 32-column slice of every stream for one tile half. self.n_box = (self.mma_tiler[0], self.epi_cols) - self.num_n_bytes = self.hc * self.n_box[0] * self.n_box[1] * 2 + # A normed stage holds `n_slots` boxes: every stream, with `wide` both halves of each. + self.n_slots = self.hc * (self.bh // self.epi_cols if self.wide else 1) + self.num_n_bytes = self.n_slots * self.n_box[0] * self.n_box[1] * 2 self.num_ab_stage = min( 8, (smem_capacity - 2048 - self.num_n_stage * self.num_n_bytes) // ab_bytes ) self.n_pre = min(self.num_ab_stage, self.rank // self.mma_tiler[2]) self.n_smem_layout_staged = utils.sm100.make_smem_layout_epi( - self.io_dtype, utils.LayoutEnum.ROW_MAJOR, self.n_box, self.num_n_stage * self.hc + self.io_dtype, utils.LayoutEnum.ROW_MAJOR, self.n_box, self.num_n_stage * self.n_slots ) self.a_smem_layout_staged = utils.sm100.make_smem_layout_a( tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage @@ -573,15 +592,27 @@ class SharedStorage: col_blocks = self.hidden // self.epi_cols while work_tile.is_valid_tile: crd = work_tile.tile_idx - for half in cutlass.range_constexpr(halves): + if cutlass.const_expr(self.wide): + # One stage a tile: box (half, stream) in slot half * C + stream. handle = n_producer.acquire_and_advance() - for s in cutlass.range_constexpr(self.hc): - cute.copy( - tma_atom_n, - tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], - tNsN[(None, handle.index * self.hc + s)], - tma_bar_ptr=handle.barrier, - ) + for half in cutlass.range_constexpr(halves): + for s in cutlass.range_constexpr(self.hc): + cute.copy( + tma_atom_n, + tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], + tNsN[(None, handle.index * self.n_slots + half * self.hc + s)], + tma_bar_ptr=handle.barrier, + ) + else: + for half in cutlass.range_constexpr(halves): + handle = n_producer.acquire_and_advance() + for s in cutlass.range_constexpr(self.hc): + cute.copy( + tma_atom_n, + tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], + tNsN[(None, handle.index * self.hc + s)], + tma_bar_ptr=handle.barrier, + ) tile_sched.advance_to_next_work() work_tile = tile_sched.get_current_work() n_producer.tail() @@ -634,7 +665,11 @@ class SharedStorage: ) tAcc_epi = cute.flat_divide(tCtAcc_base[((None, None), 0, 0, None)], epi_tile) tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) - thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + # With `wide`, warp w + 4 reads warp w's TMEM lanes (rows); it takes the other half. + etid = tidx + if cutlass.const_expr(self.wide): + etid = tidx % 128 + thr_copy_t2r = tiled_copy_t2r.get_slice(etid) tTR_tAcc_base = thr_copy_t2r.partition_S(tAcc_epi) tTR_sN = thr_copy_t2r.partition_D(sN) tTR_gX = thr_copy_t2r.partition_D( @@ -676,7 +711,35 @@ class SharedStorage: row_ok = cX_t[(0, 0, 0, 0)][0] < m acc_pipeline.consumer_wait(acc_consumer_state) - for half in cutlass.range_constexpr(halves): + if cutlass.const_expr(self.wide): + # This warp's half of every stream's columns, in the 4-warp form's arithmetic. + g = warp_idx // 4 + n_handle = n_consumer.wait_and_advance() + for s in cutlass.range_constexpr(self.hc): + cute.autovec_copy( + tTR_sN[ + (None, None, None, n_handle.index * self.n_slots + g * self.hc + s) + ], + rN, + ) + cute.copy( + tiled_copy_t2r, + tTR_tAcc[(None, None, None, s * halves + g)], + tTR_rAcc, + ) + for i in cutlass.range_constexpr(nfrag): + gate = _sigmoid(tTR_rAcc[i]) + if cutlass.const_expr(s == 0): + tTR_rSum[i] = gate * rN[i].to(self.acc_dtype) + else: + tTR_rSum[i] = tTR_rSum[i] + gate * rN[i].to(self.acc_dtype) + cute.arch.sync_warp() + n_handle.release() + for i in cutlass.range_constexpr(nfrag): + tTR_rX[i] = (tTR_rSum[i] * inv_hc).to(self.io_dtype) + if row_ok: + cute.autovec_copy(tTR_rX, gX_t[(None, None, None, g)]) + for half in cutlass.range_constexpr(0 if self.wide else halves): n_handle = n_consumer.wait_and_advance() for s in cutlass.range_constexpr(self.hc): cute.autovec_copy( @@ -736,14 +799,16 @@ class SharedStorage: _COMPILED: dict = {} -def _launcher(block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = False): +def _launcher( + block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = False, wide: bool = False +): """Compile once per static configuration; later calls reuse the TVM-FFI entry, which launches on the caller's current stream (graph-capturable). `ptrs` is `(low, up, normed, mixed tensor, xq, sf, gs)`.""" - key = (block_h, hc, hidden, rank, fp4) + key = (block_h, hc, hidden, rank, fp4, wide) fn = _COMPILED.get(key) if fn is None: - kern = HcUpGateMeanKernel(block_h, hc, hidden, rank, fp4=fp4) + kern = HcUpGateMeanKernel(block_h, hc, hidden, rank, fp4=fp4, wide=wide) cptrs = [ make_ptr(cutlass.BFloat16, p, cute.AddressSpace.gmem, assumed_align=16) for p in ptrs[:3] @@ -769,7 +834,12 @@ def _launcher(block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = F def hc_up_gate_mean( - low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int, block_h: int + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + block_h: int, + wide: bool = False, ) -> torch.Tensor: """Launch the fused kernel; `up` is the `[R, C*H]` transposed view of the `[C*H, R]` up weight, as `cublas_mm(low, up)` takes it.""" @@ -780,7 +850,7 @@ def hc_up_gate_mean( ptrs = (low.data_ptr(), up.data_ptr(), normed.data_ptr()) # The NVFP4 pointers are never read without `fp4`. unused = (mixed.data_ptr(),) * 3 - _launcher(block_h, hc_count, hidden, rank, (*ptrs, mixed, *unused))( + _launcher(block_h, hc_count, hidden, rank, (*ptrs, mixed, *unused), wide=wide)( rows, *ptrs, mixed, *unused ) return mixed @@ -869,6 +939,9 @@ class Qwen4ExpHcUpGateMean(OpWrapper): turns it off): each CTA's first tile streams its up-weight stages before `griddepcontrol.wait` and reads `low` / `normed` after it. + `wide=True` runs the epilogue on eight warps instead of four, each element in the + four-warp form's arithmetic, so the bits are the same (bf16 output only). + Certified: bf16 at the checkpoint's 4 x 2560, rank 320, at 1 and 32 rows (part of one tile), 128 (one full tile), 1024, 8192, 16384 and 16390 rows (a partial last tile); graph replay. @@ -889,9 +962,14 @@ class Qwen4ExpHcUpGateMean(OpWrapper): """ def __call__( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> torch.Tensor: - return hc_up_gate_mean(low, up, normed, hc_count, _BLOCK_H) + return hc_up_gate_mean(low, up, normed, hc_count, _BLOCK_H, wide) def reference( self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int @@ -905,8 +983,14 @@ def reference( return (gate * x).mean(dim=-2).to(normed.dtype) def is_valid( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> None: + # `wide` needs 32-column blocks and bf16: always true of this entry. rows, rank = low.shape width = normed.shape[1] assert low.dtype == up.dtype == normed.dtype == torch.bfloat16, "bf16 operands" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py new file mode 100644 index 000000000000..3109d8cef5d3 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compile a Triton kernel's launch variants before the step that would first need one. + +A launcher that picks constexprs or launch options from the step's shape compiles one variant per +value; a served step whose value the engine's warm-up did not reach would compile inside a timed +request. `warm_variants` compiles the launcher's whole finite set at its first eager call for a +geometry, with that call's arguments and only the bucketed values changed. It only compiles +(`JITFunction.warmup`): nothing is launched, so results cannot move. Calls inside a CUDA-graph +capture defer to the next eager call. +""" + +from __future__ import annotations + +from typing import Any, Hashable, Iterable, Mapping, Sequence + +import torch + +_WARMED: set = set() + + +def first_time(name: str, key: Hashable) -> bool: + """True once per `(name, key)` in a process; False inside a CUDA-graph capture (the key + stays unmarked, so the next eager call warms it).""" + if (name, key) in _WARMED: + return False + if torch.cuda.is_current_stream_capturing(): + return False + _WARMED.add((name, key)) + return True + + +def compile_only(kernel, args: Sequence[Any], **kwargs: Any) -> None: + """Compile `kernel`'s variant for `args` and `kwargs` without launching it. A stand-in + without `warmup` (a test's launch recorder) is skipped.""" + warmup = getattr(kernel, "warmup", None) + if warmup is not None: + warmup(*args, grid=(1,), **kwargs) + + +def warm_variants( + kernel, + key: Hashable, + args: Sequence[Any], + variants: Iterable[Mapping[str, Any]], + **kwargs: Any, +) -> None: + """Compile `kernel` once per `key` for every mapping of `variants` laid over `kwargs` (the + call's constexprs and launch options).""" + name = getattr(getattr(kernel, "fn", None), "__name__", None) or repr(kernel) + if first_time(name, key): + for variant in variants: + compile_only(kernel, args, **{**kwargs, **variant}) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py index 7a454035db8c..1c42339b7a81 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py @@ -16,8 +16,9 @@ from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16) -> dict: - """The checkpoint's router: 512 experts, top-10, bf16 logits.""" +def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16, capture: bool = False) -> dict: + """The checkpoint's router: 512 experts, top-10, bf16 logits; `capture`: replayed from a + captured CUDA graph over two steps' logits.""" return dict( tokens=tokens, num_experts=512, @@ -25,6 +26,7 @@ def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16) -> dict: logits_dtype=torch.bfloat16, output_dtype=output_dtype, dist="randn", + capture=capture, ) @@ -86,6 +88,20 @@ class RenormMoeRoutingOp(OpWrapper): why="coarse logits (3 values): ties decide most of the ranking", spec=dict(_ckpt(64), output_dtype=torch.float32, dist="coarse"), ), + # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). + Cell( + why="FP8 unified captured decode: 8 rows (batch 8; dep2 2 x 4, dep4 4 x 2)", + spec=_ckpt(8, capture=True), + ), + Cell( + why="FP8 unified captured decode: 72 rows (batch 72)", + spec=_ckpt(72, capture=True), + ), + Cell( + why="FP8 unified captured decode: 128 rows, the largest (batch 128; dep2 2 x 64, " + "dep4 4 x 32 gathered)", + spec=_ckpt(128, capture=True), + ), ) note = """ @@ -111,7 +127,8 @@ class RenormMoeRoutingOp(OpWrapper): every multi-row cell. The bf16 weights of a row sum to 1 within 2.44e-3 (the rounding of ten weights). CUDA-graph replay at 128 rows is bit-exact against the reference on - refilled logits (the test holds it). + refilled logits (the test holds it), and so is a second step's replay at + the `capture` cells. Preconditions, enforced loudly by the op: `router_logits` 2-D CUDA bf16 / fp16 / fp32; `output_dtype` fp32, bf16 or None (fp16 raises even for fp16 @@ -149,8 +166,9 @@ class RenormMoeRoutingOp(OpWrapper): NaN and an all `-inf` row poison the row to NaN; a `-inf` among finite logits is a soft mask). Row counts between the cells -- dep4's smallest gathered decode (4 rows), the MTP3 verify (128 x 4) -- run the same kernel - instance on a different grid size and are not cells. Router logits from - the checkpoint's own router GEMM were not driven; the cells use randn and + instance on a different grid size and are not cells. Not driven under + capture: the other captured row counts. Router logits from the + checkpoint's own router GEMM were not driven; the cells use randn and coarse logits. """ diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py index 59bc3712f697..adc049f7feb2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py @@ -29,12 +29,12 @@ _PREFILL_ROWS = 8 -def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: - # A `z` that does not cover `x` goes to the registered op, which rejects it. +def _multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: + """The registered op's multi-row branch (its `eligible`, the gates it knows). A `z` + that does not cover `x` goes to the registered op, which rejects it.""" n = x.shape[-1] return ( - x.shape[0] >= _PREFILL_MIN_ROWS - and x.shape[0] == z.shape[0] * z.shape[1] + x.shape[0] == z.shape[0] * z.shape[1] and z.shape[2] == n and x.stride(-1) == 1 and z.stride(2) == 1 @@ -45,6 +45,29 @@ def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> ) +def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: + return x.shape[0] >= _PREFILL_MIN_ROWS and _multirow(x, z, gate_activation) + + +def _direct_launch(x, z, weight, eps, gate_activation): + """The registered op's own multi-row launch (same kernel, 4 rows a program, PDL by the + op's policy), skipping only the custom-op dispatch.""" + rows, n = x.shape + out = torch.empty_like(x) + grid = triton.cdiv(rows, _layernorm_gated._MULTIROW_ROWS) + device = x.device.index + pdl = _layernorm_gated._multirow_pdl(grid, device) + with torch.cuda.device(device): + _layernorm_gated._rms_norm_gated_fwd_multirow_kernel[(grid,)]( + x, out, weight, z, None, None, x.stride(0), out.stride(0), z.stride(0), rows, eps, + N=n, ROWS=_layernorm_gated._MULTIROW_ROWS, HEADS_PER_TOK=z.shape[1], + GATE_SIGMOID=gate_activation == "sigmoid", WEIGHT_IS_DELTA=False, + LAUNCH_WITH_PDL=pdl, num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, + EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], launch_pdl=pdl, + ) # fmt: skip + return out + + def _prefill_launch(x, z, weight, eps, gate_activation): """The multi-row kernel at `_PREFILL_ROWS` rows a program (no PDL: the grid covers every SM).""" rows, n = x.shape @@ -130,7 +153,9 @@ class RmsNormGatedTokenMajor(OpWrapper): copy of `z`. PDL on the 4-row path only when the grid is smaller than the SM count and `TRTLLM_ENABLE_PDL != 0`; results are bitwise identical with it off. The 8-row programs reduce a row's mean square in another order - than the 4-row ones (one-ulp differences on some outputs). + than the 4-row ones (one-ulp differences on some outputs). With + `direct_launch=True` (the FP8 target), the 4-row launch is made here, the + registered op's own, skipping only the custom-op dispatch. Loud (measured), so not guarded: 3-D `x` (ValueError), a `z` not covering `x` (AssertionError "does not match x shape"; 2-D `z` ValueError), an unknown @@ -154,6 +179,9 @@ class RmsNormGatedTokenMajor(OpWrapper): 12 heads, token stride 4120 (a tp4/tep4 rank): 1 and 4 tokens on the 4-row kernel with PDL (grids of 3 and 12 programs); 128 and 1024 tokens on it without; 8192 tokens (98304 rows) on the 8-row kernel. + 24 heads, token stride 8240 (the FP8 target's tp2/tep2 rank, with + `direct_launch`, cells in the test): 1 and 683 tokens on the 4-row + kernel, 8192 tokens on the 8-row kernel. A one-token `z` view reports the dense token stride (1536 at 12 heads); only token 0 is read, so it computes the same. Not certified: fp8 output, the silu gate, the generic fallback kernel, @@ -186,9 +214,19 @@ def __call__( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> torch.Tensor: if fp8_scale is None and z.dim() == 3 and _prefill_multirow(x, z, gate_activation): return _prefill_launch(x, z, weight, eps, gate_activation) + if ( + direct_launch + and fp8_scale is None + and x.dim() == 2 + and z.dim() == 3 + and weight.is_contiguous() + and _multirow(x, z, gate_activation) + ): + return _direct_launch(x, z, weight, eps, gate_activation) return torch.ops.trtllm.rms_norm_gated_token_major( x, z, weight, eps, fp8_scale, gate_activation ) @@ -201,8 +239,10 @@ def reference( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> torch.Tensor: - """fp32 throughout; row `t * heads + h` pairs with `z[t, h]`. + """fp32 throughout; row `t * heads + h` pairs with `z[t, h]`. `direct_launch` does + not change the result. With `fp8_scale` the gated value is rounded to `x.dtype` first, then multiplied by the fp32 reciprocal of the scale, as the kernel does. @@ -227,6 +267,7 @@ def is_valid( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> None: assert weight.shape == (x.shape[-1],), ( f"weight must be 1-D of length x.shape[-1]={x.shape[-1]}; got " diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py index 46548cc3f797..76453c5f453d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py @@ -31,6 +31,10 @@ import triton.language as tl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + compile_only, + first_time, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, TEP4_SLOTS, @@ -39,6 +43,7 @@ PER_SEQ_MAX_ROWS, _seq_of_rows, ) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber @triton.jit @@ -47,6 +52,40 @@ def _rms(x, w, HIDDEN: tl.constexpr, eps): return x * tl.rsqrt(tl.sum(x * x, axis=0) / HIDDEN + eps) * (1.0 + w) +@triton.jit +def _injected_stream( + hc_ptr, + row, + stride_hc, + ch, + cols, + m, + stream, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, +): + """One stream of the residual, fp32. With `HAS_INJECT` the previous MoE block's injection + comes first, rounded as `qwen4_exp_hc_inject_norm` rounds it.""" + x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) + if HAS_INJECT: + b = tl.load(block_ptr + row * stride_block + cols, mask=m, other=0.0).to(tl.float32) + if HAS_SHARED: + sg = tl.sigmoid(tl.load(gate_ptr + row * stride_gate).to(tl.float32)) + sh = tl.load(sh_ptr + row * stride_sh + cols, mask=m, other=0.0) + b = (b + sg * sh.to(tl.float32)).to(block_ptr.dtype.element_ty).to(tl.float32) + g = tl.load(inj_ptr + row * stride_inj + stream).to(tl.float32) + x = (x + b * g).to(hc_ptr.dtype.element_ty).to(tl.float32) + return x + + @triton.jit def _ple_gate_kernel( hc_ptr, @@ -64,22 +103,39 @@ def _ple_gate_kernel( eps, clamp_min, sqrt_d, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, HIDDEN: tl.constexpr, BLOCK: tl.constexpr, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (row, stream): the key and query grouped norms, the gate, the gated - value and the conv-input grouped norm. Writes `gated` and `xc`.""" + value and the conv-input grouped norm. Writes `gated` and `xc`. With `PDL` the norm + weights are read before the wait, everything else after it.""" row = tl.program_id(0).to(tl.int64) stream = tl.program_id(1) cols = tl.arange(0, BLOCK) m = cols < HIDDEN ch = stream * HIDDEN + cols - x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) - kp = tl.load(key_ptr + row * stride_key + ch, mask=m, other=0.0).to(tl.float32) - v = tl.load(value_ptr + row * stride_value + cols, mask=m, other=0.0).to(tl.float32) wk = tl.load(nk_ptr + ch, mask=m, other=0.0).to(tl.float32) wq = tl.load(nq_ptr + ch, mask=m, other=0.0).to(tl.float32) wc = tl.load(nc_ptr + ch, mask=m, other=0.0).to(tl.float32) + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + x = _injected_stream(hc_ptr, row, stride_hc, ch, cols, m, stream, block_ptr, inj_ptr, sh_ptr, + gate_ptr, stride_block, stride_inj, stride_sh, stride_gate, HAS_INJECT, + HAS_SHARED) # fmt: skip + kp = tl.load(key_ptr + row * stride_key + ch, mask=m, other=0.0).to(tl.float32) + v = tl.load(value_ptr + row * stride_value + cols, mask=m, other=0.0).to(tl.float32) kn = _rms(kp, wk, HIDDEN, eps).to(tl.bfloat16).to(tl.float32) qn = _rms(x, wq, HIDDEN, eps).to(tl.bfloat16).to(tl.float32) g = tl.sum(kn * qn, axis=0) / sqrt_d @@ -91,7 +147,11 @@ def _ple_gate_kernel( tl.store(xc_ptr + row * stride_out + ch, xc, mask=m) -@triton.jit +# The per-sequence metadata may begin at any row of a mixed step's tensors. +@triton.jit( + do_not_specialize=["publish_end"], + do_not_specialize_on_alignment=["slots_ptr", "carry_ptr", "qsl_ptr"], +) def _ple_conv_kernel( xc_ptr, gated_ptr, @@ -104,6 +164,7 @@ def _ple_conv_kernel( stride_out, stride_state, pad_slot, + publish_end, KSIZE: tl.constexpr, DIL: tl.constexpr, STATE: tl.constexpr, @@ -111,16 +172,21 @@ def _ple_conv_kernel( MAX_P: tl.constexpr, ROWS: tl.constexpr, HAS_CARRY: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (sequence, block of ROWS channels): the dilated depthwise conv of each of the sequence's rows against `[state | conv inputs]`, its SiLU added - to the gated value (`ple`), and the state advanced in place. + to the gated value (`ple`), and the state advanced in place for the + sequences below `publish_end` (the others' states are left as they were). A channel's state is `STATE` contiguous columns. The publish moves it as a `[ROWS, HEAD]` tile of the first HEAD (a power of two) columns plus, when `STATE == HEAD + 1`, one `[ROWS]` vector for the last, so no lane is spent on padding and every index is a plain 2-D offset. Every load is issued before the first store.""" + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() s = tl.program_id(0) ch0 = tl.program_id(1) * ROWS begin = tl.load(qsl_ptr + s).to(tl.int64) @@ -181,12 +247,17 @@ def _ple_conv_kernel( # Every read of these state rows -- `keep` and the taps -- lands before # the first store over them. tl.debug_barrier() - tl.store(block + r[:, None] * STATE + i[None, :], head, mask=live) + publish = live & (s < publish_end) + tl.store(block + r[:, None] * STATE + i[None, :], head, mask=publish) if STATE > HEAD: - tl.store(block + r * STATE + HEAD, tail, mask=live) + tl.store(block + r * STATE + HEAD, tail, mask=publish) -@triton.jit +# The packed row and sequence counts follow the step. +@triton.jit( + do_not_specialize=["num_rows", "num_seqs"], + do_not_specialize_on_alignment=["slots_ptr", "carry_ptr", "qsl_ptr"], +) def _ple_conv_rows_kernel( xc_ptr, gated_ptr, @@ -262,25 +333,62 @@ def _ple_residual_norm_kernel( stride_hc_out, stride_normed, eps, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, HIDDEN: tl.constexpr, BLOCK: tl.constexpr, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (row, stream): the PLE output added into the stream, then the next - site's grouped norm of the result.""" + site's grouped norm of the result. With `PDL` the norm weight is read before the wait.""" row = tl.program_id(0).to(tl.int64) stream = tl.program_id(1) cols = tl.arange(0, BLOCK) m = cols < HIDDEN ch = stream * HIDDEN + cols - x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) - ple = tl.load(ple_ptr + row * stride_ple + ch, mask=m, other=0.0).to(tl.float32) wx = tl.load(nx_ptr + ch, mask=m, other=0.0).to(tl.float32) + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + x = _injected_stream(hc_ptr, row, stride_hc, ch, cols, m, stream, block_ptr, inj_ptr, sh_ptr, + gate_ptr, stride_block, stride_inj, stride_sh, stride_gate, HAS_INJECT, + HAS_SHARED) # fmt: skip + ple = tl.load(ple_ptr + row * stride_ple + ch, mask=m, other=0.0).to(tl.float32) hn = (x + ple).to(tl.bfloat16) tl.store(hc_out_ptr + row * stride_hc_out + ch, hn, mask=m) nrm = _rms(hn.to(tl.float32), wx, HIDDEN, eps) tl.store(normed_ptr + row * stride_normed + ch, nrm.to(tl.bfloat16), mask=m) +def _inject_args(hc: torch.Tensor, inject_block: Optional[tuple]) -> tuple: + """`(block, inject, shared, gate)`, their row strides, `HAS_INJECT`, `HAS_SHARED` for the gate + and residual kernels (unused operands point at `hc`).""" + if inject_block is None: + return (hc, hc, hc, hc, 0, 0, 0, 0, False, False) + block_out, inject, shared, gate_logits = inject_block + has_shared = shared is not None + return ( + block_out, + inject, + shared if has_shared else hc, + gate_logits if has_shared else hc, + block_out.stride(0), + inject.stride(0), + shared.stride(0) if has_shared else 0, + gate_logits.stride(0) if has_shared else 0, + True, + has_shared, + ) + + def _state_head(state_len: int) -> int: """The largest power of two not above `state_len`: the conv kernel's tile width; the one column past it, if any, moves as a vector.""" @@ -493,6 +601,9 @@ def __call__( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: rows, width = hc.shape hidden = width // hc_count @@ -514,6 +625,7 @@ def __call__( ) block = triton.next_power_of_2(hidden) warps = 8 if block >= 2048 else 4 + inj = _inject_args(hc, inject_block) _ple_gate_kernel[(rows, hc_count)]( hc, key, @@ -530,9 +642,14 @@ def __call__( eps, clamp_min, math.sqrt(hidden), + *inj[:-2], HIDDEN=hidden, BLOCK=block, + HAS_INJECT=inj[-2], + HAS_SHARED=inj[-1], + PDL=pdl, num_warps=warps, + launch_pdl=pdl, ) # `ple` reuses `gated`'s storage row for row: each element is read by # the program that then writes it. @@ -540,35 +657,9 @@ def __call__( has_carry = has_initial_states is not None carry = has_initial_states if has_carry else state_indices per_seq = max_rows <= PER_SEQ_MAX_ROWS - if not per_seq: - tile_ch = min(_ROWS_CHANNELS, width) - _ple_conv_rows_kernel[(triton.cdiv(rows, _ROWS_BLOCK_T), width // tile_ch)]( - xc, - gated, - gated, - conv_weight, - conv_states, - state_indices, - carry, - query_start_loc, - rows, - num_seqs, - gated.stride(0), - conv_states.stride(0), - pad_slot_id, - KSIZE=ksize, - DIL=dilation, - STATE=state_len, - BLOCK_T=_ROWS_BLOCK_T, - ROWS=tile_ch, - LOG_S=max(_MIN_LOG_S, (num_seqs - 1).bit_length()), - HAS_CARRY=has_carry, - num_warps=_ROWS_WARPS, - ) - # Per sequence: the rows (decode) and the state publish. On the - # context-shaped path it carries no rows and only publishes, after - # every program above has read the state. - _ple_conv_kernel[(num_seqs, width // conv_rows)]( + tile_ch = min(_ROWS_CHANNELS, width) + log_s = max(_MIN_LOG_S, (num_seqs - 1).bit_length()) + rows_args = ( xc, gated, gated, @@ -577,9 +668,37 @@ def __call__( state_indices, carry, query_start_loc, + rows, + num_seqs, gated.stride(0), conv_states.stride(0), pad_slot_id, + ) + rows_constexprs = dict( + KSIZE=ksize, + DIL=dilation, + STATE=state_len, + BLOCK_T=_ROWS_BLOCK_T, + ROWS=tile_ch, + LOG_S=log_s, + HAS_CARRY=has_carry, + num_warps=_ROWS_WARPS, + ) + conv_args = ( + xc, + gated, + gated, + conv_weight, + conv_states, + state_indices, + carry, + query_start_loc, + gated.stride(0), + conv_states.stride(0), + pad_slot_id, + num_seqs if publish_end is None else int(publish_end), + ) + conv_constexprs = dict( KSIZE=ksize, DIL=dilation, STATE=state_len, @@ -587,8 +706,43 @@ def __call__( MAX_P=max_rows if per_seq else 0, ROWS=conv_rows, HAS_CARRY=has_carry, + PDL=pdl, num_warps=_CONV_WARPS, + launch_pdl=pdl, ) + # `MAX_P` follows the step's longest sequence (0 past PER_SEQ_MAX_ROWS): every value, and + # the context-shaped rows kernel, compile at a geometry's first call. + if first_time( + "ple_gate_conv_inject_norm", + ( + xc.dtype, + conv_states.dtype, + carry.dtype, + state_indices.dtype, + query_start_loc.dtype, + gated.stride(0), + conv_states.stride(0), + pad_slot_id, + ksize, + dilation, + state_len, + conv_rows, + has_carry, + log_s, + pdl, + ), + ): + for max_p in range(PER_SEQ_MAX_ROWS + 1): + compile_only(_ple_conv_kernel, conv_args, **{**conv_constexprs, "MAX_P": max_p}) + compile_only(_ple_conv_rows_kernel, rows_args, **rows_constexprs) + if not per_seq: + _ple_conv_rows_kernel[(triton.cdiv(rows, _ROWS_BLOCK_T), width // tile_ch)]( + *rows_args, **rows_constexprs + ) + # Per sequence: the rows (decode) and the state publish. On the + # context-shaped path it carries no rows and only publishes, after + # every program above has read the state. + _ple_conv_kernel[(num_seqs, width // conv_rows)](*conv_args, **conv_constexprs) _ple_residual_norm_kernel[(rows, hc_count)]( hc, gated, @@ -600,9 +754,14 @@ def __call__( hc_out.stride(0), normed.stride(0), eps, + *inj[:-2], HIDDEN=hidden, BLOCK=block, + HAS_INJECT=inj[-2], + HAS_SHARED=inj[-1], + PDL=pdl, num_warps=warps, + launch_pdl=pdl, ) return hc_out, normed @@ -628,6 +787,9 @@ def reference( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """`(hc_out, normed, conv_states)` as the call leaves them: float64 throughout, rounded to the activation dtype exactly where the op (and @@ -640,6 +802,17 @@ def reference( def rnd(x): return x.to(dtype).double() + if inject_block is not None: + # The previous MoE block injected first (`qwen4_exp_hc_inject_norm`'s rounding). + block_out, inject, shared, gate_logits = inject_block + b = block_out.double() + if shared is not None: + b = rnd(b + torch.sigmoid(gate_logits.reshape(rows, 1).double()) * shared.double()) + hc = hc.double().view(rows, hc_count, hidden) + b.unsqueeze( + 1 + ) * inject.double().unsqueeze(-1) + hc = hc.reshape(rows, width).to(dtype) + def norm(x, w): w = w.double().view(hc_count, hidden) return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + w) @@ -677,7 +850,7 @@ def norm(x, w): for k in range(ksize): acc += w[:, k : k + 1] * window[:, k * dilation : k * dilation + n] conv[begin:end] = rnd(acc.t()) - if live: + if live and (publish_end is None or s < publish_end): pool[slot] = window[:, end - begin : end - begin + state_len].to(dtype) ple = rnd(gated.reshape(rows, width) + rnd(torch.nn.functional.silu(conv))) hc_out = (hc.double() + ple).to(dtype) @@ -706,8 +879,25 @@ def is_valid( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> None: rows, width = hc.shape + if inject_block is not None: + block_out, inject, shared, gate_logits = inject_block + hidden_ = width // hc_count + assert block_out.shape == (rows, hidden_) and block_out.stride(1) == 1, tuple( + block_out.shape + ) + assert inject.shape == (rows, hc_count) and inject.stride(1) == 1, tuple(inject.shape) + assert block_out.dtype == inject.dtype == hc.dtype + if shared is not None: + assert shared.shape == (rows, hidden_) and shared.stride(1) == 1, tuple( + shared.shape + ) + assert shared.dtype == hc.dtype + assert gate_logits is not None and gate_logits.numel() == rows assert width % hc_count == 0, f"width {width} not divisible by hc_count {hc_count}" hidden = width // hc_count for name, t in (("hc", hc), ("key", key), ("value", value)): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py index d09f220581a3..d84606cac32f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py @@ -52,6 +52,7 @@ #: Above this many rows per sequence (MTP verify carries at most 4) the call #: takes the context-shaped path: packed row tiles plus a separate publish. PER_SEQ_MAX_ROWS = 4 +_PER_SEQ_BLOCK_P = triton.next_power_of_2(PER_SEQ_MAX_ROWS) _ROWS_BLOCK_T = 32 #: The row-to-sequence search covers at least 2^8 sequences, so one compiled #: variant serves every batch up to that size. @@ -530,7 +531,8 @@ def __call__( BLOCK_H=triton.next_power_of_2(heads), HEAD_DIM=head_dim, BLOCK_D=triton.next_power_of_2(head_dim), - BLOCK_P=triton.next_power_of_2(max(max_rows, 1)), + # One compiled variant for every per-sequence row count (rows past `length` masked). + BLOCK_P=_PER_SEQ_BLOCK_P, BLOCK_C=triton.next_power_of_2(ngram_size - 1), HAS_CARRY=has_initial_states is not None, num_warps=4, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py index 77531fa97feb..b3fd4b0e0123 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py @@ -69,6 +69,8 @@ class Fp8Quantize1x128(OpWrapper): ARCHS = frozenset({Arch.SM_103}) + # spec: tokens, hidden, use_ue8m0, capture (False: eager; True: replayed from a captured + # CUDA graph over two steps' activations). CELLS: tuple[Cell, ...] = ( Cell(why="MTPLayer._moe decode: one token at hidden 2560", spec=dict(tokens=1, **_CKPT)), Cell( @@ -91,6 +93,20 @@ class Fp8Quantize1x128(OpWrapper): why="8192 rows: the MTP layer over one max_num_tokens context chunk", spec=dict(tokens=8192, **_CKPT), ), + # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). + Cell( + why="FP8 unified captured decode: 8 rows (batch 8; dep2 2 x 4, dep4 4 x 2)", + spec=dict(tokens=8, **_CKPT, capture=True), + ), + Cell( + why="FP8 unified captured decode: 72 rows (batch 72)", + spec=dict(tokens=72, **_CKPT, capture=True), + ), + Cell( + why="FP8 unified captured decode: 128 rows, the largest (batch 128; dep2 2 x 64, " + "dep4 4 x 32 gathered)", + spec=dict(tokens=128, **_CKPT, capture=True), + ), ) note = """ @@ -110,6 +126,8 @@ class Fp8Quantize1x128(OpWrapper): stop at `T`. At a `T` that is a multiple of 4 (128, 1024, 8192) nothing is padded and `scales` is the kernel's own buffer, not a compacting copy. CUDA-graph capture and replay are certified at 64, 128 and 1024 rows. + At the `capture` cells the replay is bitwise the eager call, and a second + step's activations replay bitwise their own eager call. Source: compiled-in C++, `torch.ops.trtllm.fp8_quantize_1x128` from `thop/fp8Quantize.cpp` (`fp8_1x128_cs` -> `launch_scale_1x128`). The @@ -129,14 +147,17 @@ class Fp8Quantize1x128(OpWrapper): T certified at 1, 20, 128, 1024, 8192 and 16384 at H 2560 (128: the drafter's decode at max_batch_size 128; 1024: the first draft over mtp7's 128 x 8 verify rows; 8192: a max_num_tokens context chunk; - 16384: max_num_tokens of the tuned config); above that not probed + 16384: max_num_tokens of the tuned config), captured at 8, 72 and + 128; above that not probed H multiple of 16 (raises otherwise); certified at 2560, the draft layer's width Not certified: `T == 0` (the op accepts it and returns `[0, H]` / `[NB, 0]`); a short tail block (`H % 128 != 0`); all-zero blocks (the `AMAX_FLOOR` scale), `inf` / NaN inputs and saturation; `use_ue8m0=True`; - the element-wise fallback kernel; the registered fake. + the element-wise fallback kernel; the registered fake. Not driven under + capture: the other captured row counts, and a count that is not a multiple + of 4 (the compacting copy). """ def __call__( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py index 70c6bec2a95b..f92723950d4f 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py @@ -10,11 +10,13 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( # noqa: E501 fused_sigmoid_mul_inplace as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "fused_sigmoid_mul_inplace requires a CUDA device" @@ -68,15 +70,48 @@ def _mtp_spec(tokens: int) -> dict: return dict(spec, tokens=tokens) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The FP8 target calls with `direct_launch=True`: 12 heads off its tp2/tep2 7424-wide row (gate +#: at column 3584), and its dep4 draft layer's 24-head `[T, 24, 512]` q_proj gate half. +_FP8_TP2_ROW = dict( + heads=12, head_dim=256, dtype=torch.bfloat16, gate="fused_row", row_stride=7424, gate_col=3584 +) +_FP8_MTP = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 decode, one token: 12 heads off the 7424-wide row, direct launch", + spec=dict(tokens=1, direct_launch=True, **_FP8_TP2_ROW), + ), + Cell( + why="fp8 tp2/tep2 prefill chunk, 8192 rows, 12 heads, masked lanes, direct launch", + spec=dict(tokens=8192, direct_launch=True, **_FP8_TP2_ROW), + ), + Cell( + why="fp8 dep4 drafter at 128 requests: 24-head q_proj gate half, direct launch", + spec=dict(tokens=128, direct_launch=True, **_FP8_MTP), + ), + Cell( + why="fp8 dep4 draft layer over an 8192-row prefill chunk, 24 heads, direct launch", + spec=dict(tokens=8192, direct_launch=True, **_FP8_MTP), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name certifying(op) out, out_buffer, gate_buffer, gate = _build(cell.spec, seed=_seed(cell.why)) gate_before = gate_buffer.clone() tail_before = out_buffer[out.numel() :].clone() - expected = op.reference(out, gate) + direct = cell.spec.get("direct_launch", False) + expected = op.reference(out, gate, direct) - result = op(out, gate) + result = op(out, gate, direct) assert result.data_ptr() == out.data_ptr() and result.shape == out.shape torch.testing.assert_close(result, expected) @@ -84,7 +119,8 @@ def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves th assert torch.equal(out_buffer[out.numel() :], tail_before), "a masked lane stored" -def test_cuda_graph_capture_and_replay() -> None: +@pytest.mark.parametrize("direct_launch", [False, True], ids=["op", "direct"]) +def test_cuda_graph_capture_and_replay(direct_launch) -> None: """The decode call sites run captured: the tp4 / tep4 form (6 heads off the 4352-wide row, two 1024-wide blocks a row) replays on refilled inputs.""" spec = next(c.spec for c in op.CELLS if c.spec.get("row_stride") == 4352) @@ -93,11 +129,11 @@ def test_cuda_graph_capture_and_replay() -> None: side = torch.cuda.Stream() side.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side): - op(out, gate) # JIT compile outside capture + op(out, gate, direct_launch) # JIT compile outside capture torch.cuda.current_stream().wait_stream(side) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): - op(out, gate) + op(out, gate, direct_launch) for seed in (11, 12): fresh_out, _, fresh_gate_buffer, _ = _build(spec, seed=seed) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py index 6606eab87e21..f20bb36722a2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py @@ -500,3 +500,105 @@ def test_rejects_metadata_shorter_than_the_rows(only_visible_blocks) -> None: args["query_positions"] = args["query_positions"][:4] with pytest.raises(ValueError, match="one entry per row"): op(**args) + + +@pytest.mark.parametrize( + "chunks", + [((67584, 8190),), ((5000, 300), (20000, 1700)), ((131072, 4093),)], + ids=["one-request-8190-at-67k", "two-requests", "ragged-4093-at-128k"], +) +def test_prefill_query_tile_is_bitwise_the_engine_launch(chunks) -> None: + """`prefill_query_tile` takes the deeper query-tiled launch, bitwise the engine's launch on + every visible column. `chunks`: `(position of the first row, rows)` per request.""" + gen = torch.Generator(device=DEV).manual_seed(len(chunks) * 7919 + chunks[0][0]) + tpb, cr, width = 32, 4, 8196 + pages = [math.ceil((p + n) / tpb) for p, n in chunks] + pool = sum(pages) + 8 + cache = (torch.randn(pool, tpb, 1, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + table = torch.full((16, width), -1, dtype=torch.int32, device=DEV) + perm = 8 + torch.randperm(sum(pages), generator=gen, device=DEV).to(torch.int32) + slots = [3 + 5 * r for r in range(len(chunks))] + start = 0 + for slot, n_pages in zip(slots, pages): + table[slot, :n_pages] = perm[start : start + n_pages] + start += n_pages + rows = sum(n for _, n in chunks) + q = (torch.randn(rows, 6, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16)[:, :4] + positions = torch.cat( + [torch.arange(p, p + n, dtype=torch.int32, device=DEV) for p, n in chunks] + ) + requests = torch.cat( + [torch.full((n,), s, dtype=torch.int32, device=DEV) for s, (_, n) in zip(slots, chunks)] + ) + args = dict( + q=q, + index_cache=cache, + block_table=table, + query_positions=positions, + request_indices=requests, + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=True, + ) + assert _entry._takes_prefill_query_tile(q, True, True, False) + with _watch_launches() as watcher: + out = op(**args, prefill_query_tile=True) + torch.cuda.synchronize() + assert watcher.branch == "query_tile" + assert watcher.grid[0] == math.ceil(rows / _entry._PREFILL_QUERY_TILE) + with _watch_launches() as watcher: + default = op(**args) + torch.cuda.synchronize() + assert watcher.branch == "query_tile" + assert watcher.grid[0] != math.ceil(rows / _entry._PREFILL_QUERY_TILE) + ref = _engine_scores(**args) + vis = _visible(args) + assert torch.equal(torch.where(vis, out, 0.0), torch.where(vis, ref, 0.0)) + assert torch.equal(torch.where(vis, default, 0.0), torch.where(vis, ref, 0.0)) + + +def test_visible_split_compiles_one_variant_up_to_a_row_tile() -> None: + """Launches of up to one row tile (`16 // heads` rows) share one compiled variant.""" + dev = torch.cuda.current_device() + kernel = _entry._qsa_index_scores_visible_split_kernel + heads, dim, tpb, cr = 4, 128, 32, 4 + g = torch.Generator(device=DEV).manual_seed(7) + cache = torch.randn(64, tpb, 1, dim, generator=g, device=DEV).to(torch.bfloat16) + table = torch.randint(0, 64, (8, 40), generator=g, device=DEV, dtype=torch.int32) + seen = [] + for rows in (4, 1, 2, 3, 5, 8): + q = torch.randn(rows, heads, dim, generator=g, device=DEV).to(torch.bfloat16) + positions = torch.randint(100, 1279, (rows,), generator=g, device=DEV, dtype=torch.int64) + requests = torch.randint(0, 8, (rows,), generator=g, device=DEV, dtype=torch.int32) + args = (q, cache, table, positions, requests, tpb, cr, True, False) + _compare(op(*args), op.reference(*args)) + seen.append(len(kernel.device_caches[dev][0])) + assert len(set(seen)) == 1, seen + + +@pytest.mark.parametrize("rows", [32, 64, 96]) +def test_visible_split_wide_batch_programs_are_bitwise_the_narrow_split(rows: int) -> None: + """`wide_split` scores every visible column bitwise as the narrow split, over ragged windows.""" + g = torch.Generator(device=DEV).manual_seed(rows) + tpb, cr, heads, dim = 32, 4, 4, 128 + ctx = torch.randint(64, 200000, (rows,), generator=g, device=DEV) + pages = (ctx + tpb - 1) // tpb + max_pages = int(pages.max()) + total = int(pages.sum()) + cache = torch.randn(total, tpb, 1, dim, generator=g, device=DEV).to(torch.bfloat16) + perm = torch.randperm(total, generator=g, device=DEV).to(torch.int32) + bt = torch.zeros(rows, max_pages, dtype=torch.int32, device=DEV) + off = 0 + for r in range(rows): + bt[r, : int(pages[r])] = perm[off : off + int(pages[r])] + off += int(pages[r]) + q = torch.randn(rows, heads, dim, generator=g, device=DEV).to(torch.bfloat16) + pos = (ctx - 1).to(torch.int64) + req = torch.arange(rows, dtype=torch.int64, device=DEV) + wide = _entry._qsa_index_scores_visible_split(q, cache, bt, pos, req, tpb, cr, False, wide=True) + narrow = _entry._qsa_index_scores_visible_split(q, cache, bt, pos, req, tpb, cr, False) + torch.cuda.synchronize() + visible = torch.clamp((pos + 1) // cr, max=wide.shape[1]) + mask = torch.arange(wide.shape[1], device=DEV)[None, :] < visible[:, None] + assert torch.equal(torch.where(mask, wide, 0.0), torch.where(mask, narrow, 0.0)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py index 5b8bebbb857b..355afd2538cd 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py @@ -139,6 +139,29 @@ def test_wide_rows_equal_the_canonical_composition(width: int, distribution: str assert torch.equal(fused, composed), "fused token expansion" +@pytest.mark.parametrize("rows", [149, 192, 288, 576]) +@pytest.mark.parametrize("distribution", ["relu_sum", "relu_sparse"]) +def test_wide_rows_past_one_wave_ordered_by_window(rows: int, distribution: str) -> None: + """`ordered_wide` past one wave of rows: block lists and token selections equal the default + launch's and the reference's, over ragged windows with ties.""" + g = torch.Generator(device=DEV).manual_seed(rows) + pos = torch.randint(1, 262143, (rows,), generator=g, device=DEV) + pos[::17] = 99999 # ties of window length + pos[3] = 262143 + spec = dict(rows=rows, pos=pos.tolist(), width=65568, **{distribution: True}) + outs = [] + for order in (True, False): + args = _build(spec, seed=rows + 1) + sel = op(*args, ordered_wide=order) + torch.cuda.synchronize() + outs.append((args[4].clone(), sel.clone())) + want_topk, want_sel = op.reference(*_build(spec, seed=rows + 1)) + assert torch.equal(outs[0][0], outs[1][0]), "block list: ordered vs two CTAs an SM" + assert torch.equal(outs[0][1], outs[1][1]), "token selection: ordered vs two CTAs an SM" + assert torch.equal(outs[0][0], want_topk), "block list vs reference" + assert torch.equal(outs[0][1], want_sel.to(torch.int32)), "token selection vs reference" + + # The decode-path shapes, captured in the decode CUDA graph: tp1's one row at 2560 columns, # the default target's relu-sparse decode and verify cells (1 and 128 rows at core's width, 1024 # at MTP's), and the wide tables' decode cells (`graph` in their spec). diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py index 1cfb886f15a2..7cec23234708 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py @@ -48,13 +48,52 @@ def _build(spec, seed): return logits, RANK, hc_count, spec.get("inject", True), spec.get("dtype", torch.bfloat16) +def _capture(args): + """Capture one call over a private copy of the step's logits after a side-stream warmup: + `(graph, static logits, static outputs)`.""" + static = args[0].clone() + rest = args[1:] + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, *rest) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, *rest) + return graph, static, out + + +def _equal(a, b) -> bool: + """Both outputs bitwise equal (`inject` may be None on both sides).""" + return all((x is None and y is None) or torch.equal(x, y) for x, y in zip(a, b)) + + @pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) - args = _build(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) + seed = zlib.crc32(cell.why.encode()) % (2**31) + args = _build(cell.spec, seed=seed) low, inject = op(*args) assert low.shape == (cell.spec["rows"], RANK) _compare((low, inject), op.reference(*args)) + if not cell.spec.get("capture"): + return + + # The replay is bitwise the eager call; a second step's logits replay bitwise their own. + graph, static, replayed = _capture(args) + graph.replay() + torch.cuda.synchronize() + assert _equal(replayed, (low, inject)), "the replay differs from the eager call it captured" + second = _build(cell.spec, seed=(seed + 1) % (2**31)) + assert not torch.equal(second[0], args[0]), "the second step brings other logits" + static.copy_(second[0]) + graph.replay() + torch.cuda.synchronize() + step2 = tuple(None if t is None else t.clone() for t in replayed) + assert _equal(step2, op(*second)), "the second replay differs from its eager call" + assert not _equal(step2, (low, inject)), "the second replay repeated the first step" + _compare(step2, op.reference(*second)) def test_guard_refuses_bf16_logits() -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py index 6c7c33fd817f..da6c074f8d63 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py @@ -64,7 +64,7 @@ def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by mixed = op(*args) assert mixed.shape == (cell.spec["rows"], cell.spec.get("hidden", 2560)) # The answer came from the CuTe DSL kernel, JIT-compiled for this device. - assert (_BLOCK_H, args[3], mixed.shape[1], args[0].shape[1], False) in _COMPILED + assert (_BLOCK_H, args[3], mixed.shape[1], args[0].shape[1], False, False) in _COMPILED _compare(mixed, op.reference(*args)) @@ -90,3 +90,22 @@ def test_cuda_graph_capture_and_replay() -> None: graph.replay() torch.cuda.synchronize() assert torch.equal(g_mixed, op(fresh[0], up, fresh[2], c)) + + +@pytest.mark.parametrize("rows", [1, 17, 96, 129, 192, 288, 384, 480]) +def test_wide_epilogue_is_bitwise_the_four_warp_one(rows) -> None: + """`wide=True` gives the four-warp epilogue's bits, eagerly and replayed from a CUDA graph.""" + low, up, normed, hc = _build(dict(rows=rows), seed=4100 + rows) + four = op(low, up, normed, hc) + eight = op(low, up, normed, hc, wide=True) + torch.cuda.synchronize() + assert torch.equal(four, eight) + static = low.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, up, normed, hc, wide=True) + fresh, _, _, _ = _build(dict(rows=rows), seed=4200 + rows) + static.copy_(fresh) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, op(fresh, up, normed, hc)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py index 332fd3e44f42..5c83bc809235 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py @@ -69,24 +69,35 @@ def _check(logits, topk, output_dtype=None): return ids, weights -# ── the certified cells ─────────────────────────────────────────────────── +def _capture(logits, topk, output_dtype): + """Capture one call over a private copy of the step's logits after a side-stream warmup: + `(graph, static logits, static outputs)`.""" + static = logits.clone() + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, topk, output_dtype) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, topk, output_dtype) + return graph, static, out -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) -def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name - certifying(op) - s = cell.spec - logits = _logits( - s["tokens"], - s["num_experts"], - s["logits_dtype"], - s["dist"], - s.get("magnitude", 1.0), - seed=zlib.crc32(cell.why.encode()), +def _same(a, b) -> bool: + """Ids and weights bitwise equal.""" + return torch.equal(a[0], b[0]) and torch.equal(a[1], b[1]) + + +def _cell_logits(s, seed): + return _logits( + s["tokens"], s["num_experts"], s["logits_dtype"], s["dist"], s.get("magnitude", 1.0), seed ) - ids, weights = _check(logits, s["topk"], s["output_dtype"]) + + +def _check_sums(weights, s) -> None: + """Renormalized: every row sums to 1.""" assert weights.dtype == s["output_dtype"] - # renormalized: every row sums to 1 torch.testing.assert_close( weights.float().sum(-1), torch.ones(s["tokens"], device=DEV), @@ -95,6 +106,38 @@ def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves th ) +# ── the certified cells ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name + certifying(op) + s = cell.spec + seed = zlib.crc32(cell.why.encode()) + logits = _cell_logits(s, seed) + ids, weights = _check(logits, s["topk"], s["output_dtype"]) + _check_sums(weights, s) + if not s.get("capture"): + return + + # The replay is bitwise the eager call; a second step's logits replay bitwise their own. + graph, static, replayed = _capture(logits, s["topk"], s["output_dtype"]) + graph.replay() + torch.cuda.synchronize() + assert _same(replayed, (ids, weights)), "the replay differs from the eager call it captured" + second = _cell_logits(s, seed + 1) + static.copy_(second) + graph.replay() + torch.cuda.synchronize() + step2 = (replayed[0].clone(), replayed[1].clone()) + assert not torch.equal(step2[0], ids), "the second step reroutes" + assert _same(step2, op(second, s["topk"], s["output_dtype"])), ( + "the second replay differs from its eager call" + ) + _compare(step2, op.reference(second, s["topk"], s["output_dtype"])) + _check_sums(step2[1], s) + + # ── what a cell cannot carry ────────────────────────────────────────────── diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py index 74f2bc8bac1d..e2830fe8f739 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py @@ -7,11 +7,13 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( # noqa: E501 rms_norm_gated_token_major as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "rms_norm_gated_token_major requires a CUDA device" @@ -58,10 +60,52 @@ def randn(*shape): z = randn(t, heads, n) weight = randn(n) - return x, z, weight, spec["eps"], None, spec["gate_activation"] - + return ( + x, + z, + weight, + spec["eps"], + None, + spec["gate_activation"], + spec.get("direct_launch", False), + ) + + +#: The FP8 target's tp2/tep2 rank, called with `direct_launch=True`: 24 heads, `z` in its +#: 8240-wide projection. +_FP8_TP2 = dict( + heads=24, + width=128, + dtype=torch.bfloat16, + eps=1e-6, + gate_activation="sigmoid", + z="in_proj", + z_row_stride=8240, + direct_launch=True, +) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2, one token: 24 heads, z at token stride 8240; 4-row kernel, PDL", + spec=dict(tokens=1, **_FP8_TP2), + ), + Cell( + why="fp8 tp2/tep2, 683 tokens (16392 rows): 4-row kernel, grid past the SM count", + spec=dict(tokens=683, **_FP8_TP2), + ), + Cell( + why="fp8 tp2/tep2 prefill at max_num_tokens 8192: 196608 rows on the 8-row kernel", + spec=dict(tokens=8192, **_FP8_TP2), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args = _build(cell.spec, seed=_seed(cell.why)) @@ -84,7 +128,7 @@ def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest res def test_guard_refuses_a_weight_of_the_wrong_length() -> None: """Unguarded, a length-192 weight is silently truncated to weight[:128].""" - x, z, weight, eps, _, act = _build(_PACKED, seed=12) + x, z, weight, eps, _, act, _ = _build(_PACKED, seed=12) long_weight = torch.cat([weight, torch.randn(64, dtype=weight.dtype, device="cuda")]) assert torch.equal( torch.ops.trtllm.rms_norm_gated_token_major(x, z, long_weight, eps, None, act), diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py index d9602c215253..044675e3f75e 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py @@ -248,3 +248,87 @@ def test_cuda_graph_capture_and_replay(spec) -> None: assert torch.equal(args[7], eager[2]) if xc is not None: assert torch.equal(xc, eager_xc) + + +@pytest.mark.parametrize("rows_per_seq", (1, 3, 4)) +def test_inject_block_is_the_injection_then_the_op(rows_per_seq) -> None: + """`inject_block` is bitwise `qwen4_exp_hc_inject_norm` then the op over its `hc_out`, the + pool too; the gate is a strided column, as the target hands it.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, + ) + + args = _build(dict(lengths=(rows_per_seq,) * 2), seed=20 + rows_per_seq) + hc = args[0] + rows, width = hc.shape + g = torch.Generator(device=DEV).manual_seed(5) + routed = (0.5 * torch.randn(rows, CKPT_HIDDEN, device=DEV, generator=g)).to(hc.dtype) + shared = (0.5 * torch.randn(rows, CKPT_HIDDEN, device=DEV, generator=g)).to(hc.dtype) + gug = (2.0 * torch.randn(rows, 1288, device=DEV, generator=g)).to(hc.dtype) + gate = gug[:, 1280:1281] + inject = (2.0 * torch.sigmoid(torch.randn(rows, CKPT_HC_COUNT, device=DEV, generator=g))).to( + hc.dtype + ) + hc_inj, _ = qwen4_exp_hc_inject_norm( + hc, routed, inject, args[10], EPS, CKPT_HC_COUNT, shared, gate + ) + want = _run([hc_inj] + list(args[1:])) + args[7] = args[7].clone() + hc_out, normed = op(*args, inject_block=(routed, inject, shared, gate)) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + # Without a shared expert (a fresh pool: the call above advanced this one). + args = _build(dict(lengths=(rows_per_seq,) * 2), seed=20 + rows_per_seq) + hc_inj, _ = qwen4_exp_hc_inject_norm(args[0], routed, inject, args[10], EPS, CKPT_HC_COUNT) + want = _run([hc_inj] + list(args[1:])) + args[7] = args[7].clone() + hc_out, normed = op(*args, inject_block=(routed, inject, None, None)) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + + +@pytest.mark.parametrize("rows_per_seq", (1, 3)) +def test_pdl_launches_are_the_plain_launches(rows_per_seq) -> None: + """`pdl=True` is bitwise the plain launches, eagerly and replayed from a CUDA graph.""" + args = _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq) + want = _run(args) + args[7] = args[7].clone() + hc_out, normed = op(*args, pdl=True) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + fresh = _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq) + pool = fresh[7].clone() + fresh[7] = pool + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + g_out, g_normed = op(*fresh, pdl=True) + pool.copy_(_build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq)[7]) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(g_out, want[0]) and torch.equal(g_normed, want[1]) + assert torch.equal(pool, want[2]) + + +def test_rows_per_sequence_variants_compile_at_the_first_call() -> None: + """Every `MAX_P` variant and the rows kernel compile at a geometry's first call: later row + counts add no variant, and outputs match the reference.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple import ( + ple_gate_conv_inject_norm as mod, + ) + + def variants(): + dev = torch.cuda.current_device() + return [ + len(k.device_caches[dev][0]) for k in (mod._ple_conv_kernel, mod._ple_conv_rows_kernel) + ] + + for carries in (None, True): + first = _build(dict(lengths=[3], carries=None if carries is None else [True]), seed=1) + _run(first) + warmed = variants() + for lengths in ([1], [2], [4], [1] * 5, [2, 4], [17], [40, 3]): + spec = dict(lengths=lengths, carries=None if carries is None else [True] * len(lengths)) + args = _build(spec, seed=len(lengths)) + out = _run(args) + _compare(out, op.reference(*args)) + assert variants() == warmed, (carries, warmed, variants()) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py index 8ed23185a6f9..4311ef2aee9a 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py @@ -9,6 +9,7 @@ and graph capture at the note's replay cells. """ +import importlib import zlib from types import SimpleNamespace @@ -38,6 +39,10 @@ ) from tensorrt_llm._utils import CUASSERT +_embed_module = importlib.import_module( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed" +) + __extra_import_path__ = ["../../../.."] from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name @@ -217,3 +222,22 @@ def test_cuda_graph_capture_and_replay(lengths, pool_slots) -> None: assert torch.equal(graph_out, want) assert torch.equal(args[1], want_pool) assert not torch.equal(args[1], pool_before) + + +def test_per_sequence_rows_compile_one_variant() -> None: + """1 to 4 rows a sequence launch one compiled variant per carry flag, matching the reference.""" + kernel = _embed_module._ple_ngram_embed_kernel + dev = torch.cuda.current_device() + seen = {} + for carries in (None, [True, False, True, True]): + for rows in (1, 2, 3, 4): + spec = dict(lengths=(rows,) * 4, pool_slots=8, carries=carries, eos_density=0.1) + args = _build(spec, seed=20 + rows) + ref, ref_pool = op.reference(*args) + out = op(*args) + torch.cuda.synchronize() + assert torch.equal(out, ref) and torch.equal(args[1], ref_pool), (rows, carries) + seen[(carries is None, rows)] = len(kernel.device_caches[dev][0]) + for flag in (True, False): + counts = {seen[(flag, rows)] for rows in (1, 2, 3, 4)} + assert len(counts) == 1, f"carry={not flag}: variants per row count {seen}" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py index 88ac87ad09de..9abd46e0e6ce 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py @@ -8,11 +8,16 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( # noqa: E501 + VOCAB, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( # noqa: E501 bf16_to_fp32 as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "bf16_to_fp32 requires a CUDA device" @@ -35,7 +40,20 @@ def _bitwise(a: torch.Tensor, b: torch.Tensor) -> bool: return a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32)) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tep2 MTP-2 verify logits, 128 requests x 3 rows", + spec=dict(shape=(384, VOCAB)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) x = _build(cell.spec, zlib.crc32(cell.why.encode())) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py index 7ef1a5d0b0d2..14cc376f9650 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py @@ -106,6 +106,42 @@ def _assert_misses_are_ties(x: torch.Tensor, data: torch.Tensor, ref_data: torch return rel +def _capture(x: torch.Tensor, use_ue8m0: bool): + """Capture one call over a private copy of the step's activations after a side-stream + warmup: `(graph, static input, static outputs)`.""" + static = x.clone() + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, use_ue8m0) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, use_ue8m0) + return graph, static, out + + +def _same(a: tuple[torch.Tensor, torch.Tensor], b: tuple[torch.Tensor, torch.Tensor]) -> bool: + """Codes and scales bitwise equal (codes compared as bytes).""" + return torch.equal(a[0].view(torch.uint8), b[0].view(torch.uint8)) and torch.equal(a[1], b[1]) + + +def _check_cell(label, x, use_ue8m0, data, scales) -> None: + """Shapes and layout, the gate against the entry's reference, and every miss a tie.""" + nb = (x.shape[1] + 127) // 128 + assert data.shape == x.shape and data.is_contiguous() and data.device == x.device + assert scales.shape == (nb, x.shape[0]) and scales.is_contiguous() + ref = op.reference(x, use_ue8m0) + _compare((data, scales), ref) + rel = _assert_misses_are_ties(x, data, ref[0]) if not use_ue8m0 else 0.0 + exact = (data.view(torch.uint8) == ref[0].view(torch.uint8)).float().mean().item() + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + print( + f" {label}: {100 * exact:.4f}% bit-exact, worst tie {rel:.3e}, " + f"{x.shape[0] * nb} groups over a {sms * 8 * 8 * 2}-group grid sweep" + ) + + # ── the certified cells ─────────────────────────────────────────────────── @@ -113,20 +149,29 @@ def _assert_misses_are_ties(x: torch.Tensor, data: torch.Tensor, ref_data: torch def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name certifying(op) spec = cell.spec - x = _input(spec["tokens"], spec["hidden"], seed=zlib.crc32(cell.why.encode())) + seed = zlib.crc32(cell.why.encode()) + x = _input(spec["tokens"], spec["hidden"], seed=seed) data, scales = op(x, spec["use_ue8m0"]) - nb = (spec["hidden"] + 127) // 128 - assert data.shape == x.shape and data.is_contiguous() and data.device == x.device - assert scales.shape == (nb, spec["tokens"]) and scales.is_contiguous() - ref = op.reference(x, spec["use_ue8m0"]) - _compare((data, scales), ref) - rel = _assert_misses_are_ties(x, data, ref[0]) if not spec["use_ue8m0"] else 0.0 - exact = (data.view(torch.uint8) == ref[0].view(torch.uint8)).float().mean().item() - sms = torch.cuda.get_device_properties(x.device).multi_processor_count - print( - f" {cell.why[:44]}: {100 * exact:.4f}% bit-exact, worst tie {rel:.3e}, " - f"{spec['tokens'] * nb} groups over a {sms * 8 * 8 * 2}-group grid sweep" + _check_cell(cell.why[:44], x, spec["use_ue8m0"], data, scales) + if not spec.get("capture"): + return + + # The replay is bitwise the eager call; a second step's activations replay bitwise their own. + graph, static, replayed = _capture(x, spec["use_ue8m0"]) + graph.replay() + torch.cuda.synchronize() + assert _same(replayed, (data, scales)), "the replay differs from the eager call it captured" + second = _input(spec["tokens"], spec["hidden"], seed=seed + 1) + assert not torch.equal(second, x), "the second step brings other activations" + static.copy_(second) + graph.replay() + torch.cuda.synchronize() + step2 = (replayed[0].clone(), replayed[1].clone()) + assert _same(step2, op(second, spec["use_ue8m0"])), ( + "the second replay differs from its eager call" ) + assert not _same(step2, (data, scales)), "the second replay repeated the first step" + _check_cell(f"{cell.why[:44]} (second replay)", second, spec["use_ue8m0"], *step2) # ── what a cell cannot carry ────────────────────────────────────────────── diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py index ae508d639dd9..f6b4a3395db9 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py @@ -7,18 +7,22 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( # noqa: E501 CKPT_DIM, CKPT_X_STRIDE, + GDN_PAGE_LAYERS, RANK_DIM, RANK_X_STRIDE, _decode_conv_ok, + _spread, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( # noqa: E501 causal_conv1d_update as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "causal_conv1d_update requires a CUDA device" @@ -94,7 +98,29 @@ def _run(args, extras): return returned, got[1] -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: A tp2/tep2 rank of the FP8 target: 5120 channels (the TRT-LLM kernel below 64 rows, the CuTe +#: one from 64), in_proj row stride 8240, over the largest pool the TRT-LLM kernel addresses (its +#: `int slot * uint32_t stride` slot offset wraps from slot 7768 at this page's slot stride). +_TP2 = dict(dim=5120, x_row_stride=8240, pool_slots=7768, pool_layers=GDN_PAGE_LAYERS) + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 decode, 1 row at 5120 / stride 8240: the top addressable slot", + spec=dict(_TP2, batch=1, slots=_spread(1, 7768)), + ), + Cell( + why="fp8 tp2/tep2, 128 rows at 5120 / stride 8240, pad rows (-1) on the CuTe kernel", + spec=dict(_TP2, batch=128, slots=_spread(128, 7768, 21)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args, extras = _build(cell.spec, seed=zlib.crc32(cell.why.encode())) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py index 638a42342065..2ab51ad4a955 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py @@ -28,6 +28,7 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm import ( gdn_decode_rows_gated_norm as rows_mod, ) @@ -39,6 +40,7 @@ ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "gdn_decode_rows_gated_norm requires a CUDA device" @@ -269,7 +271,32 @@ def _stats(out, ref_out, pool, ref_pool) -> str: ) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: A tp2/tep2 rank of the FP8 target: 8 k / 24 v heads on its 8240-wide projection, bf16 state; +#: the register variant up to 12 rows, the bulk copy from 13 (24 v heads x 13 rows > 2 x 152 SMs). +_FP8_TP2 = dict(k_heads=8, v_heads=24, proj_width=8240, state="bf16") +_FP8 = "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified" + +#: The shapes each target calls `gdn_decode_rows_gated_norm` with beyond its own cells. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + _FP8: ( + Cell( + why="fp8 tp2/tep2 decode, 1 row at 8 / 24 heads, slot 7193 of a 7194-slot bf16 page", + spec=dict(_FP8_TP2, rows=1, pool="coalesced", pool_slots=7194, member="tp2"), + ), + Cell( + why="fp8 tp2/tep2, 128 rows (the bulk copy) at 8 / 24 heads on the 7194-slot page, " + "row 1 a pad slot (-1)", + spec=dict( + _FP8_TP2, rows=128, pad=True, pool="coalesced", pool_slots=7194, member="tp2" + ), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) args, engine = _build(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) @@ -467,7 +494,24 @@ def _rows_args(spec, seed): return [proj, pool, slots, A_log, dt_bias, h, k, out], buf -@pytest.mark.parametrize("cell", rows_op.CELLS, ids=[c.why[:44] for c in rows_op.CELLS]) +#: The shapes each target calls `gdn_decode_rows` with beyond its own cells. +ROWS_TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + _FP8: ( + Cell( + why="fp8 tp2/tep2 mixed step: 3 generation rows at 8 / 24 heads into a row range", + spec=dict(_FP8_TP2, rows=3, out="row_range"), + ), + Cell( + why="fp8 tp2/tep2 mixed step: 127 generation rows (the bulk copy) at 8 / 24 heads", + spec=dict(_FP8_TP2, rows=127, out="row_range"), + ), + ), +} + +ROWS_CELLS = certified_cells(rows_op, ROWS_TARGET_CELLS) + + +@pytest.mark.parametrize("cell", ROWS_CELLS, ids=[c.why[:44] for c in ROWS_CELLS]) def test_rows_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(rows_op) args, buf = _rows_args(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py index 9973bbd6f50e..7bb2485ef6d6 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py @@ -20,6 +20,7 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( # noqa: E501 A_LOG_RANGE, CONV_WIDTH, @@ -29,6 +30,7 @@ IN_PROJ_WIDTH, NUM_K_HEADS, NUM_V_HEADS, + PAGE_GDN_LAYERS, QKV_DIM, SOFTPLUS_THRESHOLD, _engine_slots, @@ -38,6 +40,7 @@ ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "gdn_prefill_conv_post requires a CUDA device" @@ -146,7 +149,43 @@ def _compare(out, pool, want, what=""): assert torch.equal(pool, pool_r), "conv pool differs" -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +def _rank2_cell(lengths, initial, slots) -> dict: + """A tp2/tep2 rank of the FP8 target: 8 k / 24 v heads over its 8240-wide projection, on the + stock conv page at 8192 slots, a bound on every pool such a rank binds (its last slot past + 2^32 elements, its middle past 2^31).""" + hk, hv = NUM_K_HEADS // 2, NUM_V_HEADS // 2 + return dict( + num_k_heads=hk, + num_v_heads=hv, + head_dim=HEAD_K_DIM, + x_row_stride=(2 * hk + 2 * hv) * HEAD_K_DIM + 2 * hv, + lengths=lengths, + initial=initial, + pool_slots=8192, + page_layers=PAGE_GDN_LAYERS, + slots=slots, + linear_gate=True, + ) + + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 rank: 37+1+128 tokens carry/fresh/carry at slots 8191, 0, 4096", + spec=_rank2_cell((37, 1, 128), (True, False, True), (8191, 0, 4096)), + ), + Cell( + why="fp8 tp2/tep2 rank at max_num_tokens: one carrying 8192-token chunk at slot 8191", + spec=_rank2_cell((8192,), (True,), (8191,)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args = _build(cell.spec, seed=zlib.crc32(cell.why.encode())) From fcc2c6258c6c1f7f226b9794943b89e78c75e5ff Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:42:07 -0800 Subject: [PATCH 02/51] [None][feat] modeling_v2 qwen4_exp: Qwen3.8-Flash-Next-FP8 target for TP1/TP2/TEP2/DEP2/TP4/TEP4/DEP4 on GB300 One modeling_v2 target, qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified, serves Qwen3.8-Flash-Next-FP8 (routed experts FP8 with 128x128 block scales, bf16 elsewhere) on GB300 at every single-node layout routing admits for it: tp1, tp2/tp4, tep2/tep4 and dep2/dep4, with CUDA graphs, the overlap scheduler, one-model MTP-Eagle (1-7 drafts) and IndexShare, KV cache auto/FP8/NVFP4, a bf16 recurrent-state cache, chunked prefill, block reuse with recurrent-state snapshots, the breakable prefill CUDA graph under attention TP, disaggregated serving and image and video input. routing.py sends a checkpoint the engine resolves as FP8_BLOCK_SCALES to this target at the NVFP4 target's layouts plus tp2 and tep2; the target's check_deployment declines the checkpoints and engine choices it does not run. Every other checkpoint routes as before. The target's own CUDA kernels build into the family library (models/qwen4_exp/csrc): the TP-block decode MoE (8-CTA clusters, one launch per layer for <= 8 rows at tp4/tep4, opt-in through TRTLLM_MODELING_V2_DECODE_MOE_TPB=rr), the hyper-connection TP sum fused with the injection and grouped norm (one IPC Lamport launch on the stock all-reduce workspace), and the one-launch decode routing + 1x128 quantization in front of the trtllm-gen FP8 block-scale MoE runner. The prefill experts use a target copy of the CuTe blockwise GEMM with a contiguous grouped mode. The target calls the catalog's cublas_mm, thop_attention and fused_qk_norm_rope and the family's operations directly, with its shapes as TARGET_CELLS in those operations' tests; its own ops/ holds only the operations it runs differently. The TP sum and the attention-DP gather and scatter are the shared catalog/comm entries. Under attention DP the MoE round trip is the all-gather / reduce-scatter; with TRTLLM_MODELING_V2_MOE_ALLTOALL=1 it is instead an all-to-all dispatch / combine on the stock NVLink one-sided MoE all-to-all (ops/comm/moe_a2a__dp_dispatch_combine), a reference of that path. Unit tests (tests/unittest/_torch/modeling_v2/models/qwen4_exp//) cover routing, the core and pool layout per rank, rank-local weight shards, speculative rollback, the in-forward snapshot, the vision tower against Hugging Face, every target operation's certified cells and the 2- and 4-rank collectives, the all-to-all round trip included. The accuracy class TestModelingV2Qwen38FlashNextFp8Sm103Unified (tests/integration/defs/accuracy/modeling_v2/qwen4_exp//test_accuracy.py) runs GSM8K with three drafts, block reuse and chunked prefill at tep4 (FP8 KV, breakable prefill graph) and at dep4 (also with the all-to-all round trip), and disaggregated dep2, against the FP8_BLOCK_SCALES references. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../models/qwen4_exp/csrc/CMakeLists.txt | 29 + .../__init__.py | 8 + .../csrc/fp8BlockMoeTpb.cu | 1251 +++ .../csrc/fp8BlockMoeTpb.h | 75 + .../csrc/fp8BlockMoeTpbOp.cpp | 160 + .../csrc/fp8BlockScaleMoeFront.cu | 378 + .../csrc/fp8BlockScaleMoeFront.h | 72 + .../csrc/fp8BlockScaleMoeFrontOp.cpp | 325 + .../csrc/hcAllreduceInjectNorm.cu | 539 ++ .../csrc/hcAllreduceInjectNorm.h | 66 + .../csrc/hcAllreduceInjectNormOp.cpp | 124 + .../fake_ops.py | 88 + .../host_share.py | 443 ++ .../modeling.py | 6894 +++++++++++++++++ .../ops/__init__.py | 3 + .../ops/attention/__init__.py | 2 + .../attention/draft_continue_rows__decode.py | 201 + .../qsa_fp8_pages_to_bf16__prefill.py | 215 + .../qsa_index_scores_cute__prefill.py | 944 +++ .../qsa_index_scores_groups_cute__decode.py | 432 ++ .../attention/qsa_index_split_plan__decode.py | 235 + .../ops/attention/qsa_paged_index_store.py | 395 + .../ops/attention/qsa_paged_kv_store.py | 255 + .../ops/attention/qsa_paged_sparse_gqa.py | 417 + .../qsa_sparse_gqa_grouped_cute__decode.py | 2817 +++++++ .../qsa_sparse_gqa_rowtile__prefill.py | 1720 ++++ .../qsa_sparse_gqa_splitk_cute__decode.py | 1519 ++++ .../qsa_spec_step_metadata__decode.py | 331 + .../qsa_topk_expand_cluster__decode.py | 753 ++ .../attention/qsa_topk_rows_cute__prefill.py | 609 ++ .../qsa_triton_kernels__stable_jit.py | 1222 +++ .../qwen4_exp_attn_decode_prologue__decode.py | 700 ++ .../attention/spec_rollback_begin__verify.py | 187 + .../ops/attention/vision_rope_2d_packed_qk.py | 202 + .../ops/comm/__init__.py | 2 + .../ops/comm/ce_exchange__sp_pair.py | 122 + .../comm/hc_allreduce_inject_norm__tp_sum.py | 79 + .../comm/lamport_allgather__index_split.py | 252 + .../ops/comm/mnnvl_allreduce__tp_sum.py | 73 + .../ops/comm/moe_a2a__dp_dispatch_combine.py | 166 + .../ops/embedding/__init__.py | 2 + .../ops/embedding/embed_rows__decode.py | 98 + .../ops/gemm/__init__.py | 2 + .../ops/gemm/low_m_bf16_mm__decode.py | 192 + .../ops/hc/__init__.py | 2 + .../hc/qwen4_exp_hc_decode_rows_tc__decode.py | 1046 +++ .../qwen4_exp_hc_decode_tc_first__decode.py | 805 ++ .../ops/moe/__init__.py | 2 + .../ops/moe/blockwise_gemm_grouped.py | 1858 +++++ .../ops/moe/fp8_block_scale_moe_runner.py | 616 ++ .../ops/moe/moe_front_router_quant__decode.py | 225 + .../ops/moe/moe_prefill_grouped__prefill.py | 723 ++ .../ops/moe/moe_tpb_decode__decode.py | 269 + .../ops/moe/mtp_moe_combine_inject__decode.py | 132 + .../ops/ssm/__init__.py | 2 + .../ssm/causal_conv1d_token_major__verify.py | 341 + .../ssm/flashinfer_chunk_gated_delta_rule.py | 646 ++ .../ops/ssm/gdn_cached_replay_cute__verify.py | 2780 +++++++ .../ops/ssm/gdn_in_proj_conv__verify.py | 743 ++ .../ops/ssm/gdn_inforward_tail.py | 211 + .../ops/vision/__init__.py | 2 + .../vision/triton_pos_embed_interpolate.py | 158 + .../pool.py | 574 ++ .../vision.py | 617 ++ .../weights.py | 1747 +++++ .../modeling_v2/models/qwen4_exp/routing.py | 34 +- .../__init__.py | 14 + .../test_accuracy.py | 251 + .../defs/accuracy/references/gsm8k.yaml | 5 + .../test_lists/test-db/l0_b300.yml | 1 + .../integration/test_lists/test-db/l0_cpu.yml | 2 + .../test_lists/test-db/l0_gb300.yml | 4 + .../test-db/l0_gb300_multi_gpus.yml | 4 + .../test_modeling_v2_fused_qk_norm_rope.py | 28 + .../test_modeling_v2_thop_attention.py | 100 +- .../gemm/test_modeling_v2_cublas_mm.py | 48 +- .../__init__.py | 4 + .../comm/_ce_exchange__sp_pair_op_matrix.py | 113 + .../comm/_hc_ar_inject_norm_op_matrix.py | 241 + ...amport_allgather__index_split_op_matrix.py | 202 + .../comm/_launch.py | 127 + .../_mnnvl_allreduce__tp_sum_op_matrix.py | 178 + ..._moe_a2a__dp_dispatch_combine_op_matrix.py | 208 + .../comm/test_fp8_unified_op_matrices.py | 43 + .../test_attention_ops.py | 2060 +++++ .../test_hc_ops.py | 338 + .../test_inforward_snapshot.py | 360 + .../test_moe_and_dense_ops.py | 1000 +++ .../test_routing_and_layout.py | 1654 ++++ .../test_spec_rollback.py | 354 + .../test_ssm_ops.py | 1321 ++++ .../test_vision.py | 789 ++ .../test_weights.py | 600 ++ 93 files changed, 48165 insertions(+), 16 deletions(-) create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py create mode 100644 tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py create mode 100644 tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt new file mode 100644 index 000000000000..975d0627c590 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# qwen4_exp native code. Each target keeps its own in /csrc/; see the +# modeling_v2 README. +add_modeling_v2_family(qwen4_exp) + +# The FP8 target routes on SM103 only (its kernels need only SM90+ features). The +# hyper-connection all-reduce uses the stock all-reduce's IPC workspace, which +# exists only in multi-device builds. +add_modeling_v2_target( + qwen4_exp qwen3_8_flash_next_fp8__sm_103__unified + ARCHS 103a + SOURCES *.cu *.cpp + REQUIRES MULTI_DEVICE) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..4343209006aa --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Qwen3.8-Flash-Next-FP8 / sm_103 / unified: tp1, tp2, tep2, dep2, tp4, tep4 and dep4.""" + +from tensorrt_llm._torch.modeling_v2._native import load_target_ops + +# torch.ops.mv2_qwen4_exp_qwen3_8_flash_next_fp8_sm_103_unified +native_ops = load_target_ops(__name__) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu new file mode 100644 index 000000000000..477b75b44666 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu @@ -0,0 +1,1251 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A block (one expert's 128 intermediate rows) closes its FC1 -> activation -> FC2 chain inside one cluster. Every CTA +// routes every row (top-10, ties to the lower id, fp32 softmax over the ten rounded to bf16). Cluster k takes blocks +// k, k + K, ...: a producer warp streams weights into a shared-memory ring, five consumer warps run the MMAs, and an +// epilogue warp sends each FC1 item to every cluster CTA by DSMEM bulk copy, completing that CTA's block barrier. +// Past one grid-wide counter every CTA sums its share of the K cluster partials in a fixed order and stores bf16. + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h" + +#include +#include + +#include +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_moe_tpb +{ +namespace +{ + +constexpr int kH = 2560; +constexpr int kE = 512; +constexpr int kK = 10; +constexpr int kRows = 8; +constexpr int kKb = kH / 128; // 20 k-blocks of FC1 +constexpr int kBlkRows = 256; // FC1 rows of a block: 128 up, 128 gate +constexpr int kItemFc2Rows = 320; // FC2 rows of one ring item +constexpr int kMt = kItemFc2Rows / 16; // 20 m-tiles of 16 FC2 rows an item +constexpr int kConsumerWarps = 5; +constexpr int kEpiWarp = 5; +constexpr int kProducerWarp = 6; +constexpr int kThreads = 7 * 32; +constexpr int kConsumerThreads = kConsumerWarps * 32; +constexpr int kMtPerWarp = kMt / kConsumerWarps; // 4 +constexpr int kItemRowBytes = 16 * kH; // 40960: 16 FC1 rows, or 320 FC2 rows x 128 +constexpr int kStageBytes = kItemRowBytes + 128; // + the item's 20 fp32 scales +constexpr int kMaxStages = 8; +constexpr int kMaxSlots = kRows * kK; // 80 +constexpr int kMaxBlk = 2 * kMaxSlots + kShItems; // up to two blocks a routed expert, plus the shared items +constexpr int kActRow = 128 + 16; +constexpr int kMaxClusters = 40; +constexpr float kE4m3Max = 448.f; +constexpr float kAmaxFloor = 1.0e-10f; + +// Workspace: the u64 grid counter at 0, which grows by kStride a call and never needs a reset, then the fp32 partials +// [kMaxClusters][8][2560] at 4096. +constexpr unsigned long long kStride = 1024; +constexpr size_t kWsPart = 4096; +constexpr size_t kWsBytes = kWsPart + size_t(kMaxClusters) * kRows * kH * 4; +constexpr int kDefaultStages = 4; // as fast as five, and leaves ~40 KB of shared memory an SM to side-stream kernels + +struct alignas(16) Smem +{ + float4 red[2][kConsumerWarps][32]; + uint64_t fullBar[kMaxStages]; + uint64_t emptyBar[kMaxStages]; + uint64_t xbar[2]; + float topkW[kRows][kK]; + int topkId[kRows][kK]; + int expertOf[kMaxSlots]; + uint32_t bitmap[kE / 32]; + int wordBase[kE / 32]; + unsigned long long barTarget; + int numDistinct; + int numBlk; + int8_t slotOf[kMaxSlots][kRows]; + uint16_t blkJ[kMaxBlk]; + uint8_t blkD[kMaxBlk]; + alignas(16) uint8_t actC[2][kRows][kActRow]; // 16-byte rows: the FC2 B fragments are 128-bit loads + alignas(16) float stage[2][4][kRows][16]; // an FC1 item's 16 rows x T outputs, the DSMEM copy's source + float actF[2][kRows]; +}; + +struct alignas(16) SmemSh : Smem +{ + alignas(16) __nv_bfloat16 actS[2][kRows][32]; // the slice item's 32 activations of each row (FC2 B operand) + float shScale[kRows]; // sigmoid(bf16 gate logit) of each row +}; + +template +using SmemT = std::conditional_t; + +template +constexpr int kStaticBytes = (int(sizeof(SmemT)) + 127) / 128 * 128; + +constexpr int kShMark = 1024; // block-list entries >= this are shared-slice items (kShMark + item) +constexpr int kShCols = 32; // intermediate columns of a shared item +constexpr int kShRowBytes = kH * 2; // one bf16 weight / activation row + +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) +#define FP8_BLOCK_MOE_TPB_DEVICE 1 +#endif + +#ifdef FP8_BLOCK_MOE_TPB_DEVICE +__device__ __forceinline__ uint32_t smemAddr(void const* p) +{ + return static_cast(__cvta_generic_to_shared(p)); +} + +__device__ __forceinline__ void mbarInit(uint64_t* bar, uint32_t count) +{ + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smemAddr(bar)), "r"(count) : "memory"); +} + +__device__ __forceinline__ void mbarArriveExpectTx(uint64_t* bar, uint32_t bytes) +{ + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smemAddr(bar)), "r"(bytes) : "memory"); +} + +__device__ __forceinline__ void mbarArrive(uint64_t* bar) +{ + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(smemAddr(bar)) : "memory"); +} + +__device__ __forceinline__ void mbarWait(uint64_t* bar, uint32_t parity) +{ + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smemAddr(bar)), + "r"(parity) + : "memory"); +} + +// Wait with cluster-scope acquire: the peers' DSMEM stores released by their arrivals are visible after it. +__device__ __forceinline__ void mbarWaitCluster(uint64_t* bar, uint32_t parity) +{ + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smemAddr(bar)), + "r"(parity) + : "memory"); +} + +__device__ __forceinline__ uint32_t clusterCtaRank() +{ + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} + +__device__ __forceinline__ uint32_t mapaShared(uint32_t addr, uint32_t rank) +{ + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(addr), "r"(rank)); + return r; +} + +__device__ __forceinline__ void clusterSync() +{ + asm volatile("barrier.cluster.arrive.release.aligned;\nbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +__device__ __forceinline__ void fenceAcqRelGpu() +{ + asm volatile("fence.acq_rel.gpu;" ::: "memory"); +} + +__device__ __forceinline__ void fenceBarrierInit() +{ + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} + +__device__ __forceinline__ void bulkG2S(void* dst, void const* src, uint32_t bytes, uint64_t* bar) +{ + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(smemAddr(dst)), + "l"(src), "r"(bytes), "r"(smemAddr(bar)) + : "memory"); +} + +// DSMEM bulk copy into another cluster CTA, completing `bytes` on its mbarrier (`dst`, `bar`: shared::cluster). +__device__ __forceinline__ void bulkS2Cluster(uint32_t dst, void const* src, uint32_t bytes, uint32_t bar) +{ + asm volatile( + "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(dst), + "r"(smemAddr(src)), "r"(bytes), "r"(bar) + : "memory"); +} + +__device__ __forceinline__ void bulkCommit() +{ + asm volatile("cp.async.bulk.commit_group;" ::: "memory"); +} + +template +__device__ __forceinline__ void bulkWaitRead() +{ + asm volatile("cp.async.bulk.wait_group.read %0;" ::"n"(N) : "memory"); +} + +__device__ __forceinline__ void fenceProxyAsyncShared() +{ + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +__device__ __forceinline__ void namedBarSync(int id, int count) +{ + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(count) : "memory"); +} + +__device__ __forceinline__ void namedBarArrive(int id, int count) +{ + asm volatile("bar.arrive %0, %1;" ::"r"(id), "r"(count) : "memory"); +} + +__device__ __forceinline__ void gridDepWait() +{ + asm volatile("griddepcontrol.wait;" ::: "memory"); +} + +__device__ __forceinline__ void gridDepLaunchDependents() +{ + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); +} + +__device__ __forceinline__ unsigned long long ldAcquireGpu64(unsigned long long const* p) +{ + unsigned long long v; + asm volatile("ld.acquire.gpu.global.u64 %0, [%1];" : "=l"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void redAddRelease64(unsigned long long* p, unsigned long long v) +{ + asm volatile("red.release.gpu.global.add.u64 [%0], %1;" ::"l"(p), "l"(v) : "memory"); +} + +__device__ __forceinline__ uint4 lds128(uint8_t const* p) +{ + return *reinterpret_cast(p); +} + +__device__ __forceinline__ uint4 ldcg128(void const* p) +{ + return __ldcg(reinterpret_cast(p)); +} + +__device__ __forceinline__ void mmaE4m3( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm("mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ void mmaBf16( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +// A 32-wide k step from the 8 bf16 a lane holds per operand, as two k16 MMAs that permute k alike in A and B. +__device__ __forceinline__ void mma32Bf16(float (&acc)[4], uint4 const& ag, uint4 const& ag8, uint4 const& b) +{ + mmaBf16(acc, ag.x, ag8.x, ag.y, ag8.y, b.x, b.y); + mmaBf16(acc, ag.z, ag8.z, ag.w, ag8.w, b.z, b.w); +} + +__device__ __forceinline__ float bf16Round(float x) +{ + return __bfloat162float(__float2bfloat16_rn(x)); +} + +__device__ __forceinline__ float warpMaxAbs(float v) +{ + uint32_t bits = __float_as_uint(fabsf(v)); + return __uint_as_float(__reduce_max_sync(0xffffffffu, bits)); +} + +__device__ __forceinline__ float e4m3RoundTrip(float x) +{ + __nv_fp8_storage_t q = __nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3); + return __half2float(__half(__nv_cvt_fp8_to_halfraw(q, __NV_E4M3))); +} + +__device__ __forceinline__ uint32_t e4m3Code(float x) +{ + return static_cast(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3)); +} + +__device__ __forceinline__ float siluFast(float x) +{ + return __fdividef(x, 1.0f + __expf(-x)); +} + +// Row t's top-10 logits (ties to the lower id) and their softmax rounded to bf16; lane l holds experts 16l..16l+15. +__device__ __forceinline__ void routeRow(TpbParams const& p, int t, int lane, Smem& s) +{ + uint8_t const* row = static_cast(p.logits) + size_t(t) * kE * 2 + lane * 32; + uint4 r0 = ldcg128(row); + uint4 r1 = ldcg128(row + 16); + uint32_t words[8] = {r0.x, r0.y, r0.z, r0.w, r1.x, r1.y, r1.z, r1.w}; + uint32_t key[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) + { + uint32_t b = (words[i >> 1] >> ((i & 1) * 16)) & 0xffffu; + uint32_t ord = (b & 0x8000u) ? (~b & 0xffffu) : (b | 0x8000u); + key[i] = (ord << 16) | uint32_t(kE - 1 - (lane * 16 + i)); + } + uint32_t mine = 0; +#pragma unroll 1 + for (int k = 0; k < kK; ++k) + { + uint32_t m8[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + m8[i] = max(key[2 * i], key[2 * i + 1]); + } + uint32_t const m4a = max(m8[0], m8[1]), m4b = max(m8[2], m8[3]); + uint32_t const m4c = max(m8[4], m8[5]), m4d = max(m8[6], m8[7]); + uint32_t const best = max(max(m4a, m4b), max(m4c, m4d)); + uint32_t const top = __reduce_max_sync(0xffffffffu, best); +#pragma unroll + for (int i = 0; i < 16; ++i) + { + key[i] = key[i] == top ? 0u : key[i]; + } + mine = lane == k ? top : mine; + } + float v = -INFINITY; + if (lane < kK) + { + uint32_t ord = mine >> 16; + uint32_t b = (ord & 0x8000u) ? (ord & 0x7fffu) : (~ord & 0xffffu); + v = __uint_as_float(b << 16); + s.topkId[t][lane] = kE - 1 - int(mine & 0xffffu); + } + float const mx = __shfl_sync(0xffffffffu, v, 0); + float e = lane < kK ? expf(v - mx) : 0.f; + float sum = e; +#pragma unroll + for (int o = 16; o > 0; o >>= 1) + { + sum += __shfl_xor_sync(0xffffffffu, sum, o); + } + if (lane < kK) + { + s.topkW[t][lane] = __bfloat162float(__float2bfloat16_rn(__fdiv_rn(e, sum))); + } +} + +struct Ring +{ + int stage; + uint32_t phase; + int numStages; + + __device__ __forceinline__ void advance() + { + if (++stage == numStages) + { + stage = 0; + phase ^= 1u; + } + } +}; + +#endif // FP8_BLOCK_MOE_TPB_DEVICE + +// CS: CTAs a cluster (4 or 8); kSh: also fold in this rank's shared-expert slice. +template +__global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( + TpbParams const p, int const numStages, int const xbufOffset, int const ringOffset, int const shXOffset) +{ +#ifdef FP8_BLOCK_MOE_TPB_DEVICE + static_assert(!kSh || CS == 4, "the shared-expert fold takes 4-CTA clusters"); + constexpr int kCs = CS; + constexpr int kUpRows = 128 / kCs; // a CTA's up rows (and gate rows) of a block + constexpr int kNh = kUpRows / 16; // items of each half, and FC2 items + constexpr int kFc2Rows = kH / kCs; // a CTA's FC2 rows of a block + extern __shared__ __align__(128) uint8_t smemRaw[]; + SmemT& s = *reinterpret_cast*>(smemRaw); + float* xbuf = reinterpret_cast(smemRaw + xbufOffset); // [2][T][256] raw FC1 outputs of a block + uint8_t* ring = smemRaw + ringOffset; + + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int const T = p.numRows; + int const G = gridDim.x; + int const cta = blockIdx.x; + int const nSlots = T * kK; + int const crank = int(clusterCtaRank()); + int const cl = cta / kCs; + int const NC = G / kCs; + + unsigned long long* counter = static_cast(p.workspace); + float* part = reinterpret_cast(static_cast(p.workspace) + kWsPart); + + if (threadIdx.x == 0) + { + for (int i = 0; i < numStages; ++i) + { + mbarInit(&s.fullBar[i], 1); + mbarInit(&s.emptyBar[i], kConsumerWarps); + } + mbarInit(&s.xbar[0], 1); // the local arrive.expect_tx; the peers' copies complete the bytes + mbarInit(&s.xbar[1], 1); + fenceBarrierInit(); + // Read before arriving: the counter is a multiple of kStride plus fewer than all of this call's arrivals. + s.barTarget = (ldAcquireGpu64(counter) / kStride + 1) * kStride; + } + for (int i = threadIdx.x; i < kMaxSlots * kRows; i += kThreads) + { + (&s.slotOf[0][0])[i] = -1; + } + if (threadIdx.x < kE / 32) + { + s.bitmap[threadIdx.x] = 0; + } + // The peers' barriers are initialized before any CTA of the cluster arrives on them. + clusterSync(); + + gridDepWait(); + + for (int t = warp; t < T; t += kThreads / 32) + { + routeRow(p, t, lane, s); + } + __syncthreads(); + int myExpert = -1; + if (threadIdx.x < nSlots) + { + int const t = threadIdx.x / kK; + int const k = threadIdx.x % kK; + myExpert = s.topkId[t][k]; + atomicOr(&s.bitmap[myExpert >> 5], 1u << (myExpert & 31)); + } + __syncthreads(); + if (warp == 0) + { + int const c = lane < kE / 32 ? __popc(s.bitmap[lane]) : 0; + int inc = c; +#pragma unroll + for (int o = 1; o < 32; o <<= 1) + { + int const v = __shfl_up_sync(0xffffffffu, inc, o); + inc += lane >= o ? v : 0; + } + if (lane < kE / 32) + { + s.wordBase[lane] = inc - c; + } + if (lane == 31) + { + s.numDistinct = inc; + } + } + __syncthreads(); + if (threadIdx.x < nSlots) + { + int const t = threadIdx.x / kK; + int const wd = myExpert >> 5; + int const bit = myExpert & 31; + int const d = s.wordBase[wd] + __popc(s.bitmap[wd] & ((1u << bit) - 1u)); + s.expertOf[d] = myExpert; + s.slotOf[d][t] = int8_t(threadIdx.x); + } + __syncthreads(); + int const D = s.numDistinct; + // This rank's blocks: entry d is block (rank - e) % 4 of expert e = expertOf[d], then block-4 entries, then shared. + if (warp == 0) + { + int base = 0; + for (int d0 = 0; d0 < D; d0 += 32) + { + int const d = d0 + lane; + int extra = 0; + if (d < D) + { + int const e = s.expertOf[d]; + int const b0 = (p.rank - e) & (kTpbRanks - 1); + s.blkJ[d] = uint16_t(e); + s.blkD[d] = uint8_t(d); + extra = b0 == 0; + } + uint32_t const bal = __ballot_sync(0xffffffffu, extra); + if (extra) + { + int const pos = D + base + __popc(bal & ((1u << lane) - 1u)); + s.blkJ[pos] = uint16_t(kE + (s.expertOf[d] >> 2)); + s.blkD[pos] = uint8_t(d); + } + base += __popc(bal); + } + if constexpr (kSh) + { + if (lane < p.shBlocks) + { + s.blkJ[D + base + lane] = uint16_t(kShMark + lane); + s.blkD[D + base + lane] = 0; + } + } + if (lane == 0) + { + s.numBlk = D + base + (kSh ? p.shBlocks : 0); + } + } + __syncthreads(); + // Every CTA is resident by now: the next kernel may start its prologue. + gridDepLaunchDependents(); + int const nB = s.numBlk; + if (nB == 0) + { + __nv_bfloat162* o = static_cast<__nv_bfloat162*>(p.out); + for (int q = cta * kThreads + threadIdx.x; q < T * kH / 2; q += G * kThreads) + { + o[q] = __floats2bfloat162_rn(0.f, 0.f); + } + return; + } + int const NCa = min(NC, nB); + int const nMy = cl < NCa ? (nB - cl + NC - 1) / NC : 0; + + if (warp == kProducerWarp) + { + // ---- producer: per block the up / gate items (16 FC1 rows + 20 scales), then FC2 items (320 rows + 20 scales) + Ring rs{0, 0u, numStages}; + for (int n = 0; n < nMy; ++n) + { + int const j = s.blkJ[cl + n * NC]; + if (kSh && j >= kShMark) + { + // Shared item: 2 x 8 bf16 FC1 rows ([gate 4 ; up 4]), then this CTA's 640 down rows x 32 columns. + int const sb = j - kShMark; + for (int q = 0; q < 3; ++q) + { + mbarWait(&s.emptyBar[rs.stage], rs.phase ^ 1u); + if (lane == 0) + { + uint8_t* dst = ring + rs.stage * kStageBytes; + mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes); + uint8_t const* src = q < 2 + ? static_cast(p.shGu) + (size_t(sb) * 64 + crank * 16 + q * 8) * kShRowBytes + : static_cast(p.shDn) + (size_t(sb) * kH + crank * kFc2Rows) * kShCols * 2; + bulkG2S(dst, src, kItemRowBytes, &s.fullBar[rs.stage]); + } + __syncwarp(); + rs.advance(); + } + continue; + } + for (int q = 0; q < 3 * kNh; ++q) + { + int const kind = q / kNh; // 0 up, 1 gate, 2 FC2 + int const u = q % kNh; + mbarWait(&s.emptyBar[rs.stage], rs.phase ^ 1u); + if (lane == 0) + { + uint8_t* dst = ring + rs.stage * kStageBytes; + mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes + kKb * 4); + if (kind < 2) + { + size_t const row = size_t(j) * kBlkRows + kind * 128 + crank * kUpRows + u * 16; + bulkG2S(dst, p.w1 + row * kH, kItemRowBytes, &s.fullBar[rs.stage]); + bulkG2S( + dst + kItemRowBytes, p.s1 + (size_t(j) * 2 + kind) * kKb, kKb * 4, &s.fullBar[rs.stage]); + } + else + { + size_t const row = size_t(j) * kH + crank * kFc2Rows + u * kItemFc2Rows; + bulkG2S(dst, p.w2 + row * 128, kItemRowBytes, &s.fullBar[rs.stage]); + bulkG2S(dst + kItemRowBytes, p.s2 + size_t(j) * kKb, kKb * 4, &s.fullBar[rs.stage]); + } + } + __syncwarp(); + rs.advance(); + } + } + __syncwarp(); + } + else if (warp == kEpiWarp) + { + // ---- epilogue: per FC1 item, sum the consumer warps' partials in a fixed order and send them to every CTA + int const g = lane >> 2; + int const tig = lane & 3; + int const t0 = 2 * tig, t1 = 2 * tig + 1; + auto fc1Items = [&](int n) { return kSh && s.blkJ[cl + n * NC] >= kShMark ? 2 : 2 * kNh; }; + int nFc1 = 0; + for (int n = 0; n < nMy; ++n) + { + nFc1 += fc1Items(n); + } + uint32_t const xbase = smemAddr(xbuf); + uint32_t const xbarBase = smemAddr(&s.xbar[0]); + int n = 0, q = 0, nq = nMy > 0 ? fc1Items(0) : 0, prevItems = 2 * kNh; + for (int jf = 0; jf < nFc1; ++jf) + { + if (q == nq) + { + prevItems = nq; + ++n; + q = 0; + nq = fc1Items(n); + } + int const buf = jf & 1; + namedBarSync(1 + buf, kConsumerThreads + 32); + float4 v = s.red[buf][0][lane]; +#pragma unroll + for (int w = 1; w < kConsumerWarps; ++w) + { + float4 const o = s.red[buf][w][lane]; + v.x += o.x; + v.y += o.y; + v.z += o.z; + v.w += o.w; + } + if (jf + 2 < nFc1) + { + namedBarArrive(3 + buf, kConsumerThreads + 32); + } + // Stage 16 rows x T, then copy into each CTA's exchange buffer [CS][2 kNh][T][16] at (this CTA, item). + if (q == 0 && n >= 2) + { + // Block n - 2's copies (this parity's slots) must have read their source; block n - 1's may pend. + if (prevItems == 2) + { + bulkWaitRead<2>(); + } + else + { + bulkWaitRead<2 * kNh>(); + } + } + float* stg = &s.stage[n & 1][q][0][0]; + if (t0 < T) + { + stg[t0 * 16 + g] = v.x; + stg[t0 * 16 + g + 8] = v.z; + } + if (t1 < T) + { + stg[t1 * 16 + g] = v.y; + stg[t1 * 16 + g + 8] = v.w; + } + fenceProxyAsyncShared(); + __syncwarp(); + if (lane < kCs) + { + uint32_t const off = uint32_t((n & 1) * T * kBlkRows + ((crank * nq + q) * T) * 16) * 4u; + bulkS2Cluster(mapaShared(xbase + off, uint32_t(lane)), stg, uint32_t(T * 16 * 4), + mapaShared(xbarBase + uint32_t(n & 1) * 8u, uint32_t(lane))); + } + if (lane < kCs) + { + bulkCommit(); + } + ++q; + } + bulkWaitRead<0>(); // the DSMEM copies have read this CTA's staging slots + } + else + { + // ---- consumers (warps 0..4) + int const g = lane >> 2; + int const tig = lane & 3; + uint4 bx[4][2]; + float xs[4][2]; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + int const kb = warp * 4 + q; +#pragma unroll + for (int m = 0; m < 2; ++m) + { + int const kpos = kb * 128 + m * 64 + tig * 16; + bx[q][m] = g < T ? ldcg128(p.xq + size_t(g) * kH + kpos) : make_uint4(0, 0, 0, 0); + } + xs[q][0] = 2 * tig < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig) : 0.f; + xs[q][1] = 2 * tig + 1 < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig + 1) : 0.f; + } + uint8_t* xsh = smemRaw + shXOffset; // [T][2560] bf16 MoE-site rows (shared fold only) + if constexpr (kSh) + { + bool anySh = false; + for (int n = 0; n < nMy; ++n) + { + anySh |= s.blkJ[cl + n * NC] >= kShMark; + } + if (anySh) + { + for (int q = threadIdx.x; q < T * kH / 8; q += kConsumerThreads) + { + *reinterpret_cast(xsh + q * 16) = ldcg128(static_cast(p.shX) + q * 16); + } + // Gate logit of each row: bf16(x . w) as the unfused GEMM outputs it, then its sigmoid in fp32. + for (int t = warp; t < T; t += kConsumerWarps) + { + float acc = 0.f; + for (int c = lane; c < kH / 8; c += 32) + { + uint4 const xv = ldcg128(static_cast(p.shX) + size_t(t) * kShRowBytes + c * 16); + uint4 const wv = ldcg128(static_cast(p.shGl) + c * 16); + __nv_bfloat162 const* xa = reinterpret_cast<__nv_bfloat162 const*>(&xv); + __nv_bfloat162 const* wa = reinterpret_cast<__nv_bfloat162 const*>(&wv); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 const xf = __bfloat1622float2(xa[i]); + float2 const wf = __bfloat1622float2(wa[i]); + acc = fmaf(xf.x, wf.x, acc); + acc = fmaf(xf.y, wf.y, acc); + } + } +#pragma unroll + for (int o = 16; o > 0; o >>= 1) + { + acc += __shfl_xor_sync(0xffffffffu, acc, o); + } + if (lane == 0) + { + s.shScale[t] = 1.0f / (1.0f + expf(-bf16Round(acc))); + } + } + namedBarSync(9, kConsumerThreads); + } + } + Ring rs{0, 0u, numStages}; + uint32_t xphase = 0; // bit p: the phase of buffer p's barrier + float out[kNh][kMtPerWarp][4]; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { +#pragma unroll + for (int r = 0; r < 4; ++r) + { + out[u][mi][r] = 0.f; + } + } + } + int jf = 0; + for (int n = 0; n < nMy; ++n) + { + int const pb = n & 1; + int const d = s.blkD[cl + n * NC]; + if constexpr (kSh) + { + if (s.blkJ[cl + n * NC] >= kShMark) + { + // ---- shared item: bf16 FC1 of 2 x 8 rows, warp w over k [512 w, 512 w + 512) + for (int q = 0; q < 2; ++q, ++jf) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + uint4 const zero = make_uint4(0, 0, 0, 0); +#pragma unroll 4 + for (int m = 0; m < 16; m += 2) + { + int const k0 = warp * 512 + m * 32 + tig * 8; + uint4 const a0 = lds128(st + g * kShRowBytes + k0 * 2); + uint4 const a1 = lds128(st + g * kShRowBytes + (k0 + 32) * 2); + uint4 const b0 = g < T ? lds128(xsh + g * kShRowBytes + k0 * 2) : zero; + uint4 const b1 = g < T ? lds128(xsh + g * kShRowBytes + (k0 + 32) * 2) : zero; + mma32Bf16(acc, a0, zero, b0); + mma32Bf16(acc1, a1, zero, b1); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + int const buf = jf & 1; + if (jf >= 2) + { + namedBarSync(3 + buf, kConsumerThreads + 32); + } + s.red[buf][warp][lane] = make_float4(acc[0] + acc1[0], acc[1] + acc1[1], 0.f, 0.f); + namedBarArrive(1 + buf, kConsumerThreads + 32); + } + if (warp == 0 && lane == 0) + { + mbarArriveExpectTx(&s.xbar[pb], uint32_t(kCs * 2 * T * 16 * 4)); + } + mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); + xphase ^= 1u << pb; + // Column c comes from CTA c / 8, item (c % 8) / 4, rows c % 4 (gate) and 4 + c % 4 (up). + float const* xb = xbuf + pb * T * kBlkRows; + for (int i = threadIdx.x; i < T * kShCols; i += kConsumerThreads) + { + int const t = i / kShCols; + int const c = i % kShCols; + float const* src = xb + (((c / 8) * 2 + (c % 8) / 4) * T + t) * 16 + (c % 4); + float const ga = bf16Round(src[0]); + float const up = bf16Round(src[4]); + s.actS[pb][t][c] = __float2bfloat16_rn(ga / (1.0f + expf(-ga)) * up); + } + namedBarSync(8, kConsumerThreads); + // FC2 of this CTA's 640 down rows, scaled by sigmoid(gate logit), into the routed accumulators. + uint4 const zero = make_uint4(0, 0, 0, 0); + uint4 const bsh = g < T ? lds128(reinterpret_cast(&s.actS[pb][g][tig * 8])) : zero; + float const sc0 = 2 * tig < T ? s.shScale[2 * tig] : 0.f; + float const sc1 = 2 * tig + 1 < T ? s.shScale[2 * tig + 1] : 0.f; + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const row = u * kItemFc2Rows + (warp + kConsumerWarps * mi) * 16 + g; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + uint4 const ag = lds128(st + row * kShCols * 2 + tig * 16); + uint4 const ag8 = lds128(st + (row + 8) * kShCols * 2 + tig * 16); + mma32Bf16(acc, ag, ag8, bsh); + out[u][mi][0] = fmaf(acc[0], sc0, out[u][mi][0]); + out[u][mi][1] = fmaf(acc[1], sc1, out[u][mi][1]); + out[u][mi][2] = fmaf(acc[2], sc0, out[u][mi][2]); + out[u][mi][3] = fmaf(acc[3], sc1, out[u][mi][3]); + } + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + continue; + } + } + // FC1: the up items, then the gate items + for (int q = 0; q < 2 * kNh; ++q, ++jf) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float const* sc = reinterpret_cast(st + kItemRowBytes); + float t0 = 0.f, t1 = 0.f, t2 = 0.f, t3 = 0.f; +#pragma unroll + for (int qq = 0; qq < 4; ++qq) + { + int const kb = warp * 4 + qq; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + int const koff = kb * 128 + tig * 16; + uint4 const a0 = lds128(st + g * kH + koff); + uint4 const a08 = lds128(st + (g + 8) * kH + koff); + uint4 const a1 = lds128(st + g * kH + koff + 64); + uint4 const a18 = lds128(st + (g + 8) * kH + koff + 64); + mmaE4m3(acc, a0.x, a08.x, a0.y, a08.y, bx[qq][0].x, bx[qq][0].y); + mmaE4m3(acc1, a1.x, a18.x, a1.y, a18.y, bx[qq][1].x, bx[qq][1].y); + mmaE4m3(acc, a0.z, a08.z, a0.w, a08.w, bx[qq][0].z, bx[qq][0].w); + mmaE4m3(acc1, a1.z, a18.z, a1.w, a18.w, bx[qq][1].z, bx[qq][1].w); +#pragma unroll + for (int r = 0; r < 4; ++r) + { + acc[r] += acc1[r]; + } + float const sw = sc[kb]; + t0 = fmaf(acc[0], sw * xs[qq][0], t0); + t1 = fmaf(acc[1], sw * xs[qq][1], t1); + t2 = fmaf(acc[2], sw * xs[qq][0], t2); + t3 = fmaf(acc[3], sw * xs[qq][1], t3); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + int const buf = jf & 1; + if (jf >= 2) + { + namedBarSync(3 + buf, kConsumerThreads + 32); + } + s.red[buf][warp][lane] = make_float4(t0, t1, t2, t3); + namedBarArrive(1 + buf, kConsumerThreads + 32); + } + + if (warp == 0 && lane == 0) + { + mbarArriveExpectTx(&s.xbar[pb], uint32_t(kCs * 2 * kNh * T * 16 * 4)); + } + mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); + xphase ^= 1u << pb; + // Activation, one routed token a warp: requantize up and gate (1x128), siluFast(gate) * up, requantize. + float const* xb = xbuf + pb * T * kBlkRows; + for (int t = warp; t < T; t += kConsumerWarps) + { + int const sl = s.slotOf[d][t]; + if (sl < 0) + { + continue; + } + int const col = 4 * lane; + int const c = col / kUpRows; + int const u = (col % kUpRows) / 16; + float const* up = xb + ((c * 2 * kNh + u) * T + t) * 16 + (col & 15); + float const* gt = xb + ((c * 2 * kNh + kNh + u) * T + t) * 16 + (col & 15); + float4 const upv = *reinterpret_cast(up); + float4 const gtv = *reinterpret_cast(gt); + float const ux[4] = {upv.x, upv.y, upv.z, upv.w}; + float const gx[4] = {gtv.x, gtv.y, gtv.z, gtv.w}; + float amU = 0.f, amG = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + amU = fmaxf(amU, fabsf(ux[i])); + amG = fmaxf(amG, fabsf(gx[i])); + } + float const sU = __fdiv_rn(fmaxf(warpMaxAbs(amU), kAmaxFloor), kE4m3Max); + float const sG = __fdiv_rn(fmaxf(warpMaxAbs(amG), kAmaxFloor), kE4m3Max); + float const rU = __frcp_rn(sU); + float const rG = __frcp_rn(sG); + float a[4]; + float amA = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float const x1 = e4m3RoundTrip(ux[i] * rU) * sU; + float const x2 = e4m3RoundTrip(gx[i] * rG) * sG; + a[i] = siluFast(x2) * x1; + amA = fmaxf(amA, fabsf(a[i])); + } + float const sA = __fdiv_rn(fmaxf(warpMaxAbs(amA), kAmaxFloor), kE4m3Max); + float const rA = __frcp_rn(sA); + uint32_t packed = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + packed |= e4m3Code(a[i] * rA) << (8 * i); + } + *reinterpret_cast(&s.actC[pb][t][lane * 4]) = packed; + if (lane == 0) + { + s.actF[pb][t] = sA * s.topkW[t][sl % kK]; + } + } + namedBarSync(8, kConsumerThreads); + + int const sg = g < T ? s.slotOf[d][g] : -1; + int const sa = 2 * tig < T ? s.slotOf[d][2 * tig] : -1; + int const sb = 2 * tig + 1 < T ? s.slotOf[d][2 * tig + 1] : -1; + float const fa = sa >= 0 ? s.actF[pb][2 * tig] : 0.f; + float const fb = sb >= 0 ? s.actF[pb][2 * tig + 1] : 0.f; + uint4 bm0 = make_uint4(0, 0, 0, 0), bm1 = make_uint4(0, 0, 0, 0); + if (sg >= 0) + { + bm0 = lds128(&s.actC[pb][g][tig * 16]); + bm1 = lds128(&s.actC[pb][g][64 + tig * 16]); + } +#pragma unroll + for (int u = 0; u < kNh; ++u) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float const* sc2 = reinterpret_cast(st + kItemRowBytes); +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const mt = warp + kConsumerWarps * mi; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + uint8_t const* pb2 = st + mt * 16 * 128; + int const koff = tig * 16; + uint4 const a0 = lds128(pb2 + g * 128 + koff); + uint4 const a08 = lds128(pb2 + (g + 8) * 128 + koff); + uint4 const a1 = lds128(pb2 + g * 128 + koff + 64); + uint4 const a18 = lds128(pb2 + (g + 8) * 128 + koff + 64); + mmaE4m3(acc, a0.x, a08.x, a0.y, a08.y, bm0.x, bm0.y); + mmaE4m3(acc1, a1.x, a18.x, a1.y, a18.y, bm1.x, bm1.y); + mmaE4m3(acc, a0.z, a08.z, a0.w, a08.w, bm0.z, bm0.w); + mmaE4m3(acc1, a1.z, a18.z, a1.w, a18.w, bm1.z, bm1.w); +#pragma unroll + for (int r = 0; r < 4; ++r) + { + acc[r] += acc1[r]; + } + float const s2 = sc2[(crank * kFc2Rows + u * kItemFc2Rows + mt * 16) >> 7]; + float const f0 = s2 * fa; + float const f1 = s2 * fb; + out[u][mi][0] = fmaf(acc[0], f0, out[u][mi][0]); + out[u][mi][1] = fmaf(acc[1], f1, out[u][mi][1]); + out[u][mi][2] = fmaf(acc[2], f0, out[u][mi][2]); + out[u][mi][3] = fmaf(acc[3], f1, out[u][mi][3]); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + } + } + if (nMy > 0) + { + int const t0 = 2 * tig, t1 = 2 * tig + 1; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const row = crank * kFc2Rows + u * kItemFc2Rows + (warp + kConsumerWarps * mi) * 16 + g; + float* pp = part + size_t(cl) * kRows * kH + row; + if (t0 < T) + { + __stcg(pp + t0 * kH, out[u][mi][0]); + __stcg(pp + t0 * kH + 8, out[u][mi][2]); + } + if (t1 < T) + { + __stcg(pp + t1 * kH, out[u][mi][1]); + __stcg(pp + t1 * kH + 8, out[u][mi][3]); + } + } + } + } + } + + // ---- past the grid counter, each CTA sums its share of the NCa cluster partials: 16 threads a (token, 4 rows) + // unit, thread q summing partials q, q + 16, ... in order, then a fixed shuffle tree. + __syncthreads(); + if (threadIdx.x == 0) + { + // Every CTA arrives, inactive ones too, so past the wait every CTA is past its prologue. + fenceAcqRelGpu(); + redAddRelease64(counter, cta == 0 ? kStride - (unsigned long long) (G - 1) : 1ull); + unsigned long long const target = s.barTarget; + while (ldAcquireGpu64(counter) < target) + { + } + } + __syncthreads(); + { + int const nUnits = T * (kH / 4); + int const u0 = int((long long) cta * nUnits / G); + int const u1 = int((long long) (cta + 1) * nUnits / G); + int const grp = threadIdx.x >> 4; + int const q = threadIdx.x & 15; + constexpr int kGroups = kThreads / 16; + __nv_bfloat16* o = static_cast<__nv_bfloat16*>(p.out); + for (int ub = u0; ub < u1; ub += kGroups) + { + int const u = ub + grp; + bool const live = u < u1; + int const t = live ? u / (kH / 4) : 0; + int const row = live ? (u % (kH / 4)) * 4 : 0; + float const* src = part + size_t(t) * kH + row; + float4 v = make_float4(0.f, 0.f, 0.f, 0.f); + if (live) + { + for (int k = q; k < NCa; k += 16) + { + float4 const x = __ldcg(reinterpret_cast(src + size_t(k) * kRows * kH)); + v.x += x.x; + v.y += x.y; + v.z += x.z; + v.w += x.w; + } + } +#pragma unroll + for (int off = 8; off > 0; off >>= 1) + { + v.x += __shfl_xor_sync(0xffffffffu, v.x, off); + v.y += __shfl_xor_sync(0xffffffffu, v.y, off); + v.z += __shfl_xor_sync(0xffffffffu, v.z, off); + v.w += __shfl_xor_sync(0xffffffffu, v.w, off); + } + if (live && q == 0) + { + __nv_bfloat162 const lo = __floats2bfloat162_rn(v.x, v.y); + __nv_bfloat162 const hi = __floats2bfloat162_rn(v.z, v.w); + uint2 pk; + pk.x = *reinterpret_cast(&lo); + pk.y = *reinterpret_cast(&hi); + *reinterpret_cast(o + size_t(t) * kH + row) = pk; + } + } + } +#endif // FP8_BLOCK_MOE_TPB_DEVICE +} + +int smemOptinTpb(int device) +{ + int v = 0; + cudaDeviceGetAttribute(&v, cudaDevAttrMaxSharedMemoryPerBlockOptin, device); + return v; +} + +template +int activeClustersT(int device) +{ + static int cache[16] = {}; + static std::mutex mu; + std::lock_guard lock(mu); + if (device >= 0 && device < 16 && cache[device] > 0) + { + return cache[device]; + } + int const maxSmemBytes = smemOptinTpb(device); + cudaError_t err = cudaFuncSetAttribute( + fp8BlockMoeTpbKernel, cudaFuncAttributeMaxDynamicSharedMemorySize, maxSmemBytes); + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: smem attribute: ") + cudaGetErrorString(err)); + } + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(CS * kMaxClusters); + cfg.blockDim = dim3(kThreads); + cfg.dynamicSmemBytes = size_t(maxSmemBytes); + cudaLaunchAttribute attr[1]; + attr[0].id = cudaLaunchAttributeClusterDimension; + attr[0].val.clusterDim.x = CS; + attr[0].val.clusterDim.y = 1; + attr[0].val.clusterDim.z = 1; + cfg.attrs = attr; + cfg.numAttrs = 1; + int n = 0; + err = cudaOccupancyMaxActiveClusters(&n, fp8BlockMoeTpbKernel, &cfg); + if (err != cudaSuccess || n <= 0) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: occupancy: ") + cudaGetErrorString(err)); + } + n = std::min(n, kMaxClusters); + if (device >= 0 && device < 16) + { + cache[device] = n; + } + return n; +} + +// Few rows have few blocks, so give each more CTAs; from 3 rows, 4-CTA clusters keep more clusters resident. +int clusterSize(int rows) +{ + return rows <= 2 ? 8 : 4; +} + +template +void launchTpbT(TpbParams const& params, bool pdl, cudaStream_t stream) +{ + int device = 0; + cudaGetDevice(&device); + int const clusters = activeClustersT(device); + int const xbufOffset = kStaticBytes; + int const xbufBytes = (2 * params.numRows * kBlkRows * 4 + 127) / 128 * 128; + int const shXOffset = xbufOffset + xbufBytes; + int const shXBytes = kSh ? (params.numRows * kShRowBytes + 127) / 128 * 128 : 0; + int const ringOffset = shXOffset + shXBytes; + int const fit = (smemOptinTpb(device) - ringOffset) / kStageBytes; + int const stages = std::min(std::min(kDefaultStages, fit), kMaxStages); + if (stages < 3) + { + throw std::invalid_argument("fp8_block_moe_tpb: three ring stages do not fit in shared memory"); + } + size_t const smem = size_t(ringOffset) + size_t(stages) * kStageBytes; + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(clusters * CS); + cfg.blockDim = dim3(kThreads); + cfg.dynamicSmemBytes = smem; + cfg.stream = stream; + cudaLaunchAttribute attr[2]; + attr[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr[0].val.programmaticStreamSerializationAllowed = pdl ? 1 : 0; + attr[1].id = cudaLaunchAttributeClusterDimension; + attr[1].val.clusterDim.x = CS; + attr[1].val.clusterDim.y = 1; + attr[1].val.clusterDim.z = 1; + cfg.attrs = attr; + cfg.numAttrs = 2; + cudaError_t const err + = cudaLaunchKernelEx(&cfg, fp8BlockMoeTpbKernel, params, stages, xbufOffset, ringOffset, shXOffset); + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: launch: ") + cudaGetErrorString(err)); + } +} + +} // namespace + +size_t tpbWorkspaceBytes() +{ + return kWsBytes; +} + +void launchTpb(TpbParams const& params, bool pdl, cudaStream_t stream) +{ + if (params.numRows < 1 || params.numRows > kRows) + { + throw std::invalid_argument("fp8_block_moe_tpb: 1..8 rows"); + } + if (params.rank < 0 || params.rank >= kTpbRanks) + { + throw std::invalid_argument("fp8_block_moe_tpb: rank 0..3 of 4"); + } + if (params.shBlocks > 0) + { + if (params.shBlocks != kShItems || clusterSize(params.numRows) != 4 || params.shX == nullptr + || params.shGu == nullptr || params.shGl == nullptr || params.shDn == nullptr) + { + throw std::invalid_argument( + "fp8_block_moe_tpb: the shared-expert fold takes 5 slice items and 4-CTA clusters (3..8 rows)"); + } + launchTpbT<4, true>(params, pdl, stream); + } + else if (clusterSize(params.numRows) == 8) + { + launchTpbT<8, false>(params, pdl, stream); + } + else + { + launchTpbT<4, false>(params, pdl, stream); + } +} + +} // namespace kernels::fp8_block_moe_tpb + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h new file mode 100644 index 000000000000..9e1d989b1691 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_moe_tpb +{ + +// Decode routed experts over 128-row intermediate blocks (e4m3, fp32 128x128 weight and 1x128 activation scales). +// Block j of a rank is one (expert, intermediate block) pair, block-major: +// w1 [blocks, 256, 2560] e4m3 [128 up rows ; 128 gate rows], s1 [blocks, 2, 20] fp32 their block scales; +// w2 [blocks, 2560, 128] e4m3 the block's down_proj columns, s2 [blocks, 20] fp32 their block scales. +// Expert e's block b lives on attention-TP rank (e + b) % 4: rank r stores block b < 4 of e at j = e and block 4 +// (when e % 4 == r) at j = 512 + e / 4, so each rank holds 640 blocks and at least one block of every expert. +constexpr int kHidden = 2560; +constexpr int kExperts = 512; +constexpr int kMaxRows = 8; +constexpr int kTpbRanks = 4; +constexpr int kTpbBlocks = 640; // blocks a rank holds (512 * 5 / 4) +constexpr int kShItems = 5; // work items of the folded shared-expert slice +constexpr int kShItemCols = 32; // intermediate columns of one shared item + +struct TpbParams +{ + void const* logits; // [T, 512] bf16 + uint8_t const* xq; // [T, 2560] e4m3 + float const* xsf; // [20, xsfStride] fp32: the 1x128 scales of xq, transposed + int xsfStride; + uint8_t const* w1; + float const* s1; + uint8_t const* w2; + float const* s2; + int rank; // this rank of kTpbRanks + int numRows; // 1..8 + // Optional fold of this rank's TP slice of the bf16 shared expert (shBlocks = 5 items of 32 columns; 3..8 rows): + // the partial also accumulates sigmoid(gate logit) * the slice's down projection in fp32 before the bf16 rounding. + void const* shX; // [T, 2560] bf16: the MoE site's rows + void const* shGu; // [5, 64, 2560] bf16: item sb, 8 groups of 4 columns, each [gate rows (4) ; up rows (4)] + void const* shGl; // [2560] bf16: the gate-logit row + void const* shDn; // [5, 2560, 32] bf16: down_proj's columns of item sb + int shBlocks; + void* out; // [T, 2560] bf16: this rank's partial (the TP sum of the ranks' partials is the layer) + void* workspace; // tpbWorkspaceBytes(), zeroed once; its grid counter only grows (one per device) +}; + +size_t tpbWorkspaceBytes(); + +// One launch of 8-CTA clusters (up to 2 rows) or 4-CTA clusters (3..8 rows). +void launchTpb(TpbParams const& params, bool pdl, cudaStream_t stream); + +} // namespace kernels::fp8_block_moe_tpb + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp new file mode 100644 index 000000000000..019944abd398 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace torch_ext +{ + +namespace +{ + +namespace kd = TRTLLM_MV2_NS::kernels::fp8_block_moe_tpb; + +void checkAligned(at::Tensor const& t, char const* name) +{ + TORCH_CHECK(reinterpret_cast(t.data_ptr()) % 16 == 0, name, " must be 16-byte aligned"); +} + +// Returns this rank's [T, 2560] bf16 partial of the routed experts for T <= 8 rows. `xsf` holds the transposed 1x128 +// scales of `xq`; `w1`..`s2` are this `rank`'s 640 blocks (layouts in fp8BlockMoeTpb.h). `workspace` is a zeroed +// uint8 buffer of at least `fp8_block_moe_tpb_workspace_bytes()`, one per device. The optional `sh_*` operands +// (3..8 rows) fold in this rank's TP slice of the bf16 shared expert (see TpbParams). +at::Tensor fp8BlockMoeTpb(at::Tensor const& logits, at::Tensor const& xq, at::Tensor const& xsf, at::Tensor const& w1, + at::Tensor const& s1, at::Tensor const& w2, at::Tensor const& s2, int64_t rank, at::Tensor& workspace, + bool enablePdl, std::optional const& shX, std::optional const& shGu, + std::optional const& shGl, std::optional const& shDn) +{ + int64_t const rows = logits.size(0); + TORCH_CHECK(logits.is_cuda() && logits.scalar_type() == at::kBFloat16 && logits.dim() == 2 + && logits.size(1) == kd::kExperts && logits.is_contiguous(), + "logits must be contiguous bf16 [T, 512]"); + TORCH_CHECK(rows >= 1 && rows <= kd::kMaxRows, "1..8 rows"); + TORCH_CHECK( + xq.scalar_type() == at::kFloat8_e4m3fn && xq.is_contiguous() && xq.size(0) == rows && xq.size(1) == kd::kHidden, + "xq must be contiguous e4m3 [T, 2560]"); + TORCH_CHECK(xsf.scalar_type() == at::kFloat && xsf.dim() == 2 && xsf.size(0) == kd::kHidden / 128 + && xsf.size(1) >= rows && xsf.stride(1) == 1, + "xsf must be fp32 [20, >= T] with unit column stride"); + int64_t const blocks = w1.size(0); + TORCH_CHECK(w1.scalar_type() == at::kFloat8_e4m3fn && w1.is_contiguous() && w1.dim() == 3 && w1.size(1) == 256 + && w1.size(2) == kd::kHidden, + "w1 must be contiguous e4m3 [blocks, 256, 2560]"); + TORCH_CHECK( + s1.scalar_type() == at::kFloat && s1.is_contiguous() && s1.size(0) == blocks && s1.numel() == blocks * 40, + "s1 must be contiguous fp32 [blocks, 2, 20]"); + TORCH_CHECK(w2.scalar_type() == at::kFloat8_e4m3fn && w2.is_contiguous() && w2.size(0) == blocks + && w2.size(1) == kd::kHidden && w2.size(2) == 128, + "w2 must be contiguous e4m3 [blocks, 2560, 128]"); + TORCH_CHECK( + s2.scalar_type() == at::kFloat && s2.is_contiguous() && s2.size(0) == blocks && s2.numel() == blocks * 20, + "s2 must be contiguous fp32 [blocks, 20]"); + TORCH_CHECK(blocks == kd::kTpbBlocks, "a rank holds 640 blocks"); + TORCH_CHECK(rank >= 0 && rank < kd::kTpbRanks, "rank must be in [0, 4)"); + TORCH_CHECK(workspace.is_cuda() + && workspace.numel() * workspace.element_size() >= static_cast(kd::tpbWorkspaceBytes()), + "workspace too small"); + for (at::Tensor const* t : std::initializer_list{&logits, &xq, &w1, &s1, &w2, &s2, &workspace}) + { + checkAligned(*t, "operand"); + } + c10::cuda::CUDAGuard const guard(logits.device()); + int const device = logits.get_device(); + auto out = at::empty({rows, kd::kHidden}, logits.options()); + kd::TpbParams p{}; + p.logits = logits.data_ptr(); + p.xq = static_cast(xq.data_ptr()); + p.xsf = xsf.data_ptr(); + p.xsfStride = static_cast(xsf.stride(0)); + p.w1 = static_cast(w1.data_ptr()); + p.s1 = s1.data_ptr(); + p.w2 = static_cast(w2.data_ptr()); + p.s2 = s2.data_ptr(); + p.rank = static_cast(rank); + p.numRows = static_cast(rows); + if (shGu.has_value()) + { + TORCH_CHECK( + shX.has_value() && shGl.has_value() && shDn.has_value(), "the shared fold takes sh_x, sh_gu, sh_gl, sh_dn"); + TORCH_CHECK(shX->scalar_type() == at::kBFloat16 && shX->is_contiguous() && shX->size(0) == rows + && shX->size(1) == kd::kHidden, + "sh_x must be contiguous bf16 [T, 2560]"); + TORCH_CHECK(shGu->scalar_type() == at::kBFloat16 && shGu->is_contiguous() && shGu->dim() == 3 + && shGu->size(0) == kd::kShItems && shGu->size(1) == 64 && shGu->size(2) == kd::kHidden, + "sh_gu must be contiguous bf16 [5, 64, 2560]"); + TORCH_CHECK(shGl->scalar_type() == at::kBFloat16 && shGl->is_contiguous() && shGl->numel() == kd::kHidden, + "sh_gl must be contiguous bf16 [2560]"); + TORCH_CHECK(shDn->scalar_type() == at::kBFloat16 && shDn->is_contiguous() && shDn->dim() == 3 + && shDn->size(0) == kd::kShItems && shDn->size(1) == kd::kHidden && shDn->size(2) == kd::kShItemCols, + "sh_dn must be contiguous bf16 [5, 2560, 32]"); + for (at::Tensor const* t : std::initializer_list{&*shX, &*shGu, &*shGl, &*shDn}) + { + checkAligned(*t, "shared operand"); + } + p.shX = shX->data_ptr(); + p.shGu = shGu->data_ptr(); + p.shGl = shGl->data_ptr(); + p.shDn = shDn->data_ptr(); + p.shBlocks = kd::kShItems; + } + p.out = out.data_ptr(); + p.workspace = workspace.data_ptr(); + kd::launchTpb(p, enablePdl, c10::cuda::getCurrentCUDAStream(device).stream()); + return out; +} + +int64_t fp8BlockMoeTpbWorkspaceBytes() +{ + return static_cast(kd::tpbWorkspaceBytes()); +} + +} // namespace + +} // namespace torch_ext + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "fp8_block_moe_tpb(Tensor logits, Tensor xq, Tensor xsf, Tensor w1, Tensor s1, Tensor w2, Tensor s2, int rank, " + "Tensor(a!) workspace, bool enable_pdl=False, Tensor? sh_x=None, Tensor? sh_gu=None, Tensor? sh_gl=None, " + "Tensor? sh_dn=None) -> Tensor"); + m.def("fp8_block_moe_tpb_workspace_bytes() -> int"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("fp8_block_moe_tpb", &TRTLLM_MV2_NS::torch_ext::fp8BlockMoeTpb); +} + +TRTLLM_MV2_LIBRARY_IMPL(CompositeExplicitAutograd, m) +{ + m.impl("fp8_block_moe_tpb_workspace_bytes", &TRTLLM_MV2_NS::torch_ext::fp8BlockMoeTpbWorkspaceBytes); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu new file mode 100644 index 000000000000..9ed76e280cd8 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu @@ -0,0 +1,378 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h" +#include "tensorrt_llm/common/cudaTypeUtils.cuh" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/kernels/moe/trtllmGen/routing/RoutingKernel.cuh" +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_scale_moe_front +{ + +namespace +{ + +namespace cg = cooperative_groups; +namespace routing = ::moe::dev::routing; +using routing::WarpSize; + +constexpr int NumThreads = 256; +constexpr int NumWarps = NumThreads / WarpSize; +constexpr int VecSize = MaxNumExperts / WarpSize; +constexpr int ScanItemsPerThread = MaxNumExperts / NumThreads; +static_assert(MaxNumExperts % NumThreads == 0); + +// Per-device state, zeroed at allocation: two banks of per-local-expert counters (a launch counts in bank +// generation % 2 and zeroes the other), then the grid barrier's arrival count and generation flag. +constexpr int CounterInts = 2 * MaxNumExperts + 64; +constexpr int BarrierCount = 2 * MaxNumExperts; +constexpr int BarrierFlag = 2 * MaxNumExperts + 32; +constexpr int QuantIters = QuantHidden / 256; // two 1x128 groups a warp iteration + +__device__ __forceinline__ int32_t ldAcquire(int32_t const* p) +{ + int32_t v; + asm volatile("ld.acquire.gpu.global.b32 %0, [%1];\n" : "=r"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void stRelease(int32_t* p, int32_t v) +{ + asm volatile("st.release.gpu.global.b32 [%0], %1;\n" ::"l"(p), "r"(v) : "memory"); +} + +__device__ __forceinline__ int32_t atomAddAcqRel(int32_t* p, int32_t v) +{ + int32_t old; + asm volatile("atom.acq_rel.gpu.global.add.s32 %0, [%1], %2;\n" : "=r"(old) : "l"(p), "r"(v) : "memory"); + return old; +} + +// Bitwise `quantize_1x128_lane` (fp8_blockscale_gemm_kernel.cuh) for one 1x128 group over 16 lanes of 8 bf16 each. +__device__ __forceinline__ void quantizeGroup(uint4 in, __nv_fp8_e4m3* out, float* scale, int laneInGroup) +{ + union + { + uint4 pack; + __nv_bfloat16 v[8]; + } x; + + x.pack = in; + __nv_bfloat16 laneAmax = __nv_bfloat16(0); +#pragma unroll + for (int i = 0; i < 8; i++) + { + laneAmax = __hmax(laneAmax, __habs(x.v[i])); + } + float groupAmax = float(laneAmax); +#pragma unroll + for (int mask = 8; mask > 0; mask /= 2) + { + groupAmax = std::max(groupAmax, __shfl_xor_sync(0xFFFFFFFF, groupAmax, mask)); + } + __nv_bfloat16 amax = ::tensorrt_llm::common::cuda_max(__nv_bfloat16(groupAmax), __nv_bfloat16(1e-10f)); + float quantScale = 448.f / float(amax); + float dequantScale = 1.f / quantScale; + if (laneInGroup == 0) + { + *scale = dequantScale; + } + + union + { + uint2 pack; + __nv_fp8_e4m3 v[8]; + } y; + +#pragma unroll + for (int i = 0; i < 8; i++) + { + y.v[i] = __nv_fp8_e4m3(float(x.v[i]) * quantScale); + } + *reinterpret_cast(out) = y.pack; +} + +// One warp a token selects its experts with the trtllm-gen routing device functions and takes each slot's offset in +// its local expert from a global atomic counter. After a grid barrier of the routing CTAs, every CTA scans the counts +// into expert offsets and writes its own slots' permutation; CTA 0 writes the grouped-GEMM launch tables. +__global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_t* state) +{ +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + using InputT = __nv_bfloat16; + using BaseType = __nv_bfloat16; + using TypePacked = routing::PackedScoreIdx; + using Scan = cub::BlockScan; + + __shared__ typename Scan::TempStorage tempStorage; + __shared__ int32_t smemExpertOffset[MaxNumExperts]; + + int32_t const warpIdx = __shfl_sync(0xffffffff, threadIdx.x / WarpSize, 0); + int32_t const laneIdx = threadIdx.x % WarpSize; + int32_t const tokenIdx = blockIdx.x * NumWarps + warpIdx; + bool const validToken = tokenIdx < params.numTokens; + auto block = cg::this_thread_block(); + auto warp = cg::tiled_partition(block); + int32_t* barrierCount = state + BarrierCount; + int32_t* barrierFlag = state + BarrierFlag; + + if (params.usePdl) + { + cudaGridDependencySynchronize(); + } + + int32_t const routeCtas = (params.numTokens + NumWarps - 1) / NumWarps; + if (static_cast(blockIdx.x) >= routeCtas) + { + // Quantization CTAs, outside the routing CTAs' barrier: half a hidden row a warp. + int32_t const half = (blockIdx.x - routeCtas) * NumWarps + warpIdx; + int32_t const tokenQ = half / 2; + if (tokenQ < params.numTokens) + { + int32_t const it0 = (half % 2) * (QuantIters / 2); + auto const* in + = static_cast<__nv_bfloat16 const*>(params.ptrHiddenBf16) + static_cast(tokenQ) * QuantHidden; + uint4 part[QuantIters / 2]; +#pragma unroll + for (int it = 0; it < QuantIters / 2; it++) + { + part[it] = *reinterpret_cast(in + (it0 + it) * 256 + laneIdx * 8); + } + auto* out = static_cast<__nv_fp8_e4m3*>(params.ptrHiddenFp8) + static_cast(tokenQ) * QuantHidden; +#pragma unroll + for (int it = 0; it < QuantIters / 2; it++) + { + int32_t const group = 2 * (it0 + it) + laneIdx / 16; + quantizeGroup(part[it], out + (it0 + it) * 256 + laneIdx * 8, + params.ptrHiddenScale + group * params.numTokens + tokenQ, laneIdx % 16); + } + } + __syncthreads(); + if (params.usePdl) + { + cudaTriggerProgrammaticLaunchCompletion(); + } + return; + } + + BaseType const minScore = BaseType{-INFINITY}; + BaseType score[VecSize]; + int32_t idx[VecSize]; + if (validToken) + { + InputT const* scores = static_cast(params.ptrScores) + tokenIdx * params.numExperts; +#pragma unroll + for (int i = 0; i < VecSize; i++) + { + auto expertIdx = i * WarpSize + laneIdx; + score[i] = expertIdx < params.numExperts ? static_cast(scores[expertIdx]) : minScore; + idx[i] = expertIdx; + } + } + + // No CTA flips the generation before every CTA, this one included, has arrived; the previous launch flipped it + // before it completed. + int32_t const generation = *reinterpret_cast(barrierFlag); + int32_t* counters = state + (generation & 1) * MaxNumExperts; + if (blockIdx.x == 0) + { + // The other bank is the previous launch's, which is complete (this grid depends on it transitively). + int32_t* other = state + ((generation + 1) & 1) * MaxNumExperts; + for (int e = threadIdx.x; e < MaxNumExperts; e += NumThreads) + { + other[e] = 0; + } + } + + int32_t slotExpert = -1; // this lane's slot (lane < topK): local expert, or -1 + int32_t slotOffset = 0; + if (validToken) + { + BaseType warpTopKScore[MaxTopK]; + int32_t warpTopKExpertIdx[MaxTopK]; + routing::topk::reduceTopK(warp, warpTopKScore, warpTopKExpertIdx, score, idx, minScore, params.topK); + BaseType const softmaxScore = routing::calcSoftmax( + warp, laneIdx < params.topK ? warpTopKScore[laneIdx] : minScore, laneIdx, params.topK); + if (laneIdx < params.topK) + { + // Round through the packed (score, idx) as the runner's cluster kernel does. + TypePacked const packed{softmaxScore, static_cast(warpTopKExpertIdx[laneIdx])}; + int32_t const expandedIdx = tokenIdx * params.topK + laneIdx; + static_cast<__nv_bfloat16*>(params.ptrTopKWeights)[expandedIdx] = __nv_bfloat16{packed.score}; + int32_t const localExpertIdx = static_cast(packed.idx) - params.localExpertsStartIdx; + if (localExpertIdx >= 0 && localExpertIdx < params.numLocalExperts) + { + slotExpert = localExpertIdx; + slotOffset = atomicAdd(counters + localExpertIdx, 1); + } + } + } + + // Grid barrier of the routing CTAs, all resident (a few dozen CTAs, and no awaited kernel holds the SMs): one + // acq_rel arrival releases this CTA's counter atomics and acquires the others', so the counts are final after it. + __syncthreads(); + if (threadIdx.x == 0) + { + if (atomAddAcqRel(barrierCount, 1) == routeCtas - 1) + { + *barrierCount = 0; + stRelease(barrierFlag, generation + 1); + } + else + { + while (ldAcquire(barrierFlag) == generation) + { + } + } + } + __syncthreads(); + + int32_t const tileN = params.tileTokensDim; + int32_t count[ScanItemsPerThread]; + int32_t numCta[ScanItemsPerThread]; +#pragma unroll + for (int e = 0; e < ScanItemsPerThread; e++) + { + int32_t const expert = threadIdx.x * ScanItemsPerThread + e; + count[e] = expert < params.numLocalExperts ? __ldcg(counters + expert) : 0; + } +#pragma unroll + for (int e = 0; e < ScanItemsPerThread; e++) + { + numCta[e] = (count[e] + tileN - 1) / tileN; + } + int32_t ctaOffset[ScanItemsPerThread]; + int32_t numNonExitingCtas; + Scan(tempStorage).ExclusiveSum(numCta, ctaOffset, numNonExitingCtas); +#pragma unroll + for (int e = 0; e < ScanItemsPerThread; e++) + { + int32_t const expert = threadIdx.x * ScanItemsPerThread + e; + smemExpertOffset[expert] = ctaOffset[e] * tileN; + if (blockIdx.x == 0 && expert < params.numLocalExperts) + { + for (int32_t cta = 0; cta < numCta[e]; cta++) + { + params.ptrCtaIdxXyToBatchIdx[ctaOffset[e] + cta] = expert; + int32_t const mnLimit1 = (ctaOffset[e] + cta + 1) * tileN; + int32_t const mnLimit2 = ctaOffset[e] * tileN + count[e]; + params.ptrCtaIdxXyToMnLimit[ctaOffset[e] + cta] = min(mnLimit1, mnLimit2); + } + } + } + if (blockIdx.x == 0 && threadIdx.x == 0) + { + params.ptrPermutedIdxSize[0] = numNonExitingCtas * tileN; + params.ptrNumNonExitingCtas[0] = numNonExitingCtas; + } + __syncthreads(); + + if (validToken && laneIdx < params.topK) + { + int32_t const expandedIdx = tokenIdx * params.topK + laneIdx; + if (slotExpert >= 0) + { + int32_t const permutedIdx = smemExpertOffset[slotExpert] + slotOffset; + params.ptrExpandedIdxToPermutedIdx[expandedIdx] = permutedIdx; + params.ptrPermutedIdxToTokenIdx[permutedIdx] = tokenIdx; + } + else + { + params.ptrExpandedIdxToPermutedIdx[expandedIdx] = -1; + } + } + + // The consumers (activation, both grouped GEMMs, finalize) read every table written above. + __syncthreads(); + if (params.usePdl) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif +} + +int32_t* deviceCounters(int device) +{ + static std::mutex mutex; + static std::array buffers{}; + std::lock_guard lock(mutex); + TLLM_CHECK_WITH_INFO(device >= 0 && device < static_cast(buffers.size()), "device %d out of range", device); + if (buffers[device] == nullptr) + { + // May run during stream capture: allocate in relaxed mode and zero on a private stream outside the capture. + cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; + TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); + int32_t* ptr = nullptr; + TLLM_CUDA_CHECK(cudaMalloc(&ptr, CounterInts * sizeof(int32_t))); + cudaStream_t side; + TLLM_CUDA_CHECK(cudaStreamCreateWithFlags(&side, cudaStreamNonBlocking)); + TLLM_CUDA_CHECK(cudaMemsetAsync(ptr, 0, CounterInts * sizeof(int32_t), side)); + TLLM_CUDA_CHECK(cudaStreamSynchronize(side)); + TLLM_CUDA_CHECK(cudaStreamDestroy(side)); + TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); + buffers[device] = ptr; + } + return buffers[device]; +} + +} // namespace + +bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim) +{ + return numTokens > 0 && numTokens <= MaxNumTokens && numExperts > 0 && numExperts <= MaxNumExperts && topK > 0 + && topK <= MaxTopK && numLocalExperts > 0 && numLocalExperts <= MaxNumExperts && hiddenDim % 128 == 0; +} + +void run(Data const& data, cudaStream_t stream) +{ + TLLM_CHECK_WITH_INFO(supports(data.numTokens, data.numExperts, data.topK, data.numLocalExperts, data.hiddenDim), + "fp8_block_scale_moe_front: unsupported problem (tokens %d, experts %d, topK %d, local %d, hidden %d)", + data.numTokens, data.numExperts, data.topK, data.numLocalExperts, data.hiddenDim); + TLLM_CHECK_WITH_INFO(data.ptrHiddenBf16 == nullptr + || (data.ptrHiddenFp8 != nullptr && data.ptrHiddenScale != nullptr && data.hiddenDim == QuantHidden), + "fp8_block_scale_moe_front: quantization needs both outputs and hidden %d", QuantHidden); + int device = 0; + TLLM_CUDA_CHECK(cudaGetDevice(&device)); + int32_t* counters = deviceCounters(device); + + cudaLaunchConfig_t config{}; + int32_t const routeCtas = (data.numTokens + NumWarps - 1) / NumWarps; + int32_t const quantCtas = data.ptrHiddenBf16 != nullptr ? (2 * data.numTokens + NumWarps - 1) / NumWarps : 0; + config.gridDim = routeCtas + quantCtas; + config.blockDim = NumThreads; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attributes[1] = {}; + attributes[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attributes[0].val.programmaticStreamSerializationAllowed = int(data.usePdl); + config.attrs = attributes; + config.numAttrs = 1; + TLLM_CUDA_CHECK(cudaLaunchKernelEx(&config, moeFrontKernel, data, counters)); +} + +} // namespace kernels::fp8_block_scale_moe_front + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h new file mode 100644 index 000000000000..646f5c94a979 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_scale_moe_front +{ + +// Renormalize routing (top-k of the bf16 logits, softmax over those k) in front of the trtllm-gen FP8 block-scale MoE +// runner for a decode-sized batch: one launch writes the runner's routing tables, optionally fused with the 1x128 e4m3 +// quantization of the bf16 hidden states. Top-k ids, order and bf16 weights match the runner bitwise; a slot's offset +// within its expert may differ (atomic order), which no consumer depends on. Launches on one device must not overlap. +struct Data +{ + void const* ptrScores{nullptr}; // [numTokens, numExperts] bf16 logits + void const* ptrHiddenBf16{nullptr}; // optional [numTokens, hiddenDim] bf16 hidden states to quantize + int32_t numTokens{0}; + int32_t numExperts{0}; + int32_t topK{0}; + int32_t tileTokensDim{0}; + int32_t localExpertsStartIdx{0}; + int32_t numLocalExperts{0}; + int32_t hiddenDim{0}; + bool usePdl{false}; + + // Outputs: the runner's routing tables. + void* ptrTopKWeights{nullptr}; // [numTokens, topK] bf16 + int32_t* ptrExpandedIdxToPermutedIdx{nullptr}; + int32_t* ptrPermutedIdxToTokenIdx{nullptr}; + int32_t* ptrPermutedIdxSize{nullptr}; + int32_t* ptrCtaIdxXyToBatchIdx{nullptr}; + int32_t* ptrCtaIdxXyToMnLimit{nullptr}; + int32_t* ptrNumNonExitingCtas{nullptr}; + void* ptrHiddenFp8{nullptr}; // [numTokens, hiddenDim] e4m3 (with ptrHiddenBf16) + float* ptrHiddenScale{nullptr}; // [hiddenDim / 128, numTokens] fp32 (with ptrHiddenBf16) +}; + +constexpr int MaxNumExperts = 512; +constexpr int MaxTopK = 16; +constexpr int MaxNumTokens = 1024; + +constexpr int QuantHidden = 2560; // hidden width the fused quantization is compiled for + +// Whether `run` takes this problem (bf16 logits, no fused shared experts, stride-1 local window). +bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim); + +void run(Data const& data, cudaStream_t stream); + +} // namespace kernels::fp8_block_scale_moe_front + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp new file mode 100644 index 000000000000..6c5936d0f3df --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp @@ -0,0 +1,325 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The stock `fp8_block_scale_moe_runner` logits path (thop/moe/trtllmGen/fp8BlockScaleMoe.cpp) at an explicit tactic +// (tile N, config), routed by fp8BlockScaleMoeFront.h; no routing bias, precomputed top-k, shared experts or groups. + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/kernels/moe/trtllmGen/runner.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace torch_ext +{ + +namespace +{ + +namespace btg = ::batchedGemm::trtllm::gen; +namespace trtllmGenMoe = ::tensorrt_llm::kernels::trtllmGenFp8BlockScaleMoe; +namespace front = TRTLLM_MV2_NS::kernels::fp8_block_scale_moe_front; + +using trtllmGenMoe::Routing::RoutingMethodType; + +constexpr std::array kSupportedTileN{8, 16, 32, 64, 128}; // FP8BlockScaleMoERunner's mSupportedTileN + +// Built as FP8BlockScaleMoERunner builds its runner for the tile, so a tactic (tile N, config) tuned through that +// class names the same FC1 / FC2 kernels here. One per tile, built on first use and kept for the process. +trtllmGenMoe::MoE::Runner& moeRunner(int64_t tileN) +{ + static std::mutex mutex; + static std::array, kSupportedTileN.size()> runners; + auto const it = std::find(kSupportedTileN.begin(), kSupportedTileN.end(), tileN); + TORCH_CHECK(it != kSupportedTileN.end(), "fp8_block_scale_moe_front: unsupported tile_n ", tileN); + std::lock_guard lock(mutex); + auto& runner = runners[std::distance(kSupportedTileN.begin(), it)]; + if (runner == nullptr) + { + runner = std::make_unique( + btg::Dtype::E4m3, /* useDeepSeekFp8 */ true, static_cast(tileN)); + } + return *runner; +} + +void checkOperands(at::Tensor const& routing_logits, std::optional const& hidden_states_bf16, + at::Tensor const& hidden_states, at::Tensor const& hidden_states_scale, at::Tensor const& gemm1_weights, + at::Tensor const& gemm1_weights_scale, at::Tensor const& gemm2_weights, at::Tensor const& gemm2_weights_scale, + int64_t const num_experts, int64_t const top_k, int64_t const intermediate_size, int64_t const local_num_experts, + int64_t const routing_method_type) +{ + TORCH_CHECK(routing_logits.scalar_type() == at::ScalarType::BFloat16 && routing_logits.is_contiguous(), + "fp8_block_scale_moe_front: contiguous bf16 routing_logits required"); + TORCH_CHECK(routing_logits.dim() == 2, "routing_logits must be 2D."); + TORCH_CHECK(routing_logits.sizes()[1] == num_experts, "routing_logits dim1 must match num_experts."); + TORCH_CHECK(hidden_states.dim() == 2, "hidden_states must be 2D."); + TORCH_CHECK(routing_logits.sizes()[0] == hidden_states.sizes()[0], + "routing_logits and hidden_states must have the same number of tokens."); + TORCH_CHECK(routing_logits.device() == hidden_states.device(), + "routing_logits and hidden_states must be on the same device"); + auto const routingMethod = static_cast(routing_method_type); + TORCH_CHECK(routingMethod == RoutingMethodType::Renormalize || routingMethod == RoutingMethodType::RenormalizeNaive, + "fp8_block_scale_moe_front: logits-routed Renormalize without bias or fused shared experts only"); + TORCH_CHECK(top_k <= 32 && top_k > 0, + "Current routing kernel (no groups, renormalize) only supports top_k<=32 && top_k>0."); + TORCH_CHECK(num_experts > top_k, "num_experts must be greater than top_k"); + + int64_t const num_tokens = hidden_states.sizes()[0]; + int64_t const hidden_size = hidden_states.sizes()[1]; + TORCH_CHECK(hidden_states.scalar_type() == at::ScalarType::Float8_e4m3fn, "hidden_states must be fp8."); + TORCH_CHECK(hidden_states_scale.scalar_type() == at::ScalarType::Float, "hidden_states_scale must be float."); + TORCH_CHECK(hidden_states_scale.dim() == 2, "hidden_states_scale must be 2D."); + TORCH_CHECK(hidden_states_scale.sizes()[0] == hidden_size / 128, + "hidden_states_scale dim0 must match hidden_states dim1 / 128."); + TORCH_CHECK(hidden_states_scale.sizes()[1] == num_tokens, "hidden_states_scale dim1 must match num_tokens."); + TORCH_CHECK(gemm1_weights.scalar_type() == at::ScalarType::Float8_e4m3fn, "gemm1_weights must be fp8."); + TORCH_CHECK(gemm1_weights.dim() == 3, "gemm1_weights must be 3D."); + TORCH_CHECK(gemm1_weights.sizes()[0] == local_num_experts, "gemm1_weights has incorrect shape."); + TORCH_CHECK(gemm1_weights.sizes()[1] % 2 == 0, "the second dimension of weights must be even."); + TORCH_CHECK(intermediate_size == gemm1_weights.sizes()[1] / 2, "intermediate_size has incorrect shape."); + TORCH_CHECK( + gemm1_weights.sizes()[2] == hidden_size, "the third dimension of weights must be equal to hidden_size."); + TORCH_CHECK(gemm1_weights_scale.scalar_type() == at::ScalarType::Float, "gemm1_weights_scale must be float."); + TORCH_CHECK(gemm1_weights_scale.dim() == 3, "gemm1_weights_scale must be 3D."); + TORCH_CHECK(gemm1_weights_scale.sizes()[0] == local_num_experts, "gemm1_weights_scale has incorrect dim 0."); + TORCH_CHECK(intermediate_size % 128 == 0, "the second dimension of weights must be a multiple of 128."); + TORCH_CHECK( + gemm1_weights_scale.sizes()[1] == 2 * intermediate_size / 128, "gemm1_weights_scale has incorrect shape."); + TORCH_CHECK(gemm1_weights_scale.sizes()[2] == hidden_size / 128, "gemm1_weights_scale has incorrect shape."); + TORCH_CHECK(gemm2_weights.scalar_type() == at::ScalarType::Float8_e4m3fn, "gemm2_weights must be fp8."); + TORCH_CHECK(gemm2_weights.dim() == 3, "gemm2_weights must be 3D."); + TORCH_CHECK(gemm2_weights.sizes()[0] == local_num_experts, "gemm2_weights has incorrect shape."); + TORCH_CHECK(gemm2_weights.sizes()[2] == intermediate_size, + "the third dimension of weights must be equal to intermediate_size."); + TORCH_CHECK(gemm2_weights_scale.scalar_type() == at::ScalarType::Float, "gemm2_weights_scale must be float."); + TORCH_CHECK(gemm2_weights_scale.dim() == 3, "gemm2_weights_scale must be 3D."); + TORCH_CHECK(gemm2_weights_scale.sizes()[0] == local_num_experts, "gemm2_weights_scale has incorrect dim 0."); + TORCH_CHECK(gemm2_weights_scale.sizes()[1] == hidden_size / 128, "gemm2_weights_scale has incorrect shape."); + TORCH_CHECK(gemm2_weights_scale.sizes()[2] == intermediate_size / 128, "gemm2_weights_scale has incorrect shape."); + + TORCH_CHECK( + front::supports(static_cast(num_tokens), static_cast(num_experts), + static_cast(top_k), static_cast(local_num_experts), static_cast(hidden_size)), + "fp8_block_scale_moe_front: unsupported problem size"); + if (hidden_states_bf16.has_value()) + { + TORCH_CHECK(hidden_states_bf16->scalar_type() == at::ScalarType::BFloat16 && hidden_states_bf16->is_contiguous() + && hidden_states_bf16->sizes() == hidden_states.sizes(), + "fp8_block_scale_moe_front: hidden_states_bf16 must be contiguous bf16 of hidden_states' shape"); + TORCH_CHECK(hidden_states.is_contiguous() && hidden_states_scale.is_contiguous(), + "fp8_block_scale_moe_front: the quantization outputs must be contiguous"); + TORCH_CHECK(hidden_states_bf16->device() == hidden_states.device(), + "hidden_states_bf16 and hidden_states must be on the same device"); + } +} + +// Returns [T, H] bf16. Given `hidden_states_bf16` [T, H], the routing launch first writes its 1x128 e4m3 quantization +// (bitwise `fp8_quantize_1x128`) into `hidden_states` [T, H] and `hidden_states_scale` [H / 128, T] fp32. +at::Tensor fp8BlockScaleMoeFront(at::Tensor const& routing_logits, std::optional const& hidden_states_bf16, + at::Tensor& hidden_states, at::Tensor& hidden_states_scale, at::Tensor const& gemm1_weights, + at::Tensor const& gemm1_weights_scale, at::Tensor const& gemm2_weights, at::Tensor const& gemm2_weights_scale, + int64_t const num_experts, int64_t const top_k, int64_t const intermediate_size, int64_t const local_expert_offset, + int64_t const local_num_experts, int64_t const routing_method_type, int64_t const tile_n, int64_t const config) +{ + c10::cuda::CUDAGuard const deviceGuard{hidden_states.device()}; + TORCH_CHECK(::tensorrt_llm::common::isSM100Family(), "Only SM100f is supported by FP8 block scale MOE"); + TORCH_CHECK(config >= 0, "fp8_block_scale_moe_front: needs an explicit config, got ", config); + checkOperands(routing_logits, hidden_states_bf16, hidden_states, hidden_states_scale, gemm1_weights, + gemm1_weights_scale, gemm2_weights, gemm2_weights_scale, num_experts, top_k, intermediate_size, + local_num_experts, routing_method_type); + trtllmGenMoe::MoE::Runner& runner = moeRunner(tile_n); + int64_t const tile_tokens_dim = tile_n; + int64_t const num_tokens = hidden_states.sizes()[0]; + int64_t const hidden_size = hidden_states.sizes()[1]; + auto const device = hidden_states.device(); + auto const stream = at::cuda::getCurrentCUDAStream(hidden_states.get_device()); + + trtllmGenMoe::MoE::MoERunnerArgs args; + trtllmGenMoe::MoE::MoEWorkspace workspace; + + // The output is bf16 (MoERunnerArgs' default mDtypeOut). + args.mDtypeElt = btg::Dtype::E4m3; + args.mDtypeBias = btg::Dtype::Bfloat16; + args.routing_logits = routing_logits.data_ptr(); + args.hidden_states = hidden_states.data_ptr(); + args.hidden_states_scale = hidden_states_scale.data_ptr(); + args.gemm1_weights = gemm1_weights.data_ptr(); + args.gemm1_weights_scale = gemm1_weights_scale.data_ptr(); + args.gemm2_weights = gemm2_weights.data_ptr(); + args.gemm2_weights_scale = gemm2_weights_scale.data_ptr(); + args.num_tokens = static_cast(num_tokens); + args.num_experts = static_cast(num_experts); + args.hidden_size = static_cast(hidden_size); + args.top_k = static_cast(top_k); + args.num_fused_shared_experts = 0; + args.n_group = 0; + args.topk_group = 0; + args.local_expert_offset = static_cast(local_expert_offset); + args.local_num_experts = static_cast(local_num_experts); + args.routed_scaling_factor = 1.0; + args.intermediate_size = static_cast(intermediate_size); + args.mUseDeepSeekFp8 = true; + + int32_t const max_num_padded_tokens = trtllmGenMoe::Routing::getMaxPermutedPaddedCount( + args.num_tokens, args.top_k, args.num_experts, static_cast(tile_tokens_dim)); + int32_t const max_num_padded_tokens_gemm1 = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, 2 * args.intermediate_size, btg::dtypeGetNumBits(args.mDtypeElt)); + // maybeGetMinTokenCount pads to a 128 KiB floor at the given row width, so each buffer passes its own width. + int32_t const max_num_padded_tokens_activation = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, args.intermediate_size, btg::dtypeGetNumBits(args.mDtypeElt)); + int32_t const max_num_padded_tokens_gemm2 = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, args.hidden_size, btg::dtypeGetNumBits(args.mDtypeOut)); + at::Tensor total_num_padded_tokens = at::empty({}, at::TensorOptions().device(device).dtype(at::ScalarType::Int)); + at::Tensor expanded_idx_to_permuted_idx + = at::detail::empty_cuda({num_tokens * top_k}, at::ScalarType::Int, device, std::nullopt); + // getRouteMapAllocCount: the gemm1 cubins read one element past the end of the route map. + at::Tensor permuted_idx_to_token_idx + = at::detail::empty_cuda({trtllmGenMoe::Routing::getRouteMapAllocCount(max_num_padded_tokens)}, + at::ScalarType::Int, device, std::nullopt); + // The routing writes bf16 weights, which the finalize reads as mDtypeExpW against the bf16 gemm2_output. + at::Tensor expert_weights + = at::detail::empty_cuda({num_tokens, top_k}, at::ScalarType::BFloat16, device, std::nullopt); + + // The rgTma TMA-OOB descriptors use OOB_ADDR_GEN_MODE_BASE_128kB on Blackwell, so TMA may probe up to a 128 KiB + // boundary past the base: gemm1_output, activation_output and gemm2_output need 128 KiB alignment and >= 128 KiB + // mapped headroom (empty_cuda aligns to 512 B). The SF buffers are loaded via LDGSTS and exempt. + static constexpr int64_t kTmaOob128k = 128 * 1024; + std::vector tmaOobKeepAlive; // owns the backing storage for the aligned pointers + auto allocTmaOobWorkspace = [&](int64_t dataBytes) -> void* + { + int64_t const allocBytes = dataBytes + 2 * kTmaOob128k; // up-align slack + tail headroom + at::Tensor buf = at::detail::empty_cuda({allocBytes}, at::ScalarType::Byte, device, std::nullopt); + uint64_t const base = reinterpret_cast(buf.data_ptr()); + uint64_t const aligned = (base + kTmaOob128k - 1) & ~(static_cast(kTmaOob128k) - 1); + tmaOobKeepAlive.push_back(std::move(buf)); + return reinterpret_cast(aligned); + }; + void* gemm1_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_gemm1) * (2 * intermediate_size)); // e4m3 + at::Tensor gemm1_output_scale = at::detail::empty_cuda( + {2 * intermediate_size / 128, max_num_padded_tokens_gemm1}, at::ScalarType::Float, device, std::nullopt); + void* activation_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_activation) * intermediate_size); // e4m3 + at::Tensor activation_output_scale = at::detail::empty_cuda( + {intermediate_size / 128, max_num_padded_tokens_activation}, at::ScalarType::Float, device, std::nullopt); + void* gemm2_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_gemm2) * args.hidden_size * 2); // bf16 + + int32_t const max_num_ctas = trtllmGenMoe::Routing::getMaxNumCtasInBatchDim( + args.num_tokens, args.top_k, args.num_experts, static_cast(tile_tokens_dim)); + at::Tensor cta_idx_xy_to_batch_idx + = at::detail::empty_cuda({max_num_ctas}, at::ScalarType::Int, device, std::nullopt); + at::Tensor cta_idx_xy_to_mn_limit + = at::detail::empty_cuda({max_num_ctas}, at::ScalarType::Int, device, std::nullopt); + at::Tensor num_non_exiting_ctas = at::empty({}, at::TensorOptions().device(device).dtype(at::ScalarType::Int)); + + front::Data data; + data.ptrScores = routing_logits.data_ptr(); + data.numTokens = args.num_tokens; + data.numExperts = args.num_experts; + data.topK = args.top_k; + data.tileTokensDim = static_cast(tile_tokens_dim); + data.localExpertsStartIdx = args.local_expert_offset; + data.numLocalExperts = args.local_num_experts; + data.hiddenDim = args.hidden_size; + data.usePdl = ::tensorrt_llm::common::getEnvEnablePDL(); + data.ptrTopKWeights = expert_weights.data_ptr(); + data.ptrExpandedIdxToPermutedIdx = expanded_idx_to_permuted_idx.data_ptr(); + data.ptrPermutedIdxToTokenIdx = permuted_idx_to_token_idx.data_ptr(); + data.ptrPermutedIdxSize = total_num_padded_tokens.data_ptr(); + data.ptrCtaIdxXyToBatchIdx = cta_idx_xy_to_batch_idx.data_ptr(); + data.ptrCtaIdxXyToMnLimit = cta_idx_xy_to_mn_limit.data_ptr(); + data.ptrNumNonExitingCtas = num_non_exiting_ctas.data_ptr(); + if (hidden_states_bf16.has_value()) + { + data.ptrHiddenBf16 = hidden_states_bf16->data_ptr(); + data.ptrHiddenFp8 = hidden_states.data_ptr(); + data.ptrHiddenScale = hidden_states_scale.data_ptr(); + } + front::run(data, stream); + + at::Tensor output + = at::detail::empty_cuda({num_tokens, hidden_size}, at::ScalarType::BFloat16, device, std::nullopt); + + workspace.total_num_padded_tokens = total_num_padded_tokens.data_ptr(); + workspace.total_max_padded_tokens + = std::max({max_num_padded_tokens_gemm1, max_num_padded_tokens_activation, max_num_padded_tokens_gemm2}); + workspace.permuted_idx_size = total_num_padded_tokens.data_ptr(); + workspace.expanded_idx_to_permuted_idx = expanded_idx_to_permuted_idx.data_ptr(); // activation / finalize + workspace.permuted_idx_to_token_idx = permuted_idx_to_token_idx.data_ptr(); // permute + gemm1 + workspace.expert_weights = expert_weights.data_ptr(); // finalize + workspace.cta_idx_xy_to_batch_idx = cta_idx_xy_to_batch_idx.data_ptr(); + workspace.cta_idx_xy_to_mn_limit = cta_idx_xy_to_mn_limit.data_ptr(); + workspace.num_non_exiting_ctas = num_non_exiting_ctas.data_ptr(); + workspace.gemm1_output = gemm1_output_ptr; + workspace.gemm1_output_scale = gemm1_output_scale.data_ptr(); + workspace.activation_output = activation_output_ptr; + workspace.activation_output_scale = activation_output_scale.data_ptr(); + workspace.gemm2_output = gemm2_output_ptr; + workspace.gemm2_output_scale = nullptr; + args.output = output.data_ptr(); + args.output_scale = nullptr; + + auto const workspace_sizes = runner.getWorkspaceSizeInBytes(args, config); + at::Tensor workspace_fc1 + = at::detail::empty_cuda({std::get<0>(workspace_sizes)}, at::ScalarType::Char, device, std::nullopt); + at::Tensor workspace_fc2 + = at::detail::empty_cuda({std::get<1>(workspace_sizes)}, at::ScalarType::Char, device, std::nullopt); + workspace.bmm1_workspace = workspace_fc1.data_ptr(); + workspace.bmm2_workspace = workspace_fc2.data_ptr(); + + runner.run(args, workspace, hidden_states.get_device(), stream, config); + return output; +} + +} // namespace + +} // namespace torch_ext + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "fp8_block_scale_moe_front(Tensor routing_logits, Tensor? hidden_states_bf16, Tensor(a!) hidden_states, " + "Tensor(b!) hidden_states_scale, Tensor gemm1_weights, Tensor gemm1_weights_scale, Tensor gemm2_weights, " + "Tensor gemm2_weights_scale, int num_experts, int top_k, int intermediate_size, int local_expert_offset, " + "int local_num_experts, int routing_method_type, int tile_n, int config) -> Tensor"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("fp8_block_scale_moe_front", &TRTLLM_MV2_NS::torch_ext::fp8BlockScaleMoeFront); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu new file mode 100644 index 000000000000..34d0e9cf4bf4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu @@ -0,0 +1,539 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ +namespace hc_inject +{ +// This kernel runs on the workspace of the stock one-shot all-reduce `allreduce_fusion_kernel_oneshot_lamport` +// (communicationKernels/allReduceFusionKernels.cu) between its calls, so LamportComm, the helpers below and the +// push / clear / poll / update steps must match that file's Lamport protocol: +// - workspace[2 * NRanks + r] is rank r's buffer of three sub-buffers of workspace[3 * NRanks + 1][1] bytes; with +// flag = workspace[3 * NRanks][2], a call writes and polls sub-buffer flag % 3 and clears its own (flag + 2) % 3. +// - workspace[3 * NRanks + 1][0] holds the bytes the previous call wrote, cleared as 16-byte stores of -0.0f. +// - A 16-byte slot has arrived once none of its 32-bit words is -0.0f, so senders store -0.0f as +0.0f. +// - Every CTA increments workspace[3 * NRanks][0] after reading the flag; once all have, CTA 0 advances the flag to +// (flag + 1) % 3, stores the bytes this call wrote and resets the counter. +template +struct LamportComm +{ + __device__ __forceinline__ LamportComm(void** workspace, int rank) + { + counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; + flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; + // In bytes whatever the writer's dtype; readers round up to 16-byte stores (some writers store 4-byte words). + clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; + flag_value = *flag_ptr; + auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; + clear_size = *clear_ptr; + int data_offset = flag_value % 3; + int clear_offset = (flag_value + 2) % 3; + for (int r = 0; r < NRanks; ++r) + { + data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; + } + clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; + __syncthreads(); + if (threadIdx.x == 0) + { + atomicAdd(counter_ptr, 1); + } + } + + __device__ __forceinline__ void update(int64_t new_clear_size) + { + if (blockIdx.x == 0 && threadIdx.x == 0) + { + while (*reinterpret_cast(counter_ptr) != gridDim.x) + { + } + *flag_ptr = (flag_value + 1) % 3; + *clear_ptr = new_clear_size; + *counter_ptr = 0; + } + } + + int* counter_ptr; + int* flag_ptr; + int64_t* clear_ptr; + uint8_t* data_bufs[NRanks]; + uint8_t* clear_buf; + int64_t clear_size; + int flag_value; +}; + +__device__ __forceinline__ bool is_neg_zero(float v) +{ + return *reinterpret_cast(&v) == 0x80000000; +} + +__device__ __forceinline__ bool is_neg_zero(float4 v) +{ + return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); +} + +__device__ __forceinline__ float4 get_neg_zero() +{ + float4 vec; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + reinterpret_cast(&vec)[i] = 0x80000000; + } + return vec; +} + +__device__ __forceinline__ float4 ld_global_volatile(float4* addr) +{ + float4 val; + asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" + : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) + : "l"(addr)); + return val; +} + +int get_sm_count() +{ + static int sm_count = 0; + if (sm_count == 0) + { + int device_id; + TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); + cudaDeviceProp device_prop; + cudaGetDeviceProperties(&device_prop, device_id); + sm_count = device_prop.multiProcessorCount; + } + return sm_count; +} +} // namespace hc_inject + +// Thread t holds columns [8t, 8t + 8) of a row's block output and of each of its 4 streams. The Triton program runs a +// stream on G = 32 * program_warps threads over a 4096-wide block, its thread u in chunk c holding this CTA's thread +// c * G + u. Its sum of squares, reproduced bitwise: per thread and chunk the fp32 pair (fma(x0, x0, x2 * x2) + +// fma(x4, x4, x6 * x6), fma(x1, x1, x3 * x3) + fma(x5, x5, x7 * x7)), a pairwise tree over chunks (masked ones are +// zero), low + high, the warp's xor butterfly 16..1, then ((w0 + w2) + (w1 + w3)) at 4 warps or (w0 + w1) at 2. +namespace hc_inject +{ +static constexpr int kHidden = 2560; +static constexpr int kStreams = 4; +static constexpr int kThreads = kHidden / 8; + +__device__ __forceinline__ float bf16_lo(uint32_t w) +{ + return __uint_as_float(w << 16); +} + +__device__ __forceinline__ float bf16_hi(uint32_t w) +{ + return __uint_as_float(w & 0xFFFF0000u); +} + +__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) +{ + uint32_t r; + asm("cvt.rn.bf16x2.f32 %0, %1, %2;" : "=r"(r) : "f"(hi), "f"(lo)); + return r; +} + +__device__ __forceinline__ float round_bf16(float x) +{ + return bf16_lo(pack_bf16x2(x, 0.f)); +} + +__device__ __forceinline__ float ld_bf16(void const* base, int64_t elem) +{ + uint16_t v = reinterpret_cast(base)[elem]; + return __uint_as_float(static_cast(v) << 16); +} + +//! `tl.sigmoid` as the Triton kernel compiles it. +__device__ __forceinline__ float sigmoid_triton(float x) +{ + float y; + asm("{ .reg .f32 t; mul.f32 t, %1, 0fBFB8AA3B; ex2.approx.f32 t, t; add.f32 t, t, 0f3F800000; " + "div.full.f32 %0, 0f3F800000, t; }" + : "=f"(y) + : "f"(x)); + return y; +} + +//! `rsqrt(ss / 2560 + eps)` as the Triton kernel compiles it (`div.full`, `add`, `rsqrt.approx.ftz`). +__device__ __forceinline__ float rstd_triton(float ss, float eps) +{ + float y; + asm("{ .reg .f32 t; div.full.f32 t, %1, 0f45200000; add.f32 t, t, %2; rsqrt.approx.ftz.f32 %0, t; }" + : "=f"(y) + : "f"(ss), "f"(eps)); + return y; +} + +__device__ __forceinline__ float2 add2(float2 a, float2 b) +{ + return make_float2(__fadd_rn(a.x, b.x), __fadd_rn(a.y, b.y)); +} + +//! A program thread's sum of squares: the pairwise tree over its unmasked chunk pairs, then low + high. +template +__device__ __forceinline__ float thread_ssq(float2 const* leaves, int u) +{ + if constexpr (G == 128) + { + // 4 chunks of 1024 columns: chunk 2 real for u < 64, chunk 3 masked. + float2 l = add2(leaves[u], leaves[G + u]); + if (u < 64) + { + l = add2(l, leaves[2 * G + u]); + } + return __fadd_rn(l.x, l.y); + } + else + { + static_assert(G == 64, "4- or 2-warp programs"); + // 8 chunks of 512 columns: chunks 0..4 real, 5..7 masked. + float2 a = add2(leaves[u], leaves[G + u]); + float2 b = add2(leaves[2 * G + u], leaves[3 * G + u]); + float2 l = add2(add2(a, b), leaves[4 * G + u]); + return __fadd_rn(l.x, l.y); + } +} + +//! Launch forms with the same arithmetic: Var 2 asks for three CTAs an SM (one wave of one-row CTAs up to 3 x SMs), +//! Var 3 has no occupancy bound. launch() takes 3 while the rows fit its resident grid, else 2 if they fit 2's, else 3. +template +struct HcVariant +{ + static constexpr int kMinBlocks = Var == 2 ? 3 : 1; +}; + +template +__global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) + hc_allreduce_inject_norm_kernel(HcInjectNormParams params) +{ + static_assert(kHidden % (8 * G) == 0 || G == 128, "the chunk layout above"); + __shared__ float2 s_leaf[kStreams][kThreads]; + __shared__ float s_red[kStreams][G / 32]; + __shared__ uint4 s_w[kStreams][kThreads]; + int const tid = threadIdx.x; + int const lane = tid & 31; + + // No kernel of the step writes the norm weights: stage them before the PDL wait (each thread reads back its own). +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + s_w[s][tid] = reinterpret_cast(params.norm_weight)[s * kThreads + tid]; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + if constexpr (!TriggerCompletionAtEnd) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif + LamportComm comm(params.workspace, params.rank); + int const rows = params.rows; + int const tot_access = rows * kThreads; + int const clear_access = static_cast((comm.clear_size + sizeof(float4) - 1) / sizeof(float4)); + + // Push this rank's partial rows, -0.0f stored as +0.0f. + for (int row = blockIdx.x; row < rows; row += gridDim.x) + { + int const idx = row * kThreads + tid; + alignas(16) float val[4]; + *reinterpret_cast(val) = reinterpret_cast(params.partial)[idx]; + +#pragma unroll + for (int i = 0; i < 4; ++i) + { + if (is_neg_zero(val[i])) + { + val[i] = 0.f; + } + } +#pragma unroll + for (int r = 0; r < NRanks; ++r) + { + reinterpret_cast(comm.data_bufs[r])[params.rank * tot_access + idx] + = *reinterpret_cast(val); + } + } + float4 const clear_vec = get_neg_zero(); + for (int idx = blockIdx.x * kThreads + tid; idx < clear_access; idx += gridDim.x * kThreads) + { + reinterpret_cast(comm.clear_buf)[idx] = clear_vec; + } + + for (int row = blockIdx.x; row < rows; row += gridDim.x) + { + int const idx = row * kThreads + tid; + // The site's operands (issued before the poll, consumed after it). + uint4 hcw[kStreams]; + float inj[kStreams]; + auto const* hc_row = reinterpret_cast(params.hc) + row * params.hc_stride * 2; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + hcw[s] = reinterpret_cast(hc_row + s * kHidden * 2)[tid]; + inj[s] = ld_bf16(params.inject, row * params.inject_stride + s); + } + [[maybe_unused]] uint4 shw = make_uint4(0, 0, 0, 0); + [[maybe_unused]] float gate = 0.f; + if constexpr (HasShared) + { + shw = reinterpret_cast( + reinterpret_cast(params.shared) + row * params.shared_stride * 2)[tid]; + gate = ld_bf16(params.gate, row * params.gate_stride); + } + + float4 vals[NRanks]; + bool done = false; + while (!done) + { + done = true; +#pragma unroll + for (int r = 0; r < NRanks; ++r) + { + vals[r] + = ld_global_volatile(&reinterpret_cast(comm.data_bufs[params.rank])[r * tot_access + idx]); + done &= !is_neg_zero(vals[r]); + } + } + // The all-reduce's bits: fp32 sum in rank order, rounded once. + float b[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + float acc = static_cast(reinterpret_cast<__nv_bfloat16 const*>(&vals[0])[i]); +#pragma unroll + for (int r = 1; r < NRanks; ++r) + { + acc += static_cast(reinterpret_cast<__nv_bfloat16 const*>(&vals[r])[i]); + } + b[i] = round_bf16(acc); + } + if constexpr (HasShared) + { + float const sg = sigmoid_triton(gate); + uint32_t const* sw = reinterpret_cast(&shw); +#pragma unroll + for (int p = 0; p < 4; ++p) + { + b[2 * p] = round_bf16(__fmaf_rn(sg, bf16_lo(sw[p]), b[2 * p])); + b[2 * p + 1] = round_bf16(__fmaf_rn(sg, bf16_hi(sw[p]), b[2 * p + 1])); + } + } + // Injection (rounded once, stored) and the per-thread sum-of-squares pairs. + uint32_t ow[kStreams][4]; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + uint32_t const* xw = reinterpret_cast(&hcw[s]); + float e[8]; +#pragma unroll + for (int p = 0; p < 4; ++p) + { + ow[s][p] = pack_bf16x2( + __fmaf_rn(inj[s], b[2 * p], bf16_lo(xw[p])), __fmaf_rn(inj[s], b[2 * p + 1], bf16_hi(xw[p]))); + e[2 * p] = bf16_lo(ow[s][p]); + e[2 * p + 1] = bf16_hi(ow[s][p]); + } + reinterpret_cast( + reinterpret_cast(params.hc_out) + (row * params.hc_out_stride + s * kHidden) * 2)[tid] + = make_uint4(ow[s][0], ow[s][1], ow[s][2], ow[s][3]); + float const lo + = __fadd_rn(__fmaf_rn(e[0], e[0], __fmul_rn(e[2], e[2])), __fmaf_rn(e[4], e[4], __fmul_rn(e[6], e[6]))); + float const hi + = __fadd_rn(__fmaf_rn(e[1], e[1], __fmul_rn(e[3], e[3])), __fmaf_rn(e[5], e[5], __fmul_rn(e[7], e[7]))); + s_leaf[s][tid] = make_float2(lo, hi); + } + __syncthreads(); + if (tid < G) + { + float v[kStreams]; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + v[s] = thread_ssq(s_leaf[s], tid); + } +#pragma unroll + for (int off = 16; off >= 1; off >>= 1) + { +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + v[s] = __fadd_rn(v[s], __shfl_xor_sync(0xffffffffu, v[s], off)); + } + } + if (lane == 0) + { +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + s_red[s][tid / 32] = v[s]; + } + } + } + __syncthreads(); +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + float tot; + if constexpr (G == 128) + { + tot = __fadd_rn(__fadd_rn(s_red[s][0], s_red[s][2]), __fadd_rn(s_red[s][1], s_red[s][3])); + } + else + { + tot = __fadd_rn(s_red[s][0], s_red[s][1]); + } + float const rstd = rstd_triton(tot, params.eps); + uint4 const wq = s_w[s][tid]; + uint32_t const* ww = reinterpret_cast(&wq); + uint32_t nw[4]; +#pragma unroll + for (int p = 0; p < 4; ++p) + { + float const ylo = __fmul_rn(__fmul_rn(bf16_lo(ow[s][p]), rstd), __fadd_rn(1.f, bf16_lo(ww[p]))); + float const yhi = __fmul_rn(__fmul_rn(bf16_hi(ow[s][p]), rstd), __fadd_rn(1.f, bf16_hi(ww[p]))); + nw[p] = pack_bf16x2(ylo, yhi); + } + reinterpret_cast( + reinterpret_cast(params.normed) + (row * params.normed_stride + s * kHidden) * 2)[tid] + = make_uint4(nw[0], nw[1], nw[2], nw[3]); + } + } + + comm.update(static_cast(rows) * kHidden * NRanks * sizeof(__nv_bfloat16)); + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + if constexpr (TriggerCompletionAtEnd) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif +} + +template +int max_grid() +{ + static int const grid = []() + { + int per_sm = 0; + TLLM_CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &per_sm, hc_allreduce_inject_norm_kernel, kThreads, 0)); + return std::max(1, per_sm) * get_sm_count(); + }(); + return grid; +} + +template +void launch_var(HcInjectNormParams const& params) +{ + auto kernel = hc_allreduce_inject_norm_kernel; + int const max_grid = hc_inject::max_grid(); + cudaLaunchConfig_t cfg; + cudaLaunchAttribute attribute[1]; + cfg.gridDim = std::min(params.rows, max_grid); + cfg.blockDim = kThreads; + cfg.dynamicSmemBytes = 0; + cfg.stream = params.stream; + attribute[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attribute[0].val.programmaticStreamSerializationAllowed = tensorrt_llm::common::getEnvEnablePDL() ? 1 : 0; + cfg.attrs = attribute; + cfg.numAttrs = 1; + TLLM_CUDA_CHECK(cudaLaunchKernelEx(&cfg, kernel, params)); +} + +template +void launch(HcInjectNormParams const& params) +{ + int const g3 = hc_inject::max_grid(); + int const g2 = hc_inject::max_grid(); + if (params.rows > g3 && params.rows <= g2) + { + launch_var(params); + } + else + { + launch_var(params); + } +} + +template +void dispatch_flags(HcInjectNormParams const& params) +{ + bool const shared = params.shared != nullptr; + if (shared && params.trigger_completion_at_end) + { + launch(params); + } + else if (shared) + { + launch(params); + } + else if (params.trigger_completion_at_end) + { + launch(params); + } + else + { + launch(params); + } +} +} // namespace hc_inject + +void hc_allreduce_inject_norm_op(HcInjectNormParams const& params) +{ + TLLM_CHECK_WITH_INFO(params.hidden_dim == hc_inject::kHidden && params.hc_count == hc_inject::kStreams, + "hc_allreduce_inject_norm: hidden 2560 and 4 streams only"); + TLLM_CHECK_WITH_INFO(params.rows > 0, "hc_allreduce_inject_norm: no rows"); + TLLM_CHECK_WITH_INFO(params.program_warps == 4 || params.program_warps == 2, + "hc_allreduce_inject_norm: the Triton program's warps must be 4 or 2"); + TLLM_CHECK_WITH_INFO((params.shared == nullptr) == (params.gate == nullptr), + "hc_allreduce_inject_norm: shared and gate come together"); + TLLM_CHECK_WITH_INFO(params.partial != nullptr, "hc_allreduce_inject_norm: no partial"); + if (params.nranks == 2) + { + params.program_warps == 4 ? hc_inject::dispatch_flags<2, 128>(params) + : hc_inject::dispatch_flags<2, 64>(params); + } + else if (params.nranks == 4) + { + params.program_warps == 4 ? hc_inject::dispatch_flags<4, 128>(params) + : hc_inject::dispatch_flags<4, 64>(params); + } + else + { + TLLM_CHECK_WITH_INFO(false, "hc_allreduce_inject_norm: 2 or 4 ranks only"); + } +} +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h new file mode 100644 index 000000000000..8ea3e1fe3326 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ + +//! IPC one-shot Lamport all-reduce of a block output (stock `allreduce_fusion_kernel_oneshot_lamport` workspace and +//! protocol) fused with a hyper-connection site's `hc_out = bf16(fma(inject, block, hc))` and grouped RMS norm +//! `normed = bf16((hc_out * rstd) * (1 + w))`; bitwise that all-reduce followed by the Triton `_hc_inject_norm_kernel`. +//! With `shared` and `gate`, the block is first `bf16(fma(sigmoid(gate), shared, routed))`. +struct HcInjectNormParams +{ + int nranks; + int rank; + int rows; + int hidden_dim; // 2560 + int hc_count; // 4 + int program_warps; // 4 or 2: warps of the Triton program whose reduction order is reproduced + void** workspace; // the stock all-reduce fusion IPC workspace + void const* partial; // [rows, hidden] bf16, contiguous + void const* hc; // [rows, hc_count * hidden] bf16 + int64_t hc_stride; + void const* inject; // [rows, >= hc_count] bf16 + int64_t inject_stride; + void const* shared; // optional [rows, hidden] bf16 + int64_t shared_stride; + void const* gate; // optional [rows, 1] bf16 + int64_t gate_stride; + void const* norm_weight; // [hc_count * hidden] bf16 + void* hc_out; // [rows, hc_count * hidden] bf16 + int64_t hc_out_stride; + void* normed; // [rows, hc_count * hidden] bf16 + int64_t normed_stride; + float eps; + bool trigger_completion_at_end; + cudaStream_t stream; +}; + +void hc_allreduce_inject_norm_op(HcInjectNormParams const& params); + +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp new file mode 100644 index 000000000000..c2f6311a958f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace +{ + +//! Returns `(hc_out, normed)` for the block output summed over ranks; see kernels::HcInjectNormParams. +std::vector hcAllreduceInjectNorm(at::Tensor const& partial, at::Tensor workspace, int64_t const rank, + int64_t const nranks, at::Tensor const& hc, at::Tensor const& inject, std::optional const& shared, + std::optional const& gate, at::Tensor const& norm_weight, double const eps, int64_t const program_warps, + bool const trigger_completion_at_end) +{ + auto const bf16 = at::ScalarType::BFloat16; + int64_t const rows = hc.size(0); + TORCH_CHECK( + partial.dim() == 2 && partial.is_contiguous() && partial.scalar_type() == bf16 && partial.size(0) == rows, + "hc_allreduce_inject_norm: partial must be a contiguous [rows, hidden] bf16 tensor"); + int64_t const hidden = partial.size(1); + TORCH_CHECK(hc.dim() == 2 && hc.size(1) % hidden == 0 && hc.stride(1) == 1 && hc.scalar_type() == bf16, + "hc_allreduce_inject_norm: hc must be [rows, hc_count * hidden] bf16 with unit column stride"); + int64_t const hc_count = hc.size(1) / hidden; + TORCH_CHECK(hc.stride(0) % 8 == 0 && reinterpret_cast(hc.data_ptr()) % 16 == 0, + "hc_allreduce_inject_norm: hc rows must be 16-byte aligned"); + TORCH_CHECK(inject.dim() == 2 && inject.size(0) == rows && inject.size(1) >= hc_count && inject.stride(1) == 1 + && inject.scalar_type() == bf16, + "hc_allreduce_inject_norm: inject must be [rows, >= hc_count] bf16 with unit column stride"); + TORCH_CHECK( + norm_weight.numel() == hc_count * hidden && norm_weight.is_contiguous() && norm_weight.scalar_type() == bf16, + "hc_allreduce_inject_norm: norm_weight must be [hc_count * hidden] bf16"); + TORCH_CHECK(shared.has_value() == gate.has_value(), "hc_allreduce_inject_norm: shared and gate come together"); + if (shared.has_value()) + { + TORCH_CHECK(shared->dim() == 2 && shared->size(0) == rows && shared->size(1) == hidden && shared->stride(1) == 1 + && shared->stride(0) % 8 == 0 && reinterpret_cast(shared->data_ptr()) % 16 == 0 + && shared->scalar_type() == bf16, + "hc_allreduce_inject_norm: shared must be [rows, hidden] bf16 with 16-byte aligned rows"); + TORCH_CHECK(gate->dim() == 2 && gate->size(0) == rows && gate->scalar_type() == bf16, + "hc_allreduce_inject_norm: gate must be [rows, 1] bf16"); + } + auto hc_out = at::empty({rows, hc_count * hidden}, hc.options()); + auto normed = at::empty({rows, hc_count * hidden}, hc.options()); + + kernels::HcInjectNormParams params; + params.nranks = static_cast(nranks); + params.rank = static_cast(rank); + params.rows = static_cast(rows); + params.hidden_dim = static_cast(hidden); + params.hc_count = static_cast(hc_count); + params.program_warps = static_cast(program_warps); + params.workspace = reinterpret_cast(workspace.mutable_data_ptr()); + params.partial = partial.data_ptr(); + params.hc = hc.data_ptr(); + params.hc_stride = hc.stride(0); + params.inject = inject.data_ptr(); + params.inject_stride = inject.stride(0); + params.shared = shared.has_value() ? shared->data_ptr() : nullptr; + params.shared_stride = shared.has_value() ? shared->stride(0) : 0; + params.gate = gate.has_value() ? gate->data_ptr() : nullptr; + params.gate_stride = gate.has_value() ? gate->stride(0) : 0; + params.norm_weight = norm_weight.data_ptr(); + params.hc_out = hc_out.mutable_data_ptr(); + params.hc_out_stride = hc_out.stride(0); + params.normed = normed.mutable_data_ptr(); + params.normed_stride = normed.stride(0); + params.eps = static_cast(eps); + params.trigger_completion_at_end = trigger_completion_at_end; + params.stream = at::cuda::getCurrentCUDAStream(hc.get_device()); + kernels::hc_allreduce_inject_norm_op(params); + return {hc_out, normed}; +} + +} // namespace + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "hc_allreduce_inject_norm(" + "Tensor partial," + "Tensor workspace," + "int rank," + "int nranks," + "Tensor hc," + "Tensor inject," + "Tensor? shared," + "Tensor? gate," + "Tensor norm_weight," + "float eps," + "int program_warps," + "bool trigger_completion_at_end) -> Tensor[]"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("hc_allreduce_inject_norm", &TRTLLM_MV2_NS::hcAllreduceInjectNorm); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py new file mode 100644 index 000000000000..b41efcb4168b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fake (meta) implementations of this target's native ops, built from its csrc/. + +The loader imports this module right after the target is found in the family +library, so a fake for an op the library does not define fails the load. +""" + +import torch + +from tensorrt_llm._torch.modeling_v2._native import namespace_of + +_NS = namespace_of(__name__) + + +def _host_query(qualname: str): + """Ops without tensor arguments answer a host-side question at setup; tracing never runs them.""" + + def fake(*args, **kwargs): + raise NotImplementedError(f"{qualname} is a setup-time query; call it outside tracing") + + return torch.library.register_fake(qualname)(fake) + + +_host_query(f"{_NS}::fp8_block_moe_tpb_workspace_bytes") + + +@torch.library.register_fake(f"{_NS}::fp8_block_moe_tpb") +def _( + logits, + xq, + xsf, + w1, + s1, + w2, + s2, + rank, + workspace, + enable_pdl=False, + sh_x=None, + sh_gu=None, + sh_gl=None, + sh_dn=None, +): + return logits.new_empty((logits.shape[0], xq.shape[1])) + + +@torch.library.register_fake(f"{_NS}::hc_allreduce_inject_norm") +def _( + partial, + workspace, + rank, + nranks, + hc, + inject, + shared, + gate, + norm_weight, + eps, + program_warps, + trigger_completion_at_end, +): + # Two new contiguous [rows, hc_count * hidden] tensors (`hc` may carry a padded row stride). + shape = (hc.shape[0], hc.shape[1]) + return [hc.new_empty(shape), hc.new_empty(shape)] + + +@torch.library.register_fake(f"{_NS}::fp8_block_scale_moe_front") +def _( + routing_logits, + hidden_states_bf16, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + tile_n, + config, +): + # A new [rows, hidden] bf16 tensor; hidden_states / hidden_states_scale are written in place. + return hidden_states.new_empty(hidden_states.shape, dtype=torch.bfloat16) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py new file mode 100644 index 000000000000..2b56f59da34e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py @@ -0,0 +1,443 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""One host copy of a read-only table per host, shared by every rank there. + +With `PLE_SHARED_HOST_ENV` set, the ranks of one host map one allocation of the PLE n-gram table +instead of one copy each, across processes and servers (a disaggregated node's context and +generation servers share no communicator). The memory comes from a `Backing`: the engine's +`weights._VmmHostBacking` is a CUDA VMM host allocation exported as a POSIX fd, because on GB300 +registered shmem pages gather several times slower; `MemfdBacking` serves the CPU tests. + +Processes meet through `.lock` (an `flock`) and `.json` (the published descriptor: +owner pid and start time, fd number, size) in `PLE_SHARED_HOST_DIR_ENV` (default `/dev/shm`): + +* The first process to take the lock while no live descriptor exists is the **owner**: it creates + and fills the allocation, then publishes the descriptor and releases the lock. +* Every other process **attaches** by duplicating the owner's fd (`pidfd_getfd`, else + `/proc//fd/`) and mapping it, or keeps a private, unpublished copy if neither works. +* A process arriving while the owner fills is **pending**: it loads its other weights and waits + on the lock only at the table. An owner's death releases the lock; the waiter becomes owner. + +The allocation lives while any process maps it, so a crash leaks nothing. The bytes and the +gather match the per-rank copy, so output is bitwise identical. +""" + +from __future__ import annotations + +import ctypes +import errno +import fcntl +import hashlib +import json +import os +import platform +import time +from typing import Callable, Optional, Protocol + +#: `node` (one copy per host); unset, `0`, `off`, `false` or `no` keep one copy per rank. +PLE_SHARED_HOST_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST" +#: Where the rendezvous files live; must be a directory every sharing process sees (tmpfs). +PLE_SHARED_HOST_DIR_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST_DIR" +_DEFAULT_DIR = "/dev/shm" +_OFF = frozenset({"", "0", "off", "false", "no", "none"}) +_SCOPE = "node" + +OWNER = "owner" +ATTACHED = "attached" +PENDING = "pending" + +# mmap(2) / mbind(2) constants (Linux, identical on aarch64 and x86_64). +_PROT_READ = 0x1 +_PROT_WRITE = 0x2 +_MAP_SHARED = 0x01 +_MPOL_PREFERRED = 1 +_SYS_MBIND = {"aarch64": 235, "x86_64": 237} +# pidfd_open(2) / pidfd_getfd(2): the generic syscall table, shared by aarch64 and x86_64. +_SYS_PIDFD_OPEN = 434 +_SYS_PIDFD_GETFD = 438 + +_libc = ctypes.CDLL(None, use_errno=True) +_libc.mmap.restype = ctypes.c_void_p +_libc.mmap.argtypes = [ + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int, + ctypes.c_int, + ctypes.c_int, + ctypes.c_long, +] +_libc.munmap.restype = ctypes.c_int +_libc.munmap.argtypes = [ctypes.c_void_p, ctypes.c_size_t] +_libc.syscall.restype = ctypes.c_long +_MAP_FAILED = ctypes.c_void_p(-1).value + + +def shared_host_scope() -> Optional[str]: + """`node`, or None (one copy per rank) from `PLE_SHARED_HOST_ENV`.""" + value = os.environ.get(PLE_SHARED_HOST_ENV, "").strip().lower() + if value in _OFF: + return None + if value != _SCOPE: + raise ValueError(f"{PLE_SHARED_HOST_ENV}={value!r}: expected {_SCOPE!r} or off") + return _SCOPE + + +def _parse_cpulist(text: str) -> set: + cpus = set() + for part in filter(None, text.strip().split(",")): + lo, _, hi = part.partition("-") + cpus.update(range(int(lo), int(hi or lo) + 1)) + return cpus + + +def numa_node_of_cpus(cpus: set, sysfs: str = "/sys/devices/system/node") -> Optional[int]: + """The NUMA node whose CPU list holds the lowest of `cpus`, or None when sysfs says nothing.""" + if not cpus: + return None + first = min(cpus) + try: + names = os.listdir(sysfs) + except OSError: + return None + for name in sorted(names): + if not (name.startswith("node") and name[4:].isdigit()): + continue + try: + with open(os.path.join(sysfs, name, "cpulist")) as fh: + if first in _parse_cpulist(fh.read()): + return int(name[4:]) + except OSError: + continue + return None + + +def _proc_start_time(pid: int) -> Optional[int]: + """`pid`'s start time in clock ticks, None if gone; with the pid, it survives pid reuse.""" + try: + with open(f"/proc/{pid}/stat") as fh: + stat = fh.read() + except OSError: + return None + # The command name (field 2) may hold spaces and parentheses: split after its last ')'. + return int(stat[stat.rindex(")") + 2 :].split()[19]) + + +def _dup_from(pid: int, fd: int) -> int: + """Duplicate `pid`'s fd `fd`: `pidfd_getfd` (needs ptrace-attach rights over `pid`), else + reopen `/proc//fd/` (enough for a memfd, not for a CUDA export).""" + pidfd = _libc.syscall(ctypes.c_long(_SYS_PIDFD_OPEN), ctypes.c_int(pid), ctypes.c_uint(0)) + if pidfd >= 0: + try: + local = _libc.syscall( + ctypes.c_long(_SYS_PIDFD_GETFD), + ctypes.c_int(pidfd), + ctypes.c_int(fd), + ctypes.c_uint(0), + ) + if local >= 0: + return int(local) + err = ctypes.get_errno() + finally: + os.close(pidfd) + else: + err = ctypes.get_errno() + try: + return os.open(f"/proc/{pid}/fd/{fd}", os.O_RDWR | os.O_CLOEXEC) + except OSError as error: + raise OSError( + error.errno, + f"cannot duplicate fd {fd} of pid {pid}: pidfd_getfd errno {err} " + f"({os.strerror(err)}), /proc reopen {error.strerror}", + ) from None + + +class Backing(Protocol): + """Where a segment's bytes live: `create` returns the device-read address and an fd another + process duplicates and passes to `attach`; `release` undoes either.""" + + def create(self, nbytes: int) -> tuple[int, int]: ... + + def attach(self, fd: int, nbytes: int) -> int: ... + + def release(self, address: int, nbytes: int) -> None: ... + + def describe(self) -> str: ... + + +#: `register(host_pointer, nbytes, read_only) -> unregister()`: `MemfdBacking`'s device side. +Register = Callable[[int, int, bool], Callable[[], None]] + + +def _prefer_node(address: int, nbytes: int, node: int) -> bool: + """Best effort `mbind(MPOL_PREFERRED, node)` over a mapping.""" + number = _SYS_MBIND.get(platform.machine()) + if number is None or node < 0: + return False + words = node // 64 + 1 + mask = (ctypes.c_ulong * words)() + mask[node // 64] = 1 << (node % 64) + rc = _libc.syscall( + ctypes.c_long(number), + ctypes.c_void_p(address), + ctypes.c_ulong(nbytes), + ctypes.c_int(_MPOL_PREFERRED), + mask, + ctypes.c_ulong(words * 64 + 1), + ctypes.c_uint(0), + ) + return rc == 0 + + +def _mmap(fd: int, nbytes: int) -> int: + address = _libc.mmap(None, nbytes, _PROT_READ | _PROT_WRITE, _MAP_SHARED, fd, 0) + if address in (None, _MAP_FAILED): + err = ctypes.get_errno() + raise OSError(err, f"mmap of {nbytes} bytes failed: {os.strerror(err)}") + return int(address) + + +class MemfdBacking: + """Shared memory (a sealed memfd) mapped in every process and handed to `register`.""" + + def __init__(self, register: Register, node: Optional[int] = None) -> None: + self._register = register + self.node = node + self._unregister: dict[int, Callable[[], None]] = {} + self._munmap = _libc.munmap + + def create(self, nbytes: int) -> tuple[int, int]: + fd = os.memfd_create("trtllm-ple", os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING) + try: + os.ftruncate(fd, nbytes) + # Nobody may resize what everyone maps (a shrink would SIGBUS every reader). + fcntl.fcntl(fd, fcntl.F_ADD_SEALS, fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_GROW) + address = _mmap(fd, nbytes) + except BaseException: + os.close(fd) + raise + if self.node is not None: + _prefer_node(address, nbytes, self.node) + self._map(address, nbytes) + return address, fd + + def attach(self, fd: int, nbytes: int) -> int: + if os.fstat(fd).st_size != nbytes: + raise ValueError(f"the shared memfd is {os.fstat(fd).st_size} bytes, not {nbytes}") + address = _mmap(fd, nbytes) + self._map(address, nbytes) + return address + + def _map(self, address: int, nbytes: int) -> None: + try: + self._unregister[address] = self._register(address, nbytes, False) + except BaseException: + self._munmap(address, nbytes) + raise + + def release(self, address: int, nbytes: int) -> None: + unregister = self._unregister.pop(address, None) + if unregister is not None: + unregister() + self._munmap(address, nbytes) + + def describe(self) -> str: + return "memfd" + + +def segment_name(key: str, scope_label: str) -> str: + """The rendezvous file stem: per user, per table content, per scope.""" + return f"trtllm-ple-{os.getuid()}-{scope_label}-{key[:24]}" + + +def table_key(*parts) -> str: + """A content key over `parts` (bytes or anything `str`-able).""" + digest = hashlib.sha256() + for part in parts: + digest.update(part if isinstance(part, (bytes, bytearray)) else str(part).encode()) + digest.update(b"\0") + return digest.hexdigest() + + +class SharedHostSegment: + """One process's handle on the scope's shared allocation. + + `role` is `OWNER` (fills, then `publish`es), `ATTACHED` (bytes ready) or `PENDING` (`wait` + first). `pointer` is the device address (0 while pending); `private` marks an owner that could + not attach to a live segment and keeps its copy to itself.""" + + def __init__( + self, + key: str, + nbytes: int, + scope_label: str, + backing: Backing, + directory: Optional[str] = None, + log: Callable[[str], None] = print, + ) -> None: + assert nbytes > 0, nbytes + self.key = key + self.nbytes = nbytes + self.scope_label = scope_label + self.backing = backing + self._log = log + directory = directory or os.environ.get(PLE_SHARED_HOST_DIR_ENV, _DEFAULT_DIR) + stem = os.path.join(directory, segment_name(key, scope_label)) + self.lock_path = stem + ".lock" + self.descriptor_path = stem + ".json" + self.role = PENDING + self.private = False + self.pointer = 0 + self.owner_pid = 0 + self.wait_seconds = 0.0 + self._fd = -1 + # Bound now: at interpreter shutdown the modules may already be torn down. + self._close = os.close + self._lock_fd = os.open(self.lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600) + try: + fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as error: + if error.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + return # PENDING: someone holds the lock (an owner filling, or a quick reader) + self._resolve_locked() + + def _resolve_locked(self) -> None: + """With the lock held: attach to a live published segment, or become its owner.""" + if self._try_attach(): + self._unlock() + return + self._create() + + def _unlock(self) -> None: + if self._lock_fd >= 0: + fcntl.flock(self._lock_fd, fcntl.LOCK_UN) + os.close(self._lock_fd) + self._lock_fd = -1 + + def _read_descriptor(self) -> Optional[dict]: + try: + with open(self.descriptor_path) as fh: + desc = json.load(fh) + except (OSError, ValueError): + return None + if desc.get("key") != self.key or desc.get("nbytes") != self.nbytes: + return None + if _proc_start_time(int(desc["pid"])) != desc.get("start"): + return None # the owner is gone (or its pid was reused) + return desc + + def _try_attach(self) -> bool: + desc = self._read_descriptor() + if desc is None: + return False + try: + fd = _dup_from(int(desc["pid"]), int(desc["fd"])) + except OSError as error: + self._log(f"ple shared host: {error}; this process keeps a private copy") + self.private = True + return False + try: + self.pointer = self.backing.attach(fd, self.nbytes) + except Exception as error: + self._log( + f"ple shared host: attaching pid {desc['pid']}'s segment failed ({error}); " + f"this process keeps a private copy" + ) + self.private = True + return False + finally: + # The mapping holds the allocation; the duplicate is no longer needed. + os.close(fd) + self.role = ATTACHED + self.owner_pid = int(desc["pid"]) + return True + + def _create(self) -> None: + """Become the owner of a new, unfilled allocation; the lock stays held until `publish`.""" + self.pointer, self._fd = self.backing.create(self.nbytes) + self.role = OWNER + self.owner_pid = os.getpid() + + def wait(self, poll_seconds: float = 0.2, report_seconds: float = 30.0) -> None: + """Pending -> attached (the owner published) or owner (it never will: it died, or the + lock holder was only reading). Returns at once if not pending.""" + if self.role != PENDING: + return + t0 = time.monotonic() + next_report = t0 + report_seconds + while True: + try: + fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except OSError as error: + if error.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + now = time.monotonic() + if now >= next_report: + self._log( + f"ple shared host: waiting {now - t0:.0f} s for the {self.scope_label} " + f"segment's owner to fill it ({self.lock_path})" + ) + next_report = now + report_seconds + time.sleep(poll_seconds) + self.wait_seconds = time.monotonic() - t0 + self._resolve_locked() + + def publish(self) -> None: + """Owner, after the bytes are complete (device writes synchronized): make the segment + attachable and release the waiters. A private owner only releases the lock.""" + assert self.role == OWNER and self._fd >= 0, self.role + if not self.private: + desc = dict( + key=self.key, + nbytes=self.nbytes, + pid=os.getpid(), + start=_proc_start_time(os.getpid()), + fd=self._fd, + scope=self.scope_label, + backing=self.backing.describe(), + ) + tmp = f"{self.descriptor_path}.{os.getpid()}.tmp" + with open(tmp, "w") as fh: + json.dump(desc, fh) + os.replace(tmp, self.descriptor_path) + self._unlock() + + def describe(self) -> str: + extra = "" + if self.role == OWNER: + extra = ", private copy" if self.private else "" + elif self.role == ATTACHED: + extra = f" to pid {self.owner_pid}'s segment" + ( + f", waited {self.wait_seconds:.1f} s" if self.wait_seconds else "" + ) + return f"shared per {self.scope_label} ({self.backing.describe()}, {self.role}{extra})" + + def _withdraw(self) -> None: + """Owner: unlink the descriptor if it still names this pid and fd, before the fd closes.""" + try: + with open(self.descriptor_path) as fh: + desc = json.load(fh) + if desc.get("pid") == os.getpid() and desc.get("fd") == self._fd: + os.unlink(self.descriptor_path) + except (OSError, ValueError, TypeError): + pass + + def close(self) -> None: + if self.role == OWNER and self._fd >= 0 and not self.private: + self._withdraw() + pointer, self.pointer = self.pointer, 0 + if pointer: + self.backing.release(pointer, self.nbytes) + if self._fd >= 0: + fd, self._fd = self._fd, -1 + self._close(fd) + if self._lock_fd >= 0: + fd, self._lock_fd = self._lock_fd, -1 + self._close(fd) + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py new file mode 100644 index 000000000000..857e48912db2 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -0,0 +1,6894 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""ModelingV2 target: Qwen3.8-Flash-Next-FP8 (hf-236dfdf_orig) / sm_103 / unified. + +One target for every layout `routing.py` admits for this checkpoint in one node (`_FP8_LAYOUTS`): +tp1; tp2, tep2 (attention TP2) and dep2 (attention DP) at two ranks; tp4, tep4 (attention TP4) +and dep4 (attention DP) at four. The core holds this rank's share of the weights, the derived +operands and the per-manager pool views; `PrefillTarget` (any batch with context rows) and +`DecodeTarget` (generation only) run a step over it. + +The checkpoint is block FP8 (`FP8_BLOCK_SCALES`): only the routed experts are quantized (e4m3, +one scale per 128x128 weight block); everything else is bf16 (`weights.py`). Per rank: + +* **Attention and Gated-DeltaNet heads** are split over attention TP `t` (1 at tp1 and under + attention DP; `weights.rank_dims`, `pool.pool_geometry`): 24 / t q heads, 16 / t GDN key and + 48 / t value heads, and 2 / t KV heads (one, repeated on two ranks, at t = 4). The QSA indexer + is replicated. +* **The mixer output is a partial sum** under attention TP (o_proj and out_proj read this rank's + heads); one plain sum (`catalog/comm/allreduce`) completes it before the hyper-connection + inject, which every rank runs on the same rows. Under attention DP and at tp1 nothing is summed. +* **Routed experts** are whole-expert windows: rank `r` of `t` holds experts + `[512r / t, 512(r + 1) / t)` at the whole 640-row intermediate (`weights.ExpertWindow`), + whatever the engine's `Mapping` says of MoE TP and EP, because 640 / t rows do not take the + 128-row FP8 weight block. Under attention TP the plain sum completes the window partials. + Under attention DP each rank's rows are gathered (`catalog/comm/allgather`, ragged when the + counts differ), run through the window and reduce-scattered back + (`catalog/comm/reducescatter`; `_expert_sum_op`, or an all-to-all with `MOE_ALLTOALL_ENV`). +* **Replicated:** the router, the bf16 shared expert, the hyper-connections, PLE and the + embedding run on this rank's rows. `lm_head` is the shell's: vocab-parallel under attention + TP, whole under attention DP and at tp1. + +There is **no** `model.language_model.norm.weight`: the stack-final mixer is the final norm. + +**QSA sparse attention.** The index plane is written on every step so a request can cross +`dense_seq_len_threshold` later without rebuilding its prefix; above it the sparse core (index +scores, block top-k and expansion, sparse GQA) replaces the dense op. The raw index key is +stored before anything normalizes it: the compression applies the norm and the rotation itself. + +**KV storage.** The paged K/V pool holds bf16, fp8-e4m3 under `kv_cache_config.dtype: fp8` +(`QM_FP8_KV_CACHE`) or NVFP4 under `nvfp4` (`QM_NVFP4_KV_CACHE`: e2m1 with e4m3 block scales in +a second pool, second-level scales 1.0; paged-context attention only). The QSA index and +position planes keep their own dtypes under all three. + +**Breakable prefill CUDA graph.** With `prefill_cuda_graph_backend: breakable` the engine +captures the core's `forward` (the body, not the shell), pads a step up to its bucket and +replays it without calling the body. The shell therefore calls `begin_step` before the body on +every step: the asserts, the cache-manager bind, the target choice and the prefill target's host +bookkeeping live there. The graph breaks (the vision splice, the PLE block, each layer's +attention core and GDN recurrence) each write rows `[:n]` of a fixed output and zero rows `[n:]`. + +**Host offload.** The PLE n-gram table lives in device-addressable host memory, one copy per +rank, or one per host with `weights.PLE_SHARED_HOST_ENV` (`host_share.py`). + +**One-model MTP-Eagle.** With a `speculative_config` the core also holds the MTP module +(`MTPLayer`), replayed `max_draft_len` times by the spec worker; the trunk runs every generation +request's `max_draft_len + 1` verify rows. + +* The draft layer is one more full-attention QSA layer, cut like a trunk one. Under attention TP + it has a separate draft KV cache; under attention DP the trunk's manager maps it too + (`_bind_draft_cache_manager`). Its experts are the trunk's window, completed the trunk's way. +* Under attention DP `_dp_pad_rows` pads each draft step's rows by the group's row counts, so + every rank issues the same collectives at every draft step. +* Verify GDN rows run the CuTe cached replay (`ops/ssm/gdn_cached_replay_cute__verify`, the + stock Triton replay's contract): it commits the history in the kernel below + `CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE` requests and leaves it to the manager from there on. +* What a verify row writes that the runtime does not roll back (the PLE n-gram history and conv + state, every attention layer's QSA index-K plane and the shared position plane) is staged + during the forward and rewritten to the accepted prefix by `SpeculativeRollback`, which the + spec worker calls after its draft loop. +* IndexShare (`index_share_for_mtp_iteration`): the draft-extend step captures each request's + selection and the later draft steps rebuild theirs from it (`qsa_mtp_reuse_topk`). + +**Images and video.** The shell runs the replicated bf16 vision tower (`vision.VisionTower`) +over the step's media and `begin_step`'s splice break writes the rows over the token embedding; +`validate_request` refuses media the tower would assert on. + +Integration facts this file depends on: + +1. The engine rebuilds the composite config and normalizes its text tower + (`config_utils.load_pretrained_config`). The shell adopts that text config as + `model_config.pretrained_config` before building the core, because the recurrent-state and + PLE pool geometry are read from it. +2. `position_ids` arrives in the mRoPE layout (`[3, 1, T]`) even for text-only batches. The + main rope reads all three rows; the QSA index plane takes coordinate 0 only. +""" + +from __future__ import annotations + +import contextlib +import math +import os +from typing import TYPE_CHECKING, NamedTuple + +import torch +import triton +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._ipc_utils import can_access_peer +from tensorrt_llm._torch.modeling_v2._router_index import ( + NoTarget, + engine_kv_cache_quant_algo, + step_contract_enabled, +) +from tensorrt_llm._torch.modeling_v2.catalog.attention.fused_qk_norm_rope import ( + fused_qk_norm_rope, +) +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import ( + QM_FP8_KV_CACHE, + QM_NVFP4_KV_CACHE, + thop_attention, +) +from tensorrt_llm._torch.modeling_v2.catalog.comm.allgather import allgather +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import ( + allreduce, + allreduce_group_args, +) +from tensorrt_llm._torch.modeling_v2.catalog.comm.reducescatter import reducescatter +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.catalog.graph.eager_on_graph import ( + eager_on_graph, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( + fused_sigmoid_mul_inplace, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.silu_and_mul_rows import ( + silu_and_mul_rows, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_canonical_selection import ( + qsa_canonical_selection, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_expand_block_indices import ( + qsa_expand_block_indices, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_mtp_reuse_topk import ( + qsa_mtp_reuse_topk, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + _takes_prefill_query_tile, + qsa_paged_index_scores, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + _takes_visible_split as _scores_take_visible_split, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_kv_store_nvfp4 import ( + qsa_paged_kv_store_nvfp4, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa_nvfp4 import ( + qsa_paged_sparse_gqa_nvfp4, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa_nvfp4_decode import ( # noqa: E501 + qsa_paged_sparse_gqa_nvfp4_decode, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_prefill_compress import ( + qsa_prefill_compress, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( + plane_table as _plane_table, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( + qsa_rollback_gather, + qsa_rollback_restore, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + qsa_topk_expand_decode, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + supports as _topk_expand_supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_select import ( + qsa_topk_select, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.spec_rollback_commit import ( + spec_rollback_commit, + spec_rollback_stage, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + BK as _HC_RESCALE_BK, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + _pick_split as _hc_rescale_split, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + qwen4_exp_hc_first_rescale_tc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + _PDL_TRIGGER as _HC_INJECT_PDL_TRIGGER, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + _hc_inject_norm_kernel, + qwen4_exp_hc_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_rstd import ( + qwen4_exp_hc_inject_rstd, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_lowrank_act import ( + qwen4_exp_hc_lowrank_act, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_up_gate_mean import ( + qwen4_exp_hc_up_gate_mean, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_up_gate_mean_rstd import ( + qwen4_exp_hc_up_gate_mean_rstd, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_post_inject import ( + qwen4_exp_post_inject, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.fused_sigmoid_gate_mul_add import ( + fused_sigmoid_gate_mul_add, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( + renorm_moe_routing_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.flashinfer_gemma_rmsnorm import ( + flashinfer_gemma_rmsnorm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( + rms_norm_gated_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_gate_conv_inject_norm import ( + ple_gate_conv_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed import ( + ple_ngram_embed, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( + bf16_to_fp32, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.fp8_quantize_1x128 import ( + fp8_quantize_1x128, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( + causal_conv1d_update, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.cu_seqlens_from_seq_lens import ( + cu_seqlens_from_seq_lens, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_decode_rows_gated_norm import ( + gdn_decode_rows, + gdn_decode_rows_gated_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( + gdn_prefill_conv_post, +) +from tensorrt_llm._torch.attention.backends.fmha.phased import get_spec_decoding_position_offsets +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.distributed.communicator import Distributed +from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM +from tensorrt_llm._torch.modules.fla.cached_replay import CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE +from tensorrt_llm._torch.modules.linear import TensorParallelMode +from tensorrt_llm._torch.modules.logits_processor import LogitsProcessor +from tensorrt_llm._torch.pyexecutor.breakable_cuda_graph import ( + eager_on_graph as _keyed_eager_on_graph, +) +from tensorrt_llm._torch.pyexecutor.model_loader import effective_mamba_ssm_cache_dtype +from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks +from tensorrt_llm._torch.speculative import get_spec_worker, should_use_separate_draft_kv_cache +from tensorrt_llm._torch.utils import is_gdn_replay_enabled +from tensorrt_llm._utils import mpi_disabled +from tensorrt_llm.functional import AllReduceFusionOp +from tensorrt_llm.logger import logger +from tensorrt_llm.quantization.mode import QuantAlgo + +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + +from . import pool as _pool +from . import vision as _vision +from . import weights as _weights +from .ops.attention.draft_continue_rows__decode import draft_continue_select, draft_rows_gather +from .ops.attention.qsa_fp8_pages_to_bf16__prefill import qsa_fp8_pages_to_bf16 +from .ops.attention.qsa_index_scores_cute__prefill import qsa_index_scores_prefill +from .ops.attention.qsa_index_scores_cute__prefill import supports as _index_scores_cute_supports +from .ops.attention.qsa_index_scores_groups_cute__decode import ( + index_scores_groups as index_scores_groups_cute, +) +from .ops.attention.qsa_index_scores_groups_cute__decode import serves as _index_groups_cute_serves +from .ops.attention.qsa_index_split_plan__decode import index_split_plan +from .ops.attention.qsa_paged_index_store import qsa_paged_index_store +from .ops.attention.qsa_paged_kv_store import qsa_paged_kv_store +from .ops.attention.qsa_paged_sparse_gqa import qsa_paged_sparse_gqa +from .ops.attention.qsa_sparse_gqa_grouped_cute__decode import qsa_sparse_gqa_grouped_cute +from .ops.attention.qsa_sparse_gqa_rowtile__prefill import qsa_sparse_gqa_rowtile +from .ops.attention.qsa_sparse_gqa_splitk_cute__decode import qsa_sparse_gqa_splitk_cute +from .ops.attention.qsa_spec_step_metadata__decode import ( + SpecStepMetadataPrologue, + take_step_positions, +) +from .ops.attention.qsa_topk_expand_cluster__decode import qsa_topk_expand_cluster +from .ops.attention.qsa_topk_expand_cluster__decode import supports as _topk_expand_cluster_supports +from .ops.attention.qsa_topk_rows_cute__prefill import qsa_topk_rows +from .ops.attention.qsa_topk_rows_cute__prefill import supports as _topk_rows_supports +from .ops.attention.qsa_triton_kernels__stable_jit import splitk_launch_config as _gqa_splitk_config +from .ops.attention.qsa_triton_kernels__stable_jit import warm_decode_buckets +from .ops.attention.qwen4_exp_attn_decode_prologue__decode import qwen4_exp_attn_decode_prologue +from .ops.attention.spec_rollback_begin__verify import spec_rollback_begin +from .ops.comm.hc_allreduce_inject_norm__tp_sum import fits as _hc_ar_inject_operands_fit +from .ops.comm.hc_allreduce_inject_norm__tp_sum import hc_allreduce_inject_norm +from .ops.comm.lamport_allgather__index_split import build as _selection_lamport_build +from .ops.comm.lamport_allgather__index_split import lamport_allgather +from .ops.comm.mnnvl_allreduce__tp_sum import mnnvl_tp_sum, mnnvl_workspace_args +from .ops.comm.moe_a2a__dp_dispatch_combine import build_strategy as _build_moe_a2a +from .ops.comm.moe_a2a__dp_dispatch_combine import moe_a2a_combine, moe_a2a_dispatch +from .ops.embedding.embed_rows__decode import embed_rows +from .ops.gemm.low_m_bf16_mm__decode import low_m_bf16_mm +from .ops.hc.qwen4_exp_hc_decode_rows_tc__decode import MAX_ROWS as _HC_ROWS_TC_ROWS +from .ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( + qwen4_exp_hc_decode_rows_first, + qwen4_exp_hc_decode_rows_mix, +) +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import MAX_ROWS as _HC_TC_FIRST_ROWS +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import precompile as hc_tc_first_precompile +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import qwen4_exp_hc_decode_tc_first +from .ops.moe.fp8_block_scale_moe_runner import ( + fp8_block_scale_moe_front, + fp8_block_scale_moe_runner, +) +from .ops.moe.moe_front_router_quant__decode import MAX_ROWS as _MOE_FRONT_ROWS +from .ops.moe.moe_front_router_quant__decode import moe_front_router_quant +from .ops.moe.moe_prefill_grouped__prefill import ( + moe_prefill_grouped, + prefill_grouped_ready, + warm_prefill_grouped, +) +from .ops.moe.moe_tpb_decode__decode import MAX_ROWS as _MOE_TPB_ROWS +from .ops.moe.moe_tpb_decode__decode import BlockExperts as _BlockExperts +from .ops.moe.moe_tpb_decode__decode import SharedBlocks as _TpbShared +from .ops.moe.moe_tpb_decode__decode import moe_tpb_decode +from .ops.moe.mtp_moe_combine_inject__decode import mtp_moe_combine_inject +from .ops.ssm.causal_conv1d_token_major__verify import causal_conv1d_token_major +from .ops.ssm.flashinfer_chunk_gated_delta_rule import INPLACE_MAX_TOKENS as _GDN_INPLACE_MAX_TOKENS +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _chunk_carried as _gdn_chunk_carried +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _chunk_cp_inplace as _gdn_chunk_cp_inplace +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _reaches_cp as _gdn_reaches_cp +from .ops.ssm.flashinfer_chunk_gated_delta_rule import ( + chunk_split_first_pass, + cp_chunk_len_for, + flashinfer_chunk_gated_delta_rule, + split_takes_cp, +) +from .ops.ssm.gdn_cached_replay_cute__verify import GEOMETRIES as _GDN_REPLAY_CUTE_GEOMETRIES +from .ops.ssm.gdn_cached_replay_cute__verify import conv_fold_supported as gdn_conv_fold_supported +from .ops.ssm.gdn_cached_replay_cute__verify import gdn_cached_replay_cute +from .ops.ssm.gdn_in_proj_conv__verify import CONV_DIM as _GDN_IN_PROJ_CONV_DIM +from .ops.ssm.gdn_in_proj_conv__verify import N_OUT as _GDN_IN_PROJ_N_OUT +from .ops.ssm.gdn_in_proj_conv__verify import fused_tactic as _gdn_in_proj_conv_fused_tactic +from .ops.ssm.gdn_in_proj_conv__verify import gdn_in_proj_conv +from .ops.ssm.gdn_inforward_tail import gdn_inforward_conv_swap, gdn_inforward_tail + +# The certified GPU architecture, checked again because a direct instantiation bypasses routing. +_SM = (10, 3) + +# `moe_config.backend` requests served by the trunk's routed experts (`fp8_block_scale_moe_runner`). +_TRUNK_MOE_BACKENDS = ("AUTO", "TRTLLM") + +# The engine's resolved quantization of the one checkpoint this target loads. +_QUANT_ALGO = QuantAlgo.FP8_BLOCK_SCALES + +# Qwen3.8-Flash-Next's (layers, hidden, experts, attention heads, linear value heads, vocab), and +# its vision tower's (depth, hidden, heads, merge, out_hidden, num_pos, deepstack_indexes, act). +_TEXT_SHAPE = (48, 2560, 512, 24, 48, 248320) +_VISION_SHAPE = (27, 1152, 16, 2, 2560, 2304, (), "gelu_pytorch_tanh") + +# Rows per request of the widest verify step (`max_draft_len + 1`). +_MAX_VERIFY_WIDTH = _pool.MAX_VERIFY_WIDTH + +_SSM_STATE_DTYPES = _pool.SSM_STATE_DTYPES + +# QSA backend metadata buffers rebuilt every step. Their absence means the engine did not +# select the QSA metadata and the index plane would never be written. +_QSA_FIELDS = ( + "qsa_block_table", + "qsa_req_idx_per_token", + "qsa_logical_positions", + "qsa_sequence_lengths", + "qsa_visible_blocks", + "qsa_topk_indices", + "qsa_topk_row_starts", +) + +# Fields read off `attn_metadata.mamba_metadata`. Absence is silent: a smoke run passes +# with no recurrent state at all. +_MAMBA_FIELDS = ( + "state_indices", + "has_initial_states", + "query_start_loc", + "query_start_loc_long", +) + +# thop_attention's constant arguments: in-kernel RoPE off (the rope runs before the op), +# causal mask, no MLA, sparse or cross surface. +_CALL_CONSTANTS = dict( + output_sf=None, + k=None, + v=None, + is_fused_qkv=True, + update_kv_cache=True, + attention_input_type=0, + is_mla_enable=False, + mask_type=1, + q_scaling=1.0, + attention_sinks=None, + position_embedding_type=0, + rotary_inv_freq=None, + rotary_cos_sin=None, + rope_dim=0, + rope_base=10000.0, + rope_scale_type=0, + rope_scale=1.0, + rope_short_m_scale=1.0, + rope_long_m_scale=1.0, + rope_max_positions=1024, + rope_original_max_positions=1024, + out_scale=None, + latent_cache=None, + q_pe=None, + q_lora_rank=None, + kv_lora_rank=None, + qk_nope_head_dim=None, + qk_rope_head_dim=None, + v_head_dim=None, + rope_append=None, + chunked_prefill_buffer_batch_size=1, + attention_chunk_size=None, + softmax_stats_tensor=None, + sparse_kv_indices=None, + sparse_kv_offsets=None, + sparse_attn_indices=None, + sparse_attn_offsets=None, + sparse_attn_indices_block_size=0, + mrope_rotary_cos_sin=None, + mrope_position_deltas=None, + helix_position_offsets=None, + helix_is_inactive_rank=None, + cross_kv=None, + relative_attention_bias=None, + relative_attention_max_distance=0, + kv_norm_weight=None, + kv_norm_eps=1e-6, + skip_correction_threshold=0.0, +) + +_ROUTING_METHOD_RENORM = 1 +_ACT_TYPE_SWIGLU = 0 +_PAD_SLOT_ID = -1 +#: Whether a decode step's `low_m_bf16_mm` projections launch with PDL (the FlashInfer kernels +#: stream the weight before waiting on the producer; the cuBLAS branch ignores it). +_DECODE_PROJ_PDL = True +#: (request x value head) pairs a verify step up to which the CuTe cached replay launches with PDL +#: and the conv before it triggers at its start, so the replay streams the checkpoint and history +#: (nothing earlier in the step writes them) while the conv runs. +_GDN_REPLAY_PDL_MAX_HEAD_ROWS = 24 +#: A tp2/tep2 rank's GDN heads (8 key / 24 value), and the HT generation batches (from this many +#: requests: the partitioned replay) whose replay launch comes from `_GDN_REPLAY_HT_LAUNCH`. +_GDN_REPLAY_HT_GEOMETRY = (8, 24) +_GDN_REPLAY_HT_MIN_REQUESTS = 16 +#: `(max requests, pdl, CuTe replay mapping or None for its default)` for a tp2/tep2 HT verify +#: batch. The mappings (`gdn_cached_replay_cute.default_config`'s knobs) only move where a tile is +#: loaded and which CTA runs it, so every choice is bitwise the default launch. +_GDN_REPLAY_HT_LAUNCH = ( + (24, True, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (40, True, dict(tpc=8, warps=4, pf=1, minb=6)), + (56, True, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (80, True, None), + (1 << 30, False, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), +) +#: Requests up to which a tp2/tep2 HT verify batch folds the conv into the CuTe replay +#: (`conv_fold=`, bitwise `causal_conv1d_token_major`); the replay writes the per-row windows and +#: the acceptance promote advances the pool window from them. +_GDN_VERIFY_CONV_FOLD_MAX_REQUESTS = 24 +#: Whether a verify step's conv (`causal_conv1d_token_major`) is a PDL dependent of the input +#: projection: it reads the slots and the conv pool before `griddepcontrol.wait`, the q|k|v after. +_GDN_CONV_PDL_WAIT = True +#: Whether a decode step's QSA index scoring (`_decode_index_selection`) launches with PDL: its +#: kernels wait before their first read and the decode prologue triggers at its start. +_INDEX_SCORES_PDL = get_env_enable_pdl() +#: Request groups (this rank's share) from which verify index scoring takes this target's CuTe +#: kernel (`qsa_index_scores_groups_cute__decode`) over the family's Triton one. It sums in another +#: order (~7e-8 relative), so a row's top-k can differ only at a near-tie. +_INDEX_SCORES_CUTE_MIN_GROUPS = 8 +_SOFTPLUS_THRESHOLD = 20.0 +_L2_NORM_EPS = 1e-6 +_PLE_CLAMP_MIN = 1e-6 +#: Rows up to which a hyper-connection mixer's first half is one tcgen05 launch +#: (`qwen4_exp_hc_decode_tc_first`, two 128-row MMA tiles) instead of cuBLAS + lowrank_act. +_HC_TC_FIRST_MAX_ROWS = _HC_TC_FIRST_ROWS +#: Row counts over which a decode or draft hyper-connection site (`_hc_site`) runs whole as the +#: two `qwen4_exp_hc_decode_rows_*` launches. A dispatch threshold on a per-capture row count, not +#: a buffer bound; zero rows keep the other path. +_HC_ROWS_TC_MIN_ROWS = 1 +_HC_ROWS_TC_MAX_ROWS = _HC_ROWS_TC_ROWS +#: Under attention TP, a decode block's partial output of `_HC_AR_INJECT_MIN_ROWS` .. +#: `_HC_RSTD_MIN_ROWS - 1` rows skips the TP sum and the next hyper-connection site sums it in its +#: injection (`ops/comm/hc_allreduce_inject_norm__tp_sum`, bitwise the all-reduce followed by +#: `_hc_inject_norm_launch`). Smaller steps use the MNNVL sum or the decode-rows site. +_HC_AR_INJECT_MIN_ROWS = 33 +#: Prefill steps of at least this many rows store the residual and a `[T, C]` fp32 rstd +#: (`qwen4_exp_hc_inject_rstd`) instead of the `[T, C*H]` normed stream, which `_hc_mixer_rstd` +#: forms on chip. A dispatch threshold on the step's rows, not a buffer bound. +_HC_RSTD_MIN_ROWS = 3072 +#: Warps a program of `_hc_inject_norm_launch` takes below `_HC_INJECT_FEW_WARPS_MIN_ROWS` rows, +#: and two from there (as `qwen4_exp_hc_inject_norm` does). Dispatch thresholds, not buffer bounds. +_HC_INJECT_WARPS = 4 +_HC_INJECT_FEW_WARPS_MIN_ROWS = 512 +#: Rows up to which `qwen4_exp_hc_up_gate_mean` runs its epilogue on eight warps instead of four +#: (at one tile a CTA the gate's MUFU work is the critical path). Same bits. +_HC_UGM_WIDE_MAX_ROWS = 384 +#: Whether `_hc_inject_norm_launch` launches with PDL and waits for its predecessor in the kernel +#: (`griddepcontrol.wait`); off with TRTLLM_ENABLE_PDL=0. +_HC_INJECT_PDL_WAIT = _HC_INJECT_PDL_TRIGGER +_QSA_POSITION_AXES = _pool.QSA_POSITION_AXES +# The compressed index keys rotate at coordinate 0 alone, as the index query does. +_QSA_MROPE_SECTION = None +# The routed experts' activation scales are the plain fp32 `amax / 448`, not the UE8M0 +# power-of-two form: the runner reads them so (`fp8_block_scale_moe_runner`). +_MOE_USE_UE8M0 = False +# Rows per FP8 runner call under attention DP: the rows an attention-DP group gathers (up to +# its size times a rank's) run in chunks of it. +_MOE_MAX_ROWS = 8192 +#: A decode step's routed experts up to this many rows enter the runner from one front-end launch +#: (`moe_front_router_quant`: router logits + 1x128 quantization). Past 8 rows the front end +#: contends with the concurrent shared expert. +_MOE_FRONT_MAX_ROWS = min(8, _MOE_FRONT_ROWS) +#: Past `_MOE_FRONT_MAX_ROWS` an attention-TP decode step hands the runner the router's bf16 +#: logits to route itself (bitwise `renorm_moe_routing_op`), forking the shared expert only once +#: the router and quantization are queued (`before_runner`) so it overlaps the runner's GEMMs. +#: Up to this many rows (the routing kernel's bound) the runner routes and quantizes in one launch +#: (`fp8_block_scale_moe_front`) instead of the 8-CTA cluster kernel; the layer's output is bitwise +#: the cluster path's (only a slot's offset within its expert may differ; nothing reads it). +_MOE_FAST_FRONT_MAX_ROWS = 1024 +#: With the block-major expert copy (`weights.MOE_TPB_ENV`, attention TP4), decode steps up to this +#: many rows take `moe_tpb_decode` over this rank's 128-row intermediate blocks. From +#: `_MOE_TPB_SHARED_MIN_ROWS` rows it also folds in this rank's shared-expert slice (`shared=`): +#: the launch holds nearly every SM, so a side-stream shared expert would wait behind it. +_MOE_TPB_MAX_ROWS = min(8, _MOE_TPB_ROWS) +#: The fold's smallest step: at a 3-row verify the side stream's shared expert mostly hides behind +#: the routed launch and folding it in costs more. +_MOE_TPB_SHARED_MIN_ROWS = 5 +#: Rows per FP8 runner call under attention TP (or at one rank): the runner is certified to 16384 +#: rows (past its last tuning bucket), and one call reads every expert's weights once. +_MOE_MAX_ROWS_TP = 16384 +#: Smallest rows of a runner call under attention TP that run the window as two grouped FP8 +#: block-scale GEMMs (`moe_prefill_grouped`: no host synchronization, CUDA graph capturable) +#: instead of the trtllm-gen runner. 0 keeps the runner at every row count (tests set it). +_MOE_PREFILL_GROUPED_MIN_ROWS = 2048 +#: The expert id a padding row's slots are routed to: outside every rank's window, so the +#: runner skips the slot (`_routed_experts`). +_NO_EXPERT = -1 +#: Rows up to which `_mtp_front` runs its embedding branch on the side stream; from 16 rows the +#: hidden branch's GEMM (4 streams a row) fills the GPU and the concurrent branch only slows it. +_MTP_FRONT_FORK_MAX_ROWS = 8 +#: Context rows from which an attention-TP rank scores and selects only its share of a prefill +#: step's rows and gathers the others' (`_split_context_selection`), instead of every rank +#: repeating the whole unsplit index selection. Every rank holds the same bits. +_SELECTION_SPLIT_MIN_ROWS = 1024 +#: A split selection cuts rows where the ranks' summed costs are equal (`_selection_cuts`); a row +#: costs its window (`position + 1` tokens) plus this many tokens of fixed work (a measured fit). +_SELECTION_BALANCE_ROW_TOKENS = 12500 +#: Rows a balanced cut is rounded to (the index scores' query tile). +_SELECTION_CUT_ALIGN = 64 +#: Pieces a rank's share of a split selection is processed in: each piece's gather runs on the +#: comm stream while the next is scored, so only the last gather is exposed. Same bits. +_SELECTION_PIECES = 4 +#: Rows from which a context chunk's selection over a wide score table takes the row top-k kernel +#: (`qsa_topk_rows`) instead of the family's wide kernel; same bits. +_TOPK_ROWS_MIN_ROWS = 256 + + +def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, cr, context_rows): + """`qsa_paged_index_scores` (visible blocks only) of a context part of a step's rows. + + Takes the tcgen05 kernel `qsa_index_scores_prefill` where it serves the call (same bits). + """ + if _takes_prefill_query_tile(q, True, context_rows, False) and _index_scores_cute_supports( + q, index_cache, tpb, cr + ): + return qsa_index_scores_prefill(q, index_cache, block_table, positions, requests, tpb, cr) + return qsa_paged_index_scores( + q=q, + index_cache=index_cache, + block_table=block_table, + query_positions=positions, + request_indices=requests, + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=context_rows, + prefill_query_tile=True, + wide_split=True, + ) + + +#: Rows a piece of a pipelined selection keeps at least (whole query tiles of the index scores). +_SELECTION_PIECE_MIN_ROWS = 1024 +#: Breakable-graph buckets of at most this many rows replay a context-only step's GDN and sparse +#: attention breaks as CUDA graphs keyed by what fixes their launches (`_gdn_graph_key`, +#: `_attn_graph_key`); a key the warmup did not capture runs eagerly. Same bits. +_GRAPH_BREAKS_MAX_TOKENS = 4096 +#: Attention-TP group sizes whose prefill steps split the context selection and run +#: sequence-parallel (`PrefillTarget._sp_split`). Two ranks only: at four the sums would run in +#: another order than the all-reduce's. +_PREFILL_SPLIT_DEFAULT_TP = frozenset({2}) +#: Rows from which a prefill step runs sequence-parallel: a rank's share stays past the HC +#: kernels' small-row paths, so each runs the configuration the whole step would. +_SP_MIN_ROWS = 4096 +# cuBLASLt picks a bf16 projection's kernel by its exact row count up to this many rows, and +# past them by the 8-row group the row count falls in (`_projection_warm_rows`). +_KERNEL_CHOICE_EACH_ROW = 16 +_KERNEL_CHOICE_ROW_GROUP = 8 +# The logits rows `lm_head`'s kernels are loaded for: a step's requests' last tokens and its +# verify rows, up to 256 requests at MTP-3 or 128 at MTP-7. +_LM_HEAD_WARM_ROWS = 1024 +# A context launch of at least this many rows over a bf16 or fp8 pool takes +# `qsa_sparse_gqa_rowtile`; below it the per-row kernel is faster. +_ROWTILE_MIN_ROWS = 2048 # capacity: fixed: a dispatch threshold, not a buffer size +#: Requests from which MTP verify rows over an e4m3 pool take `qsa_sparse_gqa_grouped_cute` where +#: it serves them (one load and widening of each selected K/V token for a request's rows, over the +#: union of their selections) instead of the per-row `qsa_sparse_gqa_splitk_cute`. +_GROUPED_DECODE_MIN_REQUESTS = 32 # capacity: fixed: a dispatch threshold, not a buffer size +#: Rows from which a generation step with several rows a request (MTP verify, draft-extend) scores +#: its QSA index keys over request groups (one key gather serves a request's rows); low-latency +#: steps of up to 16 rows keep the entry's own choice. +_INDEX_GROUP_MIN_ROWS = 17 # capacity: fixed: a dispatch threshold, not a buffer size +#: Requests from which an attention-TP verify step splits QSA index scoring and selection by +#: request over the TP group (`_index_split_rows`) and gathers the int32 selections in rank order. +#: Bitwise the unsplit selection; needs `n % t == 0`. +_INDEX_SPLIT_MIN_REQUESTS = 16 +#: Rows up to which the split's gather is one Lamport exchange over symmetric memory +#: (`lamport_allgather__index_split`) instead of NCCL, bitwise the same; it sizes the buffers. +_INDEX_SPLIT_LAMPORT_ROWS = 768 +# Over an fp8 pool the launch reads its requests' pages widened into a bf16 scratch; a launch +# whose scratch would pass this many bytes stays on the per-row kernel. +_FP8_ROWTILE_MAX_SCRATCH = 1 << 30 # capacity: fixed: bounds a per-call scratch allocation +#: Opt-in (`1`, `true`, `yes` or `on`): a context step over an fp8 pool whose pages would widen +#: past `_FP8_ROWTILE_MAX_SCRATCH` runs one row-tile launch per group of requests that fits +#: (`_rowtile_scratch_groups`) instead of the per-row kernel. Off by default because such steps' +#: outputs then move by rounding. Read once, when the target is built. +ROWTILE_SCRATCH_GROUPS_ENV = "TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS" +#: Under attention DP, 1 runs the MoE round trip as an all-to-all dispatch / combine +#: (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one. Needs MPI and MNNVL; +#: elsewhere the all-gather / reduce-scatter stays. Read once, when the target is built. +MOE_ALLTOALL_ENV = "TRTLLM_MODELING_V2_MOE_ALLTOALL" +# Rows per FP8 runner call over an attention-DP group's gathered rows (`_routed_experts`): one +# call reads the window's weights once, bitwise the `_MOE_MAX_ROWS`-row calls; the router, its +# routing and the activation quantization still run per `_MOE_MAX_ROWS` rows. +_ADP_WINDOW_MAX_ROWS = 4 * _MOE_MAX_ROWS +# What every bind of this target prints, on every rank: the disaggregated-serving test looks for +# it in each worker's log. +_LOG_PREFIX = "modeling_v2 qwen4_exp fp8 unified target" + +# In-forward Mamba snapshots: a context chunk ending at a snapshot point P just before the prompt +# end L runs through L, leaving the GDN and PLE state at P in the live slot and staging the state +# at L (`PrefillTarget._gdn_split`). The fewest chunk tokens before P a split request takes: past +# `_GDN_INPLACE_MAX_TOKENS` (the main chunk's FlashInfer path) and covering the conv (3) and PLE +# (9 columns, 2 ids) windows that end at P. +_INFORWARD_MIN_MAIN = _GDN_INPLACE_MAX_TOKENS + 1 + +#: Rows up to which `tp_sum` takes the multicast NVLink one-shot kernel +#: (`ops/comm/mnnvl_allreduce__tp_sum`) instead of the IPC one, where the platform supports MNNVL +#: (`MNNVLAllReduce.is_mnnvl`); the two return the same bits. +_TP_SUM_MNNVL_MAX_ROWS = 32 + + +def _runtime_hooks() -> RuntimeHooks: + """Step-loop hooks of the fast paths: one execution stream, one-launch verify prologue.""" + return RuntimeHooks(loop_on_execution_stream=True, decode_prologue=SpecStepMetadataPrologue()) + + +class _Replay(NamedTuple): + """The GDN cached replay's history buffers and bookkeeping of one manager.""" + + old: dict # GDN layer -> (old_x, old_B, old_dt, old_dA_cumsum) + cache_buf_idx: torch.Tensor # [slots] int32: the history half each slot writes next + prev_num_accepted: torch.Tensor # [slots] int32: tokens pending since the checkpoint + history: int # rows a history half holds + width: int # the one verify width a step may carry per generation request + + +class _Bound(NamedTuple): + """What the core reads off one KV cache manager; valid only for that manager. + + The manager, this rank's pool views and, under MTP, the verify step's runtime-owned buffers: + the per-step conv windows (`int_conv`, by GDN layer), the rows `update_mamba_states` promotes + them from (`int_idx`), the cached replay and the in-forward snapshot staging rows. + """ + + cm: object + pools: _pool.RankPools + int_idx: torch.Tensor | None = None + int_conv: dict | None = None + replay: _Replay | None = None + stage: "_InforwardStage | None" = None + + +class _InforwardStage(NamedTuple): + """The manager's in-forward snapshot staging rows (`Qwen4ExpHybridCacheManager`). + + Each GDN layer's SSM `[rows, HV, V, K]` and conv `[rows, conv_dim, 3]` rows, and the PLE + layer's conv state `[rows, stream, 9]` and n-gram history `[rows, ngram_size - 1]` rows. The + forward stages each split request's prompt-end state in its row; the manager copies it back + after committing the snapshot at P. + """ + + ssm: dict + conv: dict + ple_conv: torch.Tensor + ple_ngram: torch.Tensor + + +class _Split(NamedTuple): + """One step's in-forward snapshot split, from `Mamba2Metadata.inforward_split`. + + `k` of the step's context requests end their chunk at the prompt end L with a snapshot point + P inside it; in the GDN recurrence each is two sequences, [.., P) and the tail [P, L). + + `cu` (int64) the split sequences' boundaries; `gather` (int32) each split sequence's live + slot and `carry` (bool) whether it starts from it (a tail starts from zeros, its final state + dropped); `main_rows` (int64) each request's [.., P) or whole sequence and `live` (int32) its + slot; `tail_tok` (int64) the tail rows, `cu_tail` (int64) their boundaries, `stage_rows` + (int64) the split requests' staging rows, `tail_slots` (int64) their live slots; `conv_win`, + `ple_win` and `ngram_win` (int64) the rows of the GDN conv (3), PLE conv (9) and n-gram id (2) + windows that end at P, oldest first, request by request; `whole` (`chunk_split_first_pass`) + the rows before every P (int64, or a host count when one request is split once), their + per-request boundaries and carry flags, or None where the first pass keeps the split + boundaries; `raw` the engine's `InforwardSnapshotSplit`, views of one persistent buffer at + fixed offsets that a break's CUDA graph reads. + """ + + k: int + cu: torch.Tensor + gather: torch.Tensor + carry: torch.Tensor + main_rows: torch.Tensor + live: torch.Tensor + tail_tok: torch.Tensor + cu_tail: torch.Tensor + stage_rows: torch.Tensor + tail_slots: torch.Tensor + conv_win: torch.Tensor + ple_win: torch.Tensor + ngram_win: torch.Tensor + whole: tuple | None = None + raw: object = None + + +def _inforward_stage(cm, gdn_layers, ple_layer: int) -> _InforwardStage | None: + """The in-forward snapshot staging rows of `cm`, or None when it plans no snapshots.""" + if getattr(cm, "inforward_snapshot_rows", 0) <= 0: + return None + _pool.require( + cm, + ("inforward_snapshot_stage_views", "inforward_ple_stage_views", "conv_state_shape"), + "stages the in-forward snapshot's prompt-end GDN and PLE state", + ) + ple = cm.inforward_ple_stage_views(ple_layer) + assert ple is not None, ( + f"the KV cache manager plans in-forward snapshots but stages no PLE state for decoder " + f"layer {ple_layer}" + ) + ssm, conv = {}, {} + for i in gdn_layers: + ssm[i], conv[i] = cm.inforward_snapshot_stage_views(i) + return _InforwardStage(ssm, conv, ple[0], ple[1]) + + +class _TpPartial(NamedTuple): + """A decode block's partial output whose TP sum is left to the next hyper-connection site.""" + + partial: torch.Tensor + + +class _HcArInject(NamedTuple): + """IPC all-reduce workspace and TP rank / size for the decode sites' fused TP sum + inject.""" + + workspace: torch.Tensor + rank: int + nranks: int + + +def _hc_ar_inject_defers(core, rows: int) -> bool: + """Whether `_tp_partial` leaves a decode block's partial of `rows` rows to the next site.""" + return ( + getattr(core, "hc_ar_inject", None) is not None + and max(_HC_AR_INJECT_MIN_ROWS, _HC_ROWS_TC_MAX_ROWS + 1) <= rows < _HC_RSTD_MIN_ROWS + ) + + +def _tp_partial(core, x: torch.Tensor): + """A decode block's partial output for the next hyper-connection site. + + Left unsummed (`_TpPartial`) where that site sums it itself, else `core.tp_sum(x)`. A core + without `hc_ar_inject` (another layout, or a test's stand-in core) always sums. + """ + if _hc_ar_inject_defers(core, x.shape[0]): + return _TpPartial(x) + return core.tp_sum(x) + + +class _DpPadding(NamedTuple): + """The group's per-rank row counts of one step's attention-DP MoE round trip (ragged split).""" + + counts: tuple[int, ...] + + +class _DraftBound(NamedTuple): + """The draft layer's planes in the manager its draft step is handed. + + Its QSA index plane, that manager's position plane, its paged K/V and, under IndexShare, its + row of the metadata's shared selection buffers. + """ + + cm: object + index_cache: torch.Tensor + position_cache: torch.Tensor + kv: _pool.KvPlanes + share_slot: int | None + + +class Qwen4ExpUnifiedModelingV2(DecoderModel): + def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig): + """`composite_config` carries the vision config and media ids the text config lacks.""" + super().__init__(model_config) + cfg = model_config.pretrained_config + mapping = model_config.mapping + assert torch.cuda.get_device_capability() == _SM, ( + f"target certified on sm_{_SM[0]}{_SM[1]}, running on " + f"sm_{''.join(map(str, torch.cuda.get_device_capability()))}" + ) + assert getattr(cfg, "model_type", None) == "qwen4_exp_text", ( + f"the core must be built on the normalized text config; got model_type " + f"{getattr(cfg, 'model_type', None)!r}" + ) + self.mapping = mapping + # Ranks the attention and recurrent heads are split over (1 under attention DP). + self.attn_tp_size = _pool.attention_tp_size(mapping) + # Under attention DP each rank serves its own requests; MoE runs over the TP group's rows. + self.dp_size = mapping.tp_size if mapping.enable_attention_dp else 1 + self.dp_rank = mapping.tp_rank + self.dp_group = list(mapping.tp_group) + # One-model MTP-Eagle is the only speculation routing admits; None without speculation. + spec_config = getattr(model_config, "spec_config", None) + self.spec_config = spec_config + self.mtp_enabled = spec_config is not None + self.max_num_tokens = model_config.max_num_tokens + self.moe_max_rows = _moe_max_rows( + self.max_num_tokens, mapping.tp_size if mapping.enable_attention_dp else 1 + ) + + self.num_layers = cfg.num_hidden_layers + self.hidden = cfg.hidden_size + self.vocab = cfg.vocab_size + self.eps = cfg.rms_norm_eps + layer_types = list(cfg.layer_types) + self.is_attn = [t == "full_attention" for t in layer_types] + self.attn_layers = [i for i, a in enumerate(self.is_attn) if a] + self.gdn_layers = [i for i, a in enumerate(self.is_attn) if not a] + + self.hc_count = cfg.hc_count + self.hc_lowrank = cfg.hc_lowrank + self.stream = self.hc_count * self.hidden + # Columns the first projection reads: the `[down; block_inject]` rows rounded up to 32. + self.hc_first_cols = -(-(self.hc_lowrank + self.hc_count) // 32) * 32 + + # RoPE: partial, neox pairs, interleaved mRoPE. + self.head_dim = cfg.head_dim + rope = getattr(cfg, "rope_parameters", None) or getattr(cfg, "rope_scaling", None) + self.theta = float(rope["rope_theta"]) + partial = float(rope.get("partial_rotary_factor", cfg.partial_rotary_factor)) + self.rotary_dim = int(self.head_dim * partial) + self.mrope_declared = str(rope.get("type", rope.get("rope_type"))) == "mrope" + section = list(rope["mrope_section"]) + self.mrope_section1 = int(section[1]) + self.mrope_section2 = int(section[2]) + self.max_positions = cfg.max_position_embeddings + + # QSA sparse index geometry, replicated on every rank. + self.index_heads = cfg.indexer_n_heads + self.index_kv_heads = cfg.indexer_kv_heads + self.index_head_dim = cfg.indexer_head_dim + self.token_topk = cfg.indexer_budget + self.compress_ratio = cfg.indexer_compress_ratio + self.block_topk = self.token_topk // self.compress_ratio + # Dense-attention cutoff; must match the engine's (graph families split on it). + self.dense_threshold = self.token_topk + sparse_cfg = getattr(model_config, "sparse_attention_config", None) + assert getattr(sparse_cfg, "algorithm", None) == "qsa", ( + f"this target is a QSA sparse-attention assembly and the engine resolved " + f"sparse_attention_config={sparse_cfg!r}; QSA is this architecture's own default" + ) + qsa_params = sparse_cfg.to_sparse_params(pretrained_config=cfg) + assert ( + qsa_params.index_n_heads, + qsa_params.index_kv_heads, + qsa_params.index_head_dim, + qsa_params.token_topk, + qsa_params.compress_ratio, + ) == ( + self.index_heads, + self.index_kv_heads, + self.index_head_dim, + self.token_topk, + self.compress_ratio, + ), ( + f"the engine lowered the QSA index geometry to {qsa_params!r}; the side caches " + "are sized from the engine's copy and the kernels are called with this one" + ) + assert qsa_params.dense_seq_len_threshold == self.dense_threshold, ( + f"the engine's resolved QSA cutoff is {qsa_params.dense_seq_len_threshold} and " + f"this forward branches at {self.dense_threshold}: the runner captures its short " + "decode graph family at the engine's number" + ) + # IndexShare for the MTP draft loop, as resolved by the engine (it sizes the buffers). + self.mtp_index_share = bool(qsa_params.mtp_index_share) + self.index_width = (self.index_heads + self.index_kv_heads) * self.index_head_dim + # `[q | k | v]` scratch for the index norm + rope; the v head is required, never read. + self.index_scratch_width = self.index_width + self.index_head_dim + self.softmax_scale = 1.0 / math.sqrt(self.head_dim) + + # The draft layer's global index (KV managers and the weight table key it there). + self.mtp_layer_idx = self.num_layers + mtp_block = getattr(cfg, "mtp", None) + self.mtp_theta = ( + float(mtp_block.get("rope_theta", self.theta)) + if isinstance(mtp_block, dict) + else self.theta + ) + + self.head_k_dim = cfg.linear_key_head_dim + self.head_v_dim = cfg.linear_value_head_dim + self.conv_kernel = cfg.linear_conv_kernel_dim + + self.num_experts = cfg.num_experts + self.topk = cfg.num_experts_per_tok + self.moe_inter = cfg.moe_intermediate_size + self.shared_inter = cfg.shared_expert_intermediate_size + requested = model_config.moe_backend_requested.upper() + assert requested in _TRUNK_MOE_BACKENDS, ( + f"moe_config.backend={model_config.moe_backend_requested!r} is not implemented by " + f"this target; the supported requests are {sorted(_TRUNK_MOE_BACKENDS)}" + ) + quant_algo = getattr(model_config.quant_config, "quant_algo", None) + assert quant_algo == _QUANT_ALGO, ( + f"this target loads the block-FP8 checkpoint ({_QUANT_ALGO}); the engine resolved " + f"quant_algo {quant_algo!r}" + ) + + # One-based in the config. + self.ple_layer = list(cfg.ple_layer_ids)[0] - 1 + self.ngram_size = cfg.ngram_size + self.ple_dilation = self.ngram_size + self.ngram_vocab_divisor = cfg.make_ngram_vocab_size_divisible_by + eos = cfg.eos_token_id + self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) + self.ple_table_dtype = _weights.PLE_TABLE_DTYPE + # One host copy per host, not per rank (`host_share.py`). + self.ple_shared_host = _weights.ple_shared_host_scope() + # Host memory behind the offloaded table; `ple_table` is a non-owning device view. + self._ple_table_host = None + + vcfg = getattr(composite_config, "vision_config", None) + assert vcfg is not None, "the composite config carries no vision_config" + self.vision_config = vcfg + self.image_token_id = int(composite_config.image_token_id) + self.video_token_id = int(composite_config.video_token_id) + + self.kv_cache_quant_algo = model_config.quant_config.kv_cache_quant_algo + assert self.kv_cache_quant_algo in (None, QuantAlgo.FP8, QuantAlgo.NVFP4), ( + f"this target stores K/V as the checkpoint's bf16, as fp8-e4m3 or as NVFP4; the " + f"engine resolved kv_cache_quant_algo {self.kv_cache_quant_algo!r}" + ) + self.quant_mode = { + None: 0, + QuantAlgo.FP8: QM_FP8_KV_CACHE, + QuantAlgo.NVFP4: QM_NVFP4_KV_CACHE, + }[self.kv_cache_quant_algo] + + # This rank's share of the heads and experts. + d = _weights.MODEL_WEIGHTS.dims(self) + self.dims = d + self.heads_q = d.heads_q + self.heads_kv = d.heads_kv + self.kv_head_start = d.kv_head_start + self.q_width = d.q_width + self.kv_width = d.kv_width + self.qkv_width = self.q_width + 2 * self.kv_width + # Fused input projection row: `[q | k | v | gate | index q, k | v slot]`. The in-place + # norm + rope only touches q and k, so every column after k counts as v to it. + self.attn_in_width = d.attn_in_rows + assert self.attn_in_width == self.qkv_width + self.q_width + self.index_scratch_width + assert self.attn_in_width % self.head_dim == 0, self.attn_in_width + self.attn_in_tail_heads = self.attn_in_width // self.head_dim - self.heads_q - self.heads_kv + self.index_col = self.qkv_width + self.q_width + self.num_k_heads = d.num_k_heads + self.num_v_heads = d.num_v_heads + self.key_dim = d.key_dim + self.value_dim = d.value_dim + self.conv_dim = d.conv_dim + # Column widths of the one GDN input projection's output: [q|k|v | z | a | b]. + self.in_proj_split = d.in_proj_split + # Its weight's rows: their sum zero-padded to `weights.IN_PROJ_ROW_ALIGN`. + self.in_proj_rows = d.in_proj_rows + # This rank's routed experts (trunk and draft): `[expert_offset, expert_offset + + # local_experts)`, each at the full `moe_inter`. + self.local_experts = d.local_experts + self.expert_offset = d.expert_offset + assert d.moe_inter == self.moe_inter, (d.moe_inter, self.moe_inter) + + # Every weight but the n-gram table; meta-init intercepts torch.empty. + self.w = _weights.MODEL_WEIGHTS.declare(d) + + # Derived once after load (`post_load_weights`). + self._gdn_gating: dict | None = None + self._cos_sin: torch.Tensor | None = None + self._hc_first: dict | None = None + self._hc_norm_w1: dict | None = None + # KV scales: fp32 [1] ones (fp8 pool), fp32 [3] ones for q, K, V (NVFP4), None (bf16). + self.kv_scale_orig_quant: torch.Tensor | None = None + self.kv_scale_quant_orig: torch.Tensor | None = None + # Streams are made after load: one created inside a graph capture is not capturable. + self._side_stream: torch.cuda.Stream | None = None + self._decode_stream: torch.cuda.Stream | None = None + self._capture_stream: torch.cuda.Stream | None = None + self._ple_stream: torch.cuda.Stream | None = None + self._side_fork: torch.cuda.Event | None = None + self._side_join: torch.cuda.Event | None = None + self.tp_sum = None + self.expert_sum = None + self._prefill: "PrefillTarget | None" = None + self._decode: "DecodeTarget | None" = None + # Everything derived from the KV cache manager (`_bind_cache_manager`). + self._bound: _Bound | None = None + self._binds = 0 + # Set by `begin_step`, cleared by `forward`. + self._step_begun = False + self._mtp: "_MtpWeights | None" = None + self._mtp_cos_sin: torch.Tensor | None = None + # The verify step's side-cache rollback, registered with the spec worker. + self.spec_rollback: SpeculativeRollback | None = ( + SpeculativeRollback() if self.mtp_enabled else None + ) + self._draft_bound: _DraftBound | None = None + + # ---------------------------------------------------------------- load --- + + def _fuse_hc_first_projections(self) -> None: + """Fuse each hyper-connection's `down` and `block_inject` into one GEMM operand. + + Each is `[rows, stream]`, zero-padded to a multiple of 128 rows; the checkpoint + parameters become row views of it.""" + w = self.w + + def fuse(*params): + rows = sum(t.shape[0] for t in params) + padded = -(-rows // 128) * 128 + pad = torch.zeros( + [padded - rows, self.stream], dtype=params[0].dtype, device=params[0].device + ) + both = torch.cat([t.data for t in params] + [pad], 0) + start = 0 + for t in params: + n = t.shape[0] + t.data = both[start : start + n] + start += n + return both + + self._hc_first = { + (i, tag): fuse(w[f"l{i}_{tag}_down"], w[f"l{i}_{tag}_inject"]) + for i in range(self.num_layers) + for tag in ("ahc", "mhc") + } + self._hc_first["fhc"] = fuse(w["fhc_down"]) + if self.mtp_enabled: + p = f"l{self.mtp_layer_idx}_mtp_" + self._hc_first["mtp_ahc"] = fuse(w[f"{p}ahc_down"], w[f"{p}ahc_inject"]) + self._hc_first["mtp_mhc"] = fuse(w[f"{p}mhc_down"], w[f"{p}mhc_inject"]) + self._hc_first["mtp_fhc"] = fuse(w[f"{p}fhc_down"]) + # Each site's fp32 `(1 + norm)`, keyed by the norm's data_ptr (`_hc_mixer_rstd`). + norms = [w[f"l{i}_{tag}_norm"] for i in range(self.num_layers) for tag in ("ahc", "mhc")] + norms.append(w["fhc_norm"]) + if self.mtp_enabled: + p = f"l{self.mtp_layer_idx}_mtp_" + norms += [w[f"{p}{tag}_norm"] for tag in ("ahc", "mhc", "fhc")] + self._hc_norm_w1 = {t.data_ptr(): (1.0 + t.float()).contiguous() for t in norms} + + def _precompile_hc_rstd(self, device) -> None: + """Pre-compile every prefill rstd kernel variant (each K split and form) over zeros.""" + hi = self.max_num_tokens + if hi < _HC_RSTD_MIN_ROWS: + return + width, hidden, rank, c = self.stream, self.hidden, self.hc_lowrank, self.hc_count + bf = torch.bfloat16 + splits = {} + for rows in range(_HC_RSTD_MIN_ROWS, hi + 1, 256): + splits.setdefault(_hc_rescale_split(rows, width // _HC_RESCALE_BK, device), rows) + splits.setdefault(_hc_rescale_split(hi, width // _HC_RESCALE_BK, device), hi) + rows = max(splits.values()) + hc = torch.zeros([rows, width], dtype=bf, device=device) + blk = torch.zeros([rows, hidden], dtype=bf, device=device) + inj = torch.zeros([rows, c], dtype=bf, device=device) + norm = torch.zeros([width], dtype=bf, device=device) + first = torch.zeros([-(-(rank + c) // 128) * 128, width], dtype=bf, device=device).t() + up = torch.zeros([width, rank], dtype=bf, device=device).t() + _, rstd = qwen4_exp_hc_inject_rstd(hc, None, None, self.eps, c) + qwen4_exp_hc_inject_rstd(hc, blk, inj, self.eps, c) + qwen4_exp_hc_inject_rstd(hc, blk, inj, self.eps, c, blk, blk[:, :1]) + for m in sorted(splits.values()): + low, _ = qwen4_exp_hc_first_rescale_tc(hc[:m], rstd[:m], norm, first, c, rank, True) + w1 = (1.0 + norm.float()).contiguous() + qwen4_exp_hc_up_gate_mean_rstd(low, up, hc[: low.shape[0]], rstd[: low.shape[0]], w1, c) + torch.cuda.synchronize(device) + + def _precompile_draft_hc_rows(self, device) -> None: + """Pre-compile the hyper-connection sites of eager draft passes at every row count. + + Decode graphs compile at capture, but an eager draft pass would compile while serving.""" + M = getattr(self, "_mtp", None) + if M is None or _HC_ROWS_TC_MIN_ROWS > _HC_ROWS_TC_MAX_ROWS: + return + width, hidden, c = self.stream, self.hidden, self.hc_count + bf = torch.bfloat16 + hi = min(_HC_ROWS_TC_MAX_ROWS, self.max_num_tokens) + hc = torch.zeros([hi, width], dtype=bf, device=device) + blk = torch.zeros([hi, hidden], dtype=bf, device=device) + inj = torch.zeros([hi, c], dtype=bf, device=device) + n_inject = self.hc_lowrank + c + for rows in range(_HC_ROWS_TC_MIN_ROWS, hi + 1): + sites = ( + (M.ahc_norm, M.ahc_first, M.ahc_up, blk[:rows], inj[:rows], n_inject), + (M.fhc_norm, M.fhc_first, M.fhc_up, None, None, self.hc_lowrank), + ) + for norm, first, up, block_out, inject, n_cols in sites: + out, low, _, rstd = qwen4_exp_hc_decode_rows_first( + hc[:rows], block_out, inject, norm, first, c, n_cols, self.eps + ) + qwen4_exp_hc_decode_rows_mix(low, rstd, up, out, norm, c) + torch.cuda.synchronize(device) + + def post_load_weights(self) -> None: + """Derive load-time tensors and build the targets; a shell hook, not an override.""" + w = self.w + device = w["embed"].device + # The delta-rule kernels read A_log and dt_bias as contiguous fp32; the checkpoint is bf16. + self._gdn_gating = { + i: ( + w[f"l{i}_A_log"].float().contiguous(), + w[f"l{i}_dt_bias"].float().contiguous(), + ) + for i in self.gdn_layers + } + # Zero the GDN in-proj alignment rows so the decode GEMM's unread pad columns are finite. + for i in self.gdn_layers: + w[f"l{i}_in_proj_w"][sum(self.in_proj_split) :].zero_() + # An index past the table's end is a device-side assert that poisons the process. + table_rows = w["ple_table"].shape[0] + used_rows = int(w["ple_offs"][-1]) + int(w["ple_vocab"][-1]) + assert used_rows <= table_rows, ( + f"the n-gram table has {table_rows} rows but its per-head offsets and vocabulary " + f"sizes address {used_rows}" + ) + assert table_rows % self.ngram_vocab_divisor == 0, (table_rows, self.ngram_vocab_divisor) + self._cos_sin = self._index_rope_table(self.theta, device) + # Before the draft layer's views, which read its fused first projections. + self._fuse_hc_first_projections() + if self.mtp_enabled: + # The draft layer's index rope uses the MTP module's own rope base. + self._mtp_cos_sin = ( + self._cos_sin + if self.mtp_theta == self.theta + else self._index_rope_table(self.mtp_theta, device) + ) + self._mtp = _mtp_views(self) + if self.quant_mode in (QM_FP8_KV_CACHE, QM_NVFP4_KV_CACHE): + # Built here, not at construction: that runs under meta init. + scales = 3 if self.quant_mode == QM_NVFP4_KV_CACHE else 1 + self.kv_scale_quant_orig = torch.ones(scales, dtype=torch.float32, device=device) + self.kv_scale_orig_quant = 1.0 / self.kv_scale_quant_orig + # The CPU-only load checks build the targets over host weights; they launch nothing. + if device.type == "cuda": + hc_tc_first_precompile(self.stream, self.hc_lowrank, self.hc_count, device) + self._precompile_hc_rstd(device) + if self.mtp_enabled: + self._precompile_draft_hc_rows(device) + self._check_ple_residency() + self._side_stream = torch.cuda.Stream(device=device) + # Generation-only steps run at the highest stream priority (`_on_decode_stream`). + self._decode_stream = torch.cuda.Stream(device=device, priority=-100) + self._capture_stream = torch.cuda.Stream(device=device) + self._ple_stream = torch.cuda.Stream(device=device) + # `_moe_block`'s side-stream fork and join, re-recorded at every layer. + self._side_fork = torch.cuda.Event() + self._side_join = torch.cuda.Event() + self.tp_sum = self._tp_sum_op() + self.expert_sum = self._expert_sum_op() + self.hc_ar_inject = self._hc_ar_inject() + self._prefill = PrefillTarget(self) + self._decode = DecodeTarget(self) + + def _tp_sum_op(self): + """Return the attention-TP sum of block partials, bit-identical on every rank.""" + if self.attn_tp_size == 1: + return lambda x: x + group = allreduce_group_args(self.mapping) + + def tp_sum(x: torch.Tensor) -> torch.Tensor: + return allreduce( + x, + None, + None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=self.eps, + trigger_completion_at_end=False, + **group, + ) + + if not MNNVLAllReduce.is_mnnvl(self.mapping, torch.bfloat16, explicitly_requested=True): + return tp_sum + # A collective over the TP group: every rank builds the workspace here, in order. + mnnvl = mnnvl_workspace_args( + self.mapping, _TP_SUM_MNNVL_MAX_ROWS, self.hidden, torch.bfloat16 + ) + ipc_sum = tp_sum + + def tp_sum(x: torch.Tensor) -> torch.Tensor: # noqa: F811 + if x.shape[0] <= _TP_SUM_MNNVL_MAX_ROWS: + return mnnvl_tp_sum(x, **mnnvl) + return ipc_sum(x) + + return tp_sum + + def _hc_ar_inject(self): + """Return the decode sites' fused TP-sum + injection state, or None where unsupported.""" + if not ( + self.attn_tp_size in (2, 4) + and self.dp_size == 1 + and self.mapping.world_size == self.attn_tp_size + and self.hidden == 2560 + and self.hc_count == 4 + ): + return None + group = allreduce_group_args(self.mapping) + if group["workspace"] is None: + return None + members = list(self.mapping.tp_group) + rank = members.index(self.mapping.rank) + return _HcArInject(group["workspace"], rank, len(members)) + + def _expert_sum_op(self): + """Return `expert_sum(run, x, padding)`: this rank's rows of a MoE block's output. + + `run(rows)` applies this rank's expert window; `padding` comes from `_dp_pad_rows`. + Attention TP: every rank holds every row and `tp_sum` completes the partial. Attention + DP: all-gather the group's rows in rank order, run the window, reduce-scatter; routing + gathered rows keeps every rank's routing of a row identical. `MOE_ALLTOALL_ENV` (MPI and + MNNVL) swaps in the NVLink all-to-all, with `run` a `_WindowRun`. Unequal per-rank + counts (`padding` set) use the ragged collectives so the window never sees padding + rows, which would all route to the same experts and stall the group on one window.""" + if self.dp_size == 1: + tp_sum = self.tp_sum + + def window_sum(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + return tp_sum(run(x)) + + return window_sum + group = self.dp_group + # With MPI disabled the collectives run over the TP process group. + process_group = self.mapping.tp_group_pg.boxed() if mpi_disabled() else None + + def round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + sizes = None if padding is None else list(padding.counts) + gathered = allgather(x, sizes, group, process_group) + return reducescatter(run(gathered), sizes, group, process_group) + + a2a = os.environ.get(MOE_ALLTOALL_ENV, "0").strip().lower() in ("1", "true", "yes") + strategy = ( + None if not a2a or mpi_disabled() else _build_moe_a2a(self.mapping, self.max_num_tokens) + ) + if strategy is None: + if a2a: + logger.warning( + f"{MOE_ALLTOALL_ENV}=1 needs MPI and MNNVL; the attention-DP MoE runs the " + "all-gather / reduce-scatter round trip" + ) + return round_trip + self.moe_a2a = strategy + + def alltoall_round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + counts = [x.shape[0]] * len(group) if padding is None else list(padding.counts) + topk_ids, topk_w = _route_rows(run.core, x, run.router, run.mm, run.pad_mask) + got = moe_a2a_dispatch(strategy, x, topk_ids, topk_w, counts) + partials = _window_prerouted( + run.core, got.rows, got.topk_ids, got.topk_weights, run.experts + ) + return moe_a2a_combine(strategy, partials) + + return alltoall_round_trip + + def _dp_pad_rows(self, md, num_tokens: int) -> _DpPadding | None: + """Return the group's per-rank token counts for the ragged MoE round trip, or None. + + None without attention DP or with equal counts, as in every captured step.""" + if self.dp_size == 1: + return None + counts = tuple(int(n) for n in md.all_rank_num_tokens) + assert len(counts) == self.dp_size, (counts, self.dp_size) + assert counts[self.dp_rank] == num_tokens, ( + f"rank {self.dp_rank} holds {num_tokens} rows but all_rank_num_tokens says " + f"{counts[self.dp_rank]} ({list(counts)}); the collectives would gather the wrong " + "split" + ) + if min(counts) == max(counts): + return None + return _DpPadding(counts) + + def _index_rope_table(self, theta: float, device) -> torch.Tensor: + """Index rope table at `theta`: `[max_positions, 2 * rotary_pairs]` fp32, cos then sin.""" + exponent = torch.arange(0, self.rotary_dim, 2, dtype=torch.float64, device=device) + inv_freq = (1.0 / (theta ** (exponent / self.rotary_dim))).to(torch.float32) + positions = torch.arange(self.max_positions, dtype=torch.float32, device=device) + angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) + return torch.cat([torch.cos(angles), torch.sin(angles)], dim=1).contiguous() + + def _check_ple_residency(self) -> None: + """Assert via the driver that the n-gram table is host-resident; outputs cannot show it.""" + table = self.w["ple_table"] + segment = self._ple_table_host + if hasattr(segment, "backing"): + # A VMM mapping: the allocation's own properties say where its pages are. + assert segment.backing.resident_on_host(segment.pointer), ( + f"the shared n-gram table ({segment.describe()}) is not host NUMA memory" + ) + else: + kind = _weights.memory_kind(table.data_ptr()) + assert kind == _weights.CUDA_MEMORY_TYPE_HOST, ( + f"the n-gram table was declared in host memory, but the driver reports " + f"cudaMemoryType {kind} for the pointer the forward gathers from" + ) + + # ---------------------------------------------------------------- bind --- + + def _bind_cache_manager(self, md, position_ids) -> None: + """Bind this rank's pool views of the step's KV cache manager and check its geometry. + + Keyed on the manager, not a first-forward flag: capacity estimation builds a throwaway + manager before the real one, and a view held across the swap would dangle.""" + cm = md.kv_cache_manager + assert cm is not None, "a paged KV cache is required" + assert not (self._binds and torch.cuda.is_current_stream_capturing()), ( + f"the KV cache manager changed to {type(cm).__name__} during CUDA graph capture" + ) + self._binds += 1 + missing = [name for name in _QSA_FIELDS if not hasattr(md, name)] + assert not missing, ( + f"metadata fields missing: {missing}; the engine did not select QSA metadata" + ) + assert position_ids.dtype == torch.int32, position_ids.dtype + assert self.mrope_declared, "this target takes the mRoPE position layout" + assert position_ids.dim() == 3 and position_ids.shape[0] == _QSA_POSITION_AXES, ( + f"expected the [{_QSA_POSITION_AXES}, 1, T] mRoPE position buffer, got " + f"{tuple(position_ids.shape)}" + ) + pools = {row[0] for row in md.host_kv_cache_pool_mapping.tolist()} + assert pools <= {0, 1}, ( + f"KV addressing over {len(pools)} pools is not certified; layer->pool ids " + f"{sorted(pools)}" + ) + assert self.quant_mode != QM_NVFP4_KV_CACHE or md.use_paged_context_fmha, ( + "an NVFP4 KV cache is read by the paged-context attention kernels only, which the " + "engine enables for chunked prefill, KV cache block reuse or speculative decoding" + ) + g = _pool.pool_geometry( + self.model_config.pretrained_config, + self.mapping, + tokens_per_block=md.tokens_per_block, + kv_cache_quant_algo=self.kv_cache_quant_algo, + dtype=self.w["embed"].dtype, + ) + assert (g.heads_kv, g.kv_head_start, g.num_k_heads, g.num_v_heads, g.conv_dim) == ( + self.heads_kv, + self.kv_head_start, + self.num_k_heads, + self.num_v_heads, + self.conv_dim, + ), ( + f"the KV cache manager's share {g} disagrees with the weights' heads " + f"(kv {self.heads_kv} from {self.kv_head_start}, GDN {self.num_k_heads} / " + f"{self.num_v_heads}, conv {self.conv_dim})" + ) + assert md.qsa_topk_indices.shape[1] == self.block_topk, ( + f"the engine sized the QSA selection buffer {md.qsa_topk_indices.shape[1]} wide " + f"and this target selects {self.block_topk} blocks per row" + ) + mm = md.mamba_metadata + assert mm is not None and mm is not False, ( + "attn_metadata.mamba_metadata is absent: the runtime allocated no recurrent-state pool" + ) + mm_missing = [name for name in _MAMBA_FIELDS if not hasattr(mm, name)] + assert not mm_missing, f"mamba metadata fields missing: {mm_missing}" + rank_pools = _pool.take_pools(cm, g, self.attn_layers, self.gdn_layers, self.ple_layer) + # No sliding window: the targets pass `max_seq_len` as the window. + windows = cm.max_attention_window_vec + attn_windows = {windows[off] for off in rank_pools.pool_idx.values()} + assert attn_windows <= {None, md.max_seq_len}, ( + f"the attention layers' pool windows are {sorted(map(repr, attn_windows))}; a " + "sliding window is not implemented here" + ) + assert md.effective_beam_width == 1, "beam search is not implemented" + assert md.cache_indirection is None, "beam search is not implemented" + assert not md.is_cross, "cross attention is not implemented" + # Printed, not logged: the executor's workers keep the info log level off. + first_attn, first_gdn = self.attn_layers[0], self.gdn_layers[0] + print( + f"{_LOG_PREFIX}: rank {self.mapping.rank} binds " + f"{rank_pools.kv[first_attn].k.shape[0]} K/V pages, " + f"{rank_pools.index_cache[first_attn].shape[0]} QSA index pages, " + f"{rank_pools.conv[first_gdn].shape[0]} recurrent-state slots and " + f"{rank_pools.ple_state[0].shape[0]} PLE slots (bind {self._binds})", + flush=True, + ) + bound = _Bound(cm, rank_pools, stage=_inforward_stage(cm, self.gdn_layers, self.ple_layer)) + if self.spec_rollback is not None: + self.spec_rollback.set_index_planes( + [rank_pools.index_cache[i] for i in self.attn_layers] + ) + if self.mtp_enabled: + bound = bound._replace(**self._bind_step_buffers(cm, g, rank_pools)) + self._bound = bound + + def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools) -> dict: + """Bind the verify step's per-step buffers and return the `_Bound` fields they add. + + Per-row conv windows feed `update_mamba_states`; the recurrent state uses cached replay.""" + _pool.require( + cm, + ("is_speculative", "mamba_layer_cache", "get_replay_state_update_metadata"), + "binds the verify step's per-step state buffers and replay metadata", + ) + assert cm.is_speculative(), ( + f"the KV cache manager {type(cm).__name__} reports is_speculative() False, so it " + "allocated none of the per-step state buffers the verify path writes" + ) + layer_caches = {i: cm.mamba_layer_cache(i) for i in self.gdn_layers} + int_conv = {i: cache.intermediate_conv_window for i, cache in layer_caches.items()} + int_idx = getattr(cm, "intermediate_state_indices", None) + g0 = self.gdn_layers[0] + _pool.check_step_buffers(g, int_conv[g0], pools.conv[g0], int_idx) + meta = cm.get_replay_state_update_metadata() + if meta is None: + raise NotImplementedError( + "the KV cache manager runs the legacy per-step recurrent verify path " + "(TRTLLM_USE_GDN_REPLAY=0), which this target does not assemble; leave " + "TRTLLM_USE_GDN_REPLAY unset for the cached replay" + ) + assert not getattr(cm, "use_gdn_flashinfer_ring_replay", False), ( + "the manager laid its replay history out for the FlashInfer ring kernel; this " + "target runs the double-buffered Triton cached replay" + ) + assert layer_caches[g0].intermediate_ssm is None, ( + "the cache manager runs the cached-replay verify path and also allocated a " + "per-step recurrent buffer; the runtime allocates one or the other" + ) + ssm0 = pools.ssm[g0] + history, width = meta.replay_history_size, meta.replay_step_width + _pool.check_replay_state( + history, width, meta.prev_num_accepted_tokens, meta.cache_buf_idx, ssm0 + ) + old = {} + for i, cache in layer_caches.items(): + old[i] = (cache.old_x, cache.old_B, cache.old_dt, cache.old_dA_cumsum) + _pool.check_replay_history(g, i, history, ssm0.shape[0], *old[i][:3]) + replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) + return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) + + def _bind_draft_cache_manager(self, md) -> None: + """Bind the draft layer's index plane, position plane and K/V in its step's manager. + + That is the separate draft KV cache under attention TP, else the trunk's manager (also + under disaggregated serving). The runner runs every captured shape eagerly first, so + the first bind never happens under capture.""" + cm = md.kv_cache_manager + assert not torch.cuda.is_current_stream_capturing(), ( + f"the draft layer is not bound to KV cache manager {type(cm).__name__} (first use, " + "or the manager changed since its last bind) and CUDA graph capture is active" + ) + g = _pool.pool_geometry( + self.model_config.pretrained_config, + self.mapping, + tokens_per_block=md.tokens_per_block, + kv_cache_quant_algo=self.kv_cache_quant_algo, + dtype=self.w["embed"].dtype, + ) + layer = self.mtp_layer_idx + draft = _pool.take_draft_pools( + cm, g, layer, separate=_pool.draft_kv_cache_is_separate(self.mapping, self.spec_config) + ) + index_cache, position_cache, kv = draft.index_cache, draft.position_cache, draft.kv + share_slot = None + if self.mtp_index_share: + assert md.sparse_metadata_params.mtp_index_share, ( + "this target resolved IndexShare on but the engine's QSA metadata did not" + ) + slots = cm.qsa_shared_topk_slots + _pool.check_shared_selection(md.qsa_shared_topk_indices, slots, layer, self.block_topk) + share_slot = slots[layer] + self._draft_bound = _DraftBound(cm, index_cache, position_cache, kv, share_slot) + + def _draft_step_bind(self, md) -> tuple[_DraftBound, int]: + """Return the draft layer's bind and pool offset for this draft step's manager. + + Checked every call: the worker swaps managers and page tables in for the draft loop.""" + cm = md.kv_cache_manager + assert cm is not None, "a paged KV cache is required" + offsets = cm.layer_offsets + assert self.mtp_layer_idx in offsets, ( + f"the MTP module's attention is global layer index {self.mtp_layer_idx}, which the " + f"cache manager {type(cm).__name__} does not map (it maps {len(offsets)} layers); " + "the engine allocated no KV layer for the draft module" + ) + layer_idx = offsets[self.mtp_layer_idx] + window = cm.max_attention_window_vec[layer_idx] + assert window is None or window == md.max_seq_len, ( + f"the MTP layer's pool window is {window!r} but this call passes max_seq_len " + f"{md.max_seq_len} as the attention window; a sliding window is not implemented" + ) + if self._draft_bound is None or cm is not self._draft_bound.cm: + self._bind_draft_cache_manager(md) + assert cm is self._bound.cm or md.qsa_block_table is getattr( + md, "qsa_draft_block_table", None + ), ( + "the draft step is bound to a separate draft KV cache manager but qsa_block_table " + "is not that manager's table: the engine's QSA metadata did not swap its page " + "table for the draft forward" + ) + return self._draft_bound, layer_idx + + # -------------------------------------------------------------- forward --- + + def _select_target(self, attn_metadata) -> "PrefillTarget | DecodeTarget": + return self._decode if attn_metadata.num_contexts == 0 else self._prefill + + def begin_step( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None, + position_ids: torch.IntTensor | None, + inputs_embeds: torch.FloatTensor | None = None, + lora_params: dict | None = None, + spec_metadata=None, + vision: tuple | None = None, + ) -> None: + """Validate the step, bind the cache manager and let the chosen target prepare it. + + All host reads happen here: a breakable prefill CUDA graph replays `forward` without + calling it. `vision` is `(rows, mm_token_indices)` when the step carries media.""" + assert self._prefill is not None and self._decode is not None, ( + "load_weights must run before forward" + ) + assert position_ids is not None + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert lora_params is None, "LoRA is not implemented by this target" + assert input_ids is not None, "this target's PLE layer hashes token ids" + assert inputs_embeds is None, ( + "this target computes its token embedding itself; an engine-supplied " + "inputs_embeds is not implemented" + ) + # One-model MTP-Eagle captures no trunk layer, so the trunk owes no capture hook. + assert (spec_metadata is not None) == self.mtp_enabled, ( + f"spec_metadata {'arrived' if spec_metadata is not None else 'is absent'} but the " + f"model was built with mtp_enabled={self.mtp_enabled}; the draft path and the " + "weights it loads are decided at construction" + ) + if self._bound is None or attn_metadata.kv_cache_manager is not self._bound.cm: + self._bind_cache_manager(attn_metadata, position_ids) + target = self._select_target(attn_metadata) + if target is self._prefill: + target.prepare(attn_metadata, input_ids, position_ids, vision, spec_metadata) + else: + assert vision is None, "a generation-only step carries vision rows" + self._step_begun = True + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + lora_params: dict | None = None, + **kwargs, + ): + """Run the step `begin_step` prepared; nothing here may read the step from the host. + + Returns the logits input, or under speculation what `split_spec_output` takes.""" + assert self._step_begun, ( + "the shell calls begin_step before every forward; this forward has no prepared step" + ) + self._step_begun = False + target = self._select_target(attn_metadata) + step = target.forward + if isinstance(target, DecodeTarget): + return _on_decode_stream( + self, + input_ids.shape[0], + step, + attn_metadata, + input_ids=input_ids, + position_ids=position_ids, + **kwargs, + ) + return step(attn_metadata, input_ids=input_ids, position_ids=position_ids, **kwargs) + + def split_spec_output(self, output) -> tuple[torch.Tensor, torch.Tensor]: + """Split speculative `forward` output into `final` `[T, hidden]` and `hc` `[T, stream]`. + + `final` feeds `lm_head`, `hc` the MTP module; the prefill target packs both in one.""" + if isinstance(output, tuple): + return output + final, hc = torch.split(output, [self.hidden, self.stream], -1) + return final, hc + + +# --------------------------------------------------------------- weight views --- + + +class _LayerWeights(NamedTuple): + """One trunk layer's op views (GEMM operands `.t()`); unused attention/GDN fields are None.""" + + ahc_norm: torch.Tensor + ahc_first: torch.Tensor + ahc_up: torch.Tensor + mhc_norm: torch.Tensor + mhc_first: torch.Tensor + mhc_up: torch.Tensor + attn_in: torch.Tensor | None + o: torch.Tensor | None + q_norm: torch.Tensor | None + k_norm: torch.Tensor | None + idx_qn: torch.Tensor | None + idx_kn: torch.Tensor | None + in_proj: torch.Tensor | None + # With the zero alignment rows (`core.in_proj_rows` columns) the decode GEMM tiles whole; + # `in_proj` is its leading `sum(in_proj_split)` columns. + in_proj_dec: torch.Tensor | None + out: torch.Tensor | None + conv_w: torch.Tensor | None + a_log: torch.Tensor | None + dt_bias: torch.Tensor | None + gnorm: torch.Tensor | None + router: torch.Tensor + experts: _weights.ExpertWindow + # `[gate | up | gate logit | pad]` (`weights._cat_rows`), and the down projection. + sh_gug: torch.Tensor + sh_dn: torch.Tensor + + +def _layer_views(core: Qwen4ExpUnifiedModelingV2) -> list[_LayerWeights]: + w = core.w + layers = [] + for i in range(core.num_layers): + attn: dict = dict(attn_in=None, o=None, q_norm=None, k_norm=None, idx_qn=None, idx_kn=None) + gdn: dict = dict( + in_proj=None, + in_proj_dec=None, + out=None, + conv_w=None, + a_log=None, + dt_bias=None, + gnorm=None, + ) + if core.is_attn[i]: + attn = dict( + attn_in=w[f"l{i}_attn_in_w"].t(), + o=w[f"l{i}_o_w"].t(), + q_norm=w[f"l{i}_q_norm"], + k_norm=w[f"l{i}_k_norm"], + idx_qn=w[f"l{i}_idx_qn"], + idx_kn=w[f"l{i}_idx_kn"], + ) + else: + a_log, dt_bias = core._gdn_gating[i] + gdn = dict( + in_proj=w[f"l{i}_in_proj_w"][: sum(core.in_proj_split)].t(), + in_proj_dec=w[f"l{i}_in_proj_w"].t(), + out=w[f"l{i}_out_w"].t(), + # [conv_dim, 1, width] -> [conv_dim, width]: a view (unit width stride). + conv_w=w[f"l{i}_conv_w"].reshape(core.conv_dim, core.conv_kernel), + a_log=a_log, + dt_bias=dt_bias, + gnorm=w[f"l{i}_gnorm"], + ) + layers.append( + _LayerWeights( + ahc_norm=w[f"l{i}_ahc_norm"], + ahc_first=core._hc_first[(i, "ahc")].t(), + ahc_up=w[f"l{i}_ahc_up"].t(), + mhc_norm=w[f"l{i}_mhc_norm"], + mhc_first=core._hc_first[(i, "mhc")].t(), + mhc_up=w[f"l{i}_mhc_up"].t(), + **attn, + **gdn, + router=w[f"l{i}_router"].t(), + experts=_weights.expert_window(w, core.dims, i), + sh_gug=w[f"l{i}_sh_gu"].t(), + sh_dn=w[f"l{i}_sh_dn"].t(), + ) + ) + return layers + + +class _StackWeights(NamedTuple): + """Weights outside the layer loop: the stack-final mixer (final norm) and the PLE path.""" + + fhc_norm: torch.Tensor + fhc_first: torch.Tensor + fhc_up: torch.Tensor + ple_key: torch.Tensor + ple_val: torch.Tensor + ple_nk_w: torch.Tensor + ple_nq_w: torch.Tensor + ple_nc_w: torch.Tensor + ple_conv_w: torch.Tensor + ple_mult: torch.Tensor + ple_vocab: torch.Tensor + ple_offs: torch.Tensor + ple_tscale: torch.Tensor + + +def _stack_views(core: Qwen4ExpUnifiedModelingV2) -> _StackWeights: + w = core.w + return _StackWeights( + fhc_norm=w["fhc_norm"], + fhc_first=core._hc_first["fhc"].t(), + fhc_up=w["fhc_up"].t(), + ple_key=w["ple_key_w"].t(), + ple_val=w["ple_val_w"].t(), + # Whole norm weights: the fused tail normalizes all three streams in one program. + ple_nk_w=w["ple_nk"], + ple_nq_w=w["ple_nq"], + ple_nc_w=w["ple_nc"], + ple_conv_w=w["ple_conv_w"], + ple_mult=w["ple_mult"], + ple_vocab=w["ple_vocab"], + ple_offs=w["ple_offs"], + # The table's dequantize scale at the gathered rows' rank (3), all 1s. + ple_tscale=w["ple_tscale"].reshape(1, 1, 1), + ) + + +class _MtpWeights(NamedTuple): + """The MTP module's views at this rank's share, keyed at layer `num_hidden_layers`. + + Laid out as the trunk's (GEMM operands `.t()`); `unit_inject` adds `x_e` as a site injection.""" + + pre_e_norm: torch.Tensor + pre_h_norm: torch.Tensor + fc_e: torch.Tensor + fc_h: torch.Tensor + ahc_norm: torch.Tensor + ahc_first: torch.Tensor + ahc_up: torch.Tensor + mhc_norm: torch.Tensor + mhc_first: torch.Tensor + mhc_up: torch.Tensor + unit_inject: torch.Tensor + # [hidden, attn_in_width]: the trunk's fused attention input layout, at this rank's heads. + attn_in: torch.Tensor + o: torch.Tensor + q_norm: torch.Tensor + k_norm: torch.Tensor + idx_qn: torch.Tensor + idx_kn: torch.Tensor + router: torch.Tensor + experts: _weights.ExpertWindow + sh_gug: torch.Tensor + sh_dn: torch.Tensor + fhc_norm: torch.Tensor + fhc_first: torch.Tensor + fhc_up: torch.Tensor + + +def _mtp_views(core: Qwen4ExpUnifiedModelingV2) -> _MtpWeights: + """Built after `_fuse_hc_first_projections`, whose `"mtp_*"` tensors it reads.""" + w = core.w + p = f"l{core.mtp_layer_idx}_mtp_" + dt = w[f"{p}fc_h"].dtype + return _MtpWeights( + pre_e_norm=w[f"{p}pre_e_norm"], + pre_h_norm=w[f"{p}pre_h_norm"], + fc_e=w[f"{p}fc_e"].t(), + fc_h=w[f"{p}fc_h"].t(), + ahc_norm=w[f"{p}ahc_norm"], + ahc_first=core._hc_first["mtp_ahc"].t(), + ahc_up=w[f"{p}ahc_up"].t(), + mhc_norm=w[f"{p}mhc_norm"], + mhc_first=core._hc_first["mtp_mhc"].t(), + mhc_up=w[f"{p}mhc_up"].t(), + unit_inject=torch.ones( + [core.max_num_tokens, core.hc_count], dtype=dt, device=w[f"{p}fc_h"].device + ), + attn_in=w[f"{p}attn_in_w"].t(), + o=w[f"{p}o_w"].t(), + q_norm=w[f"{p}q_norm"], + k_norm=w[f"{p}k_norm"], + idx_qn=w[f"{p}idx_qn"], + idx_kn=w[f"{p}idx_kn"], + router=w[f"{p}router"].t(), + experts=_weights.expert_window(w, core.dims, core.mtp_layer_idx), + sh_gug=w[f"{p}sh_gu"].t(), + sh_dn=w[f"{p}sh_dn"].t(), + fhc_norm=w[f"{p}fhc_norm"], + fhc_first=core._hc_first["mtp_fhc"].t(), + fhc_up=w[f"{p}fhc_up"].t(), + ) + + +# ------------------------------------------------------------ shared blocks --- + + +def _verify_rows(core, spec_metadata, num_seqs: int, num_rows: int) -> tuple[int, bool]: + """Return `(p, is_verify)`: rows per generation sequence and whether the step verifies. + + `p` is 1 without speculation, else `runtime_draft_len + 1`; host ints, constant per graph.""" + p = num_rows // num_seqs if num_seqs else 1 + assert p * num_seqs == num_rows, ( + f"{num_rows} generation rows do not divide over {num_seqs} generation sequences; a " + "ragged per-sequence draft length can be expressed neither to the attention call nor " + "to the multi-token recurrence" + ) + assert core.mtp_enabled or p == 1, ( + f"{num_rows} generation rows over {num_seqs} generation sequences with speculation " + "off; the base path implements one query token per generation sequence" + ) + if spec_metadata is not None and num_seqs > 0: + declared = spec_metadata.runtime_draft_len + 1 + assert p == declared, ( + f"the batch carries {p} query rows per generation sequence but the worker's " + f"runtime_draft_len says {declared}; the per-step conv snapshots and the replay " + "history are sized from the second and written from the first" + ) + _pool.check_verify_width(p) + _pool.check_replay_width(core._bound.replay, p) + return p, spec_metadata is not None and num_seqs > 0 and p > 1 + + +def _verify_state(core, mm, num_decodes: int) -> tuple: + """Return what `_gdn_verify` reads besides the layer: promotion rows and replay work items. + + Work items and the all-layer commit follow the stock Gated-DeltaNet layer's conditions.""" + assert core._bound.int_idx is not None, "the cache-manager bind has not run" + int_idx = core._bound.int_idx[:num_decodes] + partitioned = num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + all_layer = partitioned and bool( + getattr(core._bound.cm, "use_gdn_cached_replay_all_layer_commit", False) + ) + work_items = mm.replay_work_items[:num_decodes] if partitioned else None + n_writes = mm.replay_n_writes if partitioned else None + return int_idx, work_items, n_writes, all_layer + + +def _gdn_in_proj_conv_tactic(core, x, L, si_d, verify): + """Return the fused GDN in-proj + conv tactic for a verify step, or None if unsupported. + + The fused kernel is bitwise `low_m_bf16_mm` + `causal_conv1d_token_major`, one launch fewer.""" + rows = x.shape[0] + w = L.in_proj_dec + if ( + core.conv_dim != _GDN_IN_PROJ_CONV_DIM + or tuple(w.shape) != (x.shape[1], _GDN_IN_PROJ_N_OUT) + or w.stride(0) != 1 + or not x.is_contiguous() + or si_d.dtype != torch.int32 + or verify[0].dtype != torch.int32 + or not L.conv_w.is_contiguous() + or tuple(L.conv_w.shape) != (_GDN_IN_PROJ_CONV_DIM, 4) + ): + return None + return _gdn_in_proj_conv_fused_tactic(rows) + + +def _gdn_verify(core, mq, a, b, i, L, si_d, n, p, verify, mq_out=None, z=None) -> torch.Tensor: + """Run the recurrent half of a verify step: `n` generation sequences of `p` rows. + + The conv also writes each row's window to the per-step buffer `update_mamba_states` + promotes from; the cached replay rebuilds each slot's state from its checkpoint and + history, runs the `p` rows and appends them, so rejected rows cost nothing to undo. `mq` is + the `[n * p, conv_dim]` q|k|v view, `a`/`b` the raw gate columns (the kernel applies + `A_log`, `dt_bias`, q/k L2 norm). Returns `[1, n * p, num_v_heads, head_v_dim]`.""" + int_idx, work_items, n_writes, all_layer = verify + pools = core._bound.pools + replay = core._bound.replay + ssm = pools.ssm[i] + assert (core.num_k_heads, core.num_v_heads) in _GDN_REPLAY_CUTE_GEOMETRIES, ( + f"GDN heads {core.num_k_heads} / {core.num_v_heads} are not a CuTe replay geometry" + ) + pdl, mapping = _gdn_replay_launch(core, n) + fold = None + if mq_out is not None: + # `gdn_in_proj_conv` already ran the conv and advanced the pool and windows. + pass + elif _gdn_verify_folds_conv(core, n) and gdn_conv_fold_supported( + mq, L.conv_w, pools.conv[i], core._bound.int_conv[i] + ): + # The replay computes the conv rows itself from the q|k|v columns (`mq`). + fold = dict( + x=mq, + weight=L.conv_w, + pool=pools.conv[i], + window=core._bound.int_conv[i], + window_rows=int_idx, + ) + mq_out = mq + else: + # Token-major `[n*p, C]` conv output, which the replay reads q/k/v from directly. + mq_out = causal_conv1d_token_major( + mq, + n, + pools.conv[i], + L.conv_w, + si_d, + core._bound.int_conv[i], + int_idx, + _PAD_SLOT_ID, + launch_dependent_kernels=pdl, + wait_producer=_GDN_CONV_PDL_WAIT, + ) + norm = None + if z is not None: + # The gated RMS norm folded in (`_gdn_verify_folds_norm`, decided by the caller). + norm = dict( + z=torch.reshape(z, [n * p, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + ) + old_x, old_B, old_dt, old_dA = replay.old[i] + qv, kv, vv = torch.split(mq_out, [core.key_dim, core.key_dim, core.value_dim], 1) + out = gdn_cached_replay_cute( + q=torch.reshape(qv, [n, p, core.num_k_heads, core.head_k_dim]), + k=torch.reshape(kv, [n, p, core.num_k_heads, core.head_k_dim]), + v=torch.reshape(vv, [n, p, core.num_v_heads, core.head_v_dim]), + g=torch.reshape(a, [n, p, core.num_v_heads]), + beta=torch.reshape(b, [n, p, core.num_v_heads]), + ssm_states=ssm, + state_indices=si_d, + old_u=old_x, + old_k=old_B, + old_G=old_dt, + old_beta=old_dA, + cache_buf_idx=replay.cache_buf_idx, + prev_num_accepted_tokens=replay.prev_num_accepted, + history_size=replay.history, + scale=None, + use_qk_l2norm_in_kernel=True, + A_log=L.a_log, + dt_bias=L.dt_bias, + launch_with_pdl=pdl, + replay_work_items=work_items, + n_writes=n_writes, + block_v=None, + num_warps=None, + use_l2_state_cache=None, + use_l2_streaming_inputs=None, + use_l2_shared_inputs=None, + main_num_stages=None, + packed_qkv=None if fold is not None else mq_out, + use_all_layer_commit=all_layer, + output=None, + **({"config": mapping} if mapping is not None else {}), + **({"conv_fold": fold} if fold is not None else {}), + **({"gated_norm": norm} if norm is not None else {}), + ) + return torch.reshape(out, [1, n * p, core.num_v_heads, core.head_v_dim]) + + +def _gdn_verify_folds_norm(core, n: int, p: int) -> bool: + """Whether a verify step folds its gated RMS norm into the CuTe replay (bitwise equal). + + Only at tp2/tep2 HT batches (up to 4 rows a request); saves a launch and an HBM round trip.""" + return ( + (core.num_k_heads, core.num_v_heads) == _GDN_REPLAY_HT_GEOMETRY + and n >= _GDN_REPLAY_HT_MIN_REQUESTS + and p <= 4 + ) + + +def _gdn_verify_folds_conv(core, n: int) -> bool: + """Whether a verify step folds its conv into the CuTe replay (tp2/tep2 HT batches).""" + return ( + (core.num_k_heads, core.num_v_heads) == _GDN_REPLAY_HT_GEOMETRY + and _GDN_REPLAY_HT_MIN_REQUESTS <= n <= _GDN_VERIFY_CONV_FOLD_MAX_REQUESTS + ) + + +def _gdn_replay_launch(core, n: int) -> tuple: + """Return `(pdl, mapping)` for the CuTe cached replay over `n` requests.""" + heads = (core.num_k_heads, core.num_v_heads) + if heads == _GDN_REPLAY_HT_GEOMETRY and n >= _GDN_REPLAY_HT_MIN_REQUESTS: + for bound, pdl, mapping in _GDN_REPLAY_HT_LAUNCH: + if n <= bound: + return pdl, mapping + return n * core.num_v_heads <= _GDN_REPLAY_PDL_MAX_HEAD_ROWS, None + + +def _draft_positions(position_ids: torch.Tensor) -> tuple: + """Return `(pos3, pos, coords)` from `[3, 1, N]` or `[3, N]` draft-step mRoPE positions. + + `pos3` is contiguous int32 `[3, N]` (the input itself if already so; nothing writes it + during the step), `pos` its row 0 and `coords` the `[N, 3]` transpose the plane stores.""" + assert position_ids.shape[0] == _QSA_POSITION_AXES, ( + f"expected the mRoPE position layout with {_QSA_POSITION_AXES} coordinate rows, got " + f"{tuple(position_ids.shape)}" + ) + if position_ids.dtype == torch.int32 and position_ids.is_contiguous(): + pos3 = torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]) + else: + pos3 = _copy_positions(position_ids) + return pos3, pos3[0], torch.transpose(pos3, 0, 1) + + +def _draft_embedding(input_ids: torch.Tensor, embed_tokens: torch.Tensor) -> torch.Tensor: + """Embed the MTP front end's tokens; negative ids read row 0. + + Rejection sampling fills slots past a request's accepted count with -1; those rows feed + nothing the step keeps, so clamping only keeps the gather inside the table.""" + if input_ids.dim() == 1 and input_ids.is_contiguous(): + # The id clamp and gather in one launch (bitwise the composed path). + return embed_rows(input_ids, embed_tokens, 1, True) + return nn.functional.embedding(input_ids.clamp_min(0), embed_tokens) + + +def _mtp_front(core, M: _MtpWeights, embed_tokens, input_ids, hidden_states, mm=None) -> tuple: + """Run the draft layer's front end and attention-side hyper-connection. + + `fc_embedding` of the normalized next-token embedding is added to every stream of + `fc_hidden` as the trunk site's unit-weight injection; returns `(x, mixed, inject)`. `mm` + runs dense projections here and in every draft-layer and MoE helper: `cublas_mm` when + None, `_decode_projection` from `DecodeTarget`.""" + mm = mm or cublas_mm + rows = hidden_states.shape[0] + assert rows <= M.unit_inject.shape[0], ( + f"a draft step of {rows} rows exceeds max_num_tokens {M.unit_inject.shape[0]}" + ) + e = _draft_embedding(input_ids, embed_tokens) + main = torch.cuda.current_stream() + side = core._side_stream if rows <= _MTP_FRONT_FORK_MAX_ROWS else main + # Fork after the embedding is queued; `x_e` is consumed on `main` after the join. + side.wait_stream(main) + with torch.cuda.stream(side): + x_e = mm(flashinfer_gemma_rmsnorm(e, M.pre_e_norm, core.eps), M.fc_e) + _, h_n = qwen4_exp_hc_inject_norm( + hidden_states, None, None, M.pre_h_norm, core.eps, core.hc_count + ) + x_h = torch.reshape( + mm(torch.reshape(h_n, [rows * core.hc_count, core.hidden]), M.fc_h), + [rows, core.stream], + ) + main.wait_stream(side) + return _hc_site(core, x_h, x_e, M.unit_inject[:rows], M.ahc_norm, M.ahc_first, M.ahc_up, True) + + +def _mtp_attention_in(core, M: _MtpWeights, mixed, pos3, pos, with_index: bool, mm=None) -> tuple: + """Compute the draft layer's attention input as a trunk attention layer does. + + Returns `(proj, qkv, gate, k, v, index_scratch)`: `gate` is `proj`'s `[T, q_width]` gate + block, `k`/`v` the rotated heads, `index_scratch` None without `with_index`.""" + mm = mm or cublas_mm + rows = mixed.shape[0] + proj = mm(mixed, M.attn_in) + index_scratch = ( + torch.empty([rows, core.index_scratch_width], dtype=mixed.dtype, device=mixed.device) + if with_index + else None + ) + _attention_rope(core, proj, index_scratch, M, pos3, pos, theta=core.mtp_theta) + if rows == 1: + # One row: the `[q | k | v]` prefix is contiguous and not written after the rope. + qkv = proj[:, : core.qkv_width] + else: + qkv = torch.empty([rows, core.qkv_width], dtype=mixed.dtype, device=mixed.device) + qkv.copy_(proj[:, : core.qkv_width]) + _, k, v = _attn_in_heads(core, proj) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + return proj, qkv, gate, k, v, index_scratch + + +def _mtp_attention_planes( + core, M: _MtpWeights, mixed, draft: _DraftBound, qsa, rows_per_request, with_index, sparse +) -> tuple: + """Compute a generation-only draft step's attention input and write its planes. + + Returns `(qkv, gate, q_index)`. The K/V append happens here only on the sparse path (the + dense op appends itself). The draft-extend step runs everything after the projection as + the trunk's one-launch `qwen4_exp_attn_decode_prologue`.""" + req_idx, logical, block_table, tokens_per_block, pos3, pos, coords = qsa + if with_index and _decode_prologue(draft.kv, sparse): + rows = mixed.shape[0] + proj = _decode_projection(mixed, M.attn_in) + qkv = torch.empty([rows, core.qkv_width], dtype=mixed.dtype, device=mixed.device) + index_scratch = torch.empty( + [rows, core.index_scratch_width], dtype=mixed.dtype, device=mixed.device + ) + q_index = _index_query(core, index_scratch) + qwen4_exp_attn_decode_prologue( + proj, + qkv, + q_index, + pos3, + coords, + core._mtp_cos_sin, + M.q_norm, + M.k_norm, + M.idx_qn, + M.idx_kn, + draft.kv.k, + draft.kv.v, + draft.index_cache, + draft.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + core.compress_ratio, + core.heads_q, + core.heads_kv, + core.head_dim, + core.index_heads, + core.index_head_dim, + core.index_col, + core.rotary_dim, + core.eps, + core.mrope_section1, + core.mrope_section2, + rows_per_request=rows_per_request, + ) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + return qkv, gate, q_index + proj, qkv, gate, k_dst, v_dst, index_scratch = _mtp_attention_in( + core, M, mixed, pos3, pos, with_index, mm=_decode_projection + ) + q_index = None + if with_index: + q_index = _mtp_index_branch( + core, + M, + proj, + index_scratch, + draft, + (req_idx, logical, block_table, tokens_per_block, coords), + ) + if sparse: + _store_kv(core, draft.kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + return qkv, gate, q_index + + +def _mtp_index_branch(core, M: _MtpWeights, proj, index_scratch, draft: _DraftBound, qsa): + """Run the draft layer's QSA index branch on its own planes, as `_index_plane` does. + + Stores the raw index key, compresses completed groups, returns the rotated index query.""" + req_idx, logical, block_table, tokens_per_block, coords = qsa + rows = proj.shape[0] + token_k = torch.split( + torch.reshape( + proj[:, core.index_col : core.index_col + core.index_scratch_width], + [rows, core.index_scratch_width // core.index_head_dim, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, 1], + 1, + )[1] + qsa_paged_index_store( + token_k, + coords, + draft.index_cache, + draft.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + ) + qsa_prefill_compress( + logical_positions=logical, + request_indices=req_idx, + block_table=block_table, + index_cache=draft.index_cache, + position_cache=draft.position_cache, + k_norm_weight=M.idx_kn, + cos_sin=core._mtp_cos_sin, + eps=core.eps, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + mrope_section=_QSA_MROPE_SECTION, + ) + return _index_query(core, index_scratch) + + +def _moe_max_rows(max_num_tokens: int, dp_size: int) -> int: + """Return the rows per routed-experts call, never fewer than `_MOE_MAX_ROWS`. + + Attention DP: `_MOE_MAX_ROWS`. Attention TP: the step's rows, capped at `_MOE_MAX_ROWS_TP` + (the runner's certified bound) or, with the grouped prefill experts, `_ADP_WINDOW_MAX_ROWS` + (the most rows the runner takes elsewhere, for calls the grouped experts decline).""" + if dp_size > 1: + return _MOE_MAX_ROWS + if _MOE_PREFILL_GROUPED_MIN_ROWS and max_num_tokens >= _MOE_PREFILL_GROUPED_MIN_ROWS: + return min(max(max_num_tokens, _MOE_MAX_ROWS), _ADP_WINDOW_MAX_ROWS) + return min(max(max_num_tokens, _MOE_MAX_ROWS), _MOE_MAX_ROWS_TP) + + +def _runner_tune_rows(core) -> int: + """Return the FP8 runner's `tune_max_num_tokens`: `core.moe_max_rows`, clamped. + + Tuning at the rows large context chunks run beats reusing the 8192-row tactic; past + `_MOE_MAX_ROWS_TP` calls run the grouped experts. Every tactic writes the same bits.""" + return min(max(getattr(core, "moe_max_rows", _MOE_MAX_ROWS), _MOE_MAX_ROWS), _MOE_MAX_ROWS_TP) + + +def _window_runner( + core, + experts: _weights.ExpertWindow, + xq, + xsf, + topk_ids, + topk_w, + output=None, + fin_first=None, + after_first=None, +): + """Run this rank's whole-expert window over pre-routed rows. + + `xq` is 1x128 e4m3 rows with `[hidden / 128, rows]` scales `xsf`; top-k ids span the + whole routing space and an id outside the window (or -1) adds nothing. Context chunks run + `moe_prefill_grouped`, which finalizes `fin_first` rows first and hands them to + `after_first`; on the runner `after_first` sees the whole output.""" + if ( + output is None + and _MOE_PREFILL_GROUPED_MIN_ROWS + and _use_prefill_grouped(core, xq.shape[0], experts.count) + ): + split = {} if after_first is None else {"fin_first": fin_first, "after_first": after_first} + return moe_prefill_grouped( + xq, + xsf, + topk_ids, + topk_w, + experts.fc1_w, + experts.fc1_s, + experts.fc2_w, + experts.fc2_s, + experts.offset, + **split, + ) + if after_first is not None: + out = _window_runner(core, experts, xq, xsf, topk_ids, topk_w, output) + after_first(out) + return out + return fp8_block_scale_moe_runner( + routing_logits=None, + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=experts.fc1_w, + gemm1_weights_scale=experts.fc1_s, + gemm2_weights=experts.fc2_w, + gemm2_weights_scale=experts.fc2_s, + num_experts=core.num_experts, + top_k=core.topk, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=core.moe_inter, + local_expert_offset=experts.offset, + local_num_experts=experts.count, + routed_scaling_factor=None, + routing_method_type=_ROUTING_METHOD_RENORM, + topk_weights=topk_w, + topk_ids=topk_ids, + act_type=_ACT_TYPE_SWIGLU, + gemm1_clamp_limit=None, + output=output, + tune_max_num_tokens=_runner_tune_rows(core), + ) + + +def _use_prefill_grouped(core, rows: int, local: int) -> bool: + """Whether a call over `rows` rows of a `local`-expert window runs `moe_prefill_grouped`. + + A capture before the first eager call keeps the runner: capture cannot load kernels.""" + if not _MOE_PREFILL_GROUPED_MIN_ROWS or rows < _MOE_PREFILL_GROUPED_MIN_ROWS: + return False + if getattr(core, "dp_size", 1) != 1: + return False + if prefill_grouped_ready(local): + return True + if torch.cuda.is_current_stream_capturing(): + return False + warm_prefill_grouped(torch.cuda.current_device(), local) + return True + + +def _routed_experts( + core, x, router, experts: _weights.ExpertWindow, mm=None, pad_mask=None, before_runner=None +) -> torch.Tensor: + """Return this rank's partial of a MoE layer's routed experts over every row of `x`. + + Router logits, renormalized top-k (`renorm_moe_routing_op`) and 1x128 e4m3 quantization + feed this rank's window (`_window_runner`); slots outside it add nothing, so the windows + sum to the layer. Routing runs in chunks of `core.moe_max_rows` rows; an attention-DP + rank batches up to `_ADP_WINDOW_MAX_ROWS` rows of chunks per runner call (bitwise equal). + `pad_mask` (`[T, 1]` bool) routes a prefill graph bucket's padding rows to expert -1, so + they come back zero instead of all landing on the first window's ten lowest experts.""" + mm = mm or cublas_mm + rows = x.shape[0] + if pad_mask is None and mm is _decode_projection and 0 < rows <= _MOE_FRONT_MAX_ROWS: + if before_runner is not None: + before_runner() + return _routed_experts_front(core, x, router, experts) + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + if ( + pad_mask is None + and mm is _decode_projection + and getattr(core, "dp_size", 1) == 1 + and 0 < rows <= step + ): + logits = mm(x, router) + if rows <= _MOE_FAST_FRONT_MAX_ROWS: + out = _window_runner_front(core, experts, x, logits, before_runner) + if out is not None: + return out + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=_MOE_USE_UE8M0) + if before_runner is not None: + before_runner() + return _window_runner_logits(core, experts, xq, xsf, logits) + if before_runner is not None: + before_runner() + group = 1 + if getattr(core, "dp_size", 1) > 1 and rows > step: + group = max(1, _ADP_WINDOW_MAX_ROWS // step) + parts, pending = [], [] + for lo in range(0, rows, step): + chunk = x[lo : lo + step] + logits = mm(chunk, router) + topk_ids, topk_w = renorm_moe_routing_op(logits, topk=core.topk, output_dtype=chunk.dtype) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask[lo : lo + step], _NO_EXPERT) + xq, xsf = fp8_quantize_1x128(chunk, use_ue8m0=_MOE_USE_UE8M0) + pending.append((xq, xsf, topk_ids, topk_w)) + if len(pending) == group or lo + step >= rows: + if len(pending) == 1: + args = pending[0] + else: + q, sf, ids, w = zip(*pending) + codes = torch.cat([t.view(torch.uint8) for t in q]).view(q[0].dtype) + args = (codes, torch.cat(sf, 1), torch.cat(ids), torch.cat(w)) + parts.append(_window_runner(core, experts, *args)) + pending = [] + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _route_rows(core, x, router, mm=None, pad_mask=None) -> tuple: + """Route every row of `x` as `_routed_experts` does; return `(topk_ids, topk_weights)`. + + Used by the all-to-all round trip, which routes rows on their own rank.""" + mm = mm or cublas_mm + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + ids, weights = [], [] + for lo in range(0, x.shape[0], step): + chunk = x[lo : lo + step] + topk_ids, topk_w = renorm_moe_routing_op( + mm(chunk, router), topk=core.topk, output_dtype=chunk.dtype + ) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask[lo : lo + step], _NO_EXPERT) + ids.append(topk_ids) + weights.append(topk_w) + if not ids: + # A rank without rows this step: the routing op's empties. + logits = x.new_zeros(0, core.num_experts) + return renorm_moe_routing_op(logits, topk=core.topk, output_dtype=x.dtype) + return torch.cat(ids), torch.cat(weights) + + +def _window_prerouted(core, x, topk_ids, topk_w, experts: _weights.ExpertWindow) -> torch.Tensor: + """Run this rank's window over rows routed by `_route_rows`; rows with no local id are 0.""" + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + rows = x.shape[0] + group = max(1, _ADP_WINDOW_MAX_ROWS // step) if rows > step else 1 + parts, pending = [], [] + for lo in range(0, rows, step): + xq, xsf = fp8_quantize_1x128(x[lo : lo + step], use_ue8m0=_MOE_USE_UE8M0) + pending.append((xq, xsf, topk_ids[lo : lo + step], topk_w[lo : lo + step])) + if len(pending) == group or lo + step >= rows: + if len(pending) == 1: + args = pending[0] + else: + q, sf, ids, w = zip(*pending) + codes = torch.cat([t.view(torch.uint8) for t in q]).view(q[0].dtype) + args = (codes, torch.cat(sf, 1), torch.cat(ids), torch.cat(w)) + parts.append(_window_runner(core, experts, *args)) + pending = [] + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _window_runner_front(core, experts: _weights.ExpertWindow, x, logits, before_runner): + """Run `_window_runner_logits` through the one-launch decode routing with fused quantization. + + Calls `before_runner` once the router projection (and quantization) are queued. Returns + None, queuing nothing past the router, while the AutoTuner holds no explicit tactic.""" + rows, hidden = x.shape + fold = x.is_contiguous() + if fold: + xq = torch.empty((rows, hidden), dtype=torch.float8_e4m3fn, device=x.device) + xsf = torch.empty((hidden // 128, rows), dtype=torch.float32, device=x.device) + else: + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=_MOE_USE_UE8M0) + call = fp8_block_scale_moe_front( + logits, + xq, + xsf, + experts.fc1_w, + experts.fc1_s, + experts.fc2_w, + experts.fc2_s, + core.num_experts, + core.topk, + core.moe_inter, + experts.offset, + experts.count, + _ROUTING_METHOD_RENORM, + ) + if call is None: + if not fold: + if before_runner is not None: + before_runner() + return _window_runner_logits(core, experts, xq, xsf, logits) + return None + if before_runner is not None: + before_runner() + return call(x if fold else None) + + +def _window_runner_logits(core, experts: _weights.ExpertWindow, xq, xsf, logits) -> torch.Tensor: + """Run the window with the runner routing `logits` itself (bitwise `renorm_moe_routing_op`).""" + return fp8_block_scale_moe_runner( + routing_logits=logits, + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=experts.fc1_w, + gemm1_weights_scale=experts.fc1_s, + gemm2_weights=experts.fc2_w, + gemm2_weights_scale=experts.fc2_s, + num_experts=core.num_experts, + top_k=core.topk, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=core.moe_inter, + local_expert_offset=experts.offset, + local_num_experts=experts.count, + routed_scaling_factor=None, + routing_method_type=_ROUTING_METHOD_RENORM, + topk_weights=None, + topk_ids=None, + act_type=_ACT_TYPE_SWIGLU, + gemm1_clamp_limit=None, + ) + + +def _routed_experts_front( + core, x, router, experts: _weights.ExpertWindow, shared: _TpbShared | None = None +) -> torch.Tensor: + """Run `_routed_experts` for a decode step's rows (up to `_MOE_FRONT_MAX_ROWS`). + + One launch forms the bf16 router logits and the 1x128 e4m3 activation; then + `moe_tpb_decode` over block-major experts (optionally folding in `shared`) or the runner. + `router` is a `.t()` view of the checkpoint's `[512, hidden]` rows.""" + logits, xq, xsf = moe_front_router_quant(x.contiguous(), router.t()) + blocks = experts.blocks + if blocks is not None and x.shape[0] <= _MOE_TPB_MAX_ROWS: + return moe_tpb_decode( + logits, + xq, + xsf, + _BlockExperts(blocks.w1, blocks.s1, blocks.w2, blocks.s2, blocks.rank), + shared, + ) + assert shared is None, "the shared-expert fold rides the block launch" + return fp8_block_scale_moe_runner( + routing_logits=logits, + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=experts.fc1_w, + gemm1_weights_scale=experts.fc1_s, + gemm2_weights=experts.fc2_w, + gemm2_weights_scale=experts.fc2_s, + num_experts=core.num_experts, + top_k=core.topk, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=core.moe_inter, + local_expert_offset=experts.offset, + local_num_experts=experts.count, + routed_scaling_factor=None, + routing_method_type=_ROUTING_METHOD_RENORM, + topk_weights=None, + topk_ids=None, + act_type=_ACT_TYPE_SWIGLU, + gemm1_clamp_limit=None, + ) + + +class _WindowRun(NamedTuple): + """A MoE layer's `run` for `core.expert_sum`: `_routed_experts` over the given rows.""" + + core: object + router: torch.Tensor + experts: _weights.ExpertWindow + mm: object = None + pad_mask: torch.Tensor | None = None + before_runner: object = None + + def __call__(self, rows: torch.Tensor) -> torch.Tensor: + return _routed_experts( + self.core, + rows, + self.router, + self.experts, + self.mm, + self.pad_mask, + before_runner=self.before_runner, + ) + + +def _mtp_moe_parts(core, M: _MtpWeights, x, pad_rows: _DpPadding | None = None, mm=None) -> tuple: + """Return `_mtp_moe_block`'s uncombined `(routed, gate_logits, shared)`, side stream joined.""" + mm = mm or cublas_mm + main = torch.cuda.current_stream() + side = core._side_stream + side.wait_stream(main) + with torch.cuda.stream(side): + gug = mm(x, M.sh_gug) + shared = mm(silu_and_mul_rows(gug, core.shared_inter), M.sh_dn) + gate_logits = gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1] + routed = core.expert_sum(_WindowRun(core, M.router, M.experts, mm), x, pad_rows) + main.wait_stream(side) + return routed, gate_logits, shared + + +def _mtp_moe_block( + core, M: _MtpWeights, x, pad_rows: _DpPadding | None = None, mm=None +) -> torch.Tensor: + """Return the draft layer's MoE, `routed + sigmoid(gate logit) * shared`, as the trunk's. + + `core.expert_sum` completes the routed partial; the shared expert runs on the side stream.""" + return fused_sigmoid_gate_mul_add(*_mtp_moe_parts(core, M, x, pad_rows, mm)) + + +def _mtp_tail( + core, M: _MtpWeights, x, attn, inject, pad_rows: _DpPadding | None, mm=None +) -> torch.Tensor: + """Run the draft layer after its attention core: output projection, MoE-side site, MoE. + + `attn` is the `[T, q_width]` gated attention output; `pad_rows` is from `_draft_pad_rows`.""" + mm = mm or cublas_mm + x, mixed, inject = _hc_site( + core, x, _tp_partial(core, mm(attn, M.o)), inject, M.mhc_norm, M.mhc_first, M.mhc_up, True + ) + if _combine_inject_fits(x, inject): + # Combine and inject in one launch, bitwise equal to the composed ops below. + routed, gate_logits, shared = _mtp_moe_parts(core, M, mixed, pad_rows, mm) + if ( + routed.dtype == shared.dtype == torch.bfloat16 + and routed.stride(1) == shared.stride(1) == 1 + ): + return mtp_moe_combine_inject(x, routed, gate_logits, shared, inject) + return qwen4_exp_post_inject( + x, fused_sigmoid_gate_mul_add(routed, gate_logits, shared), inject + ) + return qwen4_exp_post_inject(x, _mtp_moe_block(core, M, mixed, pad_rows, mm=mm), inject) + + +def _ple_fold_fits(hc, routed, inject, shared) -> bool: + """Whether the PLE launches can take these operands: 2-D, unit column stride, `hc`'s dtype.""" + rows = hc.shape[0] + ok = ( + inject is not None + and routed.dim() == 2 + and inject.dim() == 2 + and routed.shape[0] == inject.shape[0] == rows + and routed.stride(1) == 1 + and inject.stride(1) == 1 + and routed.dtype == inject.dtype == hc.dtype + ) + if ok and shared is not None: + ok = shared.dim() == 2 and shared.stride(1) == 1 and shared.dtype == hc.dtype + return ok + + +def _combine_inject_fits(x, inject) -> bool: + """Whether `mtp_moe_combine_inject` takes `x` and `inject`: 2-D bf16, unit column stride.""" + return ( + x.dim() == 2 + and inject.dim() == 2 + and x.dtype == inject.dtype == torch.bfloat16 + and x.stride(1) == 1 + and inject.stride(1) == 1 + and x.shape[0] == inject.shape[0] + ) + + +class _DraftRowCounts(NamedTuple): + """The group's per-rank row counts of one draft step, in the field `_dp_pad_rows` reads.""" + + all_rank_num_tokens: list + + +def _draft_pad_rows(core, all_rank_num_tokens, rows: int) -> _DpPadding | None: + """Return `_dp_pad_rows` over the row counts the spec worker passes for this draft step. + + The counts are each rank's trunk tokens on the loop's first step, then its request count.""" + if core.dp_size == 1: + return None + assert all_rank_num_tokens is not None, ( + "a draft step under attention DP came without the group's row counts; the MoE round " + "trip cannot pad this rank's rows to the group's largest count" + ) + return core._dp_pad_rows(_DraftRowCounts(list(all_rank_num_tokens)), rows) + + +def _decode_projection(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor: + """Dense bf16 projection for generation-only steps and their draft layer (graph-captured). + + `low_m_bf16_mm` picks FlashInfer's low-M PDL kernels (`_DECODE_PROJ_PDL`) where its table + names the shape, else cuBLAS; they compile on the engine's warmup forward before capture.""" + return low_m_bf16_mm(mat_a, mat_b, _DECODE_PROJ_PDL) + + +def _hc_first(core, normed, first, with_inject): + """Run a hyper-connection mixer's first half: first projection, then SiLU and injection. + + Returns `(low, inject)`; `inject` is None without `with_inject`.""" + if normed.shape[0] <= _HC_TC_FIRST_MAX_ROWS: + return qwen4_exp_hc_decode_tc_first( + normed, first, core.hc_lowrank, core.hc_count, with_inject + ) + return qwen4_exp_hc_lowrank_act( + cublas_mm(normed, first[:, : core.hc_first_cols], None, torch.float32), + core.hc_lowrank, + core.hc_count, + with_inject, + normed.dtype, + ) + + +def _hc_mixer(core, normed, first, up, with_inject): + """Run one hyper-connection mixer on a normed stream; return `(mixed, inject)`.""" + low, inject = _hc_first(core, normed, first, with_inject) + wide = normed.shape[0] <= _HC_UGM_WIDE_MAX_ROWS + return qwen4_exp_hc_up_gate_mean(low, up, normed, core.hc_count, wide), inject + + +def _hc_inject_norm_launch( + hc, block_out, inject, norm, eps, hc_count, shared, gate_logits, num_warps +): + """Launch `qwen4_exp_hc_inject_norm` (no finalize fold) at `num_warps` warps with PDL. + + The residual is bitwise at any warp count; the normed stream varies only with the fp32 + sum-of-squares order. Programs read nothing before `griddepcontrol.wait`. Returns + `(hc_out, normed)`; `hc_out` is `hc` itself without `block_out`.""" + rows, width = hc.shape + hidden = width // hc_count + has_inject = block_out is not None + has_shared = shared is not None + normed = torch.empty((rows, width), dtype=hc.dtype, device=hc.device) + hc_out = torch.empty((rows, width), dtype=hc.dtype, device=hc.device) if has_inject else hc + _hc_inject_norm_kernel[(rows * hc_count,)]( + hc, + block_out if has_inject else hc, + inject if has_inject else hc, + shared if has_shared else hc, + gate_logits if has_shared else hc, + hc, + hc, + norm, + hc_out, + normed, + hc.stride(0), + block_out.stride(0) if has_inject else 0, + inject.stride(0) if has_inject else 0, + shared.stride(0) if has_shared else 0, + gate_logits.stride(0) if has_shared else 0, + 0, + 0, + hc_out.stride(0), + normed.stride(0), + eps, + HC_COUNT=hc_count, + HIDDEN=hidden, + BLOCK=triton.next_power_of_2(hidden), + HAS_INJECT=has_inject, + HAS_SHARED=has_shared, + TOPK=0, + STREAMS_PER_PROGRAM=1, + PDL_TRIGGER=_HC_INJECT_PDL_TRIGGER, + PDL_WAIT=_HC_INJECT_PDL_WAIT, + num_warps=num_warps, + launch_pdl=_HC_INJECT_PDL_WAIT, + ) + return hc_out, normed + + +def _hc_mixer_rstd(core, hc, rstd, norm, first, up, with_inject): + """Run `_hc_mixer` from the residual `hc` and its `[T, C]` rstd instead of a normed stream. + + The first projection forms `bf16(hc * rstd * (1 + norm))` on its A tiles; the up + projection's epilogue uses the fp32 value (`core._hc_norm_w1`).""" + low, inject = qwen4_exp_hc_first_rescale_tc( + hc, rstd, norm, first, core.hc_count, core.hc_lowrank, with_inject + ) + w1 = core._hc_norm_w1[norm.data_ptr()] + return qwen4_exp_hc_up_gate_mean_rstd(low, up, hc, rstd, w1, core.hc_count), inject + + +def _hc_site( + core, + hc, + block_out, + inject, + norm, + first, + up, + with_inject, + shared=None, + gate_logits=None, +): + """Run one hyper-connection site on the residual `hc`; return `(hc, mixed, inject)`. + + Injects the pending block (`block_out` at weights `inject`, plus + `sigmoid(gate_logits) * shared` after a MoE block), then the grouped norm and the mixer. + A `_TpPartial` block output is summed over the TP group inside the injection launch.""" + partial = None + if isinstance(block_out, _TpPartial): + block_out = partial = block_out.partial + if _HC_ROWS_TC_MIN_ROWS <= hc.shape[0] <= _HC_ROWS_TC_MAX_ROWS: + assert partial is None, "a deferred TP partial reached a site that does not sum it" + n_cols = core.hc_lowrank + (core.hc_count if with_inject else 0) + hc, low, inject, rstd = qwen4_exp_hc_decode_rows_first( + hc, + block_out, + inject, + norm, + first, + core.hc_count, + n_cols, + core.eps, + shared, + gate_logits, + ) + mixed = qwen4_exp_hc_decode_rows_mix(low, rstd, up, hc, norm, core.hc_count) + return hc, mixed, inject + if partial is not None: + fused = getattr(core, "hc_ar_inject", None) + if ( + fused is not None + and hc.shape[0] < _HC_RSTD_MIN_ROWS + and _hc_ar_inject_operands_fit(hc, partial, inject, shared, gate_logits) + ): + warps = 2 if hc.shape[0] >= _HC_INJECT_FEW_WARPS_MIN_ROWS else _HC_INJECT_WARPS + hc, normed = hc_allreduce_inject_norm( + partial, + fused.workspace, + fused.rank, + fused.nranks, + hc, + inject, + norm, + core.eps, + shared, + gate_logits, + warps, + ) + mixed, inject = _hc_mixer(core, normed, first, up, with_inject) + return hc, mixed, inject + block_out = core.tp_sum(block_out) + if hc.shape[0] >= _HC_RSTD_MIN_ROWS: + hc, rstd = qwen4_exp_hc_inject_rstd( + hc, block_out, inject, core.eps, core.hc_count, shared, gate_logits + ) + mixed, inject = _hc_mixer_rstd(core, hc, rstd, norm, first, up, with_inject) + return hc, mixed, inject + warps = 2 if hc.shape[0] >= _HC_INJECT_FEW_WARPS_MIN_ROWS else _HC_INJECT_WARPS + hc, normed = _hc_inject_norm_launch( + hc, block_out, inject, norm, core.eps, core.hc_count, shared, gate_logits, warps + ) + mixed, inject = _hc_mixer(core, normed, first, up, with_inject) + return hc, mixed, inject + + +def _moe_tpb_shared_folded(core, x, experts, mm, pad_rows, pad_mask, defer) -> bool: + """Whether `_moe_block` folds this rank's shared-expert slice into the block launch.""" + rows = x.shape[0] + return ( + defer + and experts.blocks is not None + and pad_rows is None + and pad_mask is None + and mm is _decode_projection + and getattr(core, "dp_size", 1) == 1 + and _MOE_TPB_SHARED_MIN_ROWS <= rows <= min(_MOE_FRONT_MAX_ROWS, _MOE_TPB_MAX_ROWS) + ) + + +def _moe_block( + core, + x, + L: _LayerWeights, + pad_rows: _DpPadding | None, + mm=None, + pad_mask=None, + defer=False, +) -> tuple: + """Run one MoE layer; return `(routed, gate_logits, shared)` uncombined. + + The next `qwen4_exp_hc_inject_norm` forms `routed + sigmoid(gate_logits) * shared`. + `routed` comes from `core.expert_sum` (`pad_rows` from `_dp_pad_rows`); the shared expert + runs on the side stream and joins before this returns. `pad_mask` keeps prefill graph + padding rows out of the window. With `defer` (attention-TP decode) `routed` may be a + `_TpPartial` the next site sums. When `_moe_tpb_shared_folded` holds, `routed` is the + whole combine and the others are None.""" + mm = mm or cublas_mm + if _moe_tpb_shared_folded(core, x, L.experts, mm, pad_rows, pad_mask, defer): + b = L.experts.blocks + shared = _TpbShared(x.contiguous(), b.sh_gu, b.sh_gl, b.sh_dn) + partial = _routed_experts_front(core, x, L.router, L.experts, shared=shared) + return _tp_partial(core, partial), None, None + main = torch.cuda.current_stream() + side = core._side_stream + forked = [] + + def fork_shared(): + # The side stream waits for `main`; its outputs are used on `main` after the join. + core._side_fork.record(main) + side.wait_event(core._side_fork) + with torch.cuda.stream(side): + gug = mm(x, L.sh_gug) + shared = mm(silu_and_mul_rows(gug, core.shared_inter), L.sh_dn) + forked.append((gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1], shared)) + + # Under attention TP a decode step forks once the router and the quantization are queued + # (`_routed_experts`' `before_runner`); every other step forks first. + late = pad_rows is None and getattr(core, "dp_size", 1) == 1 + if not late: + fork_shared() + run = _WindowRun(core, L.router, L.experts, mm, pad_mask, fork_shared if late else None) + if defer and getattr(core, "dp_size", 1) == 1: + routed = _tp_partial(core, run(x)) + else: + routed = core.expert_sum(run, x, pad_rows) + if not forked: + fork_shared() + core._side_join.record(side) + main.wait_event(core._side_join) + gate_logits, shared = forked[0] + return routed, gate_logits, shared + + +def _routed_experts_sp( + core, + x_rows, + router, + experts: _weights.ExpertWindow, + gather, + pad_mask=None, + fin_first=None, + after_first=None, +): + """This rank's window partial `[T, hidden]` over every row of a sequence-parallel step. + + Each rank routes and e4m3-quantizes only its own rows (`pad_mask` sends padding rows to + `_NO_EXPERT`); two uniform gathers carry the codes and a `[scales | ids | weights]` row.""" + topk = core.topk + logits = cublas_mm(x_rows, router) + topk_ids, topk_w = renorm_moe_routing_op(logits, topk=topk, output_dtype=x_rows.dtype) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask, _NO_EXPERT) + xq, xsf = fp8_quantize_1x128(x_rows, use_ue8m0=_MOE_USE_UE8M0) + nb = xsf.shape[0] + rows = x_rows.shape[0] + # bf16 columns: nb fp32 scales, topk int32 ids, topk bf16 weights (the width stays even). + c0, c1, c2 = 2 * nb, 2 * nb + 2 * topk, 2 * nb + 3 * topk + meta = torch.empty([rows, c2 + (c2 & 1)], dtype=torch.bfloat16, device=x_rows.device) + meta[:, :c0].view(torch.float32).copy_(xsf.t()) + meta[:, c0:c1].view(torch.int32).copy_(topk_ids) + meta[:, c1:c2].copy_(topk_w) + codes = gather(xq.view(torch.bfloat16)).view(xq.dtype) + every = gather(meta) + scales = every[:, :c0].view(torch.float32).t() + ids = every[:, c0:c1].contiguous().view(torch.int32) + weights = every[:, c1:c2].contiguous() + total = codes.shape[0] + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + if after_first is not None and total <= step: + return _window_runner( + core, + experts, + codes, + scales.contiguous(), + ids, + weights, + fin_first=fin_first, + after_first=after_first, + ) + parts = [ + _window_runner( + core, + experts, + codes[lo : lo + step], + scales[:, lo : lo + step].contiguous(), + ids[lo : lo + step], + weights[lo : lo + step], + ) + for lo in range(0, total, step) + ] + out = parts[0] if len(parts) == 1 else torch.cat(parts, 0) + if after_first is not None: + after_first(out) + return out + + +def _moe_block_sp(core, x_rows, L: _LayerWeights, gather, scatter, pad_mask=None) -> tuple: + """`_moe_block` under SP, returning `(routed, gate_logits, shared)` over this rank's rows. + + The shared expert runs on the side stream, overlapping the routed path's gathers.""" + main = torch.cuda.current_stream() + side = core._side_stream + core._side_fork.record(main) + side.wait_event(core._side_fork) + with torch.cuda.stream(side): + gug = cublas_mm(x_rows, L.sh_gug) + shared = cublas_mm(silu_and_mul_rows(gug, core.shared_inter), L.sh_dn) + gate_logits = gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1] + split = getattr(scatter, "split_finalize", None) + if split is not None: + # Two ranks: the peer's rows are finalized and sent first, overlapping this rank's. + start, finish, peer_rows = split + routed = finish( + _routed_experts_sp( + core, x_rows, L.router, L.experts, gather, pad_mask, peer_rows, start + ) + ) + else: + routed = scatter(_routed_experts_sp(core, x_rows, L.router, L.experts, gather, pad_mask)) + core._side_join.record(side) + main.wait_event(core._side_join) + return routed, gate_logits, shared + + +def _projection_warm_rows(lo: int, hi: int) -> list[int]: + """One row count of `[lo, hi]` for each kernel cuBLASLt can pick for a bf16 projection. + + Every row up to `_KERNEL_CHOICE_EACH_ROW`, then one per 8-row group, which shares a kernel.""" + each = min(hi, _KERNEL_CHOICE_EACH_ROW) + rows = list(range(lo, each + 1)) + first = max(lo, _KERNEL_CHOICE_EACH_ROW + 1) + if first <= hi: + group = _KERNEL_CHOICE_ROW_GROUP + ends = range(-(-first // group) * group, hi + group, group) + rows += [min(end, hi) for end in ends] + return rows + + +def _eager_projections(core, layers, stack: _StackWeights, mtp) -> list[tuple]: + """Every bf16 projection an eager step can hand `cublas_mm`, as `(mat_b, out_dtype, lo, hi)`. + + One entry per cuBLASLt kernel-choice key (K, N, strides, output dtype) with its row range.""" + rows = core.max_num_tokens + router_rows = min(getattr(core, "moe_max_rows", _MOE_MAX_ROWS), core.dp_size * rows) + hc_cols = core.hc_first_cols + found = {} + + def add(mat_b, hi, lo=1, out_dtype=None): + key = (tuple(mat_b.shape), mat_b.stride(), mat_b.dtype, out_dtype) + if key in found: + _, _, was_lo, was_hi = found[key] + lo, hi = min(lo, was_lo), max(hi, was_hi) + if lo <= hi: + found[key] = (mat_b, out_dtype, lo, hi) + + def mixer(first): + add(first[:, :hc_cols], rows, _HC_TC_FIRST_MAX_ROWS + 1, torch.float32) + + for L in layers: + mixer(L.ahc_first) + mixer(L.mhc_first) + for mat_b in (L.attn_in, L.o, L.in_proj, L.out, L.sh_gug, L.sh_dn): + if mat_b is not None: + add(mat_b, rows) + add(L.router, router_rows) + mixer(stack.fhc_first) + add(stack.ple_key, rows) + add(stack.ple_val, rows) + if mtp is not None: + for first in (mtp.ahc_first, mtp.mhc_first, mtp.fhc_first): + mixer(first) + for mat_b in (mtp.fc_e, mtp.attn_in, mtp.o, mtp.sh_gug, mtp.sh_dn): + add(mat_b, rows) + add(mtp.fc_h, core.hc_count * rows) + add(mtp.router, router_rows) + return list(found.values()) + + +def _load_projection_kernels(projections, device) -> int: + """Run each projection at every `_projection_warm_rows` count; return the calls made. + + A kernel's first lazy-loaded launch waits for queued work and stalls the next collective.""" + k_max = max(mat_b.shape[0] * hi for mat_b, _, _, hi in projections) + flat = torch.zeros(k_max, dtype=torch.bfloat16, device=device) + calls = 0 + for mat_b, out_dtype, lo, hi in projections: + k = mat_b.shape[0] + for m in _projection_warm_rows(lo, hi): + cublas_mm(flat[: m * k].view(m, k), mat_b, None, out_dtype) + calls += 1 + return calls + + +def _attn_in_heads(core, proj): + """The main q, k, v heads of `proj` `[T, attn_in_width]` as `[T, heads, head_dim]` views.""" + q, k, v, _ = torch.split( + torch.reshape(proj, [proj.shape[0], core.attn_in_width // core.head_dim, core.head_dim]), + [core.heads_q, core.heads_kv, core.heads_kv, core.attn_in_tail_heads - core.heads_kv], + 1, + ) + return q, k, v + + +def _packed_q(core, qkv): + """The query heads `[T, heads_q, head_dim]` of the compact `[T, qkv_width]` packed QKV.""" + heads = core.heads_q + 2 * core.heads_kv + return torch.reshape(qkv, [qkv.shape[0], heads, core.head_dim])[:, : core.heads_q] + + +def _attention_rope(core, proj, index_scratch, L, pos3, pos, theta=None) -> None: + """Both planes' Gemma norm + rope: main q/k in place in `proj`, index q/k in `index_scratch`. + + The index q/k are rotated in a copy so the raw index key stays in `proj` for the index store; + the heads after k pass as v only to set the row stride. `theta` defaults to the trunk's.""" + base = core.theta if theta is None else theta + fused_qk_norm_rope( + proj, + num_heads_q=core.heads_q, + num_heads_k=core.heads_kv, + num_heads_v=core.attn_in_tail_heads, + head_dim=core.head_dim, + rotary_dim=core.rotary_dim, + eps=core.eps, + q_weight=L.q_norm, + k_weight=L.k_norm, + base=base, + is_neox=True, + position_ids=pos3, + is_qk_norm=True, + use_gemma=True, + use_mrope=True, + mrope_section1=core.mrope_section1, + mrope_section2=core.mrope_section2, + ) + if index_scratch is None: + return + index_scratch[:, : core.index_width].copy_( + proj[:, core.index_col : core.index_col + core.index_width] + ) + fused_qk_norm_rope( + index_scratch, + num_heads_q=core.index_heads, + num_heads_k=core.index_kv_heads, + num_heads_v=core.index_kv_heads, + head_dim=core.index_head_dim, + rotary_dim=core.rotary_dim, + eps=core.eps, + q_weight=L.idx_qn, + k_weight=L.idx_kn, + base=base, + is_neox=True, + position_ids=pos, + is_qk_norm=True, + use_gemma=True, + use_mrope=False, + mrope_section1=0, + mrope_section2=0, + ) + + +def _index_query(core, index_scratch): + """The rotated index query `[T, index_heads, index_head_dim]`, a view of `index_scratch`.""" + rows = index_scratch.shape[0] + return torch.split( + torch.reshape( + index_scratch, + [rows, core.index_heads + 2 * core.index_kv_heads, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, core.index_kv_heads], + 1, + )[0] + + +def _index_plane(core, proj, i, L: _LayerWeights, qsa) -> None: + """Store layer `i`'s raw index keys and positions; compress the groups this step completes. + + The key is stored raw because the compression applies the norm and the rotation itself.""" + req_idx, logical, block_table, tokens_per_block, coords = qsa + rows = proj.shape[0] + token_k = torch.split( + torch.reshape( + proj[:, core.index_col : core.index_col + core.index_scratch_width], + [rows, core.index_scratch_width // core.index_head_dim, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, 1], + 1, + )[1] + pools = core._bound.pools + index_cache = pools.index_cache[i] + qsa_paged_index_store( + token_k, + coords, + index_cache, + pools.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + ) + qsa_prefill_compress( + logical_positions=logical, + request_indices=req_idx, + block_table=block_table, + index_cache=index_cache, + position_cache=pools.position_cache, + k_norm_weight=L.idx_kn, + cos_sin=core._cos_sin, + eps=core.eps, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + mrope_section=_QSA_MROPE_SECTION, + ) + + +def _select(core, scores, row_starts, row_ends, topk_indices) -> None: + """Each row's canonical top-`block_topk` groups of its window into `topk_indices`. + + Canonical: groups above the budget's last score, then the highest-index ties, ascending, `-1` + last (row starts are zero, so window and table indices agree).""" + qsa_topk_select(scores, row_starts, row_ends, topk_indices, index_topk=core.block_topk) + qsa_canonical_selection(scores, row_starts, row_ends, topk_indices) + + +def _index_split_group(core) -> tuple | None: + """`(group, process_group, rank, lamport)` a decode step's index scoring splits over, or None. + + One small in-order gather builds the NCCL communicator now; a first call cannot be captured.""" + if core.attn_tp_size < 2 or core.dp_size != 1 or core.tp_sum is None: + return None + group = list(core.mapping.tp_group) + process_group = core.mapping.tp_group_pg.boxed() if mpi_disabled() else None + probe = torch.zeros([1, 1], dtype=torch.int32, device=core._cos_sin.device) + allgather(probe, None, group, process_group) + return group, process_group, core.mapping.tp_rank, _index_split_lamport(core) + + +def _index_split_lamport(core): + """The split gather's Lamport IPC buffers, or None where the Lamport exchange does not apply. + + Needs 2-4 attention-TP ranks with peer access, PDL and captured batches that reach the split. + A collective: every rank builds the buffers here, before any capture.""" + graph_batches = tuple( + getattr(getattr(core, "model_config", None), "cuda_graph_batch_sizes", ()) or () + ) + if not ( + 2 <= core.attn_tp_size <= 4 + and get_env_enable_pdl() + and _INDEX_SPLIT_LAMPORT_ROWS > 0 + and (not graph_batches or max(graph_batches) >= _INDEX_SPLIT_MIN_REQUESTS) + and can_access_peer(core.mapping) + ): + return None + width = core.token_topk + core.compress_ratio - 1 + rows = min(_INDEX_SPLIT_LAMPORT_ROWS, core.max_num_tokens) + return _selection_lamport_build( + core.mapping.tp_rank, + core.attn_tp_size, + rows * width, + Distributed.get(core.mapping).tp_allgather, + ) + + +def _index_split_rows(split, rows: int, rows_per_seq: int) -> tuple | None: + """This rank's `[lo, hi)` rows of a split index scoring, or None to score every row.""" + if split is None: + return None + group, rank = split[0], split[2] + t = len(group) + requests = rows // rows_per_seq + if requests < _INDEX_SPLIT_MIN_REQUESTS or requests % t or rows % rows_per_seq: + return None + share = (requests // t) * rows_per_seq + return rank * share, (rank + 1) * share + + +def _index_split_plan(split, qsa, num_tokens: int, rows_per_seq: int): + """A work-balanced request plan for the split index selection, or None to split by position. + + Made once a step for every layer, over the Lamport gather only; splitting by position puts the + CUDA graph padding on one rank. Bitwise equal to the position split.""" + if _index_split_rows(split, num_tokens, rows_per_seq) is None: + return None + lam = split[3] if len(split) > 3 else None + req_idx, logical, seq_lens_q, visible_blocks, _, row_starts = qsa[:6] + vectors = (visible_blocks, req_idx, logical, seq_lens_q, row_starts) + if lam is None or not all(v.is_contiguous() for v in vectors): + return None + return index_split_plan( + visible_blocks[:num_tokens], + req_idx[:num_tokens], + logical[:num_tokens], + seq_lens_q[:num_tokens], + row_starts[:num_tokens], + num_tokens // rows_per_seq, + rows_per_seq, + len(split[0]), + split[2], + ) + + +def _decode_index_selection( + core, split, q, qsa, num_tokens, p, device, index_cache, plan=None, q_permuted=False +): + """A sparse decode layer's int32 `[num_tokens, token_topk + compress_ratio - 1]` selection. + + Index scores of `q` (`p` rows a request), then `_selection`, over every row or, under `split`, + over this rank's rows gathered in rank order. `qsa` is `(req_idx, logical, seq_lens_q, + visible_blocks, topk_indices, topk_row_starts, block_table, tokens_per_block)`. With a `plan`, + this rank takes its slice of the permuted order (`q_permuted`: `q` is in it already).""" + req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts = qsa[:6] + block_table, tokens_per_block = qsa[6], qsa[7] + part = _index_split_rows(split, num_tokens, p) + if part is None: + plan = None + lo, hi = (0, num_tokens) if part is None else part + rows = hi - lo + if plan is not None: + q_rows = q[lo:hi] if q_permuted else q.index_select(0, plan.perm_rows[lo:hi]) + rows_of = (plan.req_idx, plan.logical, plan.seq_lens, plan.visible, plan.row_starts) + else: + q_rows = q[lo:hi] + rows_of = ( + req_idx[lo:hi], + logical[lo:hi], + seq_lens_q[lo:hi], + visible_blocks[lo:hi], + topk_row_starts[lo:hi], + ) + r_req, r_logical, r_seq, r_visible, r_starts = rows_of + rpr = _index_rows_per_request(rows, p) + if ( + rpr > 1 + and rows >= _INDEX_SCORES_CUTE_MIN_GROUPS * rpr + and logical.dtype == torch.int64 + and req_idx.dtype == torch.int32 + and _index_groups_cute_serves(q_rows, index_cache, rpr) + ): + scores = index_scores_groups_cute( + q_rows, + index_cache, + block_table, + r_logical, + r_req, + tokens_per_block, + core.compress_ratio, + _INDEX_SCORES_PDL, + rpr, + ) + else: + scores = qsa_paged_index_scores( + q=q_rows, + index_cache=index_cache, + block_table=block_table, + query_positions=r_logical, + request_indices=r_req, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + only_visible_blocks=True, + context_rows=False, + use_pdl=_INDEX_SCORES_PDL, + rows_per_request=rpr, + wide_split=True, + ) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=device + ) + _selection( + core, + scores, + r_visible, + r_logical, + r_seq, + topk_indices[lo:hi], + r_starts, + selected, + ) + if part is None: + return selected + group, process_group = split[0], split[1] + lam = split[3] if len(split) > 3 else None + if lam is not None and lam.fits(num_tokens * selected.shape[1]): + if plan is None: + return lamport_allgather(selected, lam) + return lamport_allgather(selected, lam, plan.perm_rows) + gathered = allgather(selected, None, group, process_group) + if plan is None: + return gathered + out = torch.empty_like(gathered) + out.index_copy_(0, plan.perm_rows.long(), gathered) + return out + + +def _index_rows_per_request(rows: int, rows_per_seq: int) -> int: + """`qsa_paged_index_scores`' `rows_per_request`: grouped from `_INDEX_GROUP_MIN_ROWS` rows.""" + return rows_per_seq if rows_per_seq > 1 and rows >= _INDEX_GROUP_MIN_ROWS else 1 + + +def _draft_continue_ok(core, md, p: int, share: bool) -> bool: + """Whether `_draft_continue_rows` returns rows rather than None.""" + if p <= 1 or core.dp_size != 1 or share: + return False + return getattr(md, "qsa_mtp_num_accepted", None) is not None + + +def _draft_continue_rows(core, md, gen_seqs: int, p: int, share: bool): + """The `[gen_seqs]` int64 rows a draft-extend step's outputs are read from, or None for all. + + Each request's last accepted row; the others only leave K/V and index keys behind. Not under + attention DP (the MoE pads by the worker's rows) or IndexShare (it keeps every selection).""" + if p <= 1 or core.dp_size != 1 or share: + return None + accepted = getattr(md, "qsa_mtp_num_accepted", None) + if accepted is None: + return None + last = torch.clamp(accepted[:gen_seqs].to(torch.int64) - 1, min=0, max=p - 1) + return torch.arange(gen_seqs, dtype=torch.int64, device=last.device) * p + last + + +def _selection( + core, scores, visible_blocks, logical, seq_lens_q, topk_indices, row_starts, out, context=False +): + """The canonical block top-k and expansion of `scores`' rows into `out`, fused if possible.""" + if ( + context + and scores.shape[0] >= _TOPK_ROWS_MIN_ROWS + and _topk_rows_supports(scores, topk_indices) + ): + qsa_topk_rows( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + ) + return + if _topk_expand_cluster_supports(scores): + qsa_topk_expand_cluster( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + ) + return + if _topk_expand_supports(scores): + qsa_topk_expand_decode( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + ordered_wide=True, + ) + return + _select(core, scores, row_starts, visible_blocks, topk_indices) + out.copy_( + qsa_expand_block_indices( + topk_indices, + logical, + seq_lens_q, + compress_ratio=core.compress_ratio, + token_topk=core.token_topk, + ) + ) + + +def _prefill_split_enabled(attn_tp_size: int, dp_size: int) -> bool: + """Whether a prefill step splits its selection and row-wise work over the attention-TP group.""" + return dp_size == 1 and attn_tp_size in _PREFILL_SPLIT_DEFAULT_TP + + +def _selection_cuts(chunks, starts, world: int, row_tokens: int = 0) -> tuple: + """Row boundaries `(0, c_1, ..., c_{world-1}, rows)` of a context step's split selection. + + A row at position `x` costs `x + 1 + row_tokens`; cut `r` takes `r / world` of the total, + rounded to `_SELECTION_CUT_ALIGN`. Host arithmetic, identical on every rank.""" + rows = sum(chunks) + base = [s + 1 + row_tokens for s in starts] + # Summed cost of a request's first `m` rows: m * base + m (m - 1) / 2. + totals = [c * b + c * (c - 1) // 2 for c, b in zip(chunks, base)] + total = sum(totals) + cuts = [0] + req, row0, done = 0, 0, 0 + for r in range(1, world): + want = total * r // world + while req < len(chunks) - 1 and done + totals[req] < want: + done += totals[req] + row0 += chunks[req] + req += 1 + # The fewest rows `m` of request `req` whose cost reaches `want - done`: the root of + # m^2 / 2 + (b - 1 / 2) m = rest, refined on the integers. + b, c, rest = base[req], chunks[req], want - done + m = int(-(b - 0.5) + ((b - 0.5) ** 2 + 2 * rest) ** 0.5) + m = max(0, min(c, m)) + while m < c and m * b + m * (m - 1) // 2 < rest: + m += 1 + while m > 0 and (m - 1) * b + (m - 1) * (m - 2) // 2 >= rest: + m -= 1 + cut = int(round((row0 + m) / _SELECTION_CUT_ALIGN)) * _SELECTION_CUT_ALIGN + cuts.append(min(rows, max(cuts[-1], cut))) + cuts.append(rows) + return tuple(cuts) + + +def _split_context_selection( + split, score_and_select, selected, lo: int, hi: int, pipeline=None, cuts=None +) -> None: + """Rows `[lo, hi)` of `selected` scored and selected over the attention-TP group. + + Each rank runs `score_and_select` on its share (equal, or by `cuts`) into its own + `[per, width]` buffer; one uniform gather lays the shares out in rank order (int32 as bf16 + pairs), bitwise the unsplit result. A collective: every rank calls it with the same `hi - lo`. + `pipeline` `(pieces, comm, fork, join[, exchange])` gathers each piece on `comm` while the + next is scored.""" + rank, world, group, process_group = split + rows = hi - lo + if cuts is None: + per = -(-rows // world) + starts = [min(lo + r * per, hi) for r in range(world)] + ends = [min(s + per, hi) for s in starts] + else: + assert len(cuts) == world + 1 and cuts[0] == 0 and cuts[-1] == rows, (cuts, rows) + starts = [lo + c for c in cuts[:-1]] + ends = [lo + c for c in cuts[1:]] + per = max(e - s for s, e in zip(starts, ends)) + a, b = starts[rank], ends[rank] + mine = torch.empty([per, selected.shape[1]], dtype=selected.dtype, device=selected.device) + pieces = 1 + if pipeline is not None: + pieces = max(1, min(pipeline[0], per // _SELECTION_PIECE_MIN_ROWS)) + if pieces == 1: + if b > a: + score_and_select(a, b, True, mine[: b - a]) + gathered = allgather(mine.view(torch.bfloat16), None, group, process_group) + gathered = gathered.view(selected.dtype) + if cuts is None: + selected[lo:hi].copy_(gathered[:rows]) + return + for r in range(world): + n = ends[r] - starts[r] + if n > 0: + selected[starts[r] : ends[r]].copy_(gathered[r * per : r * per + n]) + return + _, comm, fork, join = pipeline[:4] + exchange = pipeline[4] if len(pipeline) > 4 else None + size = -(-per // pieces) + width = selected.shape[1] * selected.element_size() + if exchange is not None and (world != 2 or size * width > exchange.capacity): + exchange = None + main = torch.cuda.current_stream() + for p0 in range(0, per, size): + p1 = min(p0 + size, per) + if a + p0 < b: + score_and_select(a + p0, min(a + p1, b), True, mine[p0 : min(p1, b - a)]) + fork.record(main) + comm.wait_event(fork) + with torch.cuda.stream(comm): + if exchange is not None: + # The peer's piece is a view of the receive buffer, copied out below. + pieces_of = {rank: mine[p0:p1], 1 - rank: exchange.exchange(mine[p0:p1])} + else: + gathered = allgather(mine[p0:p1].view(torch.bfloat16), None, group, process_group) + gathered = gathered.view(selected.dtype) + pieces_of = {r: gathered[r * (p1 - p0) : (r + 1) * (p1 - p0)] for r in range(world)} + for r in range(world): + r0 = starts[r] + p0 + r1 = min(starts[r] + p1, ends[r]) + if r1 > r0: + selected[r0:r1].copy_(pieces_of[r][: r1 - r0]) + join.record(comm) + main.wait_event(join) + + +def _decode_prologue(kv, sparse: bool) -> bool: + """Whether a decode attention layer runs the fused prologue: sparse, over a bf16 or e4m3 pool. + + The dense op appends K/V itself and an NVFP4 pool needs the quantizing store.""" + return sparse and kv.k_scale is None + + +def _decode_prologue_call( + core, proj, qkv, index_scratch, i, L, qsa, rows_per_request, index_q_rows=None +) -> None: + """`qwen4_exp_attn_decode_prologue` over layer `i`'s pools. + + `index_q_rows[t]`, if given, is row `t`'s `index_scratch` row (the split's permuted order).""" + req_idx, logical, block_table, tokens_per_block, pos3, coords = qsa + pools = core._bound.pools + kv = pools.kv[i] + qwen4_exp_attn_decode_prologue( + proj, + qkv, + _index_query(core, index_scratch), + pos3, + coords, + core._cos_sin, + L.q_norm, + L.k_norm, + L.idx_qn, + L.idx_kn, + kv.k, + kv.v, + pools.index_cache[i], + pools.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + core.compress_ratio, + core.heads_q, + core.heads_kv, + core.head_dim, + core.index_heads, + core.index_head_dim, + core.index_col, + core.rotary_dim, + core.eps, + core.mrope_section1, + core.mrope_section2, + rows_per_request=rows_per_request, + index_q_rows=index_q_rows, + ) + + +def _store_kv(core, kv, k, v, request_indices, logical_positions, block_table, tokens_per_block): + """Append this step's K/V rows to the layer's paged pool. + + An fp8 pool takes e4m3 unscaled, matching the dense op's unit KV scales.""" + if kv.k_scale is None: + qsa_paged_kv_store( + k, v, kv.k, kv.v, request_indices, logical_positions, block_table, tokens_per_block + ) + return + qsa_paged_kv_store_nvfp4( + k, + v, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + request_indices, + logical_positions, + block_table, + tokens_per_block, + core.kv_scale_orig_quant, + ) + + +def _fp8_context_pages(md, tpb: int) -> int: + """Pool pages the step's context requests hold, `ceil(kv_len / tpb)` each, from host lengths.""" + lens = md.kv_lens_runtime[: md.num_contexts] + return int(((lens + (tpb - 1)) // tpb).sum()) + + +def _rowtile_takes(kv, rows: int, fp8_pages) -> bool: + """Whether `_sparse_gqa` sends a context launch of `rows` rows to the row-tile pass. + + Mirrors `_sparse_gqa_rowtile`'s bound on the host; the pass may take q as a strided view.""" + if kv.k_scale is not None or rows < _ROWTILE_MIN_ROWS: + return False + if kv.k.dtype == torch.bfloat16: + return True + return ( + kv.k.dtype == torch.float8_e4m3fn + and fp8_pages is not None + and fp8_pages * kv.k[0].numel() * 4 <= _FP8_ROWTILE_MAX_SCRATCH + ) + + +def _rowtile_scratch_groups(md, page_bytes: int) -> tuple | None: + """Row-tile launches `(first row, end row, pages)` of an fp8 context step, or None for one. + + The fewest groups of consecutive requests whose widened pages fit `_FP8_ROWTILE_MAX_SCRATCH`, + pages spread evenly; a request never splits (its pages always fit).""" + num_contexts = md.num_contexts + tpb = md.tokens_per_block + rows = [int(r) for r in md.seq_lens[:num_contexts].tolist()] + pages = [-(-int(k) // tpb) for k in md.kv_lens_runtime[:num_contexts].tolist()] + cap = _FP8_ROWTILE_MAX_SCRATCH // page_bytes + if sum(pages) <= cap: + return None + + def fill(limit: int) -> list: + # Contiguous groups, each closed before a request that would take it past `limit`. + out, first, held = [], 0, 0 + for j, p in enumerate(pages): + if held and held + p > limit: + out.append((first, j, held)) + first, held = j, 0 + held += p + out.append((first, len(pages), held)) + return out + + fewest = len(fill(cap)) + lo, hi = max(pages), cap + while lo < hi: + mid = (lo + hi) // 2 + if len(fill(mid)) <= fewest: + hi = mid + else: + lo = mid + 1 + starts = [0] + for r in rows: + starts.append(starts[-1] + r) + return tuple((starts[a], starts[b], n) for a, b, n in fill(lo)) + + +def _sparse_gqa_rowtile( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages +): + """The row-tile pass over a context launch, or None where it does not serve it. + + An fp8 pool's reached pages are widened exactly into a bf16 scratch pool; `fp8_pages` is + their host count (read from the device when None).""" + k_cache, v_cache, table = kv.k, kv.v, block_table + if kv.k.dtype == torch.float8_e4m3fn: + # Each slot's highest position, -1 if absent: a compare-and-max over `[slots, rows]` + # (a scatter max would serialize a request's rows on one address). + slots = torch.arange(block_table.shape[0], dtype=request_indices.dtype, device=q.device) + held = request_indices[None, :] == slots[:, None] + last = torch.where(held, query_positions[None, :].long(), -1).amax(1) + counts = torch.div(last + tpb, tpb, rounding_mode="floor") + if fp8_pages is None: + fp8_pages = int(counts.sum()) + if fp8_pages * kv.k[0].numel() * 4 > _FP8_ROWTILE_MAX_SCRATCH: + return None + k_cache = torch.empty([fp8_pages, *kv.k.shape[1:]], dtype=q.dtype, device=q.device) + v_cache = torch.empty_like(k_cache) + table = torch.full_like(block_table, -1) + qsa_fp8_pages_to_bf16( + kv.k, + kv.v, + block_table, + slots.int(), + counts, + torch.cumsum(counts, 0) - counts, + table, + k_cache, + v_cache, + ) + elif kv.k.dtype != q.dtype: + return None + return qsa_sparse_gqa_rowtile( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + query_positions=query_positions, + compress_ratio=core.compress_ratio, + ) + + +def _sparse_gqa( + core, + kv, + q, + block_table, + selected, + request_indices, + tpb, + query_positions, + fp8_pages=None, + output_gate=None, + rows_per_request=1, +): + """Sparse GQA over the layer's paged pool. + + `query_positions` bounds each context row's scan (None for decode rows). Decode rows only: + `output_gate` `[rows, Hq * D]` or `[rows, Hq, D]` makes the result `bf16(attn) * + sigmoid(gate)`, and `rows_per_request` is the consecutive rows of each request.""" + if kv.k_scale is None and query_positions is None: + if ( + rows_per_request > 1 + and q.shape[0] >= _GROUPED_DECODE_MIN_REQUESTS * rows_per_request + and qsa_sparse_gqa_grouped_cute.serves(q, kv.k, selected, rows_per_request) + ): + return qsa_sparse_gqa_grouped_cute( + q, + kv.k, + kv.v, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + rows_per_request, + output_gate, + ) + if qsa_sparse_gqa_splitk_cute.serves(q, kv.k, selected): + # Compile the split-K fallback's row buckets at warm-up, not on a serving step. + warm_decode_buckets( + q=q, + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + compress_ratio=core.compress_ratio, + ) + return qsa_sparse_gqa_splitk_cute( + q, + kv.k, + kv.v, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + output_gate, + ) + out = _sparse_gqa_ungated( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages + ) + if output_gate is not None: + fused_sigmoid_mul_inplace( + torch.reshape(out, [out.shape[0], -1]), output_gate, direct_launch=True + ) + return out + + +def _sparse_gqa_ungated( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages=None +): + """`_sparse_gqa` without the CuTe decode kernels and without the output gate.""" + if kv.k_scale is None: + if query_positions is not None and q.shape[0] >= _ROWTILE_MIN_ROWS: + out = _sparse_gqa_rowtile( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages + ) + if out is not None: + return out + return qsa_paged_sparse_gqa( + q=q, + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + query_positions=query_positions, + compress_ratio=None if query_positions is None else core.compress_ratio, + output_gate=None, + ) + if query_positions is None: + return qsa_paged_sparse_gqa_nvfp4_decode( + q, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + core.kv_scale_quant_orig, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + None, + ) + return qsa_paged_sparse_gqa_nvfp4( + q, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + core.kv_scale_quant_orig, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + query_positions=query_positions, + compress_ratio=core.compress_ratio, + ) + + +def _copy_positions(position_ids: torch.Tensor) -> torch.Tensor: + """A contiguous int32 `[3, N]` copy of the mRoPE positions; the rope rejects the engine's.""" + pos3 = torch.empty( + [_QSA_POSITION_AXES, position_ids.numel() // _QSA_POSITION_AXES], + dtype=torch.int32, + device=position_ids.device, + ) + pos3.copy_(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1])) + return pos3 + + +def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens) -> dict: + """thop_attention's batch state off the prepared metadata. + + Tensors are engine-owned buffers refreshed in place; ints are per-capture constants.""" + return dict( + sequence_length=md.kv_lens_cuda_runtime, + host_past_key_value_lengths=md.kv_lens_runtime, + host_total_kv_lens=md.host_total_kv_lens, + context_lengths=md.prompt_lens_cuda_runtime, + host_context_lengths=md.prompt_lens_cpu_runtime, + host_request_types=md.host_request_types_runtime, + kv_cache_block_offsets=md.kv_cache_block_offsets, + host_kv_cache_pool_pointers=md.host_kv_cache_pool_pointers, + host_kv_cache_pool_mapping=md.host_kv_cache_pool_mapping, + workspace=md.effective_workspace, + tokens_per_block=md.tokens_per_block, + max_num_requests=md.max_num_requests, + max_context_length=md.max_context_length, + max_seq_len=md.max_seq_len, + attention_window_size=md.max_seq_len, + num_contexts=num_contexts, + num_ctx_tokens=num_ctx_tokens, + trtllm_gen_jit_warmup=md.trtllm_gen_jit_warmup, + use_paged_context_fmha=md.use_paged_context_fmha, + beam_width=md.effective_beam_width, + cache_indirection=md.cache_indirection, + block_ids_per_seq=md.block_ids_per_seq, + max_context_q_len_override=md.max_context_q_len_override, + is_cross=md.is_cross, + is_spec_decoding_enabled=md.is_spec_decoding_enabled, + use_spec_decoding=md.use_spec_decoding, + is_spec_dec_tree=md.is_spec_dec_tree, + spec_decoding_generation_lengths=md.spec_decoding_generation_lengths, + spec_decoding_position_offsets=get_spec_decoding_position_offsets(md), + spec_decoding_packed_mask=md.spec_decoding_packed_mask, + spec_decoding_bl_tree_mask_offset=md.spec_decoding_bl_tree_mask_offset, + spec_decoding_bl_tree_mask=md.spec_decoding_bl_tree_mask, + spec_decoding_target_max_draft_tokens=md.max_total_draft_tokens, + spec_bl_tree_first_sparse_mask_offset_kv=md.spec_bl_tree_first_sparse_mask_offset_kv, + num_sparse_topk=md.num_sparse_topk, + flash_mla_tile_scheduler_metadata=md.flash_mla_tile_scheduler_metadata, + flash_mla_num_splits=md.flash_mla_num_splits, + max_num_sequences=md.max_num_sequences, + force_prepare_spec_dec_tree_mask=md.force_prepare_spec_dec_tree_mask, + ) + + +# ------------------------------------------------------------------ prefill --- + + +def _pad_window(num_tokens: int, rows: int) -> int: + """First row a break graph over a `rows` bucket zeroes from; part of the graph key. + + Half the bucket if the batch fills more than half (any power-of-two bucket ladder), else 0.""" + return rows // 2 if num_tokens > rows // 2 else 0 + + +class _PrefillStep(NamedTuple): + """One step's host-side state, built fresh by `PrefillTarget.prepare` and read by the breaks. + + `num_tokens` is the batch's rows `n`, `physical_tokens` the `T` the forward runs over (`n` or + its CUDA graph bucket), `dp_pad_rows` the MoE padding (attention DP without a CUDA graph).""" + + num_tokens: int + physical_tokens: int + dp_pad_rows: _DpPadding | None + input_ids: torch.Tensor + attn: dict + gdn: tuple + ple: tuple + qsa: tuple + vision: tuple | None + # A verify step: the generation rows' side-cache writes are staged for the rollback. + stage: bool = False + # fp8 pool only: `_fp8_context_pages`. + fp8_pages: int | None = None + # fp8 pool only: `_rowtile_scratch_groups`, None for one launch. + rowtile_groups: tuple | None = None + split: _Split | None = None + # Graph-replayable steps only: QSA per-row metadata over all `T` rows (padding = last row). + qsa_rows: tuple | None = None + # `_selection_cuts`, or None for equal row counts. + sel_cuts: tuple | None = None + + +class PrefillTarget: + """Any batch with context rows (`num_contexts > 0`), possibly with generation rows.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: + self.core = core + self._contract_pending = step_contract_enabled() + self._layers = _layer_views(core) + self._stack = _stack_views(core) + self._step: _PrefillStep | None = None + # `_attn_dispatch` by (rows, score-table width). + self._attn_dispatch_memo: dict = {} + self._coords_buf: torch.Tensor | None = None + # Keyed CUDA graphs on short buckets; a key without a graph runs the eager core. + self._attention_break = _keyed_eager_on_graph( + self._attention_core, + graph_key=self._attn_graph_key, + graph_body=self._attention_core_graph, + ) + self._gdn_break = _keyed_eager_on_graph( + self._gdn_core, graph_key=self._gdn_graph_key, graph_body=self._gdn_core_graph + ) + # Batch row count `n` (`[1]` int32), refreshed by `prepare` before every forward or + # replay so a CUDA graph can tell its padding rows; allocated once, at a fixed address. + self._live_rows: torch.Tensor | None = None + # `(rank, world, group, process_group)` of the attention-TP group, or None. + split = None + if _prefill_split_enabled(core.attn_tp_size, core.dp_size): + split = ( + core.mapping.tp_rank, + core.attn_tp_size, + list(core.mapping.tp_group), + core.mapping.tp_group_pg.boxed() if mpi_disabled() else None, + ) + self._selection_split = split + self._rowtile_groups = os.environ.get(ROWTILE_SCRATCH_GROUPS_ENV, "0").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + self._sequence_parallel = split + # TP sum, or reduce-scatter under SP; set by `forward`, a capture-time constant per bucket. + self._block_sum = core.tp_sum + # `(lo, hi, gather)` of a sequence-parallel forward, or None. + self._sp_rows = None + # Highest priority: in a CUDA graph a default-priority stream's gather waits for the + # main stream's persistent GEMM, which holds every SM. + self._comm_stream = None + self._comm_fork = None + self._comm_join = None + side = getattr(core, "_side_stream", None) + pipelined = self._selection_split is not None + if (self._sequence_parallel is not None or pipelined) and side is not None: + self._comm_stream = torch.cuda.Stream(device=side.device, priority=-1) + self._comm_fork = torch.cuda.Event() + self._comm_join = torch.cuda.Event() + # Two-rank sequence parallelism only: sized for half a max_num_tokens step's bf16 rows. + self._ce_exchange = None + if ( + self._comm_stream is not None + and self._sequence_parallel is not None + and self._sequence_parallel[1] == 2 + ): + from .ops.comm.ce_exchange__sp_pair import CopyEngineExchange + + rows = -(-core.max_num_tokens // 2) + self._ce_exchange = CopyEngineExchange.create(core.mapping, rows * core.hidden * 2) + + def _sp_split(self, num_tokens: int): + """`(lo, hi, gather, scatter)` of a sequence-parallel forward over `num_tokens`, or None. + + `[lo, hi)` is this rank's share; depends on `num_tokens` alone, the same on every rank.""" + sp = self._sequence_parallel + if sp is None or num_tokens < _SP_MIN_ROWS: + return None + rank, world, group, process_group = sp + if num_tokens % world: + return None + rows = num_tokens // world + + def gather(t: torch.Tensor) -> torch.Tensor: + return allgather(t, None, group, process_group) + + ce = getattr(self, "_ce_exchange", None) if world == 2 else None + + def scatter(t: torch.Tensor) -> torch.Tensor: + if ce is not None: + # Send the peer its rows, add the peer's partial of this rank's rows. + lo, hi = rank * rows, (rank + 1) * rows + peer = t[hi:] if rank == 0 else t[:lo] + return t[lo:hi] + ce.exchange(peer.contiguous()) + return reducescatter(t, None, group, process_group) + + comm = getattr(self, "_comm_stream", None) + if ce is not None and comm is not None: + # Split around the producer: `start` sends early on the comm stream, `finish` adds. + lo, hi = rank * rows, (rank + 1) * rows + peer_rows = (hi, num_tokens) if rank == 0 else (0, lo) + box = [] + + def start(t: torch.Tensor) -> None: + main = torch.cuda.current_stream() + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + with torch.cuda.stream(comm): + box.append(ce.exchange(t[peer_rows[0] : peer_rows[1]])) + + def finish(t: torch.Tensor): + self._comm_join.record(comm) + torch.cuda.current_stream().wait_event(self._comm_join) + return t[lo:hi] + box.pop() + + scatter.split_finalize = (start, finish, peer_rows) + + return rank * rows, (rank + 1) * rows, gather, scatter + + def _block_rows(self, x: torch.Tensor) -> torch.Tensor: + """A block's input over every row: `x`, or the gather of every rank's rows under SP.""" + return x if self._sp_rows is None else self._sp_rows[2](x) + + def _overlap_projection(self) -> bool: + """Whether a block's input projection runs as `_project_overlapped`.""" + return self._sp_rows is not None and self._comm_stream is not None + + def _project_overlapped(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """A sequence-parallel block's input projection over every row; `x` is this rank's rows. + + The gather on the comm stream overlaps this rank's rows; bitwise one `cublas_mm`.""" + lo, hi, gather = self._sp_rows + total = (hi - lo) * self._sequence_parallel[1] + out = torch.empty([total, w.shape[1]], dtype=x.dtype, device=x.device) + main = torch.cuda.current_stream() + comm = self._comm_stream + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + ce = getattr(self, "_ce_exchange", None) + with torch.cuda.stream(comm): + # Only the peer's rows: a view of the receive buffer, read before the next exchange. + whole = ce.exchange(x.contiguous()) if ce is not None else gather(x) + torch.mm(x, w, out=out[lo:hi]) + self._comm_join.record(comm) + main.wait_event(self._comm_join) + if ce is not None: + peer = slice(hi, total) if lo == 0 else slice(0, lo) + torch.mm(whole, w, out=out[peer]) + return out + if lo: + torch.mm(whole[:lo], w, out=out[:lo]) + if hi < total: + torch.mm(whole[hi:], w, out=out[hi:]) + return out + + def _overlap_reduce(self) -> bool: + """Whether a block's output projection reduces as `_reduce_overlapped`.""" + return ( + self._sp_rows is not None + and self._comm_stream is not None + and self._sequence_parallel[1] == 2 + ) + + def _reduce_overlapped(self, a: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """A two-rank SP block's output `sum over ranks of a @ w` on this rank's rows `[lo, hi)`. + + The peer's rows are projected first and exchanged while this rank's rows are projected.""" + lo, hi, gather = self._sp_rows + rows = hi - lo + rank = lo // rows + peer = a[hi:] if rank == 0 else a[:lo] + sent = torch.mm(peer, w) + main = torch.cuda.current_stream() + comm = self._comm_stream + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + ce = getattr(self, "_ce_exchange", None) + with torch.cuda.stream(comm): + if ce is not None: + got = ce.exchange(sent) # the peer's product of this rank's rows + else: + got = gather(sent)[(1 - rank) * rows : (2 - rank) * rows] + own = torch.mm(a[lo:hi], w) + self._comm_join.record(comm) + main.wait_event(self._comm_join) + return own + got + + def _split_context_rows(self, rows: int) -> bool: + """Whether `rows` context rows take `_split_context_selection`. + + A host decision on the batch, identical on every rank, so all take the gather or none.""" + return self._selection_split is not None and rows >= _SELECTION_SPLIT_MIN_ROWS + + def _selection_pipeline(self): + """`_split_context_selection`'s `pipeline`, or None to gather the share once.""" + if self._comm_stream is None: + return None + pipeline = (_SELECTION_PIECES, self._comm_stream, self._comm_fork, self._comm_join) + ce = getattr(self, "_ce_exchange", None) + return pipeline if ce is None else pipeline + (ce,) + + def _check_step_contract(self, md) -> None: + """Opt-in first-forward check: surfaces an upstream metadata rename before the forward.""" + if not self._contract_pending: + return + try: + self.step_args(md) + except AttributeError as exc: + raise AssertionError(f"attention metadata surface drifted: {exc}") from exc + self._contract_pending = False + + def step_args(self, md: TrtllmAttentionMetadata) -> dict: + """Project the prepared metadata onto thop_attention's batch state.""" + return _step_args(md, md.num_contexts, md.num_ctx_tokens) + + # --------------------------------------------------------------- blocks --- + + def _attention_block(self, x, i, L, proj_qkv, index_scratch, attn_out, pos3, pos): + """One gated GQA layer plus its QSA index branch, at this rank's heads. + + Norm + rope run over all `T` rows; caches, selection and attention run in the break.""" + core = self.core + if self._overlap_projection(): + proj = self._project_overlapped(x, L.attn_in) + else: + proj = cublas_mm(self._block_rows(x), L.attn_in) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + _attention_rope(core, proj, index_scratch, L, pos3, pos) + self._attention_break(proj, proj_qkv, index_scratch, attn_out, i) + fused_sigmoid_mul_inplace(attn_out, gate, direct_launch=True) + if self._overlap_reduce(): + return self._reduce_overlapped(attn_out, L.o) + return self._block_sum(cublas_mm(attn_out, L.o)) + + def _attn_dispatch(self, rows: int, width: int) -> tuple: + """The sparse context path's launch choices that depend on its row count. + + A `T`-row graph is bitwise the eager break only where these match the `n` rows'.""" + memo = self._attn_dispatch_memo + hit = memo.get((rows, width)) + if hit is not None: + return hit + core = self.core + sel = core.token_topk + core.compress_ratio - 1 + q = torch.empty((rows, core.index_heads, core.index_head_dim), device="meta") + scores = torch.empty((rows, width), device="meta") + heads_kv = core.heads_kv + gqa = "pair" if rows * heads_kv > 512 else _gqa_splitk_config(rows, heads_kv, sel) + topk = torch.empty((rows, core.block_topk), dtype=torch.int32, device="meta") + hit = ( + _scores_take_visible_split(q, True, False), + rows >= _TOPK_ROWS_MIN_ROWS and _topk_rows_supports(scores, topk), + _topk_expand_cluster_supports(scores), + _topk_expand_supports(scores), + gqa, + ) + memo[(rows, width)] = hit + return hit + + def _attn_graph_key(self, proj, proj_qkv, index_scratch, attn_out, i): + """Graph key for `_attention_core_graph`, or None to run `_attention_core` eagerly. + + Context-only sparse steps over bf16 / fp8 pools without the row-tile pass, whose launch + choices match at `n` and `T` rows. The key includes every buffer address the graph reads.""" + step = self._step + if step is None or step.qsa_rows is None: + return None + rows = proj.shape[0] + if rows != step.physical_tokens or step.attn["num_ctx_tokens"] != step.num_tokens: + return None + core = self.core + kv = core._bound.pools.kv[i] + n = step.num_tokens + if ( + kv.k_scale is not None + or _rowtile_takes(kv, n, step.fp8_pages) + or step.rowtile_groups is not None + ): + return None + block_table = step.qsa[6] + width = block_table.shape[1] * step.qsa[7] // core.compress_ratio + dispatch = self._attn_dispatch(n, width) + if dispatch != self._attn_dispatch(rows, width): + return None + pools = core._bound.pools + return ( + step.attn["num_contexts"], + _pad_window(n, rows), + dispatch, + kv.k.data_ptr(), + kv.v.data_ptr(), + pools.index_cache[i].data_ptr(), + pools.position_cache.data_ptr(), + block_table.data_ptr(), + tuple(block_table.shape), + tuple(t.data_ptr() for t in step.qsa_rows), + self._live_rows.data_ptr(), + ) + + def _attention_core_graph(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: + """`_attention_core` of a context-only sparse step as a CUDA graph over all `T` rows. + + Padding rows reuse the last row's metadata and store nothing (position -2 is skipped by the + stores and never anchors a compression); their outputs are zeroed. Rows `[:n]` are bitwise + the eager break's.""" + core = self.core + L = self._layers[i] + step = self._step + req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts, coords = ( + step.qsa_rows + ) + block_table, tokens_per_block = step.qsa[6], step.qsa[7] + rows = proj.shape[0] + pad = torch.arange(rows, dtype=torch.int32, device=proj.device) >= self._live_rows + store_positions = torch.where(pad, -2, logical) + q_rows, k_dst, v_dst = _attn_in_heads(core, proj) + _index_plane( + core, proj, i, L, (req_idx, store_positions, block_table, tokens_per_block, coords) + ) + kv = core._bound.pools.kv[i] + _store_kv(core, kv, k_dst, v_dst, req_idx, store_positions, block_table, tokens_per_block) + q_index = _index_query(core, index_scratch) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=proj.device + ) + scores = _context_index_scores( + q_index, + core._bound.pools.index_cache[i], + block_table, + logical, + req_idx, + tokens_per_block, + core.compress_ratio, + True, + ) + _selection( + core, + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + topk_row_starts, + selected, + context=True, + ) + proj_qkv.copy_(proj[:, : core.qkv_width]) + out = qsa_paged_sparse_gqa( + q=_packed_q(core, proj_qkv), + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=req_idx, + tokens_per_block=tokens_per_block, + softmax_scale=core.softmax_scale, + query_positions=logical, + compress_ratio=core.compress_ratio, + output_gate=None, + ) + attn_out.copy_(torch.reshape(out, [rows, core.q_width])) + lo = _pad_window(step.num_tokens, rows) + attn_out[lo:].masked_fill_(torch.unsqueeze(pad[lo:], 1), 0) + + def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: + """Break: index store and compression, then dense or sparse attention over `n` rows. + + Writes rows `[:n]` of `attn_out` and zeroes `[n:]`. The index plane advances on every + step so a request crossing the threshold finds its prefix compressed; a verify step stages + it for `SpeculativeRollback` first.""" + core = self.core + L = self._layers[i] + step = self._step + ( + req_idx, + logical, + seq_lens_q, + visible_blocks, + topk_indices, + topk_row_starts, + block_table, + tokens_per_block, + coords, + sparse, + ) = step.qsa + num_tokens = step.num_tokens + proj = proj[:num_tokens] + q_rows, k_dst, v_dst = _attn_in_heads(core, proj) + qkv = proj_qkv[:num_tokens] + if not sparse: + # The dense op reads a compact `[q | k | v]`. + qkv.copy_(proj[:, : core.qkv_width]) + if step.stage: + core.spec_rollback.stage_index(core._bound.pools.position_cache) + _index_plane(core, proj, i, L, (req_idx, logical, block_table, tokens_per_block, coords)) + + if sparse: + kv = core._bound.pools.kv[i] + # Nothing else writes the pool on this path; the selection reads the current token. + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + q_index = _index_query(core, index_scratch[:num_tokens]) + # Context rows come first and score as query tiles; generation rows score row by row. + num_ctx = step.attn["num_ctx_tokens"] + parts = [(0, num_ctx, True)] + if num_ctx < num_tokens: + parts.append((num_ctx, num_tokens, False)) + selected = torch.empty( + [num_tokens, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=proj.device, + ) + + def score_and_select(lo, hi, context_rows, out): + scores = _context_index_scores( + q_index[lo:hi], + core._bound.pools.index_cache[i], + block_table, + logical[lo:hi], + req_idx[lo:hi], + tokens_per_block, + core.compress_ratio, + context_rows, + ) + _selection( + core, + scores, + visible_blocks[lo:hi], + logical[lo:hi], + seq_lens_q[lo:hi], + topk_indices[lo:hi], + topk_row_starts[lo:hi], + out, + context=context_rows, + ) + + for lo, hi, context_rows in parts: + if context_rows and self._split_context_rows(hi - lo): + _split_context_selection( + self._selection_split, + score_and_select, + selected, + lo, + hi, + self._selection_pipeline(), + step.sel_cuts if lo == 0 and hi == num_ctx else None, + ) + else: + score_and_select(lo, hi, context_rows, selected[lo:hi]) + # The row-tile pass reads q in place; the other kernels need a compact copy. + launches = [] + for lo, hi, context_rows in parts: + if context_rows and step.rowtile_groups is not None: + launches += [(a, b, True, pages) for a, b, pages in step.rowtile_groups] + else: + launches.append((lo, hi, context_rows, step.fp8_pages)) + for lo, hi, context_rows, pages in launches: + if context_rows and _rowtile_takes(kv, hi - lo, pages): + q = q_rows[lo:hi] + else: + qkv[lo:hi].copy_(proj[lo:hi, : core.qkv_width]) + q = _packed_q(core, qkv)[lo:hi] + out = _sparse_gqa( + core, + kv, + q, + block_table, + selected[lo:hi], + req_idx[lo:hi], + tokens_per_block, + logical[lo:hi] if context_rows else None, + pages, + ) + attn_out[lo:hi].copy_(torch.reshape(out, [hi - lo, core.q_width])) + else: + thop_attention( + q=qkv, + output=attn_out[:num_tokens], + local_layer_idx=core._bound.pools.pool_idx[i], + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step.attn, + **_CALL_CONSTANTS, + ) + attn_out[num_tokens:].zero_() + + def _gdn_block(self, x, i, L, core_out) -> torch.Tensor: + """One Gated-DeltaNet layer over a batch with a context prefix, at this rank's heads. + + The projection and epilogue run over all `T` rows; the recurrence runs in `_gdn_core`.""" + core = self.core + if self._overlap_projection(): + proj = self._project_overlapped(x, L.in_proj) + else: + proj = cublas_mm(self._block_rows(x), L.in_proj) + z = torch.split(proj, core.in_proj_split, 1)[1] + self._gdn_break(proj, core_out, i) + return self._gdn_epilogue(core_out, z, L, proj.shape[0]) + + def _gdn_graph_key(self, proj, out, i): + """Graph key for `_gdn_core_graph`, or None to run `_gdn_core` eagerly. + + Context-only steps past the in-place threshold in short buckets. The key holds the request + count, split shape, FlashInfer's CP chunk length and every buffer address read.""" + step = self._step + if step is None or step.stage: + return None + rows = proj.shape[0] + if rows != step.physical_tokens or rows > _GRAPH_BREAKS_MAX_TOKENS: + return None + n = step.num_tokens + gdn = step.gdn + num_decodes, verify = gdn[11], gdn[13] + if num_decodes or verify is not None or n <= _GDN_INPLACE_MAX_TOKENS: + return None + si_all, hi_all, qsl_all, qsl_long = gdn[3], gdn[6], gdn[8], gdn[9] + num_seqs = si_all.shape[0] + core = self.core + pools = core._bound.pools + dev = proj.device + heads = max(core.num_k_heads, core.num_v_heads) + key = ( + num_seqs, + _pad_window(n, rows), + pools.ssm[i].data_ptr(), + pools.conv[i].data_ptr(), + si_all.data_ptr(), + hi_all.data_ptr(), + qsl_all.data_ptr(), + qsl_long.data_ptr(), + self._live_rows.data_ptr(), + ) + sp = step.split + if sp is None: + cp = _gdn_reaches_cp(num_seqs, core.num_v_heads, dev) + return ("whole", cp_chunk_len_for(n, heads, dev) if cp else 0) + key + raw = sp.raw + if raw is None or sp.whole is None and split_takes_cp(num_seqs, core.num_v_heads, dev): + return None + stage = core._bound.stage + key += ( + sp.k, + raw.tail_tok.shape[0], + raw.cu.data_ptr(), + stage.ssm[i].data_ptr(), + stage.conv[i].data_ptr(), + ) + if sp.whole is None: + return ("split",) + key + lead = sp.whole[0] + if not isinstance(lead, int) or num_seqs != 1 or sp.k != 1: + return None + return ("lead", cp_chunk_len_for(lead, heads, dev)) + key + + def _gdn_core_graph(self, proj, out, i) -> None: + """`_gdn_core` of a context-only step as a CUDA graph over all `T` rows. + + Padding rows never reach the batch's rows or states and are zeroed after, so rows `[:n]` of + `out` and the pools are bitwise the eager break's.""" + core = self.core + L = self._layers[i] + step = self._step + gdn = step.gdn + si_all, hi_all, qsl_all, qsl_long = gdn[3], gdn[6], gdn[8], gdn[9] + rows = proj.shape[0] + dev = proj.device + pools = core._bound.pools + ssm_states = pools.ssm[i] + mixed_qkv, _, a, b = torch.split(proj, core.in_proj_split, 1) + # Padding rows are finite (the splice zeroes their embedding). + q, k, v, g, beta = self._gdn_prefill_prep(mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all) + heads = max(core.num_k_heads, core.num_v_heads) + sp = step.split + if sp is None: + num_seqs = si_all.shape[0] + if _gdn_reaches_cp(num_seqs, core.num_v_heads, dev): + fi = { + "_cp_chunk_len": cp_chunk_len_for(step.num_tokens, heads, dev), + "max_seqlen": rows, + } + _gdn_chunk_cp_inplace( + q, k, v, g, beta, None, ssm_states, si_all, hi_all, qsl_long, out, True, + fi_kwargs=fi, + ) # fmt: skip + else: + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, si_all, hi_all, qsl_long, out, True, + ) # fmt: skip + else: + raw = sp.raw + stage = core._bound.stage + stage_ssm, stage_conv = stage.ssm[i], stage.conv[i] + gather = raw.gather.to(torch.int32) + live = raw.live.to(torch.int32) + if sp.whole is None: + carry = torch.zeros([gather.shape[0]], dtype=torch.bool, device=dev) + carry.index_copy_(0, raw.main_rows, hi_all.to(torch.bool)) + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, gather, carry, raw.cu, out, True, + keep_rows=raw.main_rows, keep_slots=live, use_cp=False, + ) # fmt: skip + else: + # One request split once: the first pass covers its rows before P (`raw.cu[:2]`). + fi = { + "_cp_chunk_len": cp_chunk_len_for(sp.whole[0], heads, dev), + "max_seqlen": rows, + } + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, live, hi_all.to(torch.bool), raw.cu[:2], + out, True, fi_kwargs=fi, + ) # fmt: skip + gdn_inforward_tail( + q, + k, + v, + g, + beta, + raw.tail_tok, + raw.cu_tail, + ssm_states, + raw.tail_slots, + stage_ssm, + raw.stage_rows, + out, + ) + gdn_inforward_conv_swap( + pools.conv[i], stage_conv, mixed_qkv, raw.tail_slots, raw.stage_rows, raw.conv_win + ) + lo = _pad_window(step.num_tokens, rows) + pad = torch.arange(lo, rows, dtype=torch.int32, device=dev) >= self._live_rows + out[:, lo:].masked_fill_(pad.view(1, rows - lo, 1, 1), 0) + + def _gdn_core(self, proj, out, i) -> None: + """Break: the recurrence over `n` rows; writes rows `[:n]` of `out`, zeroes rows `[n:]`. + + A context sequence with no carried state starts from zeros without clearing its slots + (`has_initial_state`); a verify step's draft chains take `_gdn_verify`.""" + core = self.core + L = self._layers[i] + step = self._step + ( + num_prefill_tokens, + num_decode_tokens, + si_p, + si_all, + si_d, + hi_p, + hi_all, + qsl_p, + qsl_all, + qsl_long, + qsl_long_p, + num_decodes, + gen_p, + verify, + ) = step.gdn + num_tokens = step.num_tokens + pools = core._bound.pools + ssm_states = pools.ssm[i] + mixed_qkv, _, a, b = torch.split(proj[:num_tokens], core.in_proj_split, 1) + + if verify is not None: + mq_p, mq_d = torch.split(mixed_qkv, [num_prefill_tokens, num_decode_tokens], 0) + a_p, a_d = torch.split(a, [num_prefill_tokens, num_decode_tokens], 0) + b_p, b_d = torch.split(b, [num_prefill_tokens, num_decode_tokens], 0) + q, k, v, g, beta = self._gdn_prefill_prep(mq_p, a_p, b_p, L, i, qsl_p, si_p, hi_p) + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_p, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long_p, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_prefill_tokens], + state_workspace=None, + has_initial_state=hi_p, + linear_gate=True, + ) + out[:, num_prefill_tokens:num_tokens].copy_( + _gdn_verify( + core, + mq_d, + a_d, + b_d, + i, + L, + si_d, + num_decodes, + gen_p, + verify, + ) + ) + elif num_decode_tokens > 0: + mq_p, mq_d = torch.split(mixed_qkv, [num_prefill_tokens, num_decode_tokens], 0) + a_p, b_p = a[:num_prefill_tokens], b[:num_prefill_tokens] + q, k, v, g, beta = self._gdn_prefill_prep(mq_p, a_p, b_p, L, i, qsl_p, si_p, hi_p) + causal_conv1d_update( + x=mq_d, + conv_state=pools.conv[i], + weight=L.conv_w, + bias=None, + silu_activation=True, + cache_seqlens=None, + conv_state_indices=si_d, + pad_slot_id=_PAD_SLOT_ID, + ) + # The two halves use disjoint slots and write disjoint row ranges of the output. + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_p, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long_p, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_prefill_tokens], + state_workspace=None, + has_initial_state=hi_p, + linear_gate=True, + ) + # Reads the conv-updated q/k/v columns; the gated norm runs later, in the epilogue. + gdn_decode_rows( + proj[num_prefill_tokens:num_tokens], + ssm_states, + si_d, + L.a_log, + L.dt_bias, + core.num_k_heads, + core.head_k_dim, + out[0, num_prefill_tokens:num_tokens].view(num_decode_tokens, core.value_dim), + ) + elif step.split is not None: + q, k, v, g, beta = self._gdn_prefill_prep( + mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all + ) + self._gdn_split(q, k, v, g, beta, mixed_qkv, i, out[:, :num_tokens]) + else: + q, k, v, g, beta = self._gdn_prefill_prep( + mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all + ) + # q and k arrive L2-normalized; the recurrence must not repeat it. + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_all, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_tokens], + state_workspace=None, + has_initial_state=hi_all, + linear_gate=True, + ) + out[:, num_tokens:].zero_() + + def _gdn_split(self, q, k, v, g, beta, mixed_qkv, i, out) -> None: + """The recurrence of a context-only step with an in-forward snapshot split (`_Split`). + + Leaves each split request's state at P in its live slot, its prompt-end state in its staging + row, and every row's output in `out` `[1, n, HV, V]`. `chunk_split_first_pass` writes the + state at P; `gdn_inforward_tail` runs the tails `[P, L)` from it into the staging rows + (fp32, FlashInfer's to rounding); `gdn_inforward_conv_swap` swaps the conv windows.""" + core = self.core + sp = self._step.split + pools = core._bound.pools + stage = core._bound.stage + ssm_states, conv_states = pools.ssm[i], pools.conv[i] + stage_ssm, stage_conv = stage.ssm[i], stage.conv[i] + chunk_split_first_pass( + q, + k, + v, + g, + beta, + ssm_states, + sp.gather, + sp.carry, + sp.cu, + sp.main_rows, + sp.live, + out, + num_whole_seqs=sp.live.shape[0], + whole=sp.whole, + ) + gdn_inforward_tail( + q, + k, + v, + g, + beta, + sp.tail_tok, + sp.cu_tail, + ssm_states, + sp.tail_slots, + stage_ssm, + sp.stage_rows, + out, + ) + gdn_inforward_conv_swap( + conv_states, stage_conv, mixed_qkv, sp.tail_slots, sp.stage_rows, sp.conv_win + ) + + def _gdn_prefill_prep(self, mq, a, b, L, i, qsl, si, hi): + """The context rows' recurrence inputs `(q, k, v, g, beta)` in one pass over `mq`. + + Publishes the conv window back to the pool; q/k come out L2-normalized and the decay as the + factor `exp(g)` (`linear_gate=True`).""" + core = self.core + return gdn_prefill_conv_post( + x=mq, + weight=L.conv_w, + conv_states=core._bound.pools.conv[i], + query_start_loc=qsl, + cache_indices=si, + has_initial_state=hi, + a=a, + b=b, + A_log=L.a_log, + dt_bias=L.dt_bias, + num_k_heads=core.num_k_heads, + head_k_dim=core.head_k_dim, + num_v_heads=core.num_v_heads, + head_v_dim=core.head_v_dim, + pad_slot_id=_PAD_SLOT_ID, + l2_norm_eps=_L2_NORM_EPS, + softplus_threshold=_SOFTPLUS_THRESHOLD, + linear_gate=True, + ) + + def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: + """The gated RMS norm (sigmoid gate, non-Gemma weight) and the reduced output projection.""" + core = self.core + o = rms_norm_gated_token_major( + torch.reshape(out, [-1, core.head_v_dim]), + torch.reshape(z, [num_tokens, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + fp8_scale=None, + gate_activation="sigmoid", + direct_launch=True, + ) + o = torch.reshape(o, [num_tokens, core.value_dim]) + if self._overlap_reduce(): + return self._reduce_overlapped(o, L.out) + return self._block_sum(cublas_mm(o, L.out)) + + def _ple_block(self, hc, hc_out, normed_out) -> None: + """Break: the n-gram side path over `n` rows, added into the stream and normalized. + + Writes rows `[:n]` of `hc_out` and `normed_out`, zeroes `[n:]`. A sequence with no history + is seeded with `eos_token_id`; a verify step stages its state for `SpeculativeRollback`.""" + core = self.core + S = self._stack + step = self._step + conv_states, ngram_pool, si_all, carry_all, qsl = step.ple + num_tokens = step.num_tokens + next_norm = self._layers[core.ple_layer].ahc_norm + conv_input = None + if step.stage: + core.spec_rollback.stage_ple_verify(ngram_pool, conv_states, step.input_ids) + conv_input = torch.empty_like(hc[:num_tokens]) + elif step.split is not None: + # The conv input rows rebuild the conv state at P (the op allocates them anyway). + conv_input = torch.empty_like(hc[:num_tokens]) + ngram = ple_ngram_embed( + step.input_ids, + ngram_pool, + si_all, + carry_all, + qsl, + core.w["ple_table"], + S.ple_tscale, + S.ple_mult, + S.ple_vocab, + S.ple_offs, + core.eos_token_id, + num_tokens, + ) + ple_gate_conv_inject_norm( + hc[:num_tokens], + cublas_mm(ngram, S.ple_key), + cublas_mm(ngram, S.ple_val), + S.ple_nk_w, + S.ple_nq_w, + S.ple_nc_w, + S.ple_conv_w, + conv_states, + si_all, + qsl, + next_norm, + core.eps, + core.hc_count, + _PLE_CLAMP_MIN, + core.ple_dilation, + _PAD_SLOT_ID, + num_tokens, + carry_all, + conv_input=conv_input, + out=(hc_out[:num_tokens], normed_out[:num_tokens]), + ) + if step.stage: + core.spec_rollback.stage_ple_conv_input(conv_input) + elif step.split is not None: + self._split_ple_states(conv_states, ngram_pool, conv_input) + hc_out[num_tokens:].zero_() + normed_out[num_tokens:].zero_() + + def _split_ple_states(self, conv_states, ngram_pool, conv_input) -> None: + """The PLE side of an in-forward snapshot split. + + Moves each split request's prompt-end state to its staging row and writes the state at P + (the last conv-input rows and ids before P, oldest first) to its slot.""" + sp = self._step.split + stage = self.core._bound.stage + stage.ple_conv.index_copy_(0, sp.stage_rows, conv_states.index_select(0, sp.tail_slots)) + window = conv_input.index_select(0, sp.ple_win) + conv_states.index_copy_( + 0, sp.tail_slots, window.view(sp.k, -1, window.shape[-1]).transpose(1, 2) + ) + stage.ple_ngram.index_copy_(0, sp.stage_rows, ngram_pool.index_select(0, sp.tail_slots)) + ids = self._step.input_ids.index_select(0, sp.ngram_win) + ngram_pool.index_copy_(0, sp.tail_slots, ids.view(sp.k, -1).to(ngram_pool.dtype)) + + def _splice(self, h) -> None: + """Break: write vision rows over their placeholders in `h`; zero rows `[n:]`.""" + step = self._step + if step.vision is not None: + rows, mm_token_indices = step.vision + h.index_copy_(0, mm_token_indices, rows) + h[step.num_tokens :].zero_() + + # -------------------------------------------------------------- forward --- + + def prepare( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor, + position_ids: torch.IntTensor, + vision: tuple | None, + spec_metadata=None, + ) -> None: + """Build the step's host-side state (`_PrefillStep`) once, before the forward; on a + breakable prefill graph replay this and the breaks are all of this target that runs.""" + core = self.core + md = attn_metadata + self._check_step_contract(md) + attn_args = self.step_args(md) + mm = md.mamba_metadata + + physical_tokens = input_ids.shape[0] + assert physical_tokens in (md.num_tokens, md.padded_num_tokens), ( + f"the engine handed {physical_tokens} activation rows for a batch its metadata " + f"describes as {md.num_tokens} tokens (padded_num_tokens={md.padded_num_tokens})" + ) + num_tokens = md.num_tokens + if self._live_rows is None: + self._live_rows = torch.zeros([1], dtype=torch.int32, device=input_ids.device) + self._live_rows.fill_(num_tokens) + # Attention DP pads the MoE rows to the group's largest count; under a prefill CUDA + # graph every rank runs one bucket, so this is None (a per-capture constant). + dp_pad_rows = core._dp_pad_rows(md, physical_tokens) + assert physical_tokens == num_tokens or dp_pad_rows is None, ( + f"a prefill CUDA graph bucket of {physical_tokens} rows with attention-DP padding " + f"{dp_pad_rows}: the graph would freeze the group's row counts" + ) + batch = md.num_seqs + num_prefills = md.num_contexts + num_decodes = batch - num_prefills + num_prefill_tokens = md.num_ctx_tokens + num_decode_tokens = num_tokens - num_prefill_tokens + gen_p, is_verify = _verify_rows(core, spec_metadata, num_decodes, num_decode_tokens) + attn_args["predicted_tokens_per_seq"] = gen_p + + si_all = mm.state_indices[:batch] + hi_all = mm.has_initial_states[:batch] + si_p, si_d = torch.split(si_all, [num_prefills, num_decodes], 0) + # The decode kernel needs a 16-byte-aligned slot vector; the generation half starts + # `num_prefills` rows in, so copy it once per step. + si_d = torch.clone(si_d) + hi_p = hi_all[:num_prefills] + # `query_start_loc_long` spans the whole batch; its prefix segments the context rows. + qsl_all = mm.query_start_loc + qsl_p = None if qsl_all is None else qsl_all[: num_prefills + 1] + qsl_long = mm.query_start_loc_long + qsl_long_p = qsl_long[: num_prefills + 1] + gdn_state = ( + num_prefill_tokens, + num_decode_tokens, + si_p, + si_all, + si_d, + hi_p, + hi_all, + qsl_p, + qsl_all, + qsl_long, + qsl_long_p, + num_decodes, + gen_p, + _verify_state(core, mm, num_decodes) if is_verify else None, + ) + + # `has_initial_states` covers the context prefix only; every generation row carries. + ple_carry = nn.functional.pad(hi_p, [0, num_decodes], mode="constant", value=1.0) + # `query_start_loc_long` mis-segments PLE when `gen_p > 1`; rebuild from `seq_lens_cuda`. + ple_qsl = ( + qsl_long + if gen_p == 1 + else cu_seqlens_from_seq_lens(md.seq_lens_cuda[:batch], torch.int64) + ) + pools = core._bound.pools + ple_state = (pools.ple_state[0], pools.ple_state[1], si_all, ple_carry, ple_qsl) + + # Dense/sparse decision from a host read: a per-capture constant. + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + # `[T, 3]` view of the `[3, 1, T]` positions, as the QSA position plane stores them. + coords = torch.transpose(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]), 0, 1) + qsa = ( + md.qsa_req_idx_per_token[:num_tokens], + md.qsa_logical_positions[:num_tokens], + md.qsa_sequence_lengths[:num_tokens], + md.qsa_visible_blocks[:num_tokens], + md.qsa_topk_indices[:num_tokens], + md.qsa_topk_row_starts[:num_tokens], + md.qsa_block_table, + md.tokens_per_block, + coords[:num_tokens], + sparse, + ) + + # Verify rows write the PLE pools and QSA side caches for every draft row; reopened every + # forward, `SpeculativeRollback` stages what its commit needs to undo rejected ones. + rollback = core.spec_rollback + if rollback is not None: + rollback.discard() + if is_verify: + rollback.begin( + num_contexts=num_prefills, + prefix_tokens=num_prefill_tokens, + num_gens=num_decodes, + rows_per_gen=gen_p, + slots=si_d, + request_indices=md.qsa_req_idx_per_token[num_prefill_tokens:num_tokens], + logical_positions=md.qsa_logical_positions[num_prefill_tokens:num_tokens], + block_table=md.qsa_block_table, + tokens_per_block=md.tokens_per_block, + ) + fp8_pages = None + if ( + sparse + and core.quant_mode == QM_FP8_KV_CACHE + and num_prefill_tokens >= _ROWTILE_MIN_ROWS + ): + fp8_pages = _fp8_context_pages(md, md.tokens_per_block) + rowtile_groups = None + if fp8_pages is not None and self._rowtile_groups and num_decodes == 0: + kv0 = core._bound.pools.kv[core.attn_layers[0]] + rowtile_groups = _rowtile_scratch_groups(md, kv0.k[0].numel() * 4) + qsa_rows = None + if ( + sparse + and num_decodes == 0 + and not is_verify + and 0 < num_tokens <= physical_tokens <= _GRAPH_BREAKS_MAX_TOKENS + ): + qsa_rows = self._qsa_padded_rows(md, coords, num_tokens, physical_tokens) + sel_cuts = None + if sparse and qsa_rows is None and self._split_context_rows(num_prefill_tokens): + chunks = [int(c) for c in md.seq_lens[:num_prefills].tolist()] + kv_lens = [int(k) for k in md.kv_lens_runtime[:num_prefills].tolist()] + if sum(chunks) == num_prefill_tokens: + sel_cuts = _selection_cuts( + chunks, + [k - c for k, c in zip(kv_lens, chunks)], + self._selection_split[1], + _SELECTION_BALANCE_ROW_TOKENS, + ) + self._step = _PrefillStep( + num_tokens=num_tokens, + physical_tokens=physical_tokens, + dp_pad_rows=dp_pad_rows, + input_ids=input_ids[:num_tokens], + attn=attn_args, + gdn=gdn_state, + ple=ple_state, + qsa=qsa, + vision=vision, + stage=is_verify, + fp8_pages=fp8_pages, + rowtile_groups=rowtile_groups, + split=self._split_of(mm, num_prefills, num_decodes, hi_p, num_tokens), + qsa_rows=qsa_rows, + sel_cuts=sel_cuts, + ) + + def _qsa_padded_rows(self, md, coords, num_tokens: int, physical_tokens: int) -> tuple: + """QSA per-row metadata over the bucket's `T` rows: rows past `n` copy the last real row, + so padded rows read only what real rows read (their stores use position -2, which every + store kernel skips). Persistent buffers: the views keep their addresses across steps.""" + n, rows = num_tokens, physical_tokens + planes = ( + md.qsa_req_idx_per_token, + md.qsa_logical_positions, + md.qsa_sequence_lengths, + md.qsa_visible_blocks, + ) + if rows > n: + for plane in planes: + plane[n:rows].copy_(plane[n - 1 : n].expand(rows - n)) + # The coordinates are a fresh reshape each step: the graph reads them from its own buffer. + buf = self._coords_buf + if buf is None or buf.shape[0] < rows or buf.dtype != coords.dtype: + buf = torch.empty( + [max(rows, self.core.max_num_tokens), coords.shape[1]], + dtype=coords.dtype, + device=coords.device, + ) + self._coords_buf = buf + buf[:rows].copy_(coords[:rows]) + return tuple(plane[:rows] for plane in planes) + ( + md.qsa_topk_indices[:rows], + md.qsa_topk_row_starts[:rows], + buf[:rows], + ) + + def _split_of( + self, mm, num_prefills: int, num_decodes: int, hi_p, num_tokens: int + ) -> _Split | None: + """The step's `_Split` from `inforward_split` (planned for context-only batches, each split + chunk at least `_INFORWARD_MIN_MAIN` tokens before P), or None. No host read.""" + split = getattr(mm, "inforward_split", None) + if split is None: + return None + stage = self.core._bound.stage + assert stage is not None, ( + "the engine split a step for in-forward snapshots but the KV cache manager stages " + "no state for them" + ) + assert num_decodes == 0, "an in-forward snapshot split in a step with generation rows" + k = split.k + dev = split.cu.device + # Main parts carry as their request does; tails start from zeros and drop their state. + carry = torch.zeros([num_prefills + k], dtype=torch.bool, device=dev) + carry.index_copy_(0, split.main_rows, hi_p.to(torch.bool)) + # The row of P in the packed batch: one past the last row of the conv window ending at P. + p_rows = split.conv_win.view(k, -1)[:, -1:] + 1 + ple_len = stage.ple_conv.shape[-1] + ngram_len = stage.ple_ngram.shape[-1] + ple_win = p_rows + torch.arange(-ple_len, 0, dtype=p_rows.dtype, device=dev) + ngram_win = p_rows + torch.arange(-ngram_len, 0, dtype=p_rows.dtype, device=dev) + whole = None + v_heads = next(iter(stage.ssm.values())).shape[1] if stage.ssm else 0 + if v_heads and split_takes_cp(num_prefills, v_heads, dev): + if num_prefills == 1 and k == 1: + # One request split once: the rows before P are a prefix (no gather). + rows = num_tokens - split.num_tail_tokens + else: + # Every non-tail row, in order (a stable sort on the tail mask; no host read). + tail_mask = torch.zeros([num_tokens], dtype=torch.int8, device=dev) + tail_mask.index_fill_(0, split.tail_tok, 1) + rows = torch.argsort(tail_mask, stable=True)[: num_tokens - split.num_tail_tokens] + before = split.cu.index_select(0, split.main_rows + 1) - split.cu.index_select( + 0, split.main_rows + ) + cu = torch.zeros([num_prefills + 1], dtype=torch.long, device=dev) + torch.cumsum(before, 0, out=cu[1:]) + whole = (rows, cu, hi_p.to(torch.bool)) + return _Split( + k=k, + cu=split.cu, + gather=split.gather.to(torch.int32), + carry=carry, + main_rows=split.main_rows, + live=split.live.to(torch.int32), + tail_tok=split.tail_tok, + cu_tail=split.cu_tail, + stage_rows=split.stage_rows, + tail_slots=split.tail_slots, + conv_win=split.conv_win, + ple_win=ple_win.reshape(-1), + ngram_win=ngram_win.reshape(-1), + whole=whole, + raw=split, + ) + + def _pad_mask(self, physical_tokens: int, device) -> torch.Tensor | None: + """`[T, 1]` bool, True at a graph bucket's padding rows (past `_live_rows`), which the MoE + blocks keep out of the expert window. Computed on device, so a replay recomputes it; None + for an eager step and under attention DP (whose round trip drops padding itself).""" + if self.core.dp_size != 1: + return None + if ( + physical_tokens == self._step.num_tokens + and not torch.cuda.is_current_stream_capturing() + ): + return None + rows = torch.arange(physical_tokens, dtype=torch.int32, device=device) + return torch.unsqueeze(rows >= self._live_rows, 1) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor, + position_ids: torch.IntTensor, + **kwargs, + ): + """Run the trunk over `T` rows: the batch's `n`, or the prefill CUDA graph bucket's. + + A breakable graph captures this body per bucket and replays it without this Python, so + nothing here reads the step: ops are row-wise over `T`, and step-dependent work runs in + the breaks (`_splice`, `_ple_block`, `_attention_core`, `_gdn_core`), each writing rows + `[:n]` of a fixed output allocated here and zeroing rows `[n:]`.""" + core = self.core + num_tokens = input_ids.shape[0] + assert num_tokens == self._step.physical_tokens, ( + f"the forward got {num_tokens} activation rows and prepare saw " + f"{self._step.physical_tokens}" + ) + pos3 = _copy_positions(position_ids) + pos = pos3[0] + + # The token embedding (replicated) with this step's vision rows spliced in. + h = nn.functional.embedding(input_ids, core.w["embed"]) + eager_on_graph(h, inner=self._splice) + dt = h.dtype + dev = h.device + # Sequence parallelism (`_sp_split`): the stream runs on this rank's rows `[lo, hi)`; + # each block gathers all rows and leaves its sum on this rank's rows. + sp = self._sp_split(num_tokens) + if sp is None: + lo, hi, gather = 0, num_tokens, None + self._block_sum = core.tp_sum + self._sp_rows = None + else: + lo, hi, gather, self._block_sum = sp + self._sp_rows = (lo, hi, gather) + + # Scratch shared by every layer that uses it; each break rewrites its whole output. + proj_qkv = torch.empty([num_tokens, core.qkv_width], dtype=dt, device=dev) + index_scratch = torch.empty([num_tokens, core.index_scratch_width], dtype=dt, device=dev) + attn_out = torch.empty([num_tokens, core.q_width], dtype=dt, device=dev) + gdn_out = torch.empty( + [1, num_tokens, core.num_v_heads, core.head_v_dim], dtype=dt, device=dev + ) + ple_hc = torch.empty([num_tokens, core.stream], dtype=dt, device=dev) + ple_normed = torch.empty([num_tokens, core.stream], dtype=dt, device=dev) + + hc_count = core.hc_count + hc = torch.cat([h[lo:hi]] * hc_count, -1) + layers, S = self._layers, self._stack + pad_rows = self._step.dp_pad_rows + pad_mask = self._pad_mask(num_tokens, dev) + # From `_HC_RSTD_MIN_ROWS` rows (fixed per bucket) sites keep rstd and the mixer norms on + # chip; the PLE site always uses the stored-normed mixer on the PLE kernel's output. + by_rstd = hc.shape[0] >= _HC_RSTD_MIN_ROWS + normed = rstd = None + if by_rstd: + _, rstd = qwen4_exp_hc_inject_rstd(hc, None, None, core.eps, hc_count) + else: + _, normed = qwen4_exp_hc_inject_norm( + hc, None, None, layers[0].ahc_norm, core.eps, hc_count + ) + for i, L in enumerate(layers): + if i == core.ple_layer: + # The n-gram path convolves along each request's rows: it runs on all rows. + whole = hc if gather is None else gather(hc) + eager_on_graph(whole, ple_hc, ple_normed, inner=self._ple_block) + hc, normed = ple_hc[lo:hi], ple_normed[lo:hi] + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + elif by_rstd: + mixed, inject = _hc_mixer_rstd( + core, hc, rstd, L.ahc_norm, L.ahc_first, L.ahc_up, True + ) + else: + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + if core.is_attn[i]: + block_out = self._attention_block( + mixed, i, L, proj_qkv, index_scratch, attn_out, pos3, pos + ) + else: + block_out = self._gdn_block(mixed, i, L, gdn_out) + if by_rstd: + hc, rstd = qwen4_exp_hc_inject_rstd(hc, block_out, inject, core.eps, hc_count) + mixed, inject = _hc_mixer_rstd( + core, hc, rstd, L.mhc_norm, L.mhc_first, L.mhc_up, True + ) + else: + hc, normed = qwen4_exp_hc_inject_norm( + hc, block_out, inject, L.mhc_norm, core.eps, hc_count + ) + mixed, inject = _hc_mixer(core, normed, L.mhc_first, L.mhc_up, True) + next_norm = layers[i + 1].ahc_norm if i + 1 < len(layers) else S.fhc_norm + # The injection forms routed + sigmoid(gate) * shared, rounded once. + if gather is None: + routed, gate_logits, shared = _moe_block( + core, mixed, L, pad_rows, pad_mask=pad_mask + ) + else: + routed, gate_logits, shared = _moe_block_sp( + core, + mixed, + L, + gather, + self._block_sum, + None if pad_mask is None else pad_mask[lo:hi], + ) + if by_rstd: + hc, rstd = qwen4_exp_hc_inject_rstd( + hc, routed, inject, core.eps, hc_count, shared, gate_logits + ) + else: + hc, normed = qwen4_exp_hc_inject_norm( + hc, routed, inject, next_norm, core.eps, hc_count, shared, gate_logits + ) + del routed, gate_logits, shared + # The stack-final mixer with no injection weight is the final norm. + if by_rstd: + final, _ = _hc_mixer_rstd(core, hc, rstd, S.fhc_norm, S.fhc_first, S.fhc_up, False) + else: + final, _ = _hc_mixer(core, normed, S.fhc_first, S.fhc_up, False) + if not core.mtp_enabled: + return final if gather is None else gather(final) + # `lm_head` reads the `[T, hidden]` mix, the MTP module the `[T, stream]` stream; packed + # because a breakable graph body returns one tensor (`split_spec_output`). + packed = torch.cat([final, hc], -1) + return packed if gather is None else gather(packed) + + # ------------------------------------------------------------- the draft --- + + def _capture_selection(self, md, topk_indices, visible_blocks) -> None: + """IndexShare: store each request's selection at the row the draft loop continues from + (a context request's last chunk row, a generation request's last accepted row), with its + complete-group count, in the draft layer's share slot.""" + batch = md.num_seqs + assert md.qsa_mtp_num_accepted is not None, ( + "IndexShare needs the worker's accepted counts to pick each request's capture row; " + "the draft loop did not supply them" + ) + seq_lens = md.seq_lens_cuda[:batch].to(torch.int64) + starts = torch.cumsum(seq_lens, 0) - seq_lens + accepted = md.qsa_mtp_num_accepted[:batch].to(torch.int64) + offsets = torch.minimum(torch.clamp(accepted - 1, min=0), seq_lens - 1) + offsets[: md.num_contexts] = seq_lens[: md.num_contexts] - 1 + rows = starts + offsets + slot = self.core._draft_bound.share_slot + md.qsa_shared_topk_indices[slot, :batch].copy_(torch.index_select(topk_indices, 0, rows)) + md.qsa_shared_topk_visible_blocks[slot, :batch].copy_( + torch.index_select(visible_blocks, 0, rows) + ) + # Host-side readiness, recorded in program order. + md.qsa_shared_topk_captured_slots.add(slot) + + def _draft_extend_query_rows(self, md, draft, qkv, q_index, num_ctx: int, rows: int): + """Draft-layer sparse attention on a draft-extend step with context rows, over only the + rows the draft loop continues from (each context request's last row, every generation + row); the worker discards the rest. Query rows score row by row, since a chunk's last row + sees every key the chunk stored. Returns `(out, query)`: `[n, q_width]`, int64 rows.""" + core = self.core + num_contexts = md.num_contexts + ends = torch.cumsum(md.seq_lens_cuda[:num_contexts].to(torch.int64), 0) - 1 + query = torch.cat( + [ends, torch.arange(num_ctx, rows, dtype=torch.int64, device=ends.device)] + ) + n = num_contexts + rows - num_ctx + logical = md.qsa_logical_positions[:rows].index_select(0, query) + req_idx = md.qsa_req_idx_per_token[:rows].index_select(0, query) + scores = qsa_paged_index_scores( + q=q_index.index_select(0, query), + index_cache=draft.index_cache, + block_table=md.qsa_block_table, + query_positions=logical, + request_indices=req_idx, + tokens_per_block=md.tokens_per_block, + compress_ratio=core.compress_ratio, + only_visible_blocks=True, + context_rows=False, + wide_split=True, + ) + selected = torch.empty( + [n, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=qkv.device + ) + _selection( + core, + scores, + md.qsa_visible_blocks[:rows].index_select(0, query), + logical, + md.qsa_sequence_lengths[:rows].index_select(0, query), + md.qsa_topk_indices[:rows].index_select(0, query), + md.qsa_topk_row_starts[:rows].index_select(0, query), + selected, + ) + out = _sparse_gqa( + core, + draft.kv, + _packed_q(core, qkv).index_select(0, query), + md.qsa_block_table, + selected, + req_idx, + md.tokens_per_block, + None, + ) + return torch.reshape(out, [n, core.q_width]), query + + def mtp_forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + """One eager draft step with context rows (the draft loop's first). Sparse attention runs + only on the rows the worker gathers (`_draft_extend_query_rows`) unless IndexShare + captures every row's selection.""" + core = self.core + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert core._bound is not None, "the trunk's cache-manager bind has not run" + md = attn_metadata + rows = hidden_states.shape[0] + assert input_ids.shape[0] == rows and hidden_states.shape[1] == core.stream, ( + f"the draft step got {input_ids.shape[0]} token ids and {tuple(hidden_states.shape)} " + f"hidden states; the MTP module reads one {core.stream}-wide stream row per token" + ) + # Per call: the worker rewrites `attn_metadata` in place between draft steps. + step = self.step_args(md) + num_ctx = md.num_ctx_tokens + gen_seqs = md.num_seqs - md.num_contexts + p, _ = _verify_rows(core, None, gen_seqs, rows - num_ctx) + step["predicted_tokens_per_seq"] = p + draft, layer_idx = core._draft_step_bind(md) + assert rows == md.num_tokens, ( + f"the draft step carries {rows} rows for a batch its metadata describes as " + f"{md.num_tokens} tokens; the QSA row mapping covers the second" + ) + pad_rows = _draft_pad_rows(core, all_rank_num_tokens, rows) + # Same host buffer as the trunk, so every draft step takes the trunk's branch. + batch = md.num_seqs + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + # IndexShare: the loop's first step runs the indexer and captures what later steps reuse. + share = core.mtp_index_share and md.qsa_in_mtp_draft_loop + assert not (share and md.qsa_indexer_skip_topk), ( + "the worker asked a draft step with context rows to reuse a selection" + ) + pos3, pos, coords = _draft_positions(position_ids) + M = core._mtp + req_idx = md.qsa_req_idx_per_token[:rows] + logical = md.qsa_logical_positions[:rows] + block_table = md.qsa_block_table + tokens_per_block = md.tokens_per_block + + x, mixed, inject = _mtp_front(core, M, embed_tokens, input_ids, hidden_states) + proj, qkv, gate, k_dst, v_dst, index_scratch = _mtp_attention_in( + core, M, mixed, pos3, pos, True + ) + # The rows the tail runs on (`_draft_extend_query_rows`), or None for every row. + tail_rows = None + q_index = _mtp_index_branch( + core, + M, + proj, + index_scratch, + draft, + (req_idx, logical, block_table, tokens_per_block, coords), + ) + if sparse and num_ctx and not share: + kv = draft.kv + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + out, query = self._draft_extend_query_rows(md, draft, qkv, q_index, num_ctx, rows) + if core.dp_size == 1: + # Only the query rows are read: the tail runs on them, the rest stay zero. + attn, tail_rows = out, query + gate, x, inject = (t.index_select(0, query) for t in (gate, x, inject)) + else: + attn = torch.zeros([rows, core.q_width], dtype=qkv.dtype, device=qkv.device) + attn.index_copy_(0, query, out) + elif sparse: + kv = draft.kv + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + topk_indices = md.qsa_topk_indices[:rows] + visible_blocks = md.qsa_visible_blocks[:rows] + seq_lens_q = md.qsa_sequence_lengths[:rows] + row_starts = md.qsa_topk_row_starts[:rows] + parts = [(0, num_ctx, True)] if num_ctx else [] + if num_ctx < rows: + parts.append((num_ctx, rows, False)) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=qkv.device, + ) + for lo, hi, context_rows in parts: + scores = qsa_paged_index_scores( + q=q_index[lo:hi], + index_cache=draft.index_cache, + block_table=block_table, + query_positions=logical[lo:hi], + request_indices=req_idx[lo:hi], + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + only_visible_blocks=True, + context_rows=context_rows, + prefill_query_tile=True, + wide_split=True, + ) + _selection( + core, + scores, + visible_blocks[lo:hi], + logical[lo:hi], + seq_lens_q[lo:hi], + topk_indices[lo:hi], + row_starts[lo:hi], + selected[lo:hi], + ) + if share: + self._capture_selection(md, topk_indices, visible_blocks) + attn = torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device) + for lo, hi, context_rows in parts: + out = _sparse_gqa( + core, + kv, + _packed_q(core, qkv)[lo:hi], + block_table, + selected[lo:hi], + req_idx[lo:hi], + tokens_per_block, + logical[lo:hi] if context_rows else None, + ) + attn[lo:hi].copy_(torch.reshape(out, [hi - lo, core.q_width])) + else: + attn = thop_attention( + q=qkv, + output=torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device), + local_layer_idx=layer_idx, + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(attn, gate, direct_launch=True) + out = _mtp_tail(core, M, x, attn, inject, pad_rows) + if tail_rows is None: + return out + full = out.new_zeros([rows, out.shape[1]]) + full.index_copy_(0, tail_rows, out) + return full + + +# ------------------------------------------------------------------- decode --- + + +#: Generation-only steps up to this many rows run on a highest-priority stream with the MoE +#: shared expert on a default-priority side stream, so the routed (critical) path gets the SMs +#: first; on larger steps the starved shared expert would gate the join. +_DECODE_HIGH_PRIORITY_MAX_ROWS = 8 # capacity: fixed: a dispatch threshold, not a buffer size + + +def _on_decode_stream(core, rows, step, *args, **kwargs): + """Run a generation-only step of `rows` rows on the core's high-priority decode stream, + ordered after the caller's stream, which then waits for it (graph edges under capture; PDL + chains stay on one stream). Above the row limit or without the stream, run in line.""" + hi = getattr(core, "_decode_stream", None) if rows <= _DECODE_HIGH_PRIORITY_MAX_ROWS else None + if hi is None: + return step(*args, **kwargs) + caller = torch.cuda.current_stream() + hi.wait_stream(caller) + with torch.cuda.stream(hi): + out = step(*args, **kwargs) + caller.wait_stream(hi) + return out + + +class DecodeTarget: + """Generation-only batches (no context rows), every op inside the captured decode graph.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: + self.core = core + self._contract_pending = step_contract_enabled() + self._layers = _layer_views(core) + self._stack = _stack_views(core) + self._index_split = _index_split_group(core) + + def _check_step_contract(self, md) -> None: + if not self._contract_pending: + return + try: + self.step_args(md) + except AttributeError as exc: + raise AssertionError(f"attention metadata surface drifted: {exc}") from exc + self._contract_pending = False + + def step_args(self, md: TrtllmAttentionMetadata) -> dict: + """See PrefillTarget.step_args. Context counts are stated, not read: routing + guarantees them, and a decode-only CUDA graph would freeze whatever it read.""" + return _step_args(md, num_contexts=0, num_ctx_tokens=0) + + def _verify_qsl(self, seq_lens, batch: int, p: int, device) -> torch.Tensor: + """PLE segmentation `[0, p, ..., batch * p]` of a generation-only verify step, cached + when built outside capture (warmup) so a captured step launches nothing for it.""" + key = (batch, p, device) + cache = self.__dict__.setdefault("_verify_qsl_cache", {}) + hit = cache.get(key) + if hit is not None: + return hit + if torch.cuda.is_current_stream_capturing(): + return cu_seqlens_from_seq_lens(seq_lens, torch.int64) + qsl = torch.arange(batch + 1, dtype=torch.int64, device=device) * p + cache[key] = qsl + return qsl + + # --------------------------------------------------------------- blocks --- + + def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, stage): + """One gated GQA layer plus its QSA index branch for a generation-only batch. The index + plane is kept on every step; above the dense threshold the sparse core replaces the dense + op, and over a bf16 or e4m3 pool all work up to the index scores is one launch. Verify + rows (`stage`) have their index slots staged for `SpeculativeRollback` before the store.""" + core = self.core + ( + req_idx, + logical, + seq_lens_q, + visible_blocks, + topk_indices, + topk_row_starts, + block_table, + tokens_per_block, + pos3, + pos, + coords, + sparse, + ) = qsa + num_tokens = x.shape[0] + proj = _decode_projection(x, L.attn_in) + qkv_view, gate, _ = torch.split( + proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1 + ) + pools = core._bound.pools + prologue = _decode_prologue(pools.kv[i], sparse) + if stage: + core.spec_rollback.stage_index(pools.position_cache) + plan = getattr(self, "_index_plan", None) if sparse else None + if prologue: + _decode_prologue_call( + core, + proj, + qkv, + index_scratch, + i, + L, + (req_idx, logical, block_table, tokens_per_block, pos3, coords), + step["predicted_tokens_per_seq"], + None if plan is None else plan.iq_rows, + ) + else: + _, k_dst, v_dst = _attn_in_heads(core, proj) + _attention_rope(core, proj, index_scratch, L, pos3, pos) + _index_plane( + core, proj, i, L, (req_idx, logical, block_table, tokens_per_block, coords) + ) + # One compact `[q | k | v]` copy: read flat by the dense op, per head by sparse GQA. + qkv.copy_(qkv_view) + + if sparse: + kv = pools.kv[i] + if not prologue: + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + selected = _decode_index_selection( + core, + getattr(self, "_index_split", None), + _index_query(core, index_scratch), + qsa[:6] + (block_table, tokens_per_block), + num_tokens, + step["predicted_tokens_per_seq"], + proj.device, + core._bound.pools.index_cache[i], + plan, + q_permuted=prologue, + ) + attn = _sparse_gqa( + core, + kv, + _packed_q(core, qkv), + block_table, + selected, + req_idx, + tokens_per_block, + None, + output_gate=gate, + rows_per_request=step["predicted_tokens_per_seq"], + ) + out = torch.reshape(attn, [num_tokens, core.q_width]) + else: + out = thop_attention( + q=qkv, + output=attn_out, + local_layer_idx=core._bound.pools.pool_idx[i], + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(out, gate, direct_launch=True) + return _tp_partial(core, _decode_projection(out, L.o)) + + def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: + """One Gated-DeltaNet layer over a generation-only batch: one GEMM for the four input + projections, the conv step in place, then recurrence and gated norm in one launch per + (row, v-head), so over a bf16 pool a row's output does not depend on its batch. A verify + step (`verify`, `gen_p` rows a sequence) takes `_gdn_verify` instead.""" + core = self.core + pools = core._bound.pools + tactic = ( + _gdn_in_proj_conv_tactic(core, x, L, si_all, verify) if verify is not None else None + ) + mq_out = None + if tactic is not None: + # The projection and the conv of its q|k|v columns in one launch. + full, mq_out = gdn_in_proj_conv( + x, + L.in_proj_dec, + L.conv_w, + pools.conv[i], + si_all, + core._bound.int_conv[i], + verify[0], + _PAD_SLOT_ID, + gen_p, + tactic, + _DECODE_PROJ_PDL, + ) + proj = full[:, : sum(core.in_proj_split)] + else: + # Its alignment columns, past `sum(in_proj_split)`, are dropped by the view. + proj = _decode_projection(x, L.in_proj_dec)[:, : sum(core.in_proj_split)] + if verify is not None: + mixed_qkv, z, a, b = torch.split(proj, core.in_proj_split, 1) + num_tokens = x.shape[0] + fold_norm = _gdn_verify_folds_norm(core, num_tokens // gen_p, gen_p) + out = _gdn_verify( + core, + mixed_qkv, + a, + b, + i, + L, + si_all, + num_tokens // gen_p, + gen_p, + verify, + mq_out=mq_out, + z=z if fold_norm else None, + ) + if fold_norm: + # The replay returned the normed, gated rows. + o = torch.reshape(out, [num_tokens, core.value_dim]) + return _tp_partial(core, _decode_projection(o, L.out)) + return self._gdn_epilogue(out, z, L, num_tokens) + mixed_qkv = torch.split(proj, core.in_proj_split, 1)[0] + causal_conv1d_update( + x=mixed_qkv, + conv_state=pools.conv[i], + weight=L.conv_w, + bias=None, + silu_activation=True, + cache_seqlens=None, + conv_state_indices=si_all, + pad_slot_id=_PAD_SLOT_ID, + ) + # Computes both gates and the q/k L2 norm itself (softplus (1, 20), eps 1e-6). + o = gdn_decode_rows_gated_norm( + proj, + pools.ssm[i], + si_all, + L.a_log, + L.dt_bias, + L.gnorm, + core.eps, + core.num_k_heads, + core.head_k_dim, + ) + return _tp_partial(core, _decode_projection(o, L.out)) + + def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: + """A verify step's gated RMS norm (sigmoid gate, plain weight) and output projection.""" + core = self.core + o = rms_norm_gated_token_major( + torch.reshape(out, [-1, core.head_v_dim]), + torch.reshape(z, [num_tokens, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + fp8_scale=None, + gate_activation="sigmoid", + direct_launch=True, + ) + o = torch.reshape(o, [num_tokens, core.value_dim]) + return _tp_partial(core, _decode_projection(o, L.out)) + + def _ple_front(self, input_ids, ple_state, stage) -> tuple: + """The PLE part that reads no residual: verify staging of both pools (before the n-gram + history is written), then the n-gram embedding's `(key, value)` projections.""" + core = self.core + S = self._stack + conv_states, ngram_pool, si_all, qsl, rows = ple_state + if stage: + core.spec_rollback.stage_ple_verify(ngram_pool, conv_states, input_ids) + ngram = ple_ngram_embed( + input_ids, + ngram_pool, + si_all, + None, + qsl, + core.w["ple_table"], + S.ple_tscale, + S.ple_mult, + S.ple_vocab, + S.ple_offs, + core.eos_token_id, + rows, + ) + return cublas_mm(ngram, S.ple_key), cublas_mm(ngram, S.ple_val) + + def _ple_block( + self, hc, input_ids, ple_state, next_norm, stage, front=None, inject_block=None + ) -> tuple: + """The n-gram side path added into the stream and normed for the attention-side site; + returns `(hc, normed)`. No carry flags: every generation sequence carries. A verify step + (`stage`) stages both pools' pre-step rows and the conv input the second launch writes.""" + core = self.core + S = self._stack + conv_states, ngram_pool, si_all, qsl, rows = ple_state + key, val = front if front is not None else self._ple_front(input_ids, ple_state, stage) + conv_input = torch.empty_like(hc) if stage else None + from_pool = self._ple_from_pool(stage) + out = ple_gate_conv_inject_norm( + hc, + key, + val, + S.ple_nk_w, + S.ple_nq_w, + S.ple_nc_w, + S.ple_conv_w, + conv_states, + si_all, + qsl, + next_norm, + core.eps, + core.hc_count, + _PLE_CLAMP_MIN, + core.ple_dilation, + _PAD_SLOT_ID, + rows, + conv_input=conv_input, + inject_block=inject_block, + # The three launches chain through PDL; the norm weights are read before the wait. + pdl=True, + # every sequence here is a generation request the commit rewrites + publish_end=0 if from_pool else None, + ) + if stage: + core.spec_rollback.stage_ple_conv_input(conv_input) + return out + + def _ple_from_pool(self, stage) -> bool: + """Whether this verify step's PLE launch leaves the conv pool rows unpublished, for the + fused rollback commit to rebuild from the pool and the staged conv input.""" + return bool(stage) and self.core.spec_rollback.fused_pending() + + # -------------------------------------------------------------- forward --- + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + **kwargs, + ): + core = self.core + md = attn_metadata + self._check_step_contract(md) + step = self.step_args(md) + spec_metadata = kwargs.get("spec_metadata") + mm = md.mamba_metadata + pos3 = take_step_positions(md, position_ids) + if pos3 is None: + pos3 = _copy_positions(position_ids) + pos = pos3[0] + coords = torch.transpose(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]), 0, 1) + + if input_ids.dim() == 1 and input_ids.is_contiguous(): + # The embedding and its replication over the streams in one launch. + hc = embed_rows(input_ids, core.w["embed"], core.hc_count, False) + else: + hc = torch.cat( + [nn.functional.embedding(input_ids, core.w["embed"])] * core.hc_count, -1 + ) + num_tokens = hc.shape[0] + dt = hc.dtype + dev = hc.device + batch = md.num_seqs + # Decode CUDA graphs pad with real dummy requests, so they are in the metadata. + assert num_tokens == md.num_tokens, ( + f"the engine handed {num_tokens} activation rows for a generation batch its " + f"metadata describes as {md.num_tokens} tokens" + ) + # None under a decode CUDA graph, which pads every attention-DP rank to one batch. + pad_rows = core._dp_pad_rows(md, num_tokens) + gen_p, is_verify = _verify_rows(core, spec_metadata, batch, num_tokens) + step["predicted_tokens_per_seq"] = gen_p + si_all = mm.state_indices[:batch] + verify = _verify_state(core, mm, batch) if is_verify else None + pools = core._bound.pools + # As in PrefillTarget.prepare: rebuild the PLE segmentation when `gen_p > 1`. + ple_qsl = ( + mm.query_start_loc_long + if gen_p == 1 + else self._verify_qsl(md.seq_lens_cuda[:batch], batch, gen_p, dev) + ) + ple_state = (pools.ple_state[0], pools.ple_state[1], si_all, ple_qsl, gen_p) + + # Rollback staging as in PrefillTarget.prepare (`fused` under a decode CUDA graph). + rollback = core.spec_rollback + if rollback is not None: + rollback.discard() + if is_verify: + rollback.begin( + num_contexts=0, + prefix_tokens=0, + num_gens=batch, + rows_per_gen=gen_p, + slots=si_all, + request_indices=md.qsa_req_idx_per_token[:num_tokens], + logical_positions=md.qsa_logical_positions[:num_tokens], + block_table=md.qsa_block_table, + tokens_per_block=md.tokens_per_block, + fused=md.is_cuda_graph, + position_cache=pools.position_cache, + ) + + # Host-read dense/sparse decision: per capture, as the short/long decode graph families. + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + qsa = ( + md.qsa_req_idx_per_token[:num_tokens], + md.qsa_logical_positions[:num_tokens], + md.qsa_sequence_lengths[:num_tokens], + md.qsa_visible_blocks[:num_tokens], + md.qsa_topk_indices[:num_tokens], + md.qsa_topk_row_starts[:num_tokens], + md.qsa_block_table, + md.tokens_per_block, + pos3, + pos, + coords, + sparse, + ) + split = getattr(self, "_index_split", None) + self._index_plan = None + if sparse and split is not None: + self._index_plan = _index_split_plan(split, qsa, num_tokens, gen_p) + + # Scratch shared by all attention layers; each layer rewrites what it reads. + qkv = torch.empty([num_tokens, core.qkv_width], dtype=dt, device=dev) + index_scratch = torch.empty([num_tokens, core.index_scratch_width], dtype=dt, device=dev) + attn_out = torch.empty([num_tokens, core.q_width], dtype=dt, device=dev) + + hc_count = core.hc_count + layers, S = self._layers, self._stack + # The PLE front reads no residual, so it runs on the PLE stream beside the first layers. + ple_stream = getattr(core, "_ple_stream", None) + ple_front = None + if ple_stream is not None and core.ple_layer > 0: + caller = torch.cuda.current_stream() + ple_stream.wait_stream(caller) + with torch.cuda.stream(ple_stream): + ple_front = self._ple_front(input_ids, ple_state, is_verify) + # Each site injects the previous MoE block's `(routed, inject, shared, gate_logits)`. + moe = (None, None, None, None) + for i, L in enumerate(layers): + if i == core.ple_layer: + # The PLE path adds into the injected stream and normalizes it itself. + routed, inject, shared, gate_logits = moe + if isinstance(routed, _TpPartial): + routed = core.tp_sum(routed.partial) + # Fold the previous MoE injection into the PLE launches (bitwise equal to + # `qwen4_exp_hc_inject_norm` followed by the PLE op). + fold = None + if routed is not None and _ple_fold_fits(hc, routed, inject, shared): + fold = (routed, inject, shared, gate_logits) + elif routed is not None or inject is not None: + hc, _ = qwen4_exp_hc_inject_norm( + hc, routed, inject, L.ahc_norm, core.eps, hc_count, shared, gate_logits + ) + if ple_front is not None: + torch.cuda.current_stream().wait_stream(ple_stream) + hc, normed = self._ple_block( + hc, + input_ids, + ple_state, + L.ahc_norm, + is_verify, + front=ple_front, + inject_block=fold, + ) + ple_front = None + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + else: + hc, mixed, inject = _hc_site( + core, + hc, + moe[0], + moe[1], + L.ahc_norm, + L.ahc_first, + L.ahc_up, + True, + *moe[2:], + ) + moe = None + if core.is_attn[i]: + block_out = self._attention_block( + mixed, i, L, qkv, index_scratch, attn_out, qsa, step, is_verify + ) + else: + block_out = self._gdn_block(mixed, i, L, si_all, verify, gen_p) + hc, mixed, inject = _hc_site( + core, hc, block_out, inject, L.mhc_norm, L.mhc_first, L.mhc_up, True + ) + routed, gate_logits, shared = _moe_block( + core, mixed, L, pad_rows, mm=_decode_projection, defer=True + ) + moe = (routed, inject, shared, gate_logits) + del routed, gate_logits, shared + hc, final, _ = _hc_site( + core, hc, moe[0], moe[1], S.fhc_norm, S.fhc_first, S.fhc_up, False, *moe[2:] + ) + del moe + if spec_metadata is None: + return final + # `lm_head` reads the `[T, hidden]` mix, the MTP module the `[T, stream]` stream. + return final, hc + + # ------------------------------------------------------------- the draft --- + + def _capture_selection(self, md, topk_indices, visible_blocks, batch, p): + """IndexShare: store each request's selection at its last accepted row (row + `num_accepted - 1` of its `p`) in the draft layer's share slot. Runs on the capture + stream and returns it (None if in line); the caller joins it before `topk_indices` / + `visible_blocks` are rewritten or the shared rows are read.""" + assert md.qsa_mtp_num_accepted is not None, ( + "IndexShare needs the worker's accepted counts to pick each request's capture row; " + "the draft loop did not supply them" + ) + stream = getattr(self.core, "_capture_stream", None) + if stream is not None: + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream) if stream is not None else contextlib.nullcontext(): + slot = self.core._draft_bound.share_slot + accepted = md.qsa_mtp_num_accepted[:batch].to(torch.int64) + rows = torch.arange(batch, dtype=torch.int64, device=topk_indices.device) * p + rows = rows + torch.clamp(accepted - 1, min=0, max=p - 1) + md.qsa_shared_topk_indices[slot, :batch].copy_( + torch.index_select(topk_indices, 0, rows) + ) + md.qsa_shared_topk_visible_blocks[slot, :batch].copy_( + torch.index_select(visible_blocks, 0, rows) + ) + # Host-side readiness, recorded in program order: under capture this runs while the + # graph is recorded, so a graph holds the reuse only after the capture it reads. + md.qsa_shared_topk_captured_slots.add(slot) + return stream + + def mtp_forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + """One draft step over generation rows only (`p` rows a request on the first step of a + batch without context, one otherwise), inside the decode graph when the trunk is. + + Under IndexShare the draft-extend step captures each request's selection and later steps + skip the index branch, rebuilding it with `qsa_mtp_reuse_topk`; the index keys they skip + lie past the committed length and the next draft-extend step rewrites them.""" + core = self.core + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert core._bound is not None, "the trunk's cache-manager bind has not run" + md = attn_metadata + rows = hidden_states.shape[0] + assert input_ids.shape[0] == rows and hidden_states.shape[1] == core.stream, ( + f"the draft step got {input_ids.shape[0]} token ids and {tuple(hidden_states.shape)} " + f"hidden states; the MTP module reads one {core.stream}-wide stream row per token" + ) + step = self.step_args(md) + gen_seqs = md.num_seqs + p, _ = _verify_rows(core, None, gen_seqs, rows) + step["predicted_tokens_per_seq"] = p + draft, layer_idx = core._draft_step_bind(md) + assert rows == md.num_tokens, ( + f"the draft step carries {rows} rows for a batch its metadata describes as " + f"{md.num_tokens} tokens; the QSA row mapping covers the second" + ) + pad_rows = _draft_pad_rows(core, all_rank_num_tokens, rows) + # The worker advances only device KV lengths, so this matches the trunk's host decision. + sparse = gen_seqs > 0 and int(md.kv_lens_runtime[:gen_seqs].max()) > core.dense_threshold + share = core.mtp_index_share and md.qsa_in_mtp_draft_loop + reuse = share and md.qsa_indexer_skip_topk + assert not share or reuse == (p == 1), ( + f"the worker's reuse flag ({md.qsa_indexer_skip_topk}) disagrees with the step's " + f"shape ({p} rows per request): only the draft decode steps after the draft-extend " + "step reuse its selection" + ) + pos3, pos, coords = _draft_positions(position_ids) + M = core._mtp + req_idx = md.qsa_req_idx_per_token[:rows] + logical = md.qsa_logical_positions[:rows] + block_table = md.qsa_block_table + tokens_per_block = md.tokens_per_block + + x, mixed, inject = _mtp_front( + core, M, embed_tokens, input_ids, hidden_states, mm=_decode_projection + ) + qkv, gate, q_index = _mtp_attention_planes( + core, + M, + mixed, + draft, + (req_idx, logical, block_table, tokens_per_block, pos3, pos, coords), + p, + not reuse, + sparse, + ) + # Rows whose outputs are read: each request's continuing row on draft-extend, else all. + cont, pre = None, {} + if _draft_continue_ok(core, md, p, share): + meta_rows = ( + md.qsa_visible_blocks, + md.qsa_sequence_lengths, + md.qsa_logical_positions, + md.qsa_req_idx_per_token, + ) + if all(t.is_contiguous() for t in (md.qsa_mtp_num_accepted, *meta_rows)): + # The rows, the metadata and the step's operands at them in two launches. + cont, meta = draft_continue_select(md.qsa_mtp_num_accepted, gen_seqs, p, meta_rows) + pre.update(zip(("visible", "seq_lens", "logical", "req_idx"), meta)) + named = {"x": x, "inject": inject} + if sparse: + named.update(qkv=qkv, gate=gate) + if q_index is not None: + named["q_index"] = q_index.reshape(rows, -1) + named = {k: t for k, t in named.items() if t.dim() == 2 and t.stride(1) == 1} + pre.update(zip(named, draft_rows_gather(cont, tuple(named.values())))) + if "q_index" in pre: + pre["q_index"] = pre["q_index"].view(gen_seqs, *q_index.shape[1:]) + else: + # The fused select needs contiguous vectors; else `sel` gathers each operand. + cont = _draft_continue_rows(core, md, gen_seqs, p, share) + n = rows if cont is None else gen_seqs + + def sel(t, name=None): + if cont is None: + return t + got = pre.get(name) + return got if got is not None else t.index_select(0, cont) + + capture = None # the stream the IndexShare capture runs on, joined below + if sparse: + kv = draft.kv + topk_indices = md.qsa_topk_indices[:n] + visible_blocks = sel(md.qsa_visible_blocks[:rows], "visible") + seq_lens_q = sel(md.qsa_sequence_lengths[:rows], "seq_lens") + q_logical, q_req_idx = sel(logical, "logical"), sel(req_idx, "req_idx") + selected = torch.empty( + [n, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=qkv.device, + ) + if reuse: + slot = draft.share_slot + assert slot in md.qsa_shared_topk_captured_slots, ( + "a draft decode step would reuse a selection that no draft-extend step of " + "this loop captured" + ) + qsa_mtp_reuse_topk( + shared_indices=md.qsa_shared_topk_indices[slot], + shared_visible_blocks=md.qsa_shared_topk_visible_blocks[slot], + request_indices=q_req_idx, + visible_blocks=visible_blocks, + output=topk_indices, + ) + # Use the expansion's output as the selection when it matches (no copy). + expanded = qsa_expand_block_indices( + topk_indices, + q_logical, + seq_lens_q, + compress_ratio=core.compress_ratio, + token_topk=core.token_topk, + ) + if expanded.shape == selected.shape and expanded.dtype == selected.dtype: + selected = expanded + else: + selected.copy_(expanded) + else: + # IndexShare's capture keeps every row's block top-k: no split there. + selected = _decode_index_selection( + core, + None if share else getattr(self, "_index_split", None), + sel(q_index, "q_index"), + (q_req_idx, q_logical, seq_lens_q, visible_blocks, topk_indices, + md.qsa_topk_row_starts[:n], block_table, tokens_per_block), + n, + n // gen_seqs, + qkv.device, + draft.index_cache, + ) # fmt: skip + if share: + capture = self._capture_selection(md, topk_indices, visible_blocks, gen_seqs, p) + attn = torch.reshape( + _sparse_gqa( + core, + kv, + _packed_q(core, sel(qkv, "qkv")), + block_table, + selected, + q_req_idx, + tokens_per_block, + None, + output_gate=sel(gate, "gate"), + rows_per_request=n // gen_seqs if n % gen_seqs == 0 else 1, + ), + [n, core.q_width], + ) + else: + attn = thop_attention( + q=qkv, + output=torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device), + local_layer_idx=layer_idx, + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(attn, gate, direct_launch=True) + attn = sel(attn) + out = _mtp_tail( + core, M, sel(x, "x"), attn, sel(inject, "inject"), pad_rows, mm=_decode_projection + ) + if capture is not None: + torch.cuda.current_stream().wait_stream(capture) + if cont is None: + return out + # The worker reads only the continuing rows; the rest are left unwritten. + full = out.new_empty([rows, out.shape[1]]) + full.index_copy_(0, cont, out) + return full + + +# ------------------------------------------------------------ the MTP module --- + + +class MTPLayer: + """The checkpoint's MTP module (one decoder layer behind `fc_embedding` / `fc_hidden`), run + once per draft step and routed to a target by each call's metadata phase. Returns the + residual un-collapsed (`shared_head` owns the mixer). Not an `nn.Module`: its weights live + in the core's `ParameterDict`, and a second registration would break load coverage.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2, logits_processor) -> None: + self.core = core + self.logits_processor = logits_processor + # The spec worker when it samples with the TP key exchange (`_sharded_head`). + self.worker = None + # The engine step's `spec_metadata`, set by the shell before the worker runs. + self.spec_metadata = None + + def __call__(self, *args, **kwargs) -> torch.Tensor: + return self.forward(*args, **kwargs) + + def _sharded_head(self, lm_head, rows: int) -> bool: + """Whether the draft logits may stay this rank's vocabulary shard (greedy tokens via the + key exchange); not under rejection sampling, which needs the gathered fp32 logits.""" + worker, md = self.worker, self.spec_metadata + if worker is None or md is None: + return False + exchange = worker.key_exchange + return bool( + exchange is not None + and not md.wants_advanced_draft_sampling + and worker.guided_decoder is None + and 1 <= rows <= exchange.MAX_ROWS + and lm_head.tp_mode == TensorParallelMode.COLUMN + and lm_head.gather_output + and lm_head.padding_size == 0 + ) + + def shared_head(self, hidden_states, lm_head, attn_metadata, return_context_logits): + """The module's own stack-final mixer, then `lm_head`; with the key exchange + (`_sharded_head`) this rank's bf16 vocabulary shard.""" + core = self.core + M = core._mtp + _, final, _ = _hc_site( + core, hidden_states, None, None, M.fhc_norm, M.fhc_first, M.fhc_up, False + ) + if self._sharded_head(lm_head, final.shape[0]): + return lm_head(final, gather_output=False) + return self.logits_processor.forward(final, lm_head, attn_metadata, return_context_logits) + + def forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + target = self.core._select_target(attn_metadata) + args = (embed_tokens, all_rank_num_tokens, input_ids, position_ids, hidden_states) + if isinstance(target, DecodeTarget): + return _on_decode_stream( + self.core, input_ids.shape[0], target.mtp_forward, *args, attn_metadata, **kwargs + ) + return target.mtp_forward(*args, attn_metadata, **kwargs) + + +class DraftModel: + """The spec worker's handle on the drafter: `mtp_layers`, `embed_tokens` and `lm_head`. + `embed_tokens` reads through the core, since materialization replaces the meta tensors.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2, lm_head, logits_processor) -> None: + self.core = core + self.mtp_layers = [MTPLayer(core, logits_processor)] + self.lm_head = lm_head + + @property + def embed_tokens(self) -> torch.Tensor: + return self.core.w["embed"] + + +def _spec_rollback_begin_ok( + request_indices, logical_positions, block_table, position_cache +) -> bool: + """Whether `spec_rollback_begin` accepts these operands (mirrors its `is_valid`).""" + return ( + logical_positions.dtype == torch.int64 + and request_indices.dtype in (torch.int32, torch.int64) + and block_table.dtype == torch.int32 + and block_table.dim() == 2 + and block_table.stride(1) == 1 + and position_cache.dtype == torch.int32 + and position_cache.dim() == 3 + and position_cache.shape[2] == 3 + and position_cache.stride(2) == 1 + ) + + +class SpeculativeRollback: + """Undoes what a verify step's rejected draft rows wrote to side caches the runtime does not + promote (it promotes the GDN state): the PLE n-gram history and conv state, each attention + layer's QSA index-K plane and the shared QSA position plane. + + For a request that accepted `a` rows (golden token included, `1 <= a <= p`) the commit sets + the n-gram row to row `a - 1`'s id window minus its oldest id, the conv row to columns + `[a, a + state_len)` of `[pre-step state | the p rows' conv input]`, and every index-K and + position slot written by an ordinal `>= a` back to its pre-verify value. + + The targets stage during the forward and the spec worker calls `commit_speculative_states` + after its draft loop in the same forward, so a captured graph holds both ends. A `fused` step + (decode graphs) stages and commits the PLE pools in one launch each.""" + + def __init__(self) -> None: + self._pending: dict | None = None + # (base addresses, page strides, plane count) of the index-K planes, once bound. + self._planes: tuple | None = None + + def set_index_planes(self, planes: list) -> None: + """Bind the attention layers' index-K planes (static for the manager's lifetime).""" + ptrs, strides = _plane_table(planes) + self._planes = (ptrs, strides, len(planes)) + + # ------------------------------------------------------------- staging --- + + def discard(self) -> None: + self._pending = None + + def begin( + self, + *, + num_contexts: int, + prefix_tokens: int, + num_gens: int, + rows_per_gen: int, + slots: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + fused: bool = False, + position_cache: torch.Tensor | None = None, + ) -> None: + """Open a verify step's staging, before the trunk writes any index plane. Generation rows + follow the `prefix_tokens` context rows as `num_gens` runs of `rows_per_gen` (row + `s * p + t` is ordinal `t` of request `s`); each row's QSA slot is resolved as + `qsa_paged_index_store` does and every index plane's slots are gathered. A `fused` step + given `position_cache` also gathers the position plane, in one launch.""" + n, p = num_gens, rows_per_gen + assert n > 0 and p > 1, ( + f"a verify step stages generation requests carrying drafts; got {n} requests of " + f"{p} rows" + ) + assert logical_positions.shape[0] == n * p, ( + f"{n} generation requests of {p} rows are {n * p} rows; got " + f"{logical_positions.shape[0]} positions" + ) + assert self._planes is not None, "the attention layers' index planes are not bound" + ptrs, strides, count = self._planes + if ( + fused + and position_cache is not None + and _spec_rollback_begin_ok( + request_indices, logical_positions, block_table, position_cache + ) + ): + pages, within, index_all, position_old = spec_rollback_begin( + request_indices, + logical_positions, + block_table, + tokens_per_block, + ptrs, + strides, + position_cache, + ) + self._pending = dict( + num_contexts=num_contexts, + prefix=prefix_tokens, + n=n, + p=p, + slots=slots.to(torch.long), + pages=pages, + within=within, + staged_layers=0, + position=(position_cache, position_old), + index_all=index_all, + fused=fused, + ngram_stream=None, + ) + return + positions = logical_positions.to(torch.long) + pages = block_table[request_indices.to(torch.long), positions // tokens_per_block] + pages = pages.to(torch.long) + within = positions % tokens_per_block + self._pending = dict( + num_contexts=num_contexts, + prefix=prefix_tokens, + n=n, + p=p, + slots=slots.to(torch.long), + pages=pages, + within=within, + staged_layers=0, + position=None, + index_all=qsa_rollback_gather(ptrs, strides, pages, within, count), + fused=fused, + ngram_stream=None, + ) + + def fused_pending(self) -> bool: + return self._pending is not None and bool(self._pending["fused"]) + + def stage_ple_verify( + self, ngram_pool: torch.Tensor, conv_pool: torch.Tensor, input_ids: torch.Tensor + ) -> None: + """Stage the PLE pools before either is written: the slots' pre-step conv rows (not on a + fused step, `DecodeTarget._ple_from_pool`) and the n-gram candidates, row `t`'s being the + last `context` ids of `[history | ids of rows 0..t]`.""" + pending = self._pending + prefix, n, p = pending["prefix"], pending["n"], pending["p"] + slots = pending["slots"] + pending["ngram_pool"] = ngram_pool + pending["conv_pool"] = conv_pool + if pending["fused"]: + pending["ngram_stream"], pending["conv_prev"] = spec_rollback_stage( + ngram_pool, + conv_pool, + slots, + input_ids[prefix : prefix + n * p], + p, + conv=False, + ) + return + pending["conv_prev"] = conv_pool[slots] + context = ngram_pool.shape[1] + ids = torch.reshape(input_ids[prefix : prefix + n * p], [n, p]).to(ngram_pool.dtype) + stream = torch.cat([ngram_pool[slots], ids], 1) + # Window k of the unfold is the history after k rows. + pending["ngram_candidates"] = stream.unfold(1, context, 1)[:, 1:] + + def stage_ple_conv_input(self, conv_input: torch.Tensor) -> None: + """Stage the generation rows' conv input (`[T, C]`, token-major); a mixed batch's rows + are copied so its context prefix is not kept alive through the draft loop.""" + pending = self._pending + prefix, n, p = pending["prefix"], pending["n"], pending["p"] + conv_x = torch.reshape(conv_input[prefix:], [n, p, -1]) + pending["conv_x"] = conv_x.clone() if prefix else conv_x + + def stage_index(self, position_cache: torch.Tensor) -> None: + """Before an attention layer's index store: on the first, stage the position slots.""" + pending = self._pending + if pending["position"] is None: + pages, within = pending["pages"], pending["within"] + pending["position"] = (position_cache, position_cache[pages, within]) + pending["staged_layers"] += 1 + + # -------------------------------------------------------------- commit --- + + def commit_speculative_states( + self, + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor, + num_contexts: int, + ) -> None: + """Rewrite the staged planes to the accepted prefix. Called by the spec worker.""" + pending, self._pending = self._pending, None + if pending is None: + return + n, p = pending["n"], pending["p"] + assert num_contexts == pending["num_contexts"], ( + f"the worker commits with {num_contexts} context requests and the forward staged " + f"{pending['num_contexts']}" + ) + assert state_indices.shape[0] == num_contexts + n, ( + f"the worker hands {state_indices.shape[0]} slots for a batch the forward staged " + f"as {num_contexts} + {n} requests" + ) + assert pending.get("conv_x") is not None and pending["staged_layers"] > 0, ( + "a verify step staged neither its PLE rows nor any attention layer; the forward " + "did not run the staging it opened" + ) + ptrs, strides, _ = self._planes + position, position_old = pending["position"] + pages, within = pending["pages"], pending["within"] + if pending["fused"]: + spec_rollback_commit( + num_accepted_tokens[num_contexts : num_contexts + n], + state_indices[num_contexts:], + pending["ngram_pool"], + pending["ngram_stream"], + pending["conv_pool"], + pending["conv_prev"], + pending["conv_x"], + position, + position_old, + pages, + within, + ptrs, + strides, + pending["index_all"], + ) + return + accepted = num_accepted_tokens[num_contexts : num_contexts + n].to(torch.long) + slots = state_indices[num_contexts:].to(torch.long) + rows = torch.arange(n, device=accepted.device) + ngram_pool, conv_pool = pending["ngram_pool"], pending["conv_pool"] + state_len = conv_pool.shape[-1] + ngram_pool[slots] = pending["ngram_candidates"][rows, accepted - 1] + # [n, C, state_len + p]; window k of the unfold is the state after k tokens. + history = torch.cat([pending["conv_prev"], torch.transpose(pending["conv_x"], 1, 2)], 2) + conv_pool[slots] = history.unfold(2, state_len, 1)[rows, :, accepted] + # [n * p]: True at every ordinal the sampler rejected. + rejected = torch.reshape( + torch.arange(p, device=accepted.device).unsqueeze(0) >= accepted.unsqueeze(1), [-1] + ) + qsa_rollback_restore(ptrs, strides, pages, within, rejected, pending["index_all"]) + position[pages, within] = torch.where( + rejected[:, None], position_old, position[pages, within] + ) + + +# -------------------------------------------------------------------- shell --- + + +class _WidenedHead: + """`lm_head` widening bf16 logits with the vectorized `bf16_to_fp32` (bit-identical to + `.float()`), so the logits processor's own `.float()` is a no-op.""" + + def __init__(self, lm_head) -> None: + self.lm_head = lm_head + + def __call__(self, hidden_states: torch.Tensor) -> torch.Tensor: + logits = self.lm_head(hidden_states) + if logits.dtype == torch.bfloat16 and logits.is_contiguous(): + return bf16_to_fp32(logits) + return logits + + +class _WideningLogitsProcessor(LogitsProcessor): + """The shell's logits processor over `_WidenedHead`, for trunk and draft logits.""" + + def forward( + self, hidden_states, lm_head, attn_metadata, return_context_logits: bool = False + ) -> torch.Tensor: + return super().forward( + hidden_states, _WidenedHead(lm_head), attn_metadata, return_context_logits + ) + + +class ModelingV2Qwen38FlashNextFp8Sm103Unified( + DecoderModelForCausalLM[Qwen4ExpUnifiedModelingV2, PretrainedConfig] +): + """Qwen3.8-Flash-Next-FP8 at tp1, tp2, tep2, dep2, tp4, tep4 or dep4 on sm_103.""" + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: + """V2: the only KV cache manager with PLE state and QSA's index and position planes.""" + return "V2" + + @staticmethod + def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: str) -> None: + """Raise `NoTarget` for a checkpoint or engine choice this target does not run. + + `route` calls it once the GPU, `layout` (routing's `_FP8_LAYOUTS`) and draft path match, + so `auto` falls back to the built-in implementation instead of failing later.""" + cfg = model_config.pretrained_config + text = getattr(cfg, "text_config", None) + vision = getattr(cfg, "vision_config", None) + if text is None or vision is None: + raise NoTarget( + f"{type(cfg).__name__} carries no text_config/vision_config; the target reads " + "both halves of the normalized Qwen4-Exp composite" + ) + shape = ( + text.num_hidden_layers, + text.hidden_size, + text.num_experts, + text.num_attention_heads, + text.linear_num_value_heads, + text.vocab_size, + ) + vshape = ( + vision.depth, + vision.hidden_size, + vision.num_heads, + vision.spatial_merge_size, + vision.out_hidden_size, + vision.num_position_embeddings, + tuple(getattr(vision, "deepstack_visual_indexes", None) or ()), + vision.hidden_act, + ) + if (shape, vshape) != (_TEXT_SHAPE, _VISION_SHAPE): + raise NoTarget( + f"text shape {shape}, vision shape {vshape}; the target is Qwen3.8-Flash-Next's " + f"{_TEXT_SHAPE}, vision {_VISION_SHAPE}" + ) + spec = model_config.spec_config + if spec is not None: + # One linear MTP chain, verified at `max_draft_len + 1` rows on the GDN cached replay. + if ( + getattr(spec, "num_nextn_predict_layers", None) not in (None, 1) + or getattr(spec, "use_dynamic_tree", False) + or getattr(spec, "eagle_choices", None) is not None + or spec.max_total_draft_tokens != spec.max_draft_len + ): + raise NoTarget( + "an MTP draft other than one linear chain over the checkpoint's single MTP " + "module" + ) + if not 1 <= spec.max_draft_len < _MAX_VERIFY_WIDTH: + raise NoTarget( + f"max_draft_len={spec.max_draft_len}; the verify step is certified at " + f"max_draft_len 1..{_MAX_VERIFY_WIDTH - 1}" + ) + if ( + getattr(spec, "draft_len_schedule", None) is not None + or getattr(spec, "acceptance_rate_threshold", None) is not None + ): + raise NoTarget( + "the draft length changes at runtime (draft_len_schedule, max_concurrency or " + "acceptance_rate_threshold), but the verify step runs at the one width " + "max_draft_len + 1" + ) + if not is_gdn_replay_enabled(): + raise NoTarget( + "MTP with TRTLLM_USE_GDN_REPLAY=0: the verify step runs the GDN cached replay, " + "not the legacy per-step recurrent states; leave TRTLLM_USE_GDN_REPLAY unset" + ) + prefill_graph = str(llm_args.prefill_cuda_graph_backend.value) + if prefill_graph == "piecewise": + raise NoTarget( + "prefill_cuda_graph_backend='piecewise' makes the engine torch.compile the model, " + "which the target does not implement; 'breakable' runs the prefill step as a " + "breakable CUDA graph" + ) + if model_config.mapping.enable_attention_dp and prefill_graph == "breakable": + raise NoTarget( + f"prefill_cuda_graph_backend='breakable' with attention DP ({layout}): the engine " + "pads every rank to one prefill bucket, so a rank holding only generation " + "requests is handed more rows than its batch, which the decode step refuses" + ) + kv_cache_quant_algo = engine_kv_cache_quant_algo(model_config, llm_args) + if kv_cache_quant_algo not in (None, QuantAlgo.FP8, QuantAlgo.NVFP4): + raise NoTarget( + f"a {kv_cache_quant_algo} KV cache; the target stores K/V as the checkpoint's " + "bf16, as FP8 or as NVFP4" + ) + if model_config.torch_dtype != torch.bfloat16: + raise NoTarget( + f"the engine runs the model in {model_config.torch_dtype}; the target is bf16" + ) + if str(model_config.moe_backend_requested).upper() not in _TRUNK_MOE_BACKENDS: + raise NoTarget( + f"moe_config.backend={model_config.moe_backend_requested!r}; the target runs the " + f"routed experts on {', '.join(_TRUNK_MOE_BACKENDS)}" + ) + state_dtype = effective_mamba_ssm_cache_dtype( + model_config, llm_args.kv_cache_config.mamba_ssm_cache_dtype + ) + if state_dtype not in _SSM_STATE_DTYPES: + served = " or ".join(str(dtype).removeprefix("torch.") for dtype in _SSM_STATE_DTYPES) + raise NoTarget( + f"a {state_dtype} recurrent-state cache; the target serves {served} " + "(kv_cache_config.mamba_ssm_cache_dtype)" + ) + + @property + def inforward_snapshot_min_main(self) -> int: + """Fewest chunk tokens before the snapshot point for an in-forward snapshot split.""" + return _INFORWARD_MIN_MAIN + + def __init__(self, model_config: ModelConfig): + cfg = model_config.pretrained_config + assert cfg is not None + # Adopt the normalized text tower as the config in place: the pools are sized from it, + # and the engine keeps its own reference to this `ModelConfig`. + text = getattr(cfg, "text_config", None) + assert text is not None, ( + "this target is the language tower of the composite Qwen4-Exp export; " + "pretrained_config carries no text_config" + ) + assert getattr(text, "architectures", None) == ["Qwen4ExpForCausalLM"], ( + f"the text tower must be the normalized Qwen4-Exp text config; got architectures " + f"{getattr(text, 'architectures', None)!r}" + ) + composite = cfg + model_config.pretrained_config = text + super().__init__( + Qwen4ExpUnifiedModelingV2(model_config, composite), + config=model_config, + hidden_size=text.hidden_size, + vocab_size=text.vocab_size, + ) + # `lm_head` is outside the quantization scheme: a dense bf16 `LMHead`. + assert self.lm_head.weight.dtype == torch.bfloat16, ( + f"expected a bf16 lm_head; the shell built {self.lm_head.weight.dtype}" + ) + self.logits_processor = _WideningLogitsProcessor() + # No separate draft checkpoint: the MTP module's weights are the target's own. + self.draft_config = None + self.spec_config = getattr(model_config, "spec_config", None) + self.draft_model = None + self.spec_worker = None + # Read by the engine (`model_engine._spec_decode_fast_paths`, `runtime_hooks_of`). + self.spec_decode_fast_paths = False + self.runtime_hooks = None + if self.spec_config is not None: + self.draft_model = DraftModel(self.model, self.lm_head, self.logits_processor) + # Must match the resource manager, or the draft step gets no draft layer mapped. + self.use_separate_draft_kv_cache = should_use_separate_draft_kv_cache(self.spec_config) + from tensorrt_llm._torch.distributed import ll_exchange + + # The engine's one-engine fast paths, written for linear one-model MTP-Eagle: the + # only speculation routing and check_deployment admit. + self.spec_decode_fast_paths = True + self.runtime_hooks = _runtime_hooks() + # Collective over the TP group (every rank constructs the model in step); + # None without attention TP or peer memory (the NCCL gathers stay). + key_exchange = ll_exchange.create_tp_key_exchange( + model_config.mapping, model_config.max_num_tokens + ) + self.spec_worker = get_spec_worker( + self.spec_config, + model_config, + model_config.mapping, + use_separate_draft_kv_cache=self.use_separate_draft_kv_cache, + key_exchange=key_exchange, + spec_decode_fast_paths=True, + ) + assert self.spec_worker is not None, ( + f"the engine resolved no spec worker for spec_dec_mode " + f"{self.spec_config.spec_dec_mode!r}" + ) + if getattr(self.spec_worker, "key_exchange", None) is not None: + self.draft_model.mtp_layers[0].worker = self.spec_worker + self.spec_worker.set_draft_model(self.draft_model) + # Nothing the runtime owns rolls back the PLE pools and QSA side caches. + self.spec_worker.register_auxiliary_state_handler(self.model.spec_rollback) + # Media placeholder ids, built lazily (`__init__` runs under meta init). + self._mm_token_ids_value = (self.model.image_token_id, self.model.video_token_id) + self._mm_token_ids_cache: torch.Tensor | None = None + # Replicated on every rank; built once the weights are real (`post_load_weights`). + self.vision_tower: _vision.VisionTower | None = None + + @property + def multimodal_token_ids(self) -> torch.Tensor: + """Placeholder ids whose rows vision embeddings replace (in vocabulary here, so the + executor's out-of-vocabulary fallback would select nothing).""" + if self._mm_token_ids_cache is None: + self._mm_token_ids_cache = torch.tensor(self._mm_token_ids_value, dtype=torch.int32) + return self._mm_token_ids_cache + + @property + def multimodal_data_device_paths(self) -> list[str]: + """`multimodal_data` tensors moved to the device; the `(t, h, w)` grids stay on the host.""" + return list(_vision.DEVICE_PATHS) + + def validate_request(self, request) -> None: + """Reject at admission a media request the vision tower would assert on.""" + _vision.validate_request(self.vision_tower, request) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + return_context_logits: bool = False, + spec_metadata=None, + lora_params: dict | None = None, + **kwargs, + ): + """One engine step: `begin_step`, the vision tower (the inherited forward drops + `multimodal_params`), then the trunk; with a spec worker the shell gathers the verified + logits rows (`spec_metadata.gather_ids`) and hands the worker the ungathered stream.""" + self.model.begin_step( + attn_metadata, + input_ids, + position_ids, + inputs_embeds=inputs_embeds, + lora_params=lora_params, + spec_metadata=spec_metadata, + vision=_vision.multimodal_embeds(self.vision_tower, kwargs), + ) + if self.spec_worker is None: + assert spec_metadata is None, ( + "spec_metadata arrived without a spec worker; this engine was built without a " + "speculative_config" + ) + return super().forward( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + lora_params, + **kwargs, + ) + assert spec_metadata is not None, "the spec worker is built but no spec_metadata came" + final, hc = self.model.split_spec_output( + self.model( + attn_metadata=attn_metadata, + input_ids=input_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + lora_params=lora_params, + spec_metadata=spec_metadata, + **kwargs, + ) + ) + rows = nn.functional.embedding(spec_metadata.gather_ids, final) + if self.spec_worker.key_exchange is not None: + # This rank's vocabulary shard for a greedy/temperature-only batch, else gathered fp32. + logits = self.spec_worker.target_logits( + rows, + self.lm_head, + self.logits_processor, + attn_metadata, + spec_metadata, + self.draft_model, + ) + else: + logits = self.logits_processor.forward(rows, self.lm_head, attn_metadata, True) + # The draft head decides from the same batch whether its logits may stay sharded. + self.draft_model.mtp_layers[0].spec_metadata = spec_metadata + # A prefill CUDA graph pads the rows; the worker reshapes per request, so trim them. + if attn_metadata.padded_num_tokens is not None: + n = attn_metadata.num_tokens + input_ids, hc = input_ids[:n], hc[:n] + position_ids = position_ids[..., :n] + return self.spec_worker( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hc, + logits=logits, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_model=self.draft_model, + resource_manager=kwargs.get("resource_manager"), + ) + + def load_weights(self, weights, *args, **kwargs): + _weights.MODEL_WEIGHTS.load(self, weights) + + def post_load_weights(self): + super().post_load_weights() + self.model.post_load_weights() + self.vision_tower = _vision.VisionTower(self.model.w, self.model.vision_config) + # The CPU-only load checks build the shell over host weights; they launch nothing. + if self.lm_head.weight.is_cuda: + self._load_eager_gemm_kernels() + + def _load_eager_gemm_kernels(self) -> None: + """Load, once per rank, every cuBLASLt kernel an eager bf16 GEMM can pick (the core's + projections and `lm_head`, at `_projection_warm_rows`) so no serving step loads one.""" + core, head = self.model, self.lm_head + device = head.weight.device + projections = _eager_projections( + core, core._prefill._layers, core._prefill._stack, core._mtp + ) + _load_projection_kernels(projections, device) + rows = min(core.max_num_tokens, _LM_HEAD_WARM_ROWS) + x = torch.zeros([rows, head.in_features], dtype=head.weight.dtype, device=device) + for m in _projection_warm_rows(1, rows): + head.apply_linear(x[:m], head.bias) + torch.cuda.synchronize(device) + # Load the grouped routed-experts path before any prefill CUDA graph is captured. + dims = getattr(core, "dims", None) + if ( + _MOE_PREFILL_GROUPED_MIN_ROWS + and dims is not None + and getattr(core, "dp_size", 1) == 1 + and core.max_num_tokens >= _MOE_PREFILL_GROUPED_MIN_ROWS + ): + warm_prefill_grouped(device, dims.local_experts) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py new file mode 100644 index 000000000000..777b6b725c00 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ops used only by this target; the family's shared ops are in `models/qwen4_exp/ops`.""" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py new file mode 100644 index 000000000000..99d7d950867c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A generation-only draft-extend step's continuing rows and their operands, in two launches. + +The draft layer's draft-extend step (`p` rows a request) runs its attention core and tail only on +each request's continuing row, `r * p + clamp(num_accepted[r] - 1, 0, p - 1)`: +`draft_continue_select` forms those rows and gathers the QSA row metadata at them; +`draft_rows_gather` gathers the step's 2-D operands at them. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Requests a program of `draft_continue_select` covers. +SEQ_BLOCK = 256 +#: Columns a program of `draft_rows_gather` copies. +COL_BLOCK = 1024 +#: Operands one `draft_rows_gather` launch takes. +MAX_OPERANDS = 6 + + +@triton.jit(do_not_specialize=["num_seqs", "rows_per_request"]) +def _continue_select_kernel( + accepted, + v0, + v1, + v2, + v3, + rows_out, + o0, + o1, + o2, + o3, + num_seqs, + rows_per_request, + BLOCK: tl.constexpr, +): + seqs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + valid = seqs < num_seqs + last = tl.load(accepted + seqs, mask=valid, other=1).to(tl.int64) - 1 + last = tl.minimum(tl.maximum(last, 0), rows_per_request - 1) + rows = seqs.to(tl.int64) * rows_per_request + last + tl.store(rows_out + seqs, rows, mask=valid) + tl.store(o0 + seqs, tl.load(v0 + rows, mask=valid), mask=valid) + tl.store(o1 + seqs, tl.load(v1 + rows, mask=valid), mask=valid) + tl.store(o2 + seqs, tl.load(v2 + rows, mask=valid), mask=valid) + tl.store(o3 + seqs, tl.load(v3 + rows, mask=valid), mask=valid) + + +@triton.jit +def _copy_row(src, dst, s_src, row, idx, width, cols): + mask = cols < width + tl.store(dst + row * width + cols, tl.load(src + idx * s_src + cols, mask=mask), mask=mask) + + +@triton.jit( + do_not_specialize=[ + "w0", "w1", "w2", "w3", "w4", "w5", "s0", "s1", "s2", "s3", "s4", "s5", + ], + do_not_specialize_on_alignment=["x0", "x1", "x2", "x3", "x4", "x5"], +) # fmt: skip +def _rows_gather_kernel( + index, + x0, x1, x2, x3, x4, x5, + y0, y1, y2, y3, y4, y5, + w0, w1, w2, w3, w4, w5, + s0, s1, s2, s3, s4, s5, + COUNT: tl.constexpr, + BLOCK: tl.constexpr, +): # fmt: skip + row = tl.program_id(0).to(tl.int64) + which = tl.program_id(1) + cols = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + idx = tl.load(index + row).to(tl.int64) + if which == 0: + _copy_row(x0, y0, s0, row, idx, w0, cols) + if COUNT > 1: + if which == 1: + _copy_row(x1, y1, s1, row, idx, w1, cols) + if COUNT > 2: + if which == 2: + _copy_row(x2, y2, s2, row, idx, w2, cols) + if COUNT > 3: + if which == 3: + _copy_row(x3, y3, s3, row, idx, w3, cols) + if COUNT > 4: + if which == 4: + _copy_row(x4, y4, s4, row, idx, w4, cols) + if COUNT > 5: + if which == 5: + _copy_row(x5, y5, s5, row, idx, w5, cols) + + +def _select_spec(seqs: int, p: int) -> dict: + return dict(seqs=seqs, rows_per_request=p) + + +class DraftContinueSelect(OpWrapper): + """`rows, (g0, g1, g2, g3) = draft_continue_select(accepted, num_seqs, p, (v0, v1, v2, v3))`: + `rows[r] = r * p + clamp(accepted[r] - 1, 0, p - 1)` (int64, `[num_seqs]`) and + `g_k[r] = v_k[rows[r]]` (fresh, `v_k`'s dtype), for `r < num_seqs`. + + - `accepted`: the worker's accepted counts (`qsa_mtp_num_accepted`, int32, contiguous); + - `v_k`: 1-D contiguous row vectors of at least `num_seqs * p` entries (the QSA row metadata). + + Bitwise `_draft_continue_rows` followed by four `index_select`s.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep2 HT draft-extend step: 192 requests (the largest batch)", spec=_select_spec(192, 3)), + Cell(why="two programs: 300 requests, 4 rows each", spec=_select_spec(300, 4)), + ) # fmt: skip + + note = """ + Source: one Triton kernel (`_continue_select_kernel`), grid `ceil(num_seqs / 256)`; the + request count and rows a request are runtime arguments (one compile per dtype set). + """ + + def __call__(self, accepted, num_seqs: int, rows_per_request: int, vectors): + dev = accepted.device + rows = torch.empty([num_seqs], dtype=torch.int64, device=dev) + outs = tuple(torch.empty([num_seqs], dtype=v.dtype, device=dev) for v in vectors) + if num_seqs: + _continue_select_kernel[(triton.cdiv(num_seqs, SEQ_BLOCK),)]( + accepted, *vectors, rows, *outs, num_seqs, rows_per_request, BLOCK=SEQ_BLOCK, + num_warps=4, + ) # fmt: skip + return rows, outs + + def reference(self, accepted, num_seqs: int, rows_per_request: int, vectors): + last = torch.clamp(accepted[:num_seqs].to(torch.int64) - 1, min=0, max=rows_per_request - 1) + rows = torch.arange(num_seqs, dtype=torch.int64, device=accepted.device) * rows_per_request + rows = rows + last + return rows, tuple(torch.index_select(v, 0, rows) for v in vectors) + + def is_valid(self, accepted, num_seqs: int, rows_per_request: int, vectors) -> None: + assert len(vectors) == 4 and rows_per_request >= 1 + assert accepted.is_contiguous() and accepted.numel() >= num_seqs + for v in vectors: + assert v.dim() == 1 and v.is_contiguous() and v.numel() >= num_seqs * rows_per_request + + +def _gather_spec(rows: int, widths: tuple) -> dict: + return dict(rows=rows, widths=widths) + + +class DraftRowsGather(OpWrapper): + """`outs = draft_rows_gather(index, operands)`: `outs[k] = operands[k][index]` for up to six + 2-D operands, each with unit column stride (any row stride), as fresh contiguous tensors of + `[index.numel(), operands[k].shape[1]]`. + + Bitwise `index_select(operands[k], 0, index)`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell( + why="tep2 HT draft-extend step at 64 requests: index query, QKV, gate, residual, inject", + spec=_gather_spec(64, (512, 2048, 1536, 10240, 4)), + ), + Cell(why="one operand, 7 rows", spec=_gather_spec(7, (300,))), + ) + + note = """ + Source: one Triton kernel (`_rows_gather_kernel`), grid `(rows, operands, ceil(max width / + 1024))`; widths and row strides are runtime arguments, the operand count a constexpr. + """ + + def __call__(self, index, operands): + count = len(operands) + assert 1 <= count <= MAX_OPERANDS + rows = index.numel() + outs = [torch.empty([rows, t.shape[1]], dtype=t.dtype, device=t.device) for t in operands] + if rows: + pad = MAX_OPERANDS - count + xs = list(operands) + [operands[0]] * pad + ys = outs + [outs[0]] * pad + ws = [t.shape[1] for t in operands] + [0] * pad + ss = [t.stride(0) for t in operands] + [0] * pad + width = max(t.shape[1] for t in operands) + _rows_gather_kernel[(rows, count, triton.cdiv(width, COL_BLOCK))]( + index, *xs, *ys, *ws, *ss, COUNT=count, BLOCK=COL_BLOCK, num_warps=4 + ) + return tuple(outs) + + def reference(self, index, operands): + return tuple(torch.index_select(t, 0, index) for t in operands) + + def is_valid(self, index, operands) -> None: + assert 1 <= len(operands) <= MAX_OPERANDS and index.dim() == 1 + for t in operands: + assert t.dim() == 2 and (t.stride(1) == 1 or t.shape[1] == 1) + assert t.device == index.device + + +draft_continue_select = DraftContinueSelect() +draft_rows_gather = DraftRowsGather() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py new file mode 100644 index 000000000000..f57416bd72bd --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A batch's pages of an e4m3 paged K/V pool, widened into a bf16 scratch pool, and the block +table that addresses them there: the context launches over a kv_fp8 pool read their requests' +pages through this scratch with the bf16 row-tile kernel. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Elements a program widens per loop step. +_CHUNK = 4096 + +#: A dep4 rank's K/V geometry and QSA table at max_seq_len 262144 (8196 columns); `slot_pages` +#: role pages a slot (24 for the 12 trunk layers, 26 with the MTP draft layer's planes last). +_CHECKPOINT = dict( + kv_heads=2, + head_dim=256, + tokens_per_block=32, + width=8196, + slots=32, + slot_pages=24, + layer=11, + holes=False, + straddle=False, +) + + +def _spec(kv_lens, **overrides) -> dict: + """`kv_lens` is one KV length per listed request, in order.""" + return {**_CHECKPOINT, "kv_lens": tuple(kv_lens), **overrides} + + +# Per-step and per-engine sizes are not specialized on (none feeds a vector access). +@triton.jit(do_not_specialize=["num_pool_pages", "listed", "bt_stride", "table_stride"]) +def _fp8_pages_to_bf16_kernel( + k_cache, + v_cache, + block_table, + slots, + counts, + offsets, + table, + k_out, + v_out, + num_pool_pages, + listed, + stride_page, + bt_stride, + table_stride, + PAGE_ELEMS: tl.constexpr, + CHUNK: tl.constexpr, +): + request = tl.program_id(0) + count = tl.load(counts + request) + base = tl.load(offsets + request).to(tl.int64) + slot = tl.load(slots + request).to(tl.int64) + for page in range(tl.program_id(1), count, tl.num_programs(1)): + dst = base + page + if dst < listed: + src = tl.load(block_table + slot * bt_stride + page).to(tl.int64) + resident = (src >= 0) & (src < num_pool_pages) + tl.store(table + slot * table_stride + page, tl.where(resident, dst, -1).to(tl.int32)) + safe = tl.where(resident, src, 0) + for start in tl.static_range(0, PAGE_ELEMS, CHUNK): + elems = start + tl.arange(0, CHUNK) + k = tl.load(k_cache + safe * stride_page + elems, mask=resident, other=0.0) + v = tl.load(v_cache + safe * stride_page + elems, mask=resident, other=0.0) + tl.store(k_out + dst * PAGE_ELEMS + elems, k.to(tl.bfloat16)) + tl.store(v_out + dst * PAGE_ELEMS + elems, v.to(tl.bfloat16)) + + +#: Programs per request; each strides the request's pages. +_PAGE_PROGRAMS = 1024 # capacity: fixed: a launch width; a request may list any page count + + +class QsaFp8PagesToBf16(OpWrapper): + """K and V of listed requests' pages of an e4m3 HND paged pool, as bf16 scratch pools, + and the block table that addresses them there. + + Request `r` lists its first `counts[r]` table pages, `block_table[slots[r], j]` for `j < + counts[r]`, at scratch pages `offsets[r] + j` (those below `listed`, the scratch's page + count):: + + k_out[offsets[r] + j] = bf16(k_cache[block_table[slots[r], j]]) + + and `v_out` likewise. Each e4m3 code widens exactly to bf16 (no scale: the pool holds + K/V at unit scale, as `qsa_paged_sparse_gqa` reads it), so a bf16 kernel over the scratch + reads the values the e4m3 kernels read. `table[slots[r], j]` becomes `offsets[r] + j`, + or -1 where the pool's entry is unmapped or out of the pool (whose scratch page is then + zeros); the call writes no other entry of `table`, which the caller fills with -1. + `offsets` is the exclusive prefix sum of `counts`. Returns `(k_out, v_out, table)`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="one request, one page", spec=_spec((17,))), + Cell( + why="dep4 deep chunk: one request at 198160 tokens (6193 pages) over a pool " + "straddling 2^31 elements of the trunk plane", + spec=_spec((198160,), straddle=True), + ), + Cell( + why="three requests, a chunked continuation and two short ones", + spec=_spec((40000 + 3192, 3000, 900)), + ), + Cell(why="holes: unmapped and out-of-pool entries", spec=_spec((4000, 2000), holes=True)), + Cell( + why="a tp4 / tep4 rank's single KV head, [128, 8196] table", + spec=_spec((9000, 2100), kv_heads=1, slots=128), + ), + Cell( + why="ht_dep4 (dep4 + MTP): the trunk's 26 role pages a slot, the draft layer's " + "planes (layer 12), one request at 196576 tokens straddling 2^31", + spec=_spec((196576,), slot_pages=26, layer=12, straddle=True), + ), + ) + + note = """ + Source: one Triton kernel (`_fp8_pages_to_bf16_kernel`), `(requests, min(width, 1024))` + programs, each striding its request's pages. Page ids are widened to int64 before the + stride product: the engine's trunk planes pass 2^31 elements. + + Preconditions: e4m3 role planes `[pages, Hkv, tpb, D]` with a page's heads, tokens and + dims contiguous (`stride(1) == tpb * D`, `stride(2) == D`, unit last stride; any page + stride), K's and V's at equal strides; `k_out` / `v_out` contiguous bf16 `[listed, Hkv, + tpb, D]`; `table` int32 of `block_table`'s shape. Bitwise deterministic (a copy). + """ + + def __call__( + self, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + slots: torch.Tensor, + counts: torch.Tensor, + offsets: torch.Tensor, + table: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + requests, listed = slots.shape[0], k_out.shape[0] + _, heads, tpb, dim = k_cache.shape + if k_cache.dtype != torch.float8_e4m3fn or v_cache.dtype != k_cache.dtype: + raise ValueError("the widening copy reads e4m3 K/V planes") + if k_cache.stride() != v_cache.stride() or k_cache.stride()[1:] != (tpb * dim, dim, 1): + raise ValueError("K/V planes must share strides with contiguous head pages") + if not (k_out.is_contiguous() and v_out.is_contiguous()): + raise ValueError("the bf16 scratch pools must be contiguous") + if k_out.dtype != torch.bfloat16 or k_out.shape[1:] != k_cache.shape[1:]: + raise ValueError("the scratch pools must be bf16 pages of the pool's geometry") + page_elems = heads * tpb * dim + if requests and listed: + _fp8_pages_to_bf16_kernel[(requests, min(block_table.shape[1], _PAGE_PROGRAMS))]( + k_cache, + v_cache, + block_table, + slots, + counts, + offsets, + table, + k_out, + v_out, + k_cache.shape[0], + listed, + k_cache.stride(0), + block_table.stride(0), + table.stride(0), + PAGE_ELEMS=page_elems, + CHUNK=min(_CHUNK, page_elems), + num_warps=8, + ) + return k_out, v_out, table + + def reference( + self, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + slots: torch.Tensor, + counts: torch.Tensor, + offsets: torch.Tensor, + table: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + k_ref, v_ref, table_ref = k_out.clone(), v_out.clone(), table.clone() + listed, pool_pages = k_out.shape[0], k_cache.shape[0] + for r in range(slots.shape[0]): + slot, base = int(slots[r]), int(offsets[r]) + count = max(0, min(int(counts[r]), block_table.shape[1], listed - base)) + if count == 0: + continue + src = block_table[slot, :count].long() + resident = (src >= 0) & (src < pool_pages) + dst = torch.arange(base, base + count, device=src.device) + table_ref[slot, :count] = torch.where(resident, dst, -1).to(table.dtype) + safe = torch.where(resident, src, 0) + keep = resident[:, None, None, None] + k_ref[base : base + count] = torch.where(keep, k_cache[safe].float(), 0.0).to( + k_out.dtype + ) + v_ref[base : base + count] = torch.where(keep, v_cache[safe].float(), 0.0).to( + v_out.dtype + ) + return k_ref, v_ref, table_ref + + +qsa_fp8_pages_to_bf16 = QsaFp8PagesToBf16() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py new file mode 100644 index 000000000000..e01fd7b76506 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py @@ -0,0 +1,944 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index scores of a context chunk's rows on SM103 tensor cores (CuTe DSL, tcgen05). + +Bitwise equal to the family's `_qsa_paged_index_scores_qtile_kernel` (`only_visible_blocks`): +columns at and past a row's visible block count are unspecified. A persistent CTA takes +(64-row tile, `CHUNK_TILES` x 128-block chunk) items; per tile one tcgen05 MMA chain +`D[block, 4 r + h]` (M 128, N 256, K 128, bf16 in, fp32 in tensor memory, two accumulators). +Warps 9-10 gather q rows and keys with `cp.async` into a `STAGES`-slot SW128 ring, warp 8 issues +the MMAs, warps 0-7 store the scores. A tile spanning several requests runs once per request. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import cutlass.cute.nvgpu.tcgen05 as tcgen05 +import cutlass.pipeline as pipeline +import cutlass.utils.blackwell_helpers as sm100_utils +import torch +from cutlass._mlir.dialects import nvvm as _nvvm +from cutlass.cute.nvgpu import OperandMajorMode, cpasync +from cutlass.cute.runtime import from_dlpack, make_ptr +from cutlass.cute.typing import Float32, Int32, Int64 +from cutlass.cutlass_dsl import dsl_user_op + +try: + from cutlass.memory import SmemAllocator +except ImportError: # DSL releases before the memory namespace split + from cutlass.utils import SmemAllocator + +BF16 = cutlass.BFloat16 +HEADS = 4 +DIM = 128 +#: Query rows a tile (the MMA's N = rows x heads). +QROWS = 64 +BN = QROWS * HEADS +#: Compressed blocks a tile (the MMA's M). +BM = 128 +#: Key ring slots (32 KB each); with the 64 KB query operand, ~192 KB of shared memory. +STAGES = 4 +CHUNK_TILES = 8 +#: Epilogue warps: two per tensor-memory lane quadrant, each taking half the accumulator. +EPI_WARPS = 8 +MMA_WARP = 8 +LOAD_WARP0 = 9 +LOAD_WARPS = 2 +#: Accumulator columns an epilogue thread loads at a time (8 rows x 4 heads). +EPI_COLS = 32 +#: Items a CTA's metadata table holds (more: each role derives an item's metadata itself). +MAX_ITEMS = 64 +THREADS = 32 * (EPI_WARPS + 1 + LOAD_WARPS) +LOAD_THREADS = 32 * LOAD_WARPS + + +@dsl_user_op +def _fence_before(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.BEFORE_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def _fence_after(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.AFTER_THREAD_SYNC, loc=loc, ip=ip) + + +class _IndexScoresKernel: + def __init__(self, *, tokens_per_block, compress_ratio): + assert tokens_per_block % compress_ratio == 0 + self.tpb = tokens_per_block + self.cr = compress_ratio + self.kpp = tokens_per_block // compress_ratio # keys a page + assert BM % self.kpp == 0 and BM // self.kpp <= 32 + assert self.kpp == 2 * (LOAD_THREADS // 16), "a thread's keys: two token rows of each page" + self.pages_per_tile = BM // self.kpp + + @cute.jit + def __call__( + self, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + pos: cute.Pointer, + req: cute.Pointer, + out_t: cute.Tensor, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + out_stride: Int32, + scale: Float32, + num_ctas: Int32, + stream, + ): + # The output arrives as a tensor, as the TVM-FFI entry takes its stream from a GPU + # tensor; the kernel addresses it through its pointer and row stride. + out = out_t.iterator + + def make_mma(): + return sm100_utils.make_trivial_tiled_mma( + BF16, + BF16, + OperandMajorMode.K, + OperandMajorMode.K, + Float32, + tcgen05.CtaGroup.ONE, + (BM, BN), + tcgen05.OperandSource.SMEM, + ) + + mma, mma_acc = make_mma(), make_mma() + a_layout = sm100_utils.make_smem_layout_a(mma, (BM, BN, DIM), BF16, STAGES) + b_layout = sm100_utils.make_smem_layout_b(mma, (BM, BN, DIM), BF16, 1) + + @cute.struct + class SharedStorage: + k_full: cute.struct.MemRange[Int64, STAGES] + k_empty: cute.struct.MemRange[Int64, STAGES] + q_full: cute.struct.MemRange[Int64, 1] + q_empty: cute.struct.MemRange[Int64, 1] + acc_full: cute.struct.MemRange[Int64, 2] + acc_empty: cute.struct.MemRange[Int64, 2] + tmem_holding: Int32 + vis: cute.struct.MemRange[Int32, QROWS] + meta_starts: cute.struct.MemRange[Int64, MAX_ITEMS] + meta_tot: cute.struct.MemRange[Int32, MAX_ITEMS] + meta_hi: cute.struct.MemRange[Int32, MAX_ITEMS] + meta_req: cute.struct.MemRange[Int32, MAX_ITEMS] + sA: cute.struct.Align[cute.struct.MemRange[BF16, cute.cosize(a_layout)], 1024] + sB: cute.struct.Align[cute.struct.MemRange[BF16, cute.cosize(b_layout)], 1024] + + self.shared_storage = SharedStorage + self.kernel( + mma, + mma_acc, + a_layout, + b_layout, + q, + cache, + table, + pos, + req, + out, + rows, + width, + q_stride_row, + cache_stride_page, + cache_stride_token, + table_stride, + out_stride, + scale, + ).launch( + grid=[num_ctas, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + min_blocks_per_mp=1, + ) + + @cute.jit + def item_geometry(self, item: Int32, nrt: Int32, rows: Int32, width: Int32): + """Items chunk-major, so a wave's CTAs share a chunk's keys in L2.""" + ch = item // nrt + rt = item - ch * nrt + row0 = rt * QROWS + rows_in = cutlass.min(rows - row0, Int32(QROWS)) + c0 = ch * (BM * CHUNK_TILES) + c1 = cutlass.min(c0 + BM * CHUNK_TILES, width) + return row0, rows_in, c0, c1 + + @cute.jit + def item_meta(self, meta, li: Int32, nmine: Int32, mReq, mPos, row0, rows_in, c0, c1, lane): + """(run starts, tiles, first run's window bound, first run's request) of the CTA's + item `li`: from the table, or derived here past it.""" + mS = cute.make_tensor(meta[0], cute.make_layout((MAX_ITEMS,))) + mT = cute.make_tensor(meta[1], cute.make_layout((MAX_ITEMS,))) + mH = cute.make_tensor(meta[2], cute.make_layout((MAX_ITEMS,))) + mR = cute.make_tensor(meta[3], cute.make_layout((MAX_ITEMS,))) + starts = Int64(0) + total = Int32(0) + hi1 = Int32(0) + req1 = Int32(0) + if nmine <= MAX_ITEMS: + starts = mS[li] + total = mT[li] + hi1 = mH[li] + req1 = mR[li] + else: + starts = self.group_starts(mReq, row0, rows_in, lane) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + if s == 0: + hi1 = cutlass.min(vis, c1) + total = total + nt + s = e + req1 = Int32(mReq[row0]) + return starts, total, hi1, req1 + + @cute.jit + def group_starts(self, mReq, row0: Int32, rows_in: Int32, lane: Int32): + """Bit j set where tile row j starts a run of one request (warp-uniform).""" + starts = Int64(0) + for half in cutlass.range_constexpr(2): + j = lane + 32 * half + flag = cutlass.Boolean(False) + if j < rows_in: + if j == 0: + flag = True + else: + flag = Int64(mReq[row0 + j]) != Int64(mReq[row0 + j - 1]) + bal = cute.arch.vote_ballot_sync(flag) + starts = starts | ((Int64(bal) & Int64(0xFFFFFFFF)) << Int64(32 * half)) + return starts + + @cute.jit + def group_end(self, starts: Int64, s: Int32, rows_in: Int32): + """The first row after `s` that starts another run, or `rows_in`.""" + rest = starts & ~((Int64(2) << Int64(s)) - Int64(1)) + e = rows_in + if rest != Int64(0): + low = rest & (Int64(0) - rest) + m = low - Int64(1) + e = Int32(cute.arch.popc(Int32(m & Int64(0xFFFFFFFF)))) + Int32( + cute.arch.popc(Int32(m >> Int64(32))) + ) + return e + + @cute.jit + def group_tiles(self, mPos, row0: Int32, e: Int32, c0: Int32, c1: Int32): + """Tiles a run ending before row `e` walks in the chunk, and its widest window.""" + vis = (Int32(mPos[row0 + e - 1]) + 1) // self.cr + hi = cutlass.min(vis, c1) + ntiles = Int32(0) + if hi > c0: + ntiles = (hi - c0 + (BM - 1)) // BM + return ntiles, vis + + # ------------------------------------------------------------------ kernel + @cute.kernel + def kernel( + self, + mma: cute.TiledMma, + mma_acc: cute.TiledMma, + a_layout: cute.ComposedLayout, + b_layout: cute.ComposedLayout, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + pos: cute.Pointer, + req: cute.Pointer, + out: cute.Pointer, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + out_stride: Int32, + scale: Float32, + ): + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + tidx, _, _ = cute.arch.thread_idx() + lane = cute.arch.lane_idx() + bid, _, _ = cute.arch.block_idx() + nctas, _, _ = cute.arch.grid_dim() + mma.set(tcgen05.Field.ACCUMULATE, False) + mma_acc.set(tcgen05.Field.ACCUMULATE, True) + + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + k_full = storage.k_full.data_ptr() + k_empty = storage.k_empty.data_ptr() + q_full = storage.q_full.data_ptr() + q_empty = storage.q_empty.data_ptr() + acc_full = storage.acc_full.data_ptr() + acc_empty = storage.acc_empty.data_ptr() + if tidx == 0: + for i in cutlass.range_constexpr(STAGES): + cute.arch.mbarrier_init(k_full + i, LOAD_THREADS) + cute.arch.mbarrier_init(k_empty + i, 1) + cute.arch.mbarrier_init(q_full, LOAD_THREADS) + cute.arch.mbarrier_init(q_empty, 1) + for i in cutlass.range_constexpr(2): + cute.arch.mbarrier_init(acc_full + i, 1) + cute.arch.mbarrier_init(acc_empty + i, 32 * EPI_WARPS) + cute.arch.mbarrier_init_fence() + + tmem_users = pipeline.NamedBarrier(barrier_id=1, num_threads=32 * (EPI_WARPS + 1)) + if warp_idx == MMA_WARP: + cute.arch.alloc_tmem(512, storage.tmem_holding.ptr) + cute.arch.relinquish_tmem_alloc_permit() + cute.arch.sync_threads() + + thr = mma.get_slice(0) + sA = storage.sA.get_tensor(a_layout.outer, swizzle=a_layout.inner) + sB = storage.sB.get_tensor(b_layout.outer, swizzle=b_layout.inner) + per_row = cute.make_layout((1 << 30,)) + mReq = cute.make_tensor(req, per_row) + mPos = cute.make_tensor(pos, per_row) + # The widest visible window of the call bounds the chunks (the table is max_seq_len wide). + sMax = cute.make_tensor(storage.vis.data_ptr(), cute.make_layout((QROWS,))) + vmax = Int32(0) + for i in cutlass.range(tidx, rows, THREADS, unroll=4): + vmax = cutlass.max(vmax, (Int32(mPos[i]) + 1) // self.cr) + vmax = cute.arch.warp_redux_sync(vmax, "max") + if lane == 0: + sMax[warp_idx] = vmax + cute.arch.sync_threads() + vmax = Int32(0) + if lane < THREADS // 32: + vmax = sMax[lane] + vmax = cute.arch.warp_redux_sync(vmax, "max") + table_width = width + width = cutlass.min(width, vmax) + cute.arch.sync_threads() + nrt = (rows + (QROWS - 1)) // QROWS + nch = (width + (BM * CHUNK_TILES - 1)) // (BM * CHUNK_TILES) + items = nrt * nch + # The CTA's items' metadata (see `item_meta`), once, a warp an item. + meta = ( + storage.meta_starts.data_ptr(), + storage.meta_tot.data_ptr(), + storage.meta_hi.data_ptr(), + storage.meta_req.data_ptr(), + ) + mS = cute.make_tensor(meta[0], cute.make_layout((MAX_ITEMS,))) + mT = cute.make_tensor(meta[1], cute.make_layout((MAX_ITEMS,))) + mH = cute.make_tensor(meta[2], cute.make_layout((MAX_ITEMS,))) + mR = cute.make_tensor(meta[3], cute.make_layout((MAX_ITEMS,))) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + if nmine <= MAX_ITEMS: + for li in cutlass.range(warp_idx, nmine, THREADS // 32, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts = self.group_starts(mReq, row0, rows_in, lane) + total = Int32(0) + hi1 = Int32(0) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + if s == 0: + hi1 = cutlass.min(vis, c1) + total = total + nt + s = e + if lane == 0: + mS[li] = starts + mT[li] = total + mH[li] = hi1 + mR[li] = Int32(mReq[row0]) + cute.arch.sync_threads() + + if warp_idx >= LOAD_WARP0: + self.load_role( + tidx - 32 * LOAD_WARP0, + lane, + bid, + nctas, + items, + nrt, + meta, + sA, + storage.sA.data_ptr(), + sB, + q, + cache, + table, + mReq, + mPos, + rows, + width, + q_stride_row, + cache_stride_page, + cache_stride_token, + table_stride, + k_full, + k_empty, + q_full, + q_empty, + ) + elif warp_idx == MMA_WARP: + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + self.mma_role( + mma, + mma_acc, + thr, + sA, + sB, + tmem_ptr, + lane, + bid, + nctas, + items, + nrt, + meta, + mReq, + mPos, + rows, + width, + k_full, + k_empty, + q_full, + q_empty, + acc_full, + acc_empty, + ) + tmem_users.arrive_and_wait() + cute.arch.dealloc_tmem(tmem_ptr, 512) + else: + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + self.epilogue_role( + thr, + tmem_ptr, + tidx, + lane, + bid, + nctas, + items, + nrt, + meta, + mReq, + mPos, + storage.vis.data_ptr(), + out, + rows, + width, + table_width, + out_stride, + scale, + acc_full, + acc_empty, + ) + tmem_users.arrive_and_wait() + + # ------------------------------------------------------------------ loaders (warps 9-10) + @cute.jit + def load_role( + self, + lt: Int32, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + sA, + sA_raw: cute.Pointer, + sB, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + mReq, + mPos, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + k_full, + k_empty, + q_full, + q_empty, + ): + atom = cute.make_copy_atom( + cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), BF16, num_bits_per_copy=128 + ) + # The query operand's element-coordinate view: (8 elements, (chunk, 64-dim block), row). + sB_c = cute.make_tensor( + sB.iterator, + cute.make_layout((8, (8, DIM // 64), BN), stride=(1, (8, BN * 64), 64)), + ) + chunk = lt % 16 + kq = lt // 16 + kstep: cutlass.Constexpr = LOAD_THREADS // 16 + toff = [] + dbase = [] + for v in cutlass.range_constexpr(self.kpp // kstep): + toff.append(((kq + kstep * v) * self.cr + self.cr - 1) * cache_stride_token + chunk * 8) + # SW128 K-major: 16-byte chunk c of row m at chunk c ^ (m % 8) of its 128-byte row, + # each 64-dim half a BM x 128-byte block. + kr = kq + kstep * v + dbase.append((chunk // 8) * (BM * 64) + kr * 64 + ((chunk % 8) ^ kr) * 8) + ring = Int32(0) + ring_phase = Int32(1) + q_phase = Int32(1) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + # The item's query rows: B row n = 4 r + h, 16 chunks of 16 bytes a row. + cute.arch.mbarrier_wait(q_empty, q_phase) + q_phase = q_phase ^ 1 + for it in cutlass.range_constexpr(BN * 16 // LOAD_THREADS): + x = lt + LOAD_THREADS * it + n = x // 16 + c = x % 16 + r = n // HEADS + h = n % HEADS + if r < rows_in: + src = cute.make_tensor( + q + + cute.assume( + Int64(row0 + r) * q_stride_row + h * DIM + c * 8, divby=8 + ), + cute.make_layout((8,)), + ) + cute.copy(atom, src, sB_c[(None, (c % 8, c // 8), n)]) + cute.arch.cp_async_mbarrier_arrive_noinc(q_full) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt = total + hi = hi1 + r_id = Int64(req1) + if starts != Int64(1): + # Several runs: each its own tiles, window and request. + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + hi = cutlass.min(vis, c1) + r_id = Int64(mReq[row0 + s]) + mTable = cute.make_tensor(table, cute.make_layout((1 << 30,))) + # Lane p holds the tile's page p; the next tile's are read during its copies. + page_next = self.tile_page(mTable, r_id, table_stride, c0, hi, lane) + for j in cutlass.range(0, nt, 1, unroll=1): + b0 = c0 + j * BM + page = page_next + if j + 1 < nt: + page_next = self.tile_page( + mTable, r_id, table_stride, b0 + BM, hi, lane + ) + cute.arch.mbarrier_wait(k_empty + ring, ring_phase) + full = b0 + BM <= hi + # Key m = kq + kstep * it: page it // 2 (a broadcast shuffle), token row + # kq + kstep * (it % 2); only a partial tile tests the bound per key. + sbase = ring * (BM * DIM) + if full: + self.copy_keys( + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + False, + ) + else: + self.copy_keys( + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + True, + ) + cute.arch.cp_async_mbarrier_arrive_noinc(k_full + ring) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + s = e + # Tail: every slot released, so no load is in flight when the CTA retires. + for i in cutlass.range_constexpr(STAGES): + cute.arch.mbarrier_wait(k_empty + ring, ring_phase) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + + @cute.jit + def copy_keys( + self, + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + bounded: cutlass.Constexpr, + ): + kstep: cutlass.Constexpr = LOAD_THREADS // 16 + for pp in cutlass.range_constexpr(BM // self.kpp): + pg = cute.arch.shuffle_sync(page, offset=pp) + pbase = Int64(pg) * cache_stride_page + for par in cutlass.range_constexpr(2): + it = 2 * pp + par + src = cute.make_tensor( + cache + cute.assume(pbase + toff[par], divby=8), cute.make_layout((8,)) + ) + dst = cute.make_tensor( + sA_raw + cute.assume(dbase[par] + sbase + pp * (self.kpp * 64), divby=8), + cute.make_layout((8,)), + ) + if cutlass.const_expr(bounded): + if b0 + kq + kstep * it < hi: + cute.copy(atom, src, dst) + else: + cute.copy(atom, src, dst) + + @cute.jit + def tile_page(self, mTable, r_id, table_stride: Int32, b0: Int32, hi: Int32, lane: Int32): + lp0 = (b0 * self.cr + self.cr - 1) // self.tpb + page = Int32(0) + if lane < self.pages_per_tile: + if b0 + lane * self.kpp < hi: + page = mTable[r_id * table_stride + lp0 + lane] + return page + + # ------------------------------------------------------------------ MMA (warp 8) + @cute.jit + def mma_role( + self, + mma: cute.TiledMma, + mma_acc: cute.TiledMma, + thr, + sA, + sB, + tmem_ptr, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + mReq, + mPos, + rows: Int32, + width: Int32, + k_full, + k_empty, + q_full, + q_empty, + acc_full, + acc_empty, + ): + tCrA = thr.make_fragment_A(sA) + tCrB = thr.make_fragment_B(sB) + tAcc_f = thr.make_fragment_C(thr.partition_shape_C((BM, BN))) + ring = Int32(0) + ring_phase = Int32(0) + q_phase = Int32(0) + t = Int32(0) # tiles so far: accumulator t % 2, its phase (t // 2) % 2 + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + cute.arch.mbarrier_wait(q_full, q_phase) + q_phase = q_phase ^ 1 + for j in cutlass.range(0, total, 1, unroll=1): + ab = t % 2 + cute.arch.mbarrier_wait(k_full + ring, ring_phase) + cute.arch.mbarrier_wait(acc_empty + ab, ((t // 2) % 2) ^ 1) + _fence_after() + tAcc = cute.make_tensor(tmem_ptr + ab * BN, tAcc_f.layout) + for kb in cutlass.range_constexpr(cute.size(tCrA, mode=[2])): + if cutlass.const_expr(kb == 0): + cute.gemm( + mma, tAcc, tCrA[None, None, kb, ring], tCrB[None, None, kb, 0], tAcc + ) + else: + cute.gemm( + mma_acc, + tAcc, + tCrA[None, None, kb, ring], + tCrB[None, None, kb, 0], + tAcc, + ) + with cute.arch.elect_one(): + tcgen05.commit(k_empty + ring) + tcgen05.commit(acc_full + ab) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + t = t + 1 + with cute.arch.elect_one(): + tcgen05.commit(q_empty) + + # ------------------------------------------------------------------ epilogue (warps 0-7) + @cute.jit + def head_sum(self, v0, v1, v2, v3): + """The family's head reduction order, bit for bit.""" + z = Float32(0.0) + r0 = cute.arch.fmax(v0, z) + r1 = cute.arch.fmax(v1, z) + r2 = cute.arch.fmax(v2, z) + r3 = cute.arch.fmax(v3, z) + return (r0 + r2) + (r1 + r3) + + @cute.jit + def epilogue_role( + self, + thr, + tmem_ptr, + tidx: Int32, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + mReq, + mPos, + vis_ptr: cute.Pointer, + out: cute.Pointer, + rows: Int32, + width: Int32, + table_width: Int32, + out_stride: Int32, + scale: Float32, + acc_full, + acc_empty, + ): + epi_bar = pipeline.NamedBarrier(barrier_id=2, num_threads=32 * EPI_WARPS) + sVis = cute.make_tensor(vis_ptr, cute.make_layout((QROWS,))) + tAcc_f = thr.make_fragment_C(thr.partition_shape_C((BM, BN))) + cols = EPI_COLS + rpc = cols // HEADS # rows a load + half = tidx // 128 # the accumulator half (rows 0-31 / 32-63) this warp reads + tAcc0 = cute.make_tensor(tmem_ptr, tAcc_f.layout) + tAcc_c = cute.make_tensor( + tAcc0.iterator, cute.composition(tAcc0.layout, cute.make_layout((BM, cols))) + ) + cAcc = cute.make_identity_tensor((BM, BN)) + tCcAcc = thr.partition_C(cAcc) + tCcAcc_c = cute.make_tensor( + tCcAcc.iterator, cute.composition(tCcAcc.layout, cute.make_layout((BM, cols))) + ) + load = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(cols)), Float32), + tAcc_c, + ) + thr_load = load.get_slice(tidx % 128) + tL = thr_load.partition_S(tAcc_c) + tLc = thr_load.partition_D(tCcAcc_c) + mOut = cute.make_tensor(out, cute.make_layout((1 << 30,))) + m = tLc[0][0] + t = Int32(0) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + # One request's 64 rows, every tile inside the table: stores unmasked. + whole = (rows_in == QROWS) & (starts == Int64(1)) & (c0 + total * BM <= table_width) + if not whole: + # The rows' visible block counts (bounded by the table width). + epi_bar.arrive_and_wait() + if tidx < QROWS: + v = Int32(0) + if tidx < rows_in: + v = cutlass.min((Int32(mPos[row0 + tidx]) + 1) // self.cr, table_width) + sVis[tidx] = v + epi_bar.arrive_and_wait() + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt = total + if starts != Int64(1): + nt, _ = self.group_tiles(mPos, row0, e, c0, c1) + for j in cutlass.range(0, nt, 1, unroll=1): + ab = t % 2 + b = c0 + j * BM + m + base = Int64(row0) * out_stride + b + cute.arch.mbarrier_wait(acc_full + ab, (t // 2) % 2) + _fence_after() + for cc in cutlass.range_constexpr(BN // 2 // cols): + rv = cute.make_rmem_tensor(tLc.shape, Float32) + col0 = half * (BN // 2) + cc * cols + cute.copy( + load, + cute.make_tensor(tL.iterator + ab * BN + col0, tL.layout), + rv, + ) + cute.arch.fence_view_async_tmem_load() + if cutlass.const_expr(cc == BN // 2 // cols - 1): + _fence_before() + cute.arch.mbarrier_arrive(acc_empty + ab) + r0 = half * (QROWS // 2) + cc * rpc + for rr in cutlass.range_constexpr(rpc): + val = ( + self.head_sum( + rv[HEADS * rr], + rv[HEADS * rr + 1], + rv[HEADS * rr + 2], + rv[HEADS * rr + 3], + ) + * scale + ) + r = r0 + rr + if whole: + mOut[base + Int64(r) * out_stride] = val + else: + if r >= s: + if r < e: + if b < sVis[r]: + mOut[base + Int64(r) * out_stride] = val + t = t + 1 + s = e + + +_COMPILED: dict = {} +_SMS: dict = {} +_INDEX = {torch.int32: Int32, torch.int64: Int64} + + +def _compiled(tpb, cr, pos_dtype, req_dtype, out): + key = (tpb, cr, pos_dtype, req_dtype) + fn = _COMPILED.get(key) + if fn is None: + kernel = _IndexScoresKernel(tokens_per_block=tpb, compress_ratio=cr) + gmem = cute.AddressSpace.gmem + + def fake(dtype, align): + return make_ptr(dtype, 16 * 1024, gmem, assumed_align=align) + + fn = _COMPILED[key] = cute.compile( + kernel, + fake(BF16, 16), + fake(BF16, 16), + fake(Int32, 4), + fake(_INDEX[pos_dtype], 4), + fake(_INDEX[req_dtype], 4), + from_dlpack(out, assumed_align=16).mark_layout_dynamic(leading_dim=1), + 1, + 1, + 8, + 8, + 8, + 8, + 8, + 1.0, + 1, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + return fn + + +def supports(q, index_cache, tokens_per_block, compress_ratio) -> bool: + """A context-row call this kernel serves: bf16 4 x 128 index heads (unit dim stride, heads + 128 apart, 16-byte rows), a `[pages, tpb, 1, 128]` bf16 pool with 256-byte token rows.""" + return ( + q.dtype == torch.bfloat16 + and index_cache.dtype == torch.bfloat16 + and q.ndim == 3 + and q.shape[1] == HEADS + and q.shape[2] == DIM + and q.stride(2) == 1 + and q.stride(1) == DIM + and q.stride(0) % 8 == 0 + and q.data_ptr() % 16 == 0 + and index_cache.ndim == 4 + and index_cache.shape[2] == 1 + and index_cache.shape[3] == DIM + and index_cache.stride(3) == 1 + and index_cache.stride(1) % 8 == 0 + and index_cache.stride(0) % 8 == 0 + and index_cache.data_ptr() % 16 == 0 + and tokens_per_block % compress_ratio == 0 + and BM % (tokens_per_block // compress_ratio) == 0 + and torch.cuda.get_device_capability(q.device) == (10, 3) + ) + + +def qsa_index_scores_prefill( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, +): + """The family's context-row index scores (`only_visible_blocks`): `[rows, max_blocks]` fp32, + columns at and past each row's visible count unspecified.""" + rows = q.shape[0] + max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + out = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + if rows == 0: + return out + fn = _compiled( + tokens_per_block, + compress_ratio, + query_positions.dtype, + request_indices.dtype, + out, + ) + dev = q.device.index if q.device.index is not None else 0 + sms = _SMS.get(dev) + if sms is None: + sms = _SMS[dev] = torch.cuda.get_device_properties(dev).multi_processor_count + nch = -(-max_blocks // (BM * CHUNK_TILES)) + items = -(-rows // QROWS) * nch + fn( + q.data_ptr(), + index_cache.data_ptr(), + block_table.data_ptr(), + query_positions.data_ptr(), + request_indices.data_ptr(), + out, + rows, + max_blocks, + q.stride(0), + index_cache.stride(0), + index_cache.stride(1), + block_table.stride(0), + out.stride(0), + float(q.shape[2] ** -0.5), + max(1, min(items, sms)), + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py new file mode 100644 index 000000000000..b801c168a6c6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py @@ -0,0 +1,432 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index scores of a verify step's request groups: a CuTe DSL kernel with a cp.async ring. + +A CTA (4 warps) walks `TILES` tiles (`BN` keys each) of one request group with `STAGES` tiles in +flight: the program's page numbers are looked up once into shared memory, each tile's keys go to +an XOR-swizzled shared ring with `cp.async` (four threads a key), and the scores come from +`mma.sync` m16n8k16 with the group's query columns as the A fragments (rows 4 t + h: verify row t, +index head h) and the keys as B. A key's score per row is the family's +(`qsa_paged_index_scores` with `rows_per_request`): ReLU of each head's dot, summed +(h0 + h1) + (h2 + h3), times 128^-0.5. The dot's accumulation order differs from the family +kernel's, so the scores differ in the last bits. + +Precondition (`serves`): verify rows, `rows_per_request` (2 or 3) consecutive rows of one request +each, 4 bf16 index heads of 128; the index pool a `[pages, tokens, 1, 128]` view with token rows +256 B apart. Columns at and past a row's window are left unwritten. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl + +from .qsa_sparse_gqa_splitk_cute__decode import ( + _asm, + _cp16z, + _f32_bits, + _ir, + _ldg1, + _ldg4, + _lds1, + _lds4, + _mma_bf16, + _ptr, + _stg1, + _sts1, +) + +HEADS = 4 +HEAD_DIM = 128 +#: Keys a tile: 64 keeps a 2-stage ring small enough for several CTAs an SM. +BN = 64 +WARPS = 4 +THREADS = 32 * WARPS +#: Bytes of a key row in the ring: unpadded, with 16-byte chunk `ch` of key `k` stored at chunk +#: `ch ^ 4 (k & 1)`, so the two keys a quarter-warp's 16-byte loads and cp.async writes touch fall +#: in disjoint bank halves without row padding. +K_LD = 256 +#: Tiles a CTA walks, and tiles in flight (the fastest of the configurations measured on GB300). +TILES = 8 +STAGES = 2 + + +@dsl_user_op +def _stg2f(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(_f32_bits(a), cutlass.Int32, loc, ip), + _ir(_f32_bits(b), cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _shfl_xor_f(x, m, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + f"shfl.sync.bfly.b32 $0, $1, {m}, 31, 0xffffffff;", + "=f,f", + loc, + ip, + side=True, + ) + ) + + +class _IdxGroupsKernel: + def __init__(self, *, rpr, tpb, cr, tiles, stages, bt_width, num_requests, pdl, bn=BN): + assert 2 <= rpr <= 3 and tpb % cr == 0 and bn in (64, 128) + self.T, self.tpb, self.cr = rpr, tpb, cr + self.bn = bn + self.tiles, self.stages = tiles, max(2, min(stages, tiles)) + self.bt_width, self.num_requests, self.pdl = bt_width, num_requests, pdl + self.k_ld = K_LD + self.o_pg = 0 + self.pg_bytes = tiles * bn * 4 + self.o_ring = -(-self.pg_bytes // 128) * 128 + self.smem = self.o_ring + self.stages * bn * self.k_ld + 128 + + @cute.jit + def __call__( + self, + q_ptr: cute.Pointer, + cache_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_t: cute.Tensor, + rows: cutlass.Int32, + q_stride_row: cutlass.Int32, + page_stride: cutlass.Int64, + out_stride: cutlass.Int32, + max_blocks: cutlass.Int32, + groups: cutlass.Int32, + col_progs: cutlass.Int32, + stream, + ): + self.kernel( + q_ptr.toint(), + cache_ptr.toint(), + bt_ptr.toint(), + pos_ptr.toint(), + req_ptr.toint(), + # the output arrives as a tensor: the TVM-FFI entry takes its stream from it + out_t.iterator.toint(), + rows, + q_stride_row, + page_stride, + out_stride, + max_blocks, + col_progs, + ).launch( + grid=[groups * col_progs, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + ) + + @cute.jit + def _issue(self, j, buf, tidx, s_pg, s_ring, cache_base, page_stride): + """cp.async of tile `j`'s `bn` keys into ring slot `buf`, one commit group: thread t serves + chunks q4, q4 + 4, q4 + 8, q4 + 12 of key t // 4 + 32 i (64 contiguous bytes for the four + threads of a key); a key past the window (page -1) gets zeros.""" + tpb: cutlass.Constexpr = self.tpb + cr: cutlass.Constexpr = self.cr + bn: cutlass.Constexpr = self.bn + k_ld: cutlass.Constexpr = self.k_ld + q4 = tidx % 4 + dst0 = s_ring + buf * (bn * k_ld) + for i in cutlass.range_constexpr(bn // (THREADS // 4)): + k = tidx // 4 + (THREADS // 4) * i + page = _lds1(s_pg + (j * bn + k) * 4) + ok = page >= 0 + n = cutlass.Int32(cutlass.select_(ok, cutlass.Int32(16), cutlass.Int32(0))) + tok = (k * cr + cr - 1) % tpb + src = ( + cache_base + + ( + cutlass.Int64(cutlass.max(page, 0)) * page_stride + + cutlass.Int64(tok * HEAD_DIM) + ) + * 2 + ) + for cc in cutlass.range_constexpr(4): + ch = q4 + 4 * cc + _cp16z(dst0 + k * k_ld + 16 * (ch ^ ((k & 1) << 2)), src + 16 * ch, n) + cute.arch.cp_async_commit_group() + + @cute.kernel + def kernel( + self, + q_base: cutlass.Int64, + cache_base: cutlass.Int64, + bt_base: cutlass.Int64, + pos_base: cutlass.Int64, + req_base: cutlass.Int64, + out_base: cutlass.Int64, + rows: cutlass.Int32, + q_stride_row: cutlass.Int32, + page_stride: cutlass.Int64, + out_stride: cutlass.Int32, + max_blocks: cutlass.Int32, + col_progs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + g = lane // 4 + c = lane % 4 + rpr: cutlass.Constexpr = self.T + tpb: cutlass.Constexpr = self.tpb + cr: cutlass.Constexpr = self.cr + tiles: cutlass.Constexpr = self.tiles + stages: cutlass.Constexpr = self.stages + bn: cutlass.Constexpr = self.bn + ntw: cutlass.Constexpr = self.bn // (8 * WARPS) # n8 tiles a warp + group = bidx // col_progs + prog = bidx % col_progs + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_pg = sbase + self.o_pg + s_ring = sbase + self.o_ring + + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + # ---- the group's rows: request, windows ------------------------------------------- + r0 = group * rpr + lead = cutlass.Int64(_ldg1(req_base + cutlass.Int64(cutlass.min(r0, rows - 1)) * 4)) + safe_req = cutlass.min( + cutlass.max(lead, cutlass.Int64(0)), cutlass.Int64(self.num_requests - 1) + ) + vis = [] + max_vis = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + r = r0 + t + p64 = cutlass.Int64(_ldg1(pos_base + cutlass.Int64(cutlass.min(r, rows - 1)) * 8)) + v = cutlass.Int32( + cutlass.select_(r < rows, cutlass.Int32((p64 + 1) // cr), cutlass.Int32(0)) + ) + vis.append(v) + max_vis = cutlass.max(max_vis, v) + max_vis = cutlass.min(max_vis, max_blocks) + first = prog * tiles + n_t = cutlass.max(cutlass.min(cutlass.Int32(tiles), (max_vis + bn - 1) // bn - first), 0) + + # ---- the program's page numbers (page of each tile key, -1 past the window) ------ + for i in cutlass.range_constexpr(-(-(tiles * bn) // THREADS)): + kk = tidx + THREADS * i + blk = first * bn + kk + lp = (blk * cr + cr - 1) // tpb + ok = (blk < max_vis) & (lp < self.bt_width) + pg = cutlass.Int32(-1) + if ok: + pg = _ldg1(bt_base + (safe_req * self.bt_width + cutlass.Int64(lp)) * 4) + if kk < tiles * bn: + _sts1(s_pg + kk * 4, pg) + + # ---- q of the group as A fragments: row m = 4 t + h; thread (g, c) holds dims + # 32 m4 + 8 c .. + 7 of rows g and g + 8 (m4 = 0..3), the keys read the same dims. ----- + qa = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Int32) + qb = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Int32) + ta = g // 4 + tb = (g + 8) // 4 + ha = g % 4 + ra = cutlass.min(r0 + ta, rows - 1) + rb = cutlass.min(r0 + tb, rows - 1) + oka = (ta < rpr) & (r0 + ta < rows) + okb = (tb < rpr) & (r0 + tb < rows) + for m4 in cutlass.range_constexpr(4): + off = cutlass.Int64((32 * m4 + 8 * c) * 2) + wa = _ldg4(q_base + (cutlass.Int64(ra) * q_stride_row + ha * HEAD_DIM) * 2 + off) + wb = _ldg4(q_base + (cutlass.Int64(rb) * q_stride_row + ha * HEAD_DIM) * 2 + off) + for e in cutlass.range_constexpr(4): + qa[m4, e] = cutlass.Int32(cutlass.select_(oka, wa[e], cutlass.Int32(0))) + qb[m4, e] = cutlass.Int32(cutlass.select_(okb, wb[e], cutlass.Int32(0))) + cute.arch.sync_threads() + + # one commit group a ring slot, empty past the program's tiles: the loop's wait_group(stages - 2) + # then always leaves exactly the newer tiles in flight + for s in cutlass.range_constexpr(stages - 1): + if s < n_t: + self._issue(s, s, tidx, s_pg, s_ring, cache_base, page_stride) + else: + cute.arch.cp_async_commit_group() + + scale = cutlass.Float32(HEAD_DIM**-0.5) + out_a = out_base + cutlass.Int64(ra) * out_stride * 4 + out_b = out_base + cutlass.Int64(rb) * out_stride * 4 + vis_a = vis[0] + vis_b = vis[0] + for t in cutlass.range_constexpr(rpr): + vis_a = cutlass.Int32(cutlass.select_(ta == t, vis[t], vis_a)) + vis_b = cutlass.Int32(cutlass.select_(tb == t, vis[t], vis_b)) + for j in cutlass.range(n_t): + buf = j % stages + cute.arch.cp_async_wait_group(stages - 2) + cute.arch.sync_threads() + if j + stages - 1 < n_t: + self._issue( + j + stages - 1, + (j + stages - 1) % stages, + tidx, + s_pg, + s_ring, + cache_base, + page_stride, + ) + else: + cute.arch.cp_async_commit_group() + rk = s_ring + buf * (bn * self.k_ld) + for nt in cutlass.range_constexpr(ntw): + key = warp * (8 * ntw) + nt * 8 + g + zero = cutlass.Float32(0.0) + d = [zero, zero, zero, zero] + for m4 in cutlass.range_constexpr(4): + # chunk 4 m4 + c of key `key` (parity g & 1) sits at chunk (4 m4 + c) ^ 4 (g & 1) + b = _lds4(rk + key * self.k_ld + 16 * ((4 * m4 + c) ^ ((g & 1) << 2))) + a_lo = (qa[m4, 0], qb[m4, 0], qa[m4, 1], qb[m4, 1]) + a_hi = (qa[m4, 2], qb[m4, 2], qa[m4, 3], qb[m4, 3]) + d = _mma_bf16(a_lo, (b[0], b[1]), d) + d = _mma_bf16(a_hi, (b[2], b[3]), d) + # d0, d1: row g (t = g // 4, head g % 4), keys 2c, 2c + 1 of the n8 tile; + # d2, d3: row g + 8. A row's heads are lanes 4 apart: (h0 + h1) + (h2 + h3). + rr = [cutlass.max(x, zero) for x in d] + pr = [rr[e] + _shfl_xor_f(rr[e], 4) for e in range(4)] + sc = [pr[e] + _shfl_xor_f(pr[e], 8) for e in range(4)] + col = (first + j) * bn + warp * (8 * ntw) + nt * 8 + 2 * c + if ha == 0: + # both keys inside the row's window: one 8-byte store; only the first: 4 bytes + if oka & (col + 1 < vis_a): + _stg2f(out_a + cutlass.Int64(col) * 4, sc[0] * scale, sc[1] * scale) + if oka & (col + 1 == vis_a): + _stg1(out_a + cutlass.Int64(col) * 4, _f32_bits(sc[0] * scale)) + if okb & (col + 1 < vis_b): + _stg2f(out_b + cutlass.Int64(col) * 4, sc[2] * scale, sc[3] * scale) + if okb & (col + 1 == vis_b): + _stg1(out_b + cutlass.Int64(col) * 4, _f32_bits(sc[2] * scale)) + + +_COMPILED: dict = {} + + +def serves(q: torch.Tensor, index_cache: torch.Tensor, rows_per_request: int) -> bool: + """Verify rows this kernel serves: 2 or 3 rows a request, 4 bf16 index heads of 128 with unit + dim stride and 16-byte aligned rows, a `[pages, tokens, 1, 128]` bf16 pool view with token rows + 256 B apart and 16-byte aligned pages.""" + return ( + rows_per_request in (2, 3) + and q.ndim == 3 + and q.shape[1] == HEADS + and q.shape[2] == HEAD_DIM + and q.dtype == torch.bfloat16 + and q.stride(2) == 1 + and q.stride(1) == HEAD_DIM + and (q.stride(0) * 2) % 16 == 0 + and q.data_ptr() % 16 == 0 + and q.shape[0] % rows_per_request == 0 + and index_cache.ndim == 4 + and index_cache.shape[2] == 1 + and index_cache.shape[3] == HEAD_DIM + and index_cache.dtype == torch.bfloat16 + and index_cache.stride(3) == 1 + and index_cache.stride(1) == HEAD_DIM + and (index_cache.stride(0) * 2) % 16 == 0 + and index_cache.data_ptr() % 16 == 0 + ) + + +def index_scores_groups( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, +) -> torch.Tensor: + """`qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=False, + rows_per_request=...)`'s scores to the last bits (`[rows, max_blocks]` fp32, columns at and + past a row's window unspecified).""" + rows = q.shape[0] + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1] + assert query_positions.dtype == torch.int64 and request_indices.dtype == torch.int32 + max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + out = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + if rows == 0: + return out + groups = rows // rows_per_request + col_progs = -(-max_blocks // (BN * TILES)) + pdl = bool(use_pdl and get_env_enable_pdl()) + key = (rows_per_request, tokens_per_block, compress_ratio, tuple(block_table.shape), pdl) + args = ( + _ptr(q, cutlass.BFloat16), + _ptr(index_cache, cutlass.BFloat16), + _ptr(block_table, cutlass.Int32, 4), + _ptr(query_positions, cutlass.Int64, 8), + _ptr(request_indices, cutlass.Int32, 4), + from_dlpack(out, assumed_align=16).mark_layout_dynamic(leading_dim=1), + cutlass.Int32(rows), + cutlass.Int32(q.stride(0)), + cutlass.Int64(index_cache.stride(0)), + cutlass.Int32(out.stride(0)), + cutlass.Int32(max_blocks), + cutlass.Int32(groups), + cutlass.Int32(col_progs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + fn = _COMPILED.get(key) + if fn is None: + kern = _IdxGroupsKernel( + rpr=rows_per_request, + tpb=tokens_per_block, + cr=compress_ratio, + tiles=TILES, + stages=STAGES, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + pdl=pdl, + ) + fn = _COMPILED[key] = cute.compile(kern, *args, options="--opt-level 2 --enable-tvm-ffi") + fn( + q.data_ptr(), + index_cache.data_ptr(), + block_table.data_ptr(), + query_positions.data_ptr(), + request_indices.data_ptr(), + out, + rows, + q.stride(0), + index_cache.stride(0), + out.stride(0), + max_blocks, + groups, + col_progs, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py new file mode 100644 index 000000000000..3f0b46e974fc --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Which requests each TP rank scores in a decode step's split QSA index selection, by work. + +A tep decode step splits its QSA index scoring and block top-k by request over the TP group +(`_decode_index_selection`). A split by position would put the CUDA graph padding (always at the +tail) on one rank, so one launch assigns the requests by work, on the device (a graph replay's +batch is not known to the host at capture): + +- each request's cost is its first row's visible-block count; +- the requests in descending cost (ties to the lower request index) are dealt to the ranks in + snake order (0, 1, .., t - 1, t - 1, .., 0, ...): every rank gets exactly `n / t` requests, so + the kernels' shapes stay the position split's; +- in the permuted order rank `r`'s requests are positions `[r * n / t, (r + 1) * n / t)`: the + request at sorted position `s = lap * t + seat` goes to permuted position + `owner * n / t + local(lap)`, where `local` interleaves the laps from both ends + (0, last, 1, last - 1, ...) so long and short requests alternate in a rank's share. + +Outputs, for `rows = n * p` rows of `p` a request: + +- `iq_rows` `[rows]` int32: each row's position in the permuted order (where the attention + prologue stores the row's index query, so each rank's queries are one contiguous slice); +- `perm_rows` `[rows]` int32: the row at each permuted position; +- this rank's rows of the four QSA row vectors (request index, logical position, sequence length, + visible blocks) and the top-k row starts, in permuted order, `[rows / t]` each, fresh. + +Every rank computes the same permutation from the same replicated metadata, so the gathered +selection, scattered back by `perm_rows`, is bitwise the position split's. +""" + +from __future__ import annotations + +from typing import NamedTuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Requests compared at once in the ranking (a `[BLOCK, CHUNK]` tile). +_CHUNK = 32 + + +class SplitPlan(NamedTuple): + """`index_split_plan`'s outputs (see the module docstring).""" + + iq_rows: torch.Tensor + perm_rows: torch.Tensor + req_idx: torch.Tensor + logical: torch.Tensor + seq_lens: torch.Tensor + visible: torch.Tensor + row_starts: torch.Tensor + + +@triton.jit(do_not_specialize=["num_requests", "rows_per_request", "share", "rank"]) +def _split_plan_kernel( + visible, + req_idx, + logical, + seq_lens, + row_starts, + iq_rows, + perm_rows, + o_req, + o_logical, + o_seq, + o_visible, + o_starts, + num_requests, + rows_per_request, + share, + rank, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, + CHUNK: tl.constexpr, +): + i = tl.arange(0, BLOCK) + live = i < num_requests + first = i.to(tl.int64) * rows_per_request + cost = tl.load(visible + first, mask=live, other=0) + # Stable descending rank: requests strictly costlier, or as costly with a lower index. + pos = tl.zeros([BLOCK], dtype=tl.int32) + for j0 in range(0, num_requests, CHUNK): + j = j0 + tl.arange(0, CHUNK) + cj = tl.load(visible + j.to(tl.int64) * rows_per_request, mask=j < num_requests, other=0) + ahead = (cj[None, :] > cost[:, None]) | ( + (cj[None, :] == cost[:, None]) & (j[None, :] < i[:, None]) + ) + ahead = ahead & (j[None, :] < num_requests) + pos += tl.sum(ahead.to(tl.int32), axis=1) + lap = pos // WORLD + seat = pos % WORLD + owner = tl.where(lap % 2 == 0, seat, WORLD - 1 - seat) + half = (share + 1) // 2 + local_lap = tl.where(lap < half, 2 * lap, 2 * (share - 1 - lap) + 1) + slot = owner * share + local_lap + mine = live & (owner == rank) + for t in range(0, rows_per_request): + g = first + t + k = slot.to(tl.int64) * rows_per_request + t + tl.store(iq_rows + g, k.to(tl.int32), mask=live) + tl.store(perm_rows + k, g.to(tl.int32), mask=live) + local = local_lap.to(tl.int64) * rows_per_request + t + tl.store(o_req + local, tl.load(req_idx + g, mask=mine), mask=mine) + tl.store(o_logical + local, tl.load(logical + g, mask=mine), mask=mine) + tl.store(o_seq + local, tl.load(seq_lens + g, mask=mine), mask=mine) + tl.store(o_visible + local, tl.load(visible + g, mask=mine), mask=mine) + tl.store(o_starts + local, tl.load(row_starts + g, mask=mine), mask=mine) + + +def _spec(requests: int, p: int, world: int, rank: int, kind: str) -> dict: + return dict(requests=requests, rows_per_request=p, world=world, rank=rank, kind=kind) + + +class IndexSplitPlan(OpWrapper): + """`plan = index_split_plan(visible, req_idx, logical, seq_lens, row_starts, num_requests, + rows_per_request, world, rank)`: the work-balanced request assignment of a decode step's split + index selection (module docstring), a `SplitPlan`. + + - `visible`, `req_idx`, `seq_lens`, `row_starts`: int32, `logical`: int64, each a contiguous + `[>= rows]` QSA row vector (`rows = num_requests * rows_per_request`, `p` rows a request, + consecutive); + - `num_requests % world == 0`, `num_requests <= 1024`. + + Bitwise `reference`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep2 HT verify step: 64 requests x 3 rows, rank 1, 3 padding requests at the tail", + spec=_spec(64, 3, 2, 1, "padded")), + Cell(why="the largest batch (192), ties everywhere (equal windows)", spec=_spec(192, 3, 2, 1, "equal")), + Cell(why="tep4: 128 requests over 4 ranks, rank 2", spec=_spec(128, 3, 4, 2, "served")), + ) # fmt: skip + + note = """ + Source: one Triton program (`_split_plan_kernel`, 4 warps), `BLOCK` the next power of two of + the request count (a compile per block size and world); counts are runtime arguments. The + ranking compares every pair of requests in `[BLOCK, CHUNK]` tiles. + """ + + def __call__( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> SplitPlan: + rows = num_requests * rows_per_request + local = rows // world + dev = visible.device + plan = SplitPlan( + iq_rows=torch.empty([rows], dtype=torch.int32, device=dev), + perm_rows=torch.empty([rows], dtype=torch.int32, device=dev), + req_idx=torch.empty([local], dtype=req_idx.dtype, device=dev), + logical=torch.empty([local], dtype=logical.dtype, device=dev), + seq_lens=torch.empty([local], dtype=seq_lens.dtype, device=dev), + visible=torch.empty([local], dtype=visible.dtype, device=dev), + row_starts=torch.empty([local], dtype=row_starts.dtype, device=dev), + ) + if num_requests: + block = max(_CHUNK, triton.next_power_of_2(num_requests)) + _split_plan_kernel[(1,)]( + visible, req_idx, logical, seq_lens, row_starts, *plan, + num_requests, rows_per_request, num_requests // world, rank, + WORLD=world, BLOCK=block, CHUNK=_CHUNK, num_warps=4, + ) # fmt: skip + return plan + + def reference( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> SplitPlan: + p, n, t = rows_per_request, num_requests, world + share = n // t + cost = [int(c) for c in visible[: n * p : p].tolist()] + order = sorted(range(n), key=lambda r: (-cost[r], r)) + slot = [0] * n + half = (share + 1) // 2 + for s, r in enumerate(order): + lap, seat = divmod(s, t) + owner = seat if lap % 2 == 0 else t - 1 - seat + local = 2 * lap if lap < half else 2 * (share - 1 - lap) + 1 + slot[r] = owner * share + local + dev = visible.device + iq = torch.tensor( + [slot[g // p] * p + g % p for g in range(n * p)], dtype=torch.int32, device=dev + ) + perm = torch.empty_like(iq) + perm[iq.long()] = torch.arange(n * p, dtype=torch.int32, device=dev) + mine = perm[rank * share * p : (rank + 1) * share * p].long() + return SplitPlan( + iq_rows=iq, + perm_rows=perm, + req_idx=req_idx.index_select(0, mine), + logical=logical.index_select(0, mine), + seq_lens=seq_lens.index_select(0, mine), + visible=visible.index_select(0, mine), + row_starts=row_starts.index_select(0, mine), + ) + + def is_valid( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> None: + rows = num_requests * rows_per_request + assert world >= 1 and 0 <= rank < world and num_requests % world == 0 + assert num_requests <= 1024 and rows_per_request >= 1 + for v in (visible, req_idx, logical, seq_lens, row_starts): + assert v.dim() == 1 and v.is_contiguous() and v.numel() >= rows + + +index_split_plan = IndexSplitPlan() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py new file mode 100644 index 000000000000..8870b8e9aaec --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py @@ -0,0 +1,395 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index side-cache store: raw index keys and position coordinates into their pages. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_index_store`.""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +_INTEGER_DTYPES = (torch.int32, torch.int64) + + +# Row count, pool/table extents and the strides that vary per engine are not specialized on, so +# a new batch or pool does not recompile; none of them is a vector access worth a variant. +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_requests", + "table_width", + "num_pages", + "stride_c_axis", + "stride_t_request", + ], + do_not_specialize_on_alignment=[ + "coords", + "request_indices", + "logical_positions", + "block_table", + ], +) +def _qsa_paged_index_store_kernel( + token_k, + coords, + index_cache, + position_cache, + request_indices, + logical_positions, + block_table, + num_rows, + num_requests, + table_width, + num_pages, + stride_k_row, + stride_k_head, + stride_k_dim, + stride_c_row, + stride_c_axis, + stride_i_page, + stride_i_slot, + stride_i_head, + stride_i_dim, + stride_p_page, + stride_p_slot, + stride_p_axis, + stride_request, + stride_position, + stride_t_request, + stride_t_column, + HKV: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + AXES: tl.constexpr, + BLOCK_A: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + # Indices are widened to int64 before any arithmetic: plane offsets pass 2^31 elements. + rows = (tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)).to(tl.int64) + live = rows < num_rows + request = tl.load(request_indices + rows * stride_request, mask=live, other=-1).to(tl.int64) + position = tl.load(logical_positions + rows * stride_position, mask=live, other=-1).to(tl.int64) + column = position // TOKENS_PER_BLOCK + slot = position % TOKENS_PER_BLOCK + live &= (request >= 0) & (request < num_requests) & (position >= 0) & (column < table_width) + page = tl.load( + block_table + request * stride_t_request + column * stride_t_column, mask=live, other=-1 + ).to(tl.int64) + live &= (page >= 0) & (page < num_pages) + + dims = tl.arange(0, BLOCK_D) + key_mask = live[:, None] & (dims < HEAD_DIM)[None, :] + for head in tl.static_range(HKV): + key = tl.load( + token_k + + rows[:, None] * stride_k_row + + head * stride_k_head + + dims[None, :] * stride_k_dim, + mask=key_mask, + other=0.0, + ) + tl.store( + index_cache + + page[:, None] * stride_i_page + + slot[:, None] * stride_i_slot + + head * stride_i_head + + dims[None, :] * stride_i_dim, + key.to(index_cache.dtype.element_ty), + mask=key_mask, + ) + + axes = tl.arange(0, BLOCK_A) + axis_mask = live[:, None] & (axes < AXES)[None, :] + coordinate = tl.load( + coords + rows[:, None] * stride_c_row + axes[None, :] * stride_c_axis, + mask=axis_mask, + other=0, + ) + tl.store( + position_cache + + page[:, None] * stride_p_page + + slot[:, None] * stride_p_slot + + axes[None, :] * stride_p_axis, + coordinate.to(position_cache.dtype.element_ty), + mask=axis_mask, + ) + + +def _launch_rows(rows: int) -> Tuple[int, int]: + """Rows per program and warps: one row per program up to 256 rows, then tiles of 16.""" + if rows <= 256: + return 1, 1 + return 16, 4 + + +#: Index geometry: `token_k` is the k head of the packed `[T, 4 + 1, 128]` index projection; the +#: coordinates are the `[T, 3]` transpose of the engine's `[3, max_num_tokens]` position buffer. +_CKPT = dict( + hkv=1, + head_dim=128, + index_heads=4, + tokens_per_block=32, + axes=3, + slots=32, + width=260, + key_dtype=torch.bfloat16, + request_dtype=torch.int32, + position_dtype=torch.int64, + key_layout="packed", + cache_layout="plain", + hole=None, +) + +#: ISL 8192 + OSL 1024 (positions to 9215), 128 requests, 16384 rows in one step. +_WIDE = dict(slots=160, width=300) +_PREFILL_16K = ((0, 8192), (0, 8192)) + +#: 32 MTP7 verify blocks (8 rows each), some straddling a page edge. +_MTP7_32 = tuple((29 + 61 * i, 8) for i in range(32)) + + +def _spec(segments, pages, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CKPT, "segments": segments, "pages": pages, **overrides} + + +#: One rank of a member. `token_k` is the index key head of the fused input projection row +#: (`weights.py`'s `attn_in_rows`): 7424 wide under attention TP2, 4352 under TP4, 14080 under +#: DP and tp1. The bf16 index pool coalesces the 12 sparse layers' planes: `[pages, 32, 1, 128]` +#: at page stride 12 x 4096; the position plane is a contiguous `[pages, 32, 3]` int32 pool. +#: `pages` is the NVFP4 tep4 engine's pool (245835 pages of 32); the table is 128 x 292. +#: `straddle` puts the last row on the pool's last page, past 2^31 elements. +_DEFAULT = dict( + _CKPT, + key_layout="fused_row", + cache_layout="coalesced", + layers=12, + layer=11, + page_stride=49152, + slots=128, + width=292, + pages=245835, + straddle=True, +) + +#: 128 decode rows on both sides of page edges; the last at 9215. +_DECODE_128 = tuple((31 + 72 * i, 1) for i in range(127)) + ((9215, 1),) +#: 128 MTP3 verify blocks of 4 rows, some across a page edge, the last ending at 9215. +_MTP3_128 = tuple((29 + 71 * i, 4) for i in range(127)) + ((9212, 4),) + + +class QsaPagedIndexStore(OpWrapper): + """Write each row's raw index key and position coordinates into its paged slot. + + For row t, with `page = block_table[request_indices[t], logical_positions[t] + // tokens_per_block]` and `slot = logical_positions[t] % tokens_per_block`:: + + index_cache[page, slot] = token_k[t] # cast to the cache dtype + position_cache[page, slot, :A] = position_coordinates[t] # A = coordinates' width + + A row that resolves to no slot (negative or out-of-table request, position or page column, + or a table entry that is `-1` or past the pool) is skipped. The key is stored raw (un-normed, + un-rotated): `qsa_prefill_compress` applies the norm and rotation itself. Returns + `(index_cache, position_cache)`, written in place. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="decode MTP7: 32 requests x 8 rows (256), the last one-row-per-program " + "launch, blocks straddling pages", + spec=_spec(_MTP7_32, pages=1200, slots=40), + ), + Cell( + why="prefill 16384 rows (max_num_tokens), 16 rows per program, token_k off the " + "fused projection row", + spec=_spec(_PREFILL_16K, pages=600, key_layout="fused_row", **_WIDE), + ), + Cell( + why="NVFP4-KV side planes: index and position planes as strided views of one " + "per-slot buffer holding 12 index-K pages and the position page; one row " + "on an unallocated (-1) table entry is skipped", + spec=_spec( + ((0, 1), (31, 1), (32, 1), (33, 40)), + pages=8, + cache_layout="shared_page", + layers=12, + page_stride=49344, + position_strides=(24672, 3, 1), + hole="unallocated_page", + ), + ), + Cell( + why="default tep4 decode: one row at position 9215 on the pool's last page, token_k " + "off the 4352-wide fused row, the coalesced bf16 index plane", + spec=dict(_DEFAULT, member="tep4", segments=((9215, 1),), row_width=4352), + ), + Cell( + why="tp2 decode: 128 requests one row each (the largest captured batch), 7424-wide row", + spec=dict(_DEFAULT, member="tp2", segments=_DECODE_128, row_width=7424), + ), + Cell( + why="dep4 MTP3 verify: 128 requests x 4 rows (512), 16 rows per program, " + "14080-wide row", + spec=dict(_DEFAULT, member="dep4", segments=_MTP3_128, row_width=14080), + ), + ) + + note = """ + Source: this target's Triton kernel `_qsa_paged_index_store_kernel` (Triton JIT). + + The target passes `qsa_req_idx_per_token` (int32) and `qsa_logical_positions` (int64) sliced + to the step's rows, `qsa_block_table` unsliced, the layer's `get_index_k_buffer(i)` and the + shared `get_qsa_position_buffer()` (rewritten idempotently by every sparse layer). Every + operand is read through its own strides. Under fp8 and NVFP4 KV both planes are strided views + of one per-slot buffer (index page stride 12 x 4096 + 192, position strides (24672, 3, 1)); + the cells check that a page's other index-K roles and layer planes survive the call. + + Preconditions: + - destination slots are unique across the batch; two rows naming one (page, slot) both + store, unordered. The engine's mapping cannot collide. + - coordinates fit the position plane's dtype: wider values wrap, floats truncate. + Shape, rank and dtype mismatches raise `ValueError` before launching; `rows == 0` returns + before launching. CUDA-graph capturable; a replay follows buffers rewritten in place. + + Not certified: an fp8 index plane, keys other than bf16, index geometry other than 1 head x + 128 at 32-token pages, batches other than the cells', strided metadata or tables, stride-0 + coordinates, multiple devices, concurrent streams onto one pool, duplicate destinations, + skip conditions other than a `-1` table entry, tep2/dep2/tp1 at their own cells (certified + through tp2 and dep4 at the same row widths), each member's own pool size (the cells use the + NVFP4 tep4 engine's 245835 pages), and positions past 9215. + """ + + def __call__( + self, + token_k: torch.Tensor, + position_coordinates: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + rows = request_indices.shape[0] + if request_indices.dim() != 1 or logical_positions.shape != request_indices.shape: + raise ValueError( + f"request_indices {tuple(request_indices.shape)} and logical_positions " + f"{tuple(logical_positions.shape)} must be one-dimensional and equally long" + ) + if request_indices.dtype not in _INTEGER_DTYPES or ( + logical_positions.dtype not in _INTEGER_DTYPES + or block_table.dtype not in _INTEGER_DTYPES + ): + raise ValueError("QSA index store metadata and block table must be int32 or int64") + if token_k.dim() != 3 or token_k.shape[0] != rows: + raise ValueError(f"token_k {tuple(token_k.shape)} must be [{rows}, heads, dim]") + if position_coordinates.dim() != 2 or position_coordinates.shape[0] != rows: + raise ValueError( + f"position_coordinates {tuple(position_coordinates.shape)} must be [{rows}, axes]" + ) + if index_cache.dim() != 4 or index_cache.shape[1:] != ( + tokens_per_block, + *token_k.shape[1:], + ): + raise ValueError( + f"index_cache {tuple(index_cache.shape)} must be [pages, {tokens_per_block}, " + f"{token_k.shape[1]}, {token_k.shape[2]}]" + ) + if ( + position_cache.dim() != 3 + or position_cache.shape[1] != tokens_per_block + or position_cache.shape[2] < position_coordinates.shape[1] + ): + raise ValueError( + f"position_cache {tuple(position_cache.shape)} must be [pages, " + f"{tokens_per_block}, >= {position_coordinates.shape[1]}]" + ) + if block_table.dim() != 2: + raise ValueError(f"block_table {tuple(block_table.shape)} must be two-dimensional") + if rows == 0: + return index_cache, position_cache + block_rows, warps = _launch_rows(rows) + axes = position_coordinates.shape[1] + _qsa_paged_index_store_kernel[(triton.cdiv(rows, block_rows),)]( + token_k, + position_coordinates, + index_cache, + position_cache, + request_indices, + logical_positions, + block_table, + rows, + block_table.shape[0], + block_table.shape[1], + index_cache.shape[0], + *token_k.stride(), + *position_coordinates.stride(), + *index_cache.stride(), + *position_cache.stride(), + request_indices.stride(0), + logical_positions.stride(0), + *block_table.stride(), + HKV=token_k.shape[1], + HEAD_DIM=token_k.shape[2], + BLOCK_D=triton.next_power_of_2(token_k.shape[2]), + AXES=axes, + BLOCK_A=triton.next_power_of_2(axes), + TOKENS_PER_BLOCK=tokens_per_block, + BLOCK_ROWS=block_rows, + num_warps=warps, + ) + return index_cache, position_cache + + def reference( + self, + token_k: torch.Tensor, + position_coordinates: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Both planes after the store, as fresh tensors; the inputs are not touched. + + Destinations are decided in host Python integers, independent of the kernel's arithmetic. + """ + expected_index, expected_position = index_cache.clone(), position_cache.clone() + table = block_table.tolist() + pool = index_cache.shape[0] + kept, pages, slots = [], [], [] + rows = zip(request_indices.tolist(), logical_positions.tolist()) + for row, (request, position) in enumerate(rows): + if not 0 <= request < len(table) or position < 0: + continue + column, slot = divmod(position, tokens_per_block) + if column >= len(table[request]) or not 0 <= table[request][column] < pool: + continue + kept.append(row) + pages.append(table[request][column]) + slots.append(slot) + if not kept: + return expected_index, expected_position + device = index_cache.device + kept_t, page_t, slot_t = ( + torch.tensor(v, dtype=torch.long, device=device) for v in (kept, pages, slots) + ) + axes = position_coordinates.shape[-1] + expected_index[page_t, slot_t] = token_k[kept_t].to(index_cache.dtype) + expected_position[page_t, slot_t, :axes] = position_coordinates[kept_t].to( + position_cache.dtype + ) + return expected_index, expected_position + + +qsa_paged_index_store = QsaPagedIndexStore() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py new file mode 100644 index 000000000000..b9f10ccdc806 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse path's paged K/V append: this step's rows into the HND main pool. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_kv_store`.""" + +from __future__ import annotations + +from typing import Tuple + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from . import qsa_triton_kernels__stable_jit as _qsa_kernels + +#: Main K/V geometry: k and v are head slices of the packed `[T, 24 + 2 + 2, 256]` QKV row; +#: `slots` x `width` is `qsa_block_table` (max_num_sequences x max_blocks_per_seq). +_CKPT = dict( + hkv=2, + head_dim=256, + heads_q=24, + tokens_per_block=32, + slots=32, + width=260, + row_dtype=torch.bfloat16, + cache_dtype=torch.bfloat16, + request_dtype=torch.int32, + position_dtype=torch.int64, + hole=None, + # Heads after v in each row: 27 for the fused projection row [q | k | v | gate | index]. + row_pad_heads=0, +) + +_F8 = torch.float8_e4m3fn + +_DECODE = ((0, 1), (31, 1), (32, 1), (33, 1), (100, 1), (255, 1), (7, 1), (64, 1)) + +#: 32 MTP7 verify blocks (8 rows each), some straddling a page edge. +_MTP7_32 = tuple((29 + 61 * i, 8) for i in range(32)) + +#: One rank's pool as `Qwen4ExpHybridCacheManager` lays it out: a page holds the K and V planes +#: of all 12 attention layers (24 role pages of `[hkv, 32, 256]`), so a layer's +#: `get_buffers(i, kv_layout="HND")` view steps 24 role pages a page; the cells drive layer 47 +#: (`layer_slot` 11). The table is 128 x 292 at `max_seq_len` 9216. +_ENGINE = dict(_CKPT, pool="engine", role_pages=24, layer_slot=11, slots=128, width=292) + +#: tp4 / tep4 rank: fused row `[q 6 | k 1 | v 1 | gate 6 | index 3]` (17 heads, stride 4352). +_TP4 = dict(_ENGINE, hkv=1, heads_q=6, row_pad_heads=9, row_stride=4352) + +#: dep4 rank: tp1 row `[q 24 | k 2 | v 2 | gate 24 | index 3]` (55 heads, stride 14080), e4m3 pool. +_DEP4_FP8 = dict(_ENGINE, row_pad_heads=27, row_stride=14080, cache_dtype=torch.float8_e4m3fn) + +#: tp2 / tep2 rank: fused row `[q 12 | k 1 | v 1 | gate 12 | index 3]` (29 heads, stride 7424). +_TP2 = dict(_ENGINE, hkv=1, heads_q=12, row_pad_heads=15, row_stride=7424) + +#: dep4 rank over a bf16 pool. +_DEP4 = dict(_ENGINE, row_pad_heads=27, row_stride=14080) + +#: dep4 with one-model MTP: the draft layer joins the trunk's manager (13 layers, 26 role pages). +_DEP4_MTP = dict(_DEP4, role_pages=26) + +#: Attention pool pages per rank (`maxNumBlocks` less the recurrent slots) of the NVFP4 +#: checkpoint's engines at KV fraction 0.6, `max_batch_size` 128, `max_seq_len` 9216; the FP8 +#: cells reuse them at the same per-page shape. +PAGES_TEP4 = 257109 +PAGES_DEP4_FP8 = 171362 +PAGES_DEP4 = 119680 + +#: 128 decode rows: page-aligned, mid-page, the last position (9215), then a stride-577 walk. +_DECODE_128 = ((0, 1), (31, 1), (32, 1), (9215, 1)) + tuple( + ((i * 577 + 45) % 9216, 1) for i in range(124) +) + +#: An 8192-row context step: a chunk from mid-page to 9215, and a request's first 1023 tokens. +_PREFILL_8192 = ((2047, 7169), (0, 1023)) + +#: 128 MTP3 verify blocks (512 rows) ending at position 9215, at every offset in a page. +_MTP3_128 = tuple(((29 + 71 * i) % 9212, 4) for i in range(127)) + ((9212, 4),) + + +def _spec(segments, pages, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CKPT, "segments": segments, "pages": pages, **overrides} + + +def _engine(base: dict, member: str, segments, pages: int, **overrides) -> dict: + """An engine-pool cell: `rows` is the step's row count, `member` whose pool `pages` is.""" + rows = sum(n for _, n in segments) + return { + **base, + "member": member, + "segments": segments, + "pages": pages, + "rows": rows, + **overrides, + } + + +class QsaPagedKvStore(OpWrapper): + """Scatter every row's K and V into its request's page of the HND pool. + + One Triton launch over `(rows, Hkv)`. For row t:: + + page = block_table[request_indices[t], logical_positions[t] // tokens_per_block] + slot = logical_positions[t] % tokens_per_block + k_cache[page, :, slot, :] = k[t] v_cache[page, :, slot, :] = v[t] + + `logical_positions` is absolute, so prefill, decode and MTP rows are the same call. Values + are converted once to the pool's dtype (an fp8-e4m3 pool takes no scale: readers use kv + scale 1.0). A row that resolves to no slot (negative or out-of-table request, position or + page column, or a table entry that is `-1` or past the pool) is skipped. Returns + `(k_cache, v_cache)`, written in place. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="decode: 8 rows off the fused projection row (row stride 14080), " + "at aligned, mid and page-crossing positions, plus one row on an " + "unallocated (-1) table entry that is skipped", + spec=_spec(_DECODE, pages=64, row_pad_heads=27, hole="unallocated_page"), + ), + Cell( + why="fp8 decode MTP7: 32 requests x 8 rows (256), blocks straddling pages", + spec=_spec(_MTP7_32, pages=1200, slots=40, cache_dtype=_F8), + ), + # Engine pools at one rank's page count. + Cell( + why="tep4 decode, 1 row: hkv 1 off the 4352-wide row, position 9215, " + "257109-page bf16 pool", + spec=_engine(_TP4, "tep4", ((9215, 1),), PAGES_TEP4), + ), + Cell( + why="tp4 context, 8192 rows (max_num_tokens): a mid-page chunk to 9215 and a " + "request's first 1023", + spec=_engine(_TP4, "tp4", _PREFILL_8192, PAGES_TEP4), + ), + Cell( + why="tp2 decode, 128 rows (max_batch_size) off the 7424-wide row, aligned, mid " + "and page-crossing", + spec=_engine(_TP2, "tp2", _DECODE_128, PAGES_TEP4), + ), + Cell( + why="dep4 MTP3 verify, 128 requests x 4 rows (512) to position 9215: hkv 2 off " + "the 14080-wide row, 26 role pages a page (the draft layer's planes beside the " + "trunk's)", + spec=_engine(_DEP4_MTP, "dep4", _MTP3_128, PAGES_DEP4), + ), + Cell( + why="dep4 kv_fp8 decode, 128 rows (max_batch_size): hkv 2 off the 14080-wide row " + "into a 171362-page e4m3 pool", + spec=_engine(_DEP4_FP8, "dep4", _DECODE_128, PAGES_DEP4_FP8), + ), + Cell( + why="dep4 context, 8192 rows (max_num_tokens): hkv 2 off the 14080-wide row, a " + "mid-page chunk to 9215 and a request's first 1023", + spec=_engine(_DEP4, "dep4", _PREFILL_8192, PAGES_DEP4), + ), + ) + + note = """ + Source: `qsa_triton_kernels__stable_jit._qsa_kv_store_stable_kernel` (Triton JIT). + + Preconditions: + - destination slots are unique across the batch; two rows naming one (page, slot) both + store, unordered. The engine's mapping cannot collide. + - `request_indices` and `logical_positions` are contiguous (`is_valid`): the kernel gets + no stride for them. + - both are cast to int32 in the kernel, so positions must stay below 2^31 (the engine's + stay below 2^14). + Pool/table extents, the table's row stride and the row count are runtime ints, so one + compiled kernel serves every pool and batch; CUDA-graph capturable, and a replay follows + positions rewritten in place. Page ids are widened to int64 before the stride product. + + Not certified: duplicate destinations, multiple devices, a scaled fp8 pool, fp8 values past + 448, the packed-QKV-row layout over a bf16 pool or at one KV head, fp16 rows, geometry other + than D 256 / Hkv 1-2 / 32-token pages, int32 positions or int64 request indices, head-major + rows, sliced or transposed tables, `rows == 0`, unresolvable rows other than a `-1` entry + or a negative position, positions past 9215, tables wider than 292 columns, and the FP8 + checkpoint's engines at their own page counts. + """ + + def __call__( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + _qsa_kernels.qsa_paged_kv_store( + k=k, + v=v, + k_cache=k_cache, + v_cache=v_cache, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + return k_cache, v_cache + + def reference( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Both caches after the scatter, as fresh tensors; the inputs are not touched.""" + expected_k, expected_v = k_cache.clone(), v_cache.clone() + requests = request_indices.long() + positions = logical_positions.long() + columns = positions.div(tokens_per_block, rounding_mode="floor") + table_rows, table_width = block_table.shape + addressable = ( + (requests >= 0) & (requests < table_rows) & (positions >= 0) & (columns < table_width) + ) + pages = block_table[ + requests.clamp(0, table_rows - 1), columns.clamp(0, table_width - 1) + ].long() + addressable &= (pages >= 0) & (pages < k_cache.shape[0]) + page, slot = pages[addressable], (positions % tokens_per_block)[addressable] + expected_k[page, :, slot, :] = k[addressable].to(expected_k.dtype) + expected_v[page, :, slot, :] = v[addressable].to(expected_v.dtype) + return expected_k, expected_v + + def is_valid( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> None: + # A strided view lands rows at other rows' slots, with nothing raised. + assert request_indices.is_contiguous(), "request_indices must be a contiguous 1-D tensor" + assert logical_positions.is_contiguous(), ( + "logical_positions must be a contiguous 1-D tensor" + ) + + +qsa_paged_kv_store = QsaPagedKvStore() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py new file mode 100644 index 000000000000..6114b460c996 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA: attention over an explicitly selected set of paged K/V tokens. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa.py`, +launching this target's stable-JIT kernels.""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from .qsa_triton_kernels__stable_jit import ( + can_fuse_splitk_output_gate as _can_fuse_qsa_splitk_output_gate, +) +from .qsa_triton_kernels__stable_jit import qsa_paged_sparse_gqa as _triton_qsa_paged_sparse_gqa + +_CHECKPOINT = dict( + q_heads=24, + kv_heads=2, + head_dim=256, + tokens_per_block=32, + compress_ratio=4, + token_topk=2048, + cache_dtype=torch.bfloat16, + q_dtype=torch.bfloat16, + # q is the first 24 heads of the packed [T, 24 + 2 + 2, 256] QKV buffer, or ("fused_row") + # of the fused input projection row. + q_layout="qkv_slice", + request_dtype=torch.int32, + pages=260, +) + +_SPLIT_K = ("splitk", "merge") +_PREFILL = ("prefill",) +_FP8 = dict(cache_dtype=torch.float8_e4m3fn) + + +def _spec(**overrides) -> dict: + """A cell spec: this checkpoint's geometry unless the cell says otherwise. + + `segments` is `(first position, rows)` per request, in row order; + `prefill_pair` says whether `query_positions` / `compress_ratio` are + passed, which the target does exactly when the step has a context request. + """ + return {**_CHECKPOINT, **overrides} + + +def _decode(segments, launches=_SPLIT_K, **overrides) -> dict: + base = dict(segments=segments, slots=32, prefill_pair=False, launches=launches) + return _spec(**{**base, **overrides}) + + +#: The engine's trunk layout (`Qwen4ExpHybridCacheManager.get_buffers`): a pool page holds the +#: K and V planes of the 12 attention layers (24 role pages); layer 11's planes are the last. +_TRUNK = dict(slot_pages=24, layer=11) + +#: The element offset the trunk cells' pages straddle in the K and V planes. +_OFFSET_EDGE = 2**31 + +#: One tp4 / tep4 rank: 6 q heads and one KV head; a 128-row table of 292 columns +#: (max_seq_len 9216), 516 at MTP's 16384. +_TP4 = dict(q_heads=6, kv_heads=1, slots=128, pages=292, **_TRUNK) +_TP4_MTP = dict(_TP4, pages=516) +#: One dep4 rank: 24 / 2 heads and a 32-row table. +_DEP4 = dict(q_heads=24, kv_heads=2, slots=32, pages=292, **_TRUNK) +_DEP4_FP8 = dict(_DEP4, **_FP8) + +#: q and the output gate off one rank's fused input projection row `[q | k | v | gate | index]` +#: in heads of 256: 17 heads at a tp4 / tep4 rank, 55 at dep4. +_TP4_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=17) +_DEP4_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=55) + +#: One tp2 / tep2 rank: 12 q heads and one KV head, a 128-row table of 292 columns. +_TP2 = dict(q_heads=12, kv_heads=1, slots=128, pages=292, **_TRUNK) +#: A tp2 / tep2 rank's fused row: 29 heads. +_TP2_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=29) +#: One tp1 or dep2 rank: 24 / 2 heads and a 128-row table. +_TP1 = dict(q_heads=24, kv_heads=2, slots=128, pages=292, **_TRUNK) +_TP1_FP8 = dict(_TP1, **_FP8) + + +def _straddling(segments, kv_heads: int) -> int: + """The pool page count that puts a cell's requests (the pool's last pages) on both sides of + `_OFFSET_EDGE` elements of the K plane.""" + tpb, d, slot_pages, layer = 32, 256, _TRUNK["slot_pages"], _TRUNK["layer"] + needed = sum(math.ceil((s + n) / tpb) for s, n in segments) + 7 + role_page = kv_heads * tpb * d + edge_page = math.ceil((_OFFSET_EDGE - 2 * layer * role_page) / (slot_pages * role_page)) + return max(edge_page + needed // 2, needed) + + +def _default(segments, member: dict, launches, **overrides) -> dict: + """A cell at `member`'s heads, table and trunk layout, its pool straddling `_OFFSET_EDGE`.""" + return _decode( + segments, + launches=launches, + rows=sum(n for _, n in segments), + pool_pages=_straddling(segments, member["kv_heads"]), + offset_edge=_OFFSET_EDGE, + **{**member, **overrides}, + ) + + +def _spread(count: int, rows: int, last_start: int) -> tuple: + """`count` requests of `rows` rows: one starting at `last_start` (its last row at the + envelope's last position), the others spread from position 0 up.""" + span = last_start + 1 + return tuple(((5 + 1129 * g) % span, rows) for g in range(count - 1)) + ((last_start, rows),) + + +class QsaPagedSparseGqa(OpWrapper): + """Sparse GQA over the selected tokens of an HND paged K/V pool. + + For row `r`, query head `h`, KV head `h // (Hq // Hkv)` (contiguous groups), over the + columns `j` of `selected_tokens[r]` that name a resident token:: + + t = selected_tokens[r, j] # logical position + page = block_table[request_indices[r], t // tokens_per_block] + s_j = (q[r, h] . k_cache[page, kv, t % tpb]) * softmax_scale + out[r, h] = sum_j softmax(s)_j * v_cache[page, kv, t % tpb] + + Non-resident columns (negative index, page past the table, negative or out-of-pool table + entry, request outside the table) contribute nothing; a row left with nothing returns zeros. + Indices are not deduplicated. Returns a fresh contiguous `[rows, Hq, D]` tensor of `q.dtype`. + + Two kernels, chosen inside the call: with `query_positions` and `compress_ratio` given and + `rows * Hkv > 512`, a causal-prefix prefill kernel; otherwise split-K, plus a merge kernel + when it used more than one split. + + **The prefill pair is a bound, not a mask.** The prefill kernel reads only the + `min(visible // cr, block_topk) * cr + visible % cr` leading columns of a row + (`visible = query_positions[r] + 1`); a selection with valid entries past the bound is + silently truncated. QSA's top-k + expansion emit exactly that prefix. + + **The output gate rides the merge when `output_gate` is passed:** each merged head row is + rounded to `q.dtype`, multiplied in fp32 by `sigmoid(output_gate)` and rounded again (the + ungated call followed by `fused_sigmoid_mul_inplace`). `output_gate` is `[rows, Hq * D]` or + `[rows, Hq, D]` in `q.dtype` with unit last stride. Only a multi-split launch has a merge to + carry it; `fuses_output_gate` says whether a call does, and the op raises `ValueError` if + handed a gate it cannot carry. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + # ---- tp4 / tep4 / dep4 ranks: 6 / 1 and 24 / 2 heads, trunk-layout pools ---------- + Cell( + why="tp4/tep4 decode, 128 requests x 1 row (the largest captured batch): " + "6 q / 1 kv heads, 8-way split-K + merge, [128, 292] table", + spec=_default(_spread(128, 1, 9215), _TP4, _SPLIT_K), + ), + Cell( + why="tep4 MTP7 verify, 128 requests x 8 rows = 1024: 6 q / 1 kv heads, one split " + "and no merge, [128, 516] table (max_seq_len 16384), rows to position 16383", + spec=_default(_spread(128, 8, 16376), _TP4_MTP, ("splitk",), rows_per_request=8), + ), + # ---- the gated decode / verify site, one cell per 6 / 1 launch bucket ------------- + Cell( + why="tep4 gated decode 1 row in a CUDA graph: q and the gate off the 17-head " + "(4352-wide) fused row, 64-way split-K + gated merge with PDL, captured, refilled " + "and replayed", + spec=_default( + _spread(1, 1, 9215), _TP4, _SPLIT_K, splits=64, pdl=True, capture=True, **_TP4_ROW + ), + ), + Cell( + why="tep4 gated decode 3 rows (PDL's 2..4-row branch): 64-way split-K + gated merge " + "with PDL, 17-head fused row", + spec=_default(_spread(3, 1, 9215), _TP4, _SPLIT_K, splits=64, pdl=True, **_TP4_ROW), + ), + Cell( + why="tep4 gated decode 16 rows (the 5..31-row branch): 32-way split-K + gated merge, " + "17-head fused row", + spec=_default(_spread(16, 1, 9215), _TP4, _SPLIT_K, splits=32, **_TP4_ROW), + ), + Cell( + why="tep4 gated decode 128 x 1 row in a CUDA graph (the largest captured batch): " + "8-way split-K + gated merge, 17-head fused row, captured, refilled and replayed", + spec=_default( + _spread(128, 1, 9215), _TP4, _SPLIT_K, splits=8, capture=True, **_TP4_ROW + ), + ), + Cell( + why="tep4 MTP3 gated verify 128 x 4 rows = 512 (the 257..512-row branch): 4-way " + "split-K + gated merge, [128, 516] table, rows to position 16383, 17-head fused row", + spec=_default( + _spread(128, 4, 16380), + _TP4_MTP, + _SPLIT_K, + splits=4, + rows_per_request=4, + **_TP4_ROW, + ), + ), + Cell( + why="dep4 kv_fp8 context 8192 rows from position 0 with the prefill pair: 24 q / 2 " + "kv heads over an e4m3 pool on the prefill kernel, [32, 292] table", + spec=_default(((0, 8192),), _DEP4_FP8, _PREFILL, prefill_pair=True), + ), + Cell( + why="tep4 8192-row context step (max_num_tokens) with the prefill pair, q off the " + "17-head fused row: a context request 0..8064 and 127 generation rows to position " + "9215, all on the prefill kernel", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP4, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=17, + ), + ), + # ---- this target's tp2 / tep2 ranks: 12 / 1 heads off the 29-head fused row -------- + Cell( + why="tp2/tep2 gated decode 1 row at position 9215 in a CUDA graph: 12 q / 1 kv heads, " + "q and the gate off the 29-head (7424-wide) fused row, 64-way split-K + gated merge " + "with PDL, [128, 292] table, captured, refilled and replayed", + spec=_default( + _spread(1, 1, 9215), _TP2, _SPLIT_K, splits=64, pdl=True, capture=True, **_TP2_ROW + ), + ), + Cell( + why="tp2/tep2 8192-row step (max_num_tokens) with the prefill pair, q off the 29-head " + "fused row: a context request 0..8064 and 127 generation rows to position 9215, all " + "on the prefill kernel, 12 / 1 heads", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP2, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=29, + ), + ), + # ---- this target's tp1 / dep2 ranks: 24 / 2 heads off the 55-head fused row -------- + Cell( + why="tp1/dep2 gated decode 128 x 1 row in a CUDA graph: 24 q / 2 kv heads, q and the " + "gate off the 55-head (14080-wide) fused row, 8-way split-K + gated merge, [128, 292] " + "table", + spec=_default( + _spread(128, 1, 9215), _TP1, _SPLIT_K, splits=8, capture=True, **_DEP4_ROW + ), + ), + Cell( + why="tp1/dep2 MTP3 verify 128 x 4 = 512 rows in a CUDA graph: at 24 / 2 heads that is " + "1024 programs, so one split and no merge, and the gate off the 55-head row cannot " + "ride it (fuses_output_gate False); q off that row, rows to position 9215", + spec=_default( + _spread(128, 4, 9212), + _TP1, + ("splitk",), + splits=1, + capture=True, + rows_per_request=4, + gate_fused=False, + **_DEP4_ROW, + ), + ), + Cell( + why="kv_fp8 gated decode 128 x 1 row in a CUDA graph: 24 / 2 heads over an e4m3 pool, " + "8-way split-K + gated merge, 55-head fused row, [128, 292] table", + spec=_default( + _spread(128, 1, 9215), _TP1_FP8, _SPLIT_K, splits=8, capture=True, **_DEP4_ROW + ), + ), + Cell( + why="tp1/dep2 8192-row step (max_num_tokens) with the prefill pair, q off the 55-head " + "fused row: a context request 0..8064 and 127 generation rows to position 9215, all " + "on the prefill kernel, 24 / 2 heads, [128, 292] table", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP1, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=55, + ), + ), + ) + + note = """ + Source: `_qsa_prefill_pair_stable_kernel`, `_qsa_splitk_stable_kernel` and + `_qsa_merge_stable_kernel` in `qsa_triton_kernels__stable_jit` (Triton JIT), copies of the + shared QSA kernels with pool and table extents, row count and metadata alignment + unspecialized: one compile per call site layout, pool dtype and launch bucket + (`splitk_launch_config`), whatever the engine's pool or batch. + + The target passes `qsa_block_table` (physical page ids), `qsa_expand_block_indices`' + output as `selected_tokens` (logical positions, `-1`-padded) and the layer's role planes. + + Preconditions: + - whenever the prefill kernel can run, each selection row is packed at the front with + exactly the bound's count of valid entries (`qsa_topk_select` + expansion satisfy it); + passed because some rows are context, the pair puts the generation rows on the + prefill kernel too. + - an fp8 pool is read at unit scale (there is no KV scale argument). + - `D` a power of two; `Hq % Hkv == 0`. + Inputs are validated loudly (`ValueError`), so `is_valid` adds nothing. Page ids are + widened to int64 before the stride product (the trunk cells straddle 2^31 elements). + The 1-row and 128-row decodes and the one-split verify are captured in a CUDA graph and + replayed over inputs refilled in place. + + Not certified: head geometries other than 24/2, 12/1 and 6/1 x 256; `tokens_per_block` + other than 32; more than 8192 rows; `rows == 0`; capture of the prefill kernel and of the + 1024-row one-split verify at 6/1; gated decode at 24/2 below 128 rows; a trunk with the MTP + draft layer appended (26 role pages); an fp8 pool at 6/1 or 12/1; positions past 16383; + rows whose selection resolves no token; an fp16 pool or q; int64 request indices; the + engines' own pool page counts (the cells' pools hold 5592 to 33167 pages). + """ + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: Optional[torch.Tensor] = None, + compress_ratio: Optional[int] = None, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return _triton_qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + @staticmethod + def fuses_output_gate( + q: torch.Tensor, + k_cache: torch.Tensor, + selected_tokens: torch.Tensor, + output_gate: torch.Tensor, + ) -> bool: + """Whether a call with these operands and no prefill pair carries `output_gate`.""" + return _can_fuse_qsa_splitk_output_gate(q, k_cache, selected_tokens, output_gate) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: Optional[torch.Tensor] = None, + compress_ratio: Optional[int] = None, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """fp32 sparse GQA over every resident selected token, rounded once; with + `output_gate`, then gated in fp32 and rounded again. `query_positions` and + `compress_ratio` are ignored (on a prefix-packed selection the bound reads exactly the + valid entries).""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads = k_cache.shape[0], k_cache.shape[1] + groups = num_q_heads // num_kv_heads + num_requests, table_width = block_table.shape + requests = request_indices.long() + out = torch.empty(rows, num_q_heads, head_dim, dtype=torch.float32, device=q.device) + for start in range(0, rows, 32): + stop = min(start + 32, rows) + chunk = selected_tokens[start:stop].long() + logical_page = chunk.clamp_min(0) // tokens_per_block + in_page = chunk.clamp_min(0) % tokens_per_block + request = requests[start:stop] + owned = (request >= 0) & (request < num_requests) + physical = block_table[ + request.clamp(0, num_requests - 1)[:, None].expand_as(logical_page), + logical_page.clamp(max=table_width - 1), + ].long() + resident = ( + (chunk >= 0) + & owned[:, None] + & (logical_page < table_width) + & (physical >= 0) + & (physical < num_pages) + ) + physical = physical.clamp(0, num_pages - 1) + # What a non-resident column gathers (a clamped page) is not part of the answer: + # zeroed, so a pool holding NaN outside the mapped pages cannot reach the sum. + absent = ~resident[:, :, None, None] + keys = k_cache[physical, :, in_page, :].float().masked_fill(absent, 0.0) + values = v_cache[physical, :, in_page, :].float().masked_fill(absent, 0.0) + grouped = q[start:stop].reshape(-1, num_kv_heads, groups, head_dim).float() + scores = torch.einsum("bhgd,bkhd->bhgk", grouped, keys) * softmax_scale + scores = scores.masked_fill(~resident[:, None, None, :], float("-inf")) + # A row with nothing resident softmaxes over all -inf: NaN, i.e. zeros. + weights = torch.nan_to_num(torch.softmax(scores, dim=-1), nan=0.0) + pooled = torch.einsum("bhgk,bkhd->bhgd", weights, values) + out[start:stop] = pooled.reshape(stop - start, num_q_heads, head_dim) + if output_gate is None: + return out.to(q.dtype) + gate = output_gate.reshape(rows, num_q_heads, head_dim).float() + return (out.to(q.dtype).float() * torch.sigmoid(gate)).to(q.dtype) + + +qsa_paged_sparse_gqa = QsaPagedSparseGqa() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py new file mode 100644 index 000000000000..fd2e6eb20b33 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py @@ -0,0 +1,2817 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA over an e4m3 paged K/V pool for MTP verify rows, request-grouped: CuTe DSL +(tcgen05) split-K over the union of a request's rows' selections, then a row merge. + +Derived from `qsa_paged_sparse_gqa_nvfp4_verify` (same warp roles, tcgen05 pipeline, softmax, +epilogue and row merge) with an exact e4m3 -> f16 widening, and the union built in a shared hash +table instead of a token bitmap, so any block-table width is served. + +One CTA serves one (group of T = `rows_per_request` rows, KV head, split): 128 MMA rows, row +m = t * G + h (verify row t, q head h of the KV head's G; rows past T * G idle), over the group's +union selection, so each selected K/V token is loaded and widened once for the group's rows. The +union lists the first row's four-token groups in column order, then each later row's new ones +(deterministic whatever order the hash atomics ran in); per (tile, verify row) a 64-bit selection +word masks positions the row did not select to -inf, so the softmax offset is the row's own max +and causality is the selection's. Splits take interleaved tiles of the union. + +Roles (11 warps): 0-3 softmax and epilogue (one thread an MMA row), 4 the tcgen05 MMAs, 5 / 6 the +K / V bulk copies into the raw ring, 7-10 the e4m3 -> f16 widening. Tensor memory: O in columns +0-255, S/P buffers at 256 and 320, Q at 384. The hash lives in raw slot 0 and the staged q rows +and the union list in raw slot 1 until the tiles start: one CTA an SM. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple, Optional + +import cutlass +import cutlass.cute as cute +import cutlass.cute.nvgpu.tcgen05 as tcgen05 +import cutlass.utils.blackwell_helpers as sm100_utils +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass._mlir.dialects import nvvm as _nvvm +from cutlass.cute.nvgpu import OperandMajorMode +from cutlass.cute.runtime import from_dlpack +from cutlass.cute.typing import Float32, Int32, Int64 +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl + +from .qsa_sparse_gqa_splitk_cute__decode import ( + MAX_SPLITS, + _asm, + _bf2_hi, + _bf2_lo, + _bf2_pack, + _bf2_to_f2, + _bits_f32, + _cp16z, + _e4m3x4, + _f32_bits, + _ir, + _ldg1, + _ldg4, + _lds1, + _lds4, + _ptr, + _round_bf16, + _stg1, + _sts1, + _sts4, + qsa_sparse_gqa_splitk_cute, +) + +try: + from cutlass.memory import SmemAllocator +except ImportError: # DSL releases before the memory namespace split + from cutlass.utils import SmemAllocator + +F16 = cutlass.Float16 +LOG2E = 1.4426950408889634 +HEAD_DIM = 256 +#: MMA rows a CTA (T verify rows x the KV head's q heads) and positions a tile. +BM = 128 +BN = 64 +#: Four-token groups a tile. +GPT = BN // 4 +#: Bytes of a raw e4m3 K or V row in shared memory: unpadded (a four-token group lands as one +#: 1 KB bulk copy; the widening rotates its reads instead). +RAW_LD = 256 +RAW_K = BN * RAW_LD +#: One raw slot: K rows, V rows. +RAW_BUF = 2 * RAW_K +#: Bytes of one f16 K or V slot (64 positions x 256 dims). +F16_SLOT = BN * HEAD_DIM * 2 +#: The rings: raw e4m3 tiles in flight, f16 K slots, f16 V slots (32 KB each, 192 KB in all). +#: The gather is bound by the bytes in flight an SM, so the raw ring takes the room; one f16 slot +#: each suffices once the widening runs a tile ahead of the MMA (fastest split measured on GB300). +RAW_SLOTS = 4 +K_SLOTS = 1 +V_SLOTS = 1 +#: Shared memory a CTA may claim (227 KB), less barriers, scan scratch and alignment. +SMEM_BUDGET = 232448 - 4096 +#: A row's reference max moves only past this many log2 units (P <= 2^8, exact in f16 range). +RESCALE_TH = 8.0 +SOFT_WARPS = 4 +MMA_WARP = 4 +LOAD_WARP0 = 5 +#: Warps issuing the bulk copies: warp 5 the K groups, warp 6 the V groups (lanes 0-15, one group +#: a lane); a warp's bulk copies issue one lane after another, so two warps halve the issue time. +LOAD_WARPS = 2 +DQ_WARP0 = 7 +DQ_WARPS = 4 +THREADS = 32 * (DQ_WARP0 + DQ_WARPS) +TMEM_O = 0 +TMEM_S0 = 256 +TMEM_Q = 384 +TMEM_COLS = 512 +CHUNK_COLS = 32 +#: Bytes of a staged q row (bf16, 512 + 16 of padding). +Q_LD = 528 +#: Hash slots of the union (keys, then masks, in raw slot 0): twice the most groups T = 4 rows +#: name, so a probe chain stays short. +HASH = 4096 +#: Most rows a group (the hash mask holds four bits a row). +MAX_RPR = 4 +#: Most splits of one (group, KV head) union. +MAX_PLAN_SPLITS = 32 +#: Most splits of a last-wave unit (tail mode). +MAX_TAIL_SPLITS = 4 +#: Fewer request groups than this take the per-row kernel. +MIN_GROUPS = 8 +#: Scan scratch words: (row, column round) x warp counts. +SCAN_WORDS = 256 +# --- inline PTX beyond the per-row kernel's --- + + +@dsl_user_op +def _popc(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(T.i32(), [_ir(x, cutlass.Int32, loc, ip)], "popc.b32 $0, $1;", "=r,r", loc, ip) + ) + + +@dsl_user_op +def _ctz(x, *, loc=None, ip=None): + """Trailing zeros of a nonzero word.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b32 t; brev.b32 t, $1; clz.b32 $0, t; }", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _ballot(x, *, loc=None, ip=None): + """`vote.sync.ballot` of `x != 0` over the full warp.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .pred p; setp.ne.u32 p, $1, 0; vote.sync.ballot.b32 $0, p, 0xffffffff; }", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _sts_u16(saddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; cvt.u16.u32 h, $2; st.shared.u16 [$1], h; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds_u16(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; ld.shared.u16 h, [$1]; cvt.u32.u16 $0, h; }", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _cas_shared(saddr, cmp, val, *, loc=None, ip=None): + """`atom.shared.cas.b32`: the old word at `saddr`.""" + return cutlass.Int32( + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(cmp, cutlass.Int32, loc, ip), + _ir(val, cutlass.Int32, loc, ip), + ], + "atom.shared.cas.b32 $0, [$1], $2, $3;", + "=r,r,r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _red_or_shared(saddr, v, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "{ red.shared.or.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s(saddr, gaddr, nbytes, mbar, *, loc=None, ip=None): + """`cp.async.bulk` global -> this CTA's shared memory, completing `nbytes` on `mbar`.""" + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + ], + "{ cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], $3, [$4]; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _arrive_expect_tx(mbar, nbytes, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(mbar, cutlass.Int32, loc, ip), _ir(nbytes, cutlass.Int32, loc, ip)], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg4(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(v, cutlass.Int32, loc, ip) for v in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg4f(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(v, cutlass.Float32, loc, ip) for v in (a, b, c, d)], + "{ st.global.v4.f32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,f,f,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg2(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v2.b32 {$0, $1}, [$2];", + "=r,=r,l", + loc, + ip, + side=True, + ) + return [cutlass.Int32(llvm.extractvalue(T.i32(), res, [i], loc=loc, ip=ip)) for i in range(2)] + + +@dsl_user_op +def _stg2(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Int32, loc, ip), + _ir(b, cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _red_shared(op, saddr, v, *, loc=None, ip=None): + """red.shared..s32 (op: add, min, max).""" + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "{ red.shared.%s.s32 [$1], $2; mov.u32 $0, 0; }" % op, + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg1cg(gaddr, *, loc=None, ip=None): + """4-byte global load at L2 (`.cg`): data another CTA of this launch wrote.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.cg.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg4cg(gaddr, *, loc=None, ip=None): + """16-byte global load at L2 (`.cg`): data another CTA of this launch wrote.""" + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.cg.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return [cutlass.Int32(llvm.extractvalue(T.i32(), res, [i], loc=loc, ip=ip)) for i in range(4)] + + +@dsl_user_op +def _atom_add_acqrel_g(gaddr, v, *, loc=None, ip=None): + """atomicAdd on a global int32, acq_rel at GPU scope (after a CTA barrier it releases every + thread's prior stores, CUTLASS's generic-barrier pattern); returns the old value.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "atom.acq_rel.gpu.global.add.u32 $0, [$1], $2;", + "=r,l,r", + loc, + ip, + side=True, + ) + ) + + +def _spread4(nib): + """Bits 0..3 of `nib` to bits 0, 8, 16, 24.""" + return (nib & 1) | ((nib & 2) << 7) | ((nib & 4) << 14) | ((nib & 8) << 21) + + +@dsl_user_op +def _tc_fence_before(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.BEFORE_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def _tc_fence_after(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.AFTER_THREAD_SYNC, loc=loc, ip=ip) + + +#: Swizzle: B -> the descriptor's layout type (none, 32B, 64B, 128B). +_SW_TYPES = {0: 0, 1: 6, 2: 4, 3: 2} +#: Layout type -> swizzle atom size along MN (16-byte units). +_SW_ATOM_MN = {0: 1, 6: 2, 4: 4, 2: 8} + + +def _instr_desc(op) -> int: + """The 32-bit instruction descriptor of an f16/bf16 x f16/bf16 -> f32 tcgen05 MMA op.""" + fmt = {cutlass.Float16: 0, cutlass.BFloat16: 1} + a_major = 0 if op.a_major_mode == tcgen05.OperandMajorMode.K else 1 + b_major = 0 if op.b_major_mode == tcgen05.OperandMajorMode.K else 1 + m, n = op.shape_mnk[0], op.shape_mnk[1] + assert op.acc_dtype is cutlass.Float32 and m in (64, 128) and n % 8 == 0 and 8 <= n <= 256 + d = 0 + d |= 1 << 4 # c_format f32 + d |= fmt[op.a_dtype] << 7 + d |= fmt[op.b_dtype] << 10 + d |= a_major << 15 + d |= b_major << 16 + d |= (n >> 3) << 17 + d |= (m >> 4) << 24 + return d + + +def _smem_desc_base(layout_u128, swizzle, k_major: bool) -> int: + """The 64-bit shared-memory descriptor without its start address, for an operand whose + (MN, K) layout `layout_u128` is in 16-byte units (CUTLASS `mma_sm100_desc.hpp`).""" + assert swizzle.num_base == 4 and swizzle.num_shift == 3 + ltype = _SW_TYPES[swizzle.num_bits] + atom_mn = _SW_ATOM_MN[ltype] + if not k_major: + canon = cute.logical_divide(layout_u128, (atom_mn, 8)) + s00 = canon.stride[0][0] + s10 = canon.stride[1][0] + assert (ltype == 0 or s00 == 1) and s10 == atom_mn, "not a canonical UMMA MN layout" + s01, s11 = canon.stride[0][1], canon.stride[1][1] + sbo, lbo = (s01, s11) if ltype == 0 else (s11, s01) + else: + canon = cute.logical_divide(layout_u128, (8, 2)) + s00 = canon.stride[0][0] + s10 = canon.stride[1][0] + assert s00 == atom_mn and (ltype == 0 or s10 == 1), "not a canonical UMMA K layout" + sbo, lbo = canon.stride[0][1], s10 + d = 0 + d |= (lbo & 0x3FFF) << 16 + d |= (sbo & 0x3FFF) << 32 + d |= 1 << 46 # descriptor version + d |= (ltype & 0x7) << 61 + return d + + +def _gemm_ts(op, acc_tmem, a_tmem, tCrA, tCrB, sB, zero_init: bool): + """acc (+)= A B over every K step: A in tensor memory at `a_tmem` (fragment `tCrA`), B the + shared-memory operand `sB` (fragment `tCrB`, one stage). Issued by one elected thread of + the calling warp; the K loop unrolls into one PTX block (precomputed descriptors).""" + idesc = _instr_desc(op) + k_major_b = op.b_major_mode == tcgen05.OperandMajorMode.K + base_b = _smem_desc_base( + cute.recast_layout(128, op.b_dtype.width, sB.layout[0]), + sB.iterator.type.swizzle_type, + k_major_b, + ) + b_lo = base_b & 0xFFFFFFFF + b_hi = (base_b >> 32) & 0xFFFFFFFF + a_layout = cute.recast_layout(32, tCrA.element_type.width, tCrA.layout) + n_k = cute.size(tCrA.shape[2]) + off_a = [cute.crd2idx((0, 0, k), a_layout) for k in range(n_k)] + off_b = [cute.crd2idx((0, 0, k), tCrB.layout) for k in range(n_k)] + start_b = Int32(b_lo) | Int32((sB[None, None, 0].iterator.toint() & 0x3FFFF) >> 4) + body = "{\n\t.reg .pred leader;\n\t.reg .b32 db_lo;\n\t.reg .b64 db;\n\telect.sync _|leader, -1;\n\t" + for k in range(n_k): + acc = "0" if (zero_init and k == 0) else "1" + body += ( + f"add.u32 db_lo, $2, {hex(off_b[k])};\n\t" + f"mov.b64 db, {{db_lo, {hex(b_hi)}}};\n\t" + f"@leader tcgen05.mma.cta_group::1.kind::f16 [$0], [$1 + {hex(off_a[k])}], db, " + f"{hex(idesc)}, {acc};\n\t" + ) + body += "}\n" + llvm.inline_asm( + None, + [ + Int32(cute.arch.make_warp_uniform(acc_tmem)).ir_value(), + Int32(cute.arch.make_warp_uniform(a_tmem)).ir_value(), + Int32(cute.arch.make_warp_uniform(start_b)).ir_value(), + ], + body, + "r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + ) + + +# --- the grouped kernel --- + + +class _Layout(NamedTuple): + """The shared-memory plan of one compile: the union's group capacity, local tiles a CTA may + own in one pass, the union-list offset in raw slot 1 and the table size in int32 words.""" + + ug_max: int + tmax: int + u_ug: int + tab_words: int + + +def _layout(rpr: int, group: int, width: int, splits: int, rings=None): + """The kernel's shared-memory plan for `rpr` rows a group over a `width`-column expanded + selection, or None where it does not fit: the hash (keys and masks) fills raw slot 0, the + staged q rows and the union list share raw slot 1.""" + raw_n, ks_n, vs_n = rings or (RAW_SLOTS, K_SLOTS, V_SLOTS) + if rpr < 1 or rpr > MAX_RPR or rpr * group > BM: + return None + # an expanded selection names at most ceil(width / 4) four-token groups a row + ug_max = rpr * -(-width // 4) + tmax = -(-(-(-ug_max // GPT)) // splits) + # the union list: raw slot 2 where the ring has one, else after the q rows in raw slot 1 + u_ug = RAW_BUF if raw_n >= 3 else -(-(rpr * group * Q_LD) // 16) * 16 + tab_words = 2 * tmax * GPT + 2 * tmax * rpr + fits = ( + 2 * HASH * 4 <= RAW_BUF + and 5 * ug_max <= 3 * HASH + and rpr * group * Q_LD <= RAW_BUF + and u_ug + ug_max * 2 <= RAW_BUF * (2 if raw_n >= 3 else 1) + and rpr * -(-(-(-width // 4)) // THREADS) <= SCAN_WORDS // (THREADS // 32) + and 4 * tab_words + 4 * SCAN_WORDS + raw_n * RAW_BUF + (ks_n + vs_n) * F16_SLOT + <= SMEM_BUDGET + and raw_n >= 2 + and rpr * group * Q_LD <= ks_n * F16_SLOT + ) + return _Layout(ug_max, tmax, u_ug, tab_words) if fits else None + + +def _sigmoid_fast(x): + """1 / (1 + 2^(-x log2 e)) on ex2.approx and rcp.approx (relative error ~1e-6).""" + return cute.arch.rcp_approx(1.0 + cute.math.exp2(-x * LOG2E, fastmath=True)) + + +def _tree_fmax(vals): + """Max of a short list, pairwise (traced as Python).""" + while len(vals) > 1: + vals = [ + cute.arch.fmax(vals[j], vals[j + 1]) if j + 1 < len(vals) else vals[j] + for j in range(0, len(vals), 2) + ] + return vals[0] + + +def _tree_sum(vals): + """Sum of a short list, pairwise (traced as Python).""" + while len(vals) > 1: + vals = [ + vals[j] + vals[j + 1] if j + 1 < len(vals) else vals[j] for j in range(0, len(vals), 2) + ] + return vals[0] + + +def _phase(g, n): + """Parity of use `g` of a ring of `n` slots.""" + return (g // n) % 2 + + +class _GroupedKernel: + def __init__( + self, + *, + rpr, + group, + kv_heads, + width, + tpb, + splits, + tail, + scale, + num_pages, + bt_width, + num_requests, + k_strides, + q_strides, + req64, + gated, + pdl, + bal=False, + bal_ctas=0, + ): + self.RS, self.KS, self.VS = RAW_SLOTS, K_SLOTS, V_SLOTS + lay = _layout( + rpr, group, width, 1 if (tail or bal) else splits, (self.RS, self.KS, self.VS) + ) + assert lay is not None, "no shared-memory plan for this geometry (see `_layout`)" + assert tpb & (tpb - 1) == 0 and tpb % 4 == 0, "a power-of-two page of whole groups" + self.T = rpr + self.group, self.kvh, self.width, self.tpb = group, kv_heads, width, tpb + self.splits = splits + #: tail mode: CTAs below `n_whole` serve whole units, the rest `splits`-way splits + self.tail = tail + #: balanced mode: each CTA's unit and split from `_bal_schedule`; split units merge in + #: `fixup` + self.bal = bal + self.bal_ctas = bal_ctas + #: histogram bins of the schedule: a unit's union tiles are at most rpr * ceil(width / 4) + #: groups / GPT + self.hmax = -(-(-(-(rpr * -(-width // 4)) // GPT) + 1) // 32) * 32 + self.scale_log2 = scale * LOG2E + self.num_pages, self.bt_width, self.num_requests = num_pages, bt_width, num_requests + self.ks, self.qs = k_strides, q_strides + self.req64, self.gated, self.pdl = req64, gated, pdl + self.heads = group * kv_heads + self.rows_m = rpr * group + #: softmax warps holding live MMA rows (the rest only keep the barriers' counts) + self.live_warps = -(-self.rows_m // 32) + self.ug_max, self.tmax = lay.ug_max, lay.tmax + self.raw_bytes = self.RS * RAW_BUF + # union scratch: the hash in raw slot 0, the q rows then the union list in raw slot 1 + self.u_hk = 0 + self.u_hm = HASH * 4 + self.u_q = RAW_BUF + self.u_ug = RAW_BUF + lay.u_ug + # tables (int32 words). Per group: its first token's pool row (page * tpb + slot, -1 + # when it does not resolve) and its (token, row) selection bytes; per (tile, row): the + # 64-bit selection word. + self.t_tok = 0 + self.t_gmask = self.tmax * GPT + self.t_bits = 2 * self.tmax * GPT + self.tab_words = lay.tab_words + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + q_ptr: cute.Pointer, + k_ptr: cute.Pointer, + v_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_ptr: cute.Pointer, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + gate_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + n_whole: cutlass.Int32, + tail_row0: cutlass.Int32, + n_ctas: cutlass.Int32, + tab_ptr: cute.Pointer, + cnt_ptr: cute.Pointer, + est_ptr: cute.Pointer, + stream, + ): + def qk(): + return sm100_utils.make_trivial_tiled_mma( + F16, + F16, + OperandMajorMode.K, + OperandMajorMode.K, + Float32, + tcgen05.CtaGroup.ONE, + (BM, BN), + tcgen05.OperandSource.TMEM, + ) + + def pv(): + return sm100_utils.make_trivial_tiled_mma( + F16, + F16, + OperandMajorMode.K, + OperandMajorMode.MN, + Float32, + tcgen05.CtaGroup.ONE, + (BM, HEAD_DIM), + tcgen05.OperandSource.TMEM, + ) + + qk_mma, qk_mma_acc, pv_mma_acc = qk(), qk(), pv() + q_layout = sm100_utils.make_smem_layout_a(qk_mma, (BM, BN, HEAD_DIM), F16, 1) + k_layout = sm100_utils.make_smem_layout_b(qk_mma, (BM, BN, HEAD_DIM), F16, self.KS) + vk_layout = sm100_utils.make_smem_layout_b(qk_mma, (BM, BN, HEAD_DIM), F16, self.VS) + v_layout = sm100_utils.make_smem_layout_b(pv_mma_acc, (BM, HEAD_DIM, BN), F16, self.VS) + p_layout = sm100_utils.make_smem_layout_a(pv_mma_acc, (BM, HEAD_DIM, BN), F16, 1) + + @cute.struct + class SharedStorage: + q_full: cute.struct.MemRange[Int64, 1] + raw_full: cute.struct.MemRange[Int64, self.RS] + raw_empty: cute.struct.MemRange[Int64, self.RS] + k_full: cute.struct.MemRange[Int64, self.KS] + k_empty: cute.struct.MemRange[Int64, self.KS] + v_full: cute.struct.MemRange[Int64, self.VS] + v_empty: cute.struct.MemRange[Int64, self.VS] + s_full: cute.struct.MemRange[Int64, 2] + p_full: cute.struct.MemRange[Int64, 2] + o_full: cute.struct.MemRange[Int64, 1] + o_final: cute.struct.MemRange[Int64, 1] + tmem_holding: Int32 + scan: cute.struct.MemRange[Int32, SCAN_WORDS] + tab: cute.struct.Align[cute.struct.MemRange[Int32, self.tab_words], 16] + raw: cute.struct.Align[cute.struct.MemRange[cutlass.Int8, self.raw_bytes], 128] + sK: cute.struct.Align[cute.struct.MemRange[F16, cute.cosize(k_layout)], 1024] + sV: cute.struct.Align[cute.struct.MemRange[F16, cute.cosize(vk_layout)], 1024] + + self.shared_storage = SharedStorage + self.vk_layout = vk_layout + bt = cute.make_tensor( + bt_ptr, cute.make_layout((self.num_requests, self.bt_width), stride=(self.bt_width, 1)) + ) + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel( + qk_mma, + qk_mma_acc, + pv_mma_acc, + q_layout, + k_layout, + vk_layout, + v_layout, + p_layout, + sel_t.iterator.toint(), + q_ptr.toint(), + k_ptr.toint(), + v_ptr.toint(), + bt, + req, + out_ptr.toint(), + part_ptr.toint(), + lse_ptr.toint(), + gate_ptr.toint(), + rows, + sel_stride, + gate_stride, + gate_hs, + n_whole, + tail_row0, + tab_ptr.toint(), + cnt_ptr.toint(), + est_ptr.toint(), + ).launch( + grid=[n_ctas, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + min_blocks_per_mp=1, + use_pdl=self.pdl, + ) + + @cute.jit + def _bal_schedule(self, tidx, warp, lane, bidx, rows, est_base, tab_base, s_w): + """Balanced mode, every thread: this CTA's (unit, split, unit's CTAs, unit's first CTA): + unit u takes ceil(e_u / L) CTAs, L the least load in [ceil(max e / MAX_BAL_SPLITS), max e] + whose counts fit `bal_ctas`, units on consecutive CTAs; unit -1 past the plan. Scratch + `s_w` is raw slot 0 (free until the first union).""" + rpr: cutlass.Constexpr = self.T + kvh_n: cutlass.Constexpr = self.kvh + n_ctas: cutlass.Constexpr = self.bal_ctas + hmax: cutlass.Constexpr = self.hmax + nw: cutlass.Constexpr = -(-n_ctas // 32) + s_res = s_w # 16-byte aligned (raw slot 0) + s_emax = s_w + 16 + s_l = s_w + 20 + s_tot = s_w + 32 + s_h = s_w + 32 + 32 * 4 + units = ((rows + rpr - 1) // rpr) * kvh_n + for i in cutlass.range_constexpr(-(-hmax // THREADS)): + k = tidx + i * THREADS + if k < hmax: + _sts1(s_h + k * 4, cutlass.Int32(0)) + if tidx == 0: + _sts1(s_emax, cutlass.Int32(1)) + _sts1(s_l, cutlass.Int32(hmax)) + _sts4(s_res, cutlass.Int32(-1), cutlass.Int32(0), cutlass.Int32(1), cutlass.Int32(0)) + live_u = tidx < units + e = cutlass.Int32(0) + if live_u: + e = cutlass.min( + cutlass.max(_ldg1(est_base + cutlass.Int64(tidx // kvh_n) * 4), cutlass.Int32(1)), + cutlass.Int32(hmax - 1), + ) + cute.arch.sync_threads() + if live_u: + _red_shared("add", s_h + e * 4, cutlass.Int32(1)) + _red_shared("max", s_emax, e) + cute.arch.sync_threads() + emax = _lds1(s_emax) + lo = cutlass.max((emax + MAX_BAL_SPLITS - 1) // MAX_BAL_SPLITS, cutlass.Int32(1)) + # thread t tests L = t + 1: sum over the histogram of h[x] * ceil(x / L), ceil by a + # 16-bit reciprocal (exact: numerators below 2^9, L below 2^8) + cand = tidx + 1 + recip = cutlass.Int32(65536) // cand + 1 + cnt = cutlass.Int32(0) + for x4 in cutlass.range_constexpr(hmax // 4): + h4 = _lds4(s_h + x4 * 16) + for k in cutlass.range_constexpr(4): + x = 4 * x4 + k + if const_expr(x > 0): + cnt = cnt + h4[k] * (((x + cand - 1) * recip) >> 16) + if (cand >= lo) & (cand <= emax) & (cnt <= n_ctas): + _red_shared("min", s_l, cand) + cute.arch.sync_threads() + load = _lds1(s_l) + n = cutlass.Int32(0) + if live_u: + n = (e + load - 1) // load + # exclusive scan of n over units (threads 0 .. n_ctas - 1, warp scans + warp totals) + inc = n + for i in cutlass.range_constexpr(5): + up = cute.arch.shuffle_sync_up(inc, offset=1 << i) + if lane >= (1 << i): + inc = inc + up + if lane == 31: + _sts1(s_tot + warp * 4, inc) + cute.arch.sync_threads() + before = cutlass.Int32(0) + for w in cutlass.range_constexpr(nw): + if warp > w: + before = before + _lds1(s_tot + w * 4) + base = before + inc - n + if live_u & (bidx >= base) & (bidx < base + n): + _sts4(s_res, tidx, bidx - base, n, base) + cute.arch.sync_threads() + res = _lds4(s_res) + if tidx == 0: + _stg4(tab_base + cutlass.Int64(bidx) * 16, res[0], res[1], res[2], res[3]) + cute.arch.sync_threads() + return res + + # ---- pieces every role runs (every thread of the CTA, barrier 0) ----------------------- + + @cute.jit + def _pass_rows(self, rq: cute.Tensor, handled): + lead = cutlass.Int32(-1) + for t in tuple(range(self.T - 1, -1, -1)): + lead = cutlass.Int32(cutlass.select_(((handled >> t) & 1) == 0, rq[t], lead)) + act = cutlass.Int32(0) + for t in cutlass.range_constexpr(self.T): + act = act | cutlass.Int32( + cutlass.select_( + (((handled >> t) & 1) == 0) & (rq[t] == lead), + cutlass.Int32(1 << t), + cutlass.Int32(0), + ) + ) + return lead, act + + @cute.jit + def _hash_insert(self, s_hk, s_hm, gid, bits): + """Insert group `gid` (linear probing over `HASH` slots; a key never moves once + placed) and OR `bits` into its mask; returns the slot.""" + slot = ((gid * cutlass.Int32(-1640531535)) >> 20) & (HASH - 1) + probing = cutlass.Int32(1) + while probing != 0: + old = _cas_shared(s_hk + slot * 4, cutlass.Int32(-1), gid) + if (old == -1) | (old == gid): + probing = cutlass.Int32(0) + else: + slot = (slot + 1) & (HASH - 1) + _red_or_shared(s_hm + slot * 4, bits) + return slot + + @cute.jit + def _union( + self, act, r0, rows, sel_base, sel_stride, tidx, warp, lane, s_hk, s_hm, s_scan, s_ug + ): + """The union of the pass's rows' groups: hash slots of the owned groups, in (row, + column) order, at `s_ug`; returns their count.""" + rpr: cutlass.Constexpr = self.T + width: cutlass.Constexpr = self.width + n_q: cutlass.Constexpr = -(-width // 4) # four-column quads a row + n_i: cutlass.Constexpr = -(-n_q // THREADS) + nwarps: cutlass.Constexpr = THREADS // 32 + # clear the hash: keys -1, masks 0 + for i in cutlass.range_constexpr(-(-(HASH // 4) // THREADS)): + idx = tidx + i * THREADS + if idx < HASH // 4: + _sts4(s_hk + idx * 16, -1, -1, -1, -1) + _sts4(s_hm + idx * 16, 0, 0, 0, 0) + # thread: columns 4q .. 4q + 3 of each row, every load first + toks = [] + for t in cutlass.range_constexpr(rpr): + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + for i in cutlass.range_constexpr(n_i): + for u in cutlass.range_constexpr(4): + colc = cutlass.min(4 * (tidx + i * THREADS) + u, width - 1) + toks.append(_ldg1(row_sel + cutlass.Int64(colc * 4))) + cute.arch.sync_threads() + gids = [] + slots = [] + for t in cutlass.range_constexpr(rpr): + live_t = ((act >> t) & 1) != 0 + for i in cutlass.range_constexpr(n_i): + q4 = tidx + i * THREADS + gid = cutlass.Int32(-1) + bits = cutlass.Int32(0) + for u in cutlass.range_constexpr(4): + tok = toks[(t * n_i + i) * 4 + u] + ok = live_t & (4 * q4 + u < width) & (tok >= 0) + gid = cutlass.Int32(cutlass.select_(ok & (gid < 0), tok >> 2, gid)) + bits = bits | cutlass.Int32( + cutlass.select_(ok, cutlass.Int32(1) << (tok & 3), cutlass.Int32(0)) + ) + slot = cutlass.Int32(-1) + if gid >= 0: + slot = self._hash_insert(s_hk, s_hm, gid, bits << (4 * t)) + gids.append(gid) + slots.append(slot) + cute.arch.sync_threads() + # owner: the lowest row naming the group; counts per (row, round) and warp + owns = [] + for t in cutlass.range_constexpr(rpr): + for i in cutlass.range_constexpr(n_i): + k = t * n_i + i + own = cutlass.Int32(0) + if slots[k] >= 0: + msk = _lds1(s_hm + slots[k] * 4) + own = cutlass.Int32( + cutlass.select_((_ctz(msk) >> 2) == t, cutlass.Int32(1), cutlass.Int32(0)) + ) + owns.append(own) + ball = _ballot(own) + if lane == 0: + _sts1(s_scan + (k * nwarps + warp) * 4, _popc(ball)) + cute.arch.sync_threads() + # place: rounds in (row, round) order, warps within a round, lanes within a warp + lt_mask = (cutlass.Int32(1) << lane) - 1 + total = cutlass.Int32(0) + for k in cutlass.range_constexpr(rpr * n_i): + cnt_lane = cutlass.Int32(0) + if lane < nwarps: + cnt_lane = _lds1(s_scan + (k * nwarps + lane) * 4) + before_w = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, cnt_lane, cutlass.Int32(0))) + ) + round_tot = cute.arch.warp_reduction_sum(cnt_lane) + ball = _ballot(owns[k]) + pos = total + before_w + _popc(ball & lt_mask) + if owns[k] != 0: + if pos < self.ug_max: + _sts_u16(s_ug + pos * 2, slots[k]) + total = total + round_tot + cute.arch.sync_threads() + return cutlass.min(total, self.ug_max) + + @cute.jit + def _resolve(self, lead, split, nsplit, n_loc, n_ug, tidx, s_hk, s_hm, s_ug, s_tab, mBT): + """Every local tile's groups (pool row of the group's first token, (token, row) + selection bytes), then each (tile, verify row)'s 64-bit selection word.""" + rpr: cutlass.Constexpr = self.T + tmax: cutlass.Constexpr = self.tmax + tpb: cutlass.Constexpr = self.tpb + n_rs: cutlass.Constexpr = -(-(tmax * GPT) // THREADS) + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + s_bits = s_tab + self.t_bits * 4 + pages = [] + gids = [] + msks = [] + for i in cutlass.range_constexpr(n_rs): + slot = tidx + i * THREADS + kl = slot // GPT + ugi = (split + kl * nsplit) * GPT + slot % GPT + ok = (kl < n_loc) & (ugi < n_ug) + hs = cutlass.Int32(0) + if ok: + hs = _lds_u16(s_ug + cutlass.min(ugi, self.ug_max - 1) * 2) + gid = cutlass.Int32(cutlass.select_(ok, _lds1(s_hk + hs * 4), cutlass.Int32(0))) + msk = cutlass.Int32(cutlass.select_(ok, _lds1(s_hm + hs * 4), cutlass.Int32(0))) + col = cutlass.min(cutlass.max((gid * 4) // tpb, 0), self.bt_width - 1) + pg = mBT[cutlass.max(lead, 0), col] + pages.append(cutlass.Int32(cutlass.select_(ok, pg, cutlass.Int32(-1)))) + gids.append(gid) + msks.append(msk) + for i in cutlass.range_constexpr(n_rs): + slot = tidx + i * THREADS + page = pages[i] + gid = gids[i] + msk = msks[i] + mw = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + mw = mw | (_spread4((msk >> (4 * t)) & 0xF) << t) + okp = (page >= 0) & (page < self.num_pages) & (msk != 0) + tok = page * tpb + (gid * 4) % tpb + tok = cutlass.Int32(cutlass.select_(okp, tok, cutlass.Int32(-1))) + mw = cutlass.Int32(cutlass.select_(okp, mw, cutlass.Int32(0))) + if slot < tmax * GPT: + _sts1(s_tok + slot * 4, tok) + _sts1(s_gmask + slot * 4, mw) + cute.arch.sync_threads() + for i in cutlass.range_constexpr(-(-(tmax * rpr) // THREADS)): + w = tidx + i * THREADS + if w < tmax * rpr: + kl = w // rpr + t = w % rpr + lo = cutlass.Int32(0) + hi = cutlass.Int32(0) + for gq in cutlass.range_constexpr(GPT): + nb = (_lds1(s_gmask + (kl * GPT + gq) * 4) >> t) & 0x01010101 + nb = (nb | (nb >> 7) | (nb >> 14) | (nb >> 21)) & 0xF + if const_expr(gq < 8): + lo = lo | (nb << (4 * gq)) + else: + hi = hi | (nb << (4 * (gq - 8))) + _sts1(s_bits + w * 8, lo) + _sts1(s_bits + w * 8 + 4, hi) + cute.arch.sync_threads() + + @cute.jit + def _pass_tiles( + self, + act, + lead, + r0, + rows, + sel_base, + sel_stride, + split, + nsplit, + tidx, + warp, + lane, + s_raw, + s_scan, + s_tab, + mBT, + ): + """Union + resolve of one pass; returns the CTA's local tile count.""" + s_hk = s_raw + self.u_hk + s_hm = s_raw + self.u_hm + s_ug = s_raw + self.u_ug + n_ug = self._union( + act, r0, rows, sel_base, sel_stride, tidx, warp, lane, s_hk, s_hm, s_scan, s_ug + ) + n_tiles = (n_ug + GPT - 1) // GPT + n_loc = cutlass.Int32( + cutlass.select_( + split < n_tiles, + (n_tiles - split + nsplit - 1) // nsplit, + cutlass.Int32(0), + ) + ) + self._resolve(lead, split, nsplit, n_loc, n_ug, tidx, s_hk, s_hm, s_ug, s_tab, mBT) + return n_loc + + @cute.kernel + def kernel( + self, + qk_mma: cute.TiledMma, + qk_mma_acc: cute.TiledMma, + pv_mma_acc: cute.TiledMma, + q_layout: cute.ComposedLayout, + k_layout: cute.ComposedLayout, + vk_layout: cute.ComposedLayout, + v_layout: cute.ComposedLayout, + p_layout: cute.ComposedLayout, + sel_base: cutlass.Int64, + q_base: cutlass.Int64, + k_base: cutlass.Int64, + v_base: cutlass.Int64, + mBT: cute.Tensor, + mReq: cute.Tensor, + out_base: cutlass.Int64, + part_base: cutlass.Int64, + lse_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + n_whole: cutlass.Int32, + tail_row0: cutlass.Int32, + tab_base: cutlass.Int64, + cnt_base: cutlass.Int64, + est_base: cutlass.Int64, + ): + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + qk_mma.set(tcgen05.Field.ACCUMULATE, False) + qk_mma_acc.set(tcgen05.Field.ACCUMULATE, True) + pv_mma_acc.set(tcgen05.Field.ACCUMULATE, True) + rpr: cutlass.Constexpr = self.T + group: cutlass.Constexpr = self.group + kvh_n: cutlass.Constexpr = self.kvh + splits: cutlass.Constexpr = self.splits + if const_expr(self.tail): + # units (group, KV head) below n_whole: one CTA each; the rest: `splits` CTAs each + is_whole = bidx < n_whole + t_cta = bidx - n_whole + unit = cutlass.Int32(cutlass.select_(is_whole, bidx, n_whole + t_cta // splits)) + split = cutlass.Int32(cutlass.select_(is_whole, cutlass.Int32(0), t_cta % splits)) + nsplit = cutlass.Int32( + cutlass.select_(is_whole, cutlass.Int32(1), cutlass.Int32(splits)) + ) + elif const_expr(self.bal): + # read from the plan table after the dependency wait + unit = cutlass.Int32(0) + split = cutlass.Int32(0) + nsplit = cutlass.Int32(1) + else: + unit = bidx // splits + split = bidx % splits + nsplit = cutlass.Int32(splits) + gi = unit // kvh_n + kvh = unit % kvh_n + r0 = gi * rpr + + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + q_full = storage.q_full.data_ptr() + raw_full = storage.raw_full.data_ptr() + raw_empty = storage.raw_empty.data_ptr() + k_full = storage.k_full.data_ptr() + k_empty = storage.k_empty.data_ptr() + v_full = storage.v_full.data_ptr() + v_empty = storage.v_empty.data_ptr() + s_full = storage.s_full.data_ptr() + p_full = storage.p_full.data_ptr() + o_full = storage.o_full.data_ptr() + o_final = storage.o_final.data_ptr() + s_scan = cutlass.Int32(storage.scan.data_ptr().toint()) + s_tab = cutlass.Int32(storage.tab.data_ptr().toint()) + s_raw = cutlass.Int32(storage.raw.data_ptr().toint()) + if tidx == 0: + cute.arch.mbarrier_init(q_full, 32 * SOFT_WARPS) + for i in cutlass.range_constexpr(self.RS): + cute.arch.mbarrier_init(raw_full + i, LOAD_WARPS) + cute.arch.mbarrier_init(raw_empty + i, 32 * DQ_WARPS) + for i in cutlass.range_constexpr(self.KS): + cute.arch.mbarrier_init(k_full + i, BN) + cute.arch.mbarrier_init(k_empty + i, 1) + for i in cutlass.range_constexpr(self.VS): + cute.arch.mbarrier_init(v_full + i, BN) + cute.arch.mbarrier_init(v_empty + i, 1) + for i in cutlass.range_constexpr(2): + cute.arch.mbarrier_init(s_full + i, 1) + cute.arch.mbarrier_init(p_full + i, 32 * SOFT_WARPS) + cute.arch.mbarrier_init(o_full, 1) + cute.arch.mbarrier_init(o_final, 1) + cute.arch.mbarrier_init_fence() + if warp == MMA_WARP: + cute.arch.alloc_tmem(TMEM_COLS, storage.tmem_holding.ptr) + cute.arch.relinquish_tmem_alloc_permit() + cute.arch.sync_threads() + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + + qk_thr = qk_mma.get_slice(0) + pv_thr = pv_mma_acc.get_slice(0) + sK = storage.sK.get_tensor(k_layout.outer, swizzle=k_layout.inner) + sVr = storage.sV.get_tensor(vk_layout.outer, swizzle=vk_layout.inner) + sV = cute.make_tensor(cute.recast_ptr(sVr.iterator, v_layout.inner), v_layout.outer) + s_k16 = cutlass.Int32(storage.sK.data_ptr().toint()) + s_v16 = cutlass.Int32(storage.sV.data_ptr().toint()) + + # ---- softmax-side tensor-memory views (warps 0-3: one thread an MMA row) ------------- + st = tidx + tStS = qk_thr.make_fragment_C(qk_thr.partition_shape_C((BM, BN))) + tStS0 = cute.make_tensor(tmem_ptr + TMEM_S0, tStS.layout) + tOtO_f = pv_thr.make_fragment_C(pv_thr.partition_shape_C((BM, HEAD_DIM))) + tOtO = cute.make_tensor(tmem_ptr + TMEM_O, tOtO_f.layout) + tScS = qk_thr.partition_C(cute.make_identity_tensor((BM, BN))) + load_s = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), Float32), + tStS0, + ) + thr_load_s = load_s.get_slice(st) + tLtS0 = thr_load_s.partition_S(tStS0) + tLcS = thr_load_s.partition_D(tScS) + p_cols = BN // 2 + tStP0 = cute.make_tensor( + tStS0.iterator, cute.composition(tStS0.layout, cute.make_layout((BM, p_cols))) + ) + tScP = cute.make_tensor( + tScS.iterator, cute.composition(tScS.layout, cute.make_layout((BM, p_cols))) + ) + store_p = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(p_cols)), Float32), + tStP0, + ) + thr_store_p = store_p.get_slice(st) + tPtP0 = thr_store_p.partition_D(tStP0) + tPcP = thr_store_p.partition_S(tScP) + tOcO = pv_thr.partition_C(cute.make_identity_tensor((BM, HEAD_DIM))) + tOtO_c = cute.make_tensor( + tOtO.iterator, cute.composition(tOtO.layout, cute.make_layout((BM, CHUNK_COLS))) + ) + tOcO_c = cute.make_tensor( + tOcO.iterator, cute.composition(tOcO.layout, cute.make_layout((BM, CHUNK_COLS))) + ) + load_o = tcgen05.make_tmem_copy( + cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(CHUNK_COLS)), Float32 + ), + tOtO_c, + ) + store_o = tcgen05.make_tmem_copy( + cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(CHUNK_COLS)), Float32 + ), + tOtO_c, + ) + thr_load_o = load_o.get_slice(st) + thr_store_o = store_o.get_slice(st) + tLtO = thr_load_o.partition_S(tOtO_c) + tLcO = thr_load_o.partition_D(tOcO_c) + tStO = thr_store_o.partition_D(tOtO_c) + tview = (load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO) + + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + + idle = cutlass.Int32(0) + bal_base = cutlass.Int32(0) + if const_expr(self.bal): + # this CTA's (unit, split, unit's CTAs, unit's first CTA); unit -1: past the plan + ent = self._bal_schedule(tidx, warp, lane, bidx, rows, est_base, tab_base, s_raw) + idle = cutlass.Int32(cutlass.select_(ent[0] < 0, cutlass.Int32(1), cutlass.Int32(0))) + unit = cutlass.max(ent[0], cutlass.Int32(0)) + split = ent[1] + nsplit = ent[2] + bal_base = ent[3] + gi = unit // kvh_n + kvh = unit % kvh_n + r0 = gi * rpr + m = tidx + t_row = m // group + h = m % group + live = (m < self.rows_m) & (r0 + t_row < rows) + if const_expr(self.bal): + live = live & (idle == 0) + safe_row = cutlass.min(r0 + t_row, rows - 1) + qs0, qs1 = self.qs + q_row = q_base + (cutlass.Int64(safe_row) * qs0 + cutlass.Int64(kvh * group + h) * qs1) * 2 + q_s = s_raw + self.u_q + cutlass.min(m, self.rows_m - 1) * Q_LD + + if warp < SOFT_WARPS: + # O = 0; this row's q copied into raw slot 1 now, into tensor memory (f16) after the + # first pass's resolve. Dead rows stage zeros. + rZ = cute.make_rmem_tensor(tLcO.shape, Float32) + rZ.fill(0.0) + for c in cutlass.range_constexpr(HEAD_DIM // CHUNK_COLS): + cute.copy( + store_o, rZ, cute.make_tensor(tStO.iterator + c * CHUNK_COLS, tStO.layout) + ) + cute.arch.fence_view_async_tmem_store() + if m < self.rows_m: + qn = cutlass.Int32(cutlass.select_(live, cutlass.Int32(16), cutlass.Int32(0))) + for i in cutlass.range_constexpr(HEAD_DIM // 8): + _cp16z(q_s + 16 * i, q_row + cutlass.Int64(16 * i), qn) + cute.arch.cp_async_commit_group() + + rq = cute.make_rmem_tensor(cute.make_layout((rpr,), stride=(1,)), cutlass.Int32) + handled = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + row = r0 + t + xr = cutlass.Int64(mReq[cutlass.min(row, rows - 1)]) + okr = (row < rows) & (xr >= 0) & (xr < self.num_requests) + rq[t] = cutlass.Int32(cutlass.select_(okr, cutlass.Int32(xr), cutlass.Int32(-1))) + handled = handled | cutlass.Int32( + cutlass.select_(okr, cutlass.Int32(0), cutlass.Int32(1 << t)) + ) + full = (1 << rpr) - 1 + if const_expr(self.bal): + handled = cutlass.Int32(cutlass.select_(idle != 0, cutlass.Int32(full), handled)) + scale_log2 = Float32(self.scale_log2) + neg_inf = -Float32.inf + row_max = neg_inf + row_sum = Float32(0.0) + q_pending = cutlass.Int32(1) + g = cutlass.Int32(0) + for _pass in cutlass.range(rpr): + if handled != full: + lead, act = self._pass_rows(rq, handled) + n_loc = self._pass_tiles( + act, + lead, + r0, + rows, + sel_base, + sel_stride, + split, + nsplit, + tidx, + warp, + lane, + s_raw, + s_scan, + s_tab, + mBT, + ) + if warp >= DQ_WARP0: + self.widen_tiles( + tidx - 32 * DQ_WARP0, + g, + n_loc, + s_raw, + s_tab, + s_k16, + s_v16, + raw_full, + raw_empty, + k_full, + k_empty, + v_full, + v_empty, + ) + elif warp >= LOAD_WARP0: + self.load_tiles( + tidx - 32 * LOAD_WARP0, + g, + n_loc, + kvh, + s_raw, + s_tab, + k_base, + v_base, + raw_full, + raw_empty, + q_full, + ) + elif warp == MMA_WARP: + self.mma_tiles( + g, + n_loc, + qk_mma, + pv_mma_acc, + qk_thr, + pv_thr, + q_layout, + p_layout, + sK, + sV, + tmem_ptr, + q_full, + k_full, + k_empty, + v_full, + v_empty, + s_full, + p_full, + o_full, + ) + else: + if q_pending != 0: + self.stage_q(q_s, live, tview, q_full) + q_pending = cutlass.Int32(0) + row_max, row_sum = self.softmax_tiles( + g, + n_loc, + row_max, + row_sum, + live, + t_row, + warp, + s_tab, + scale_log2, + tview, + s_full, + p_full, + o_full, + ) + g = g + n_loc + cute.arch.sync_threads() # [P] end of the pass + handled = handled | act + + if warp == MMA_WARP: + with cute.arch.elect_one(): + tcgen05.commit(o_final) + if warp < SOFT_WARPS: + cute.arch.cp_async_wait_group(0) + epi = ( + row_max, + row_sum, + live, + warp, + m, + r0, + rows, + kvh, + bidx if self.bal else split, # balanced: the partial slot is the CTA + nsplit, + tail_row0, + scale_log2, + out_base, + part_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + tview, + o_final, + tidx, + s_k16, + ) + if const_expr(self.bal): + if idle == 0: + self.epilogue(*epi) + else: + self.epilogue(*epi) + # every role done with tensor memory (the softmax warps read O last) + cute.arch.sync_threads() + if warp == MMA_WARP: + cute.arch.dealloc_tmem(tmem_ptr, TMEM_COLS) + if const_expr(self.bal): + if (idle == 0) & (nsplit > 1): + self.fixup( + tidx, + unit, + kvh, + r0, + rows, + nsplit, + bal_base, + cnt_base, + part_base, + lse_base, + out_base, + gate_base, + gate_stride, + gate_hs, + s_scan, + s_raw, + ) + + # ------------------------------------------------------------------ loaders (warps 5-6) + @cute.jit + def load_tiles( + self, lt, g, n_loc, kvh, s_raw, s_tab, k_base, v_base, raw_full, raw_empty, q_full + ): + """Raw tile copies: lane l < 16 of warp 5 (K) and of warp 6 (V) bulk-copies group l's four + rows (1 KB, contiguous in the pool); each warp's lane 0 first arms the slot's mbarrier with + its half of the tile's bytes. A group that does not resolve is not copied (the widening + zeroes it).""" + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + tpb: cutlass.Constexpr = self.tpb + ks0, ks1, ks2 = self.ks + if lt < 32 * LOAD_WARPS: + gi = lt % 16 + is_v = lt >= 32 + base = cutlass.select_(is_v, v_base, k_base) + cutlass.Int64(kvh * ks1) + dst0 = ( + s_raw + + cutlass.Int32(cutlass.select_(is_v, cutlass.Int32(RAW_K), cutlass.Int32(0))) + + gi * (4 * RAW_LD) + ) + rf0 = cutlass.Int32(raw_full.toint()) + for kl in cutlass.range(n_loc): + gg = g + kl + r = gg % self.RS + cute.arch.mbarrier_wait(raw_empty + r, _phase(gg, self.RS) ^ 1) + if r == 1: + cute.arch.mbarrier_wait(q_full, 0) + slot = kl * GPT + gi + tok0 = _lds1(s_tok + slot * 4) + mw = _lds1(s_gmask + slot * 4) + ok = (tok0 >= 0) & (mw != 0) & ((lt % 32) < 16) + nb = cutlass.Int32( + cutlass.select_(ok, cutlass.Int32(4 * HEAD_DIM), cutlass.Int32(0)) + ) + total = cute.arch.warp_reduction_sum(nb) + if (lt % 32) == 0: + _arrive_expect_tx(rf0 + r * 8, total) + cute.arch.sync_warp() + if ok: + off = cutlass.Int64(tok0 // tpb) * ks0 + cutlass.Int64((tok0 % tpb) * ks2) + _bulk_g2s(dst0 + r * RAW_BUF, base + off, nb, rf0 + r * 8) + + # ------------------------------------------------------------------ widen (warps 7-10) + @cute.jit + def widen_tiles( + self, + dt, + g, + n_loc, + s_raw, + s_tab, + s_k16, + s_v16, + raw_full, + raw_empty, + k_full, + k_empty, + v_full, + v_empty, + ): + """Thread `dt` < 64 widens K row `dt` of each tile, the others V row `dt - 64`: the raw + row into registers (the raw slot is then free), then into the f16 SW128 slot once the + MMA has released it.""" + is_v = dt >= BN + n = dt % BN + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + # bulk copies land unpadded rows: chunk c of the read order is the row's chunk (c + n) % 16 + rot = n % 16 + for kl in cutlass.range(n_loc): + gg = g + kl + r = gg % self.RS + slot = kl * GPT + n // 4 + tok0 = _lds1(s_tok + slot * 4) + mw = _lds1(s_gmask + slot * 4) + live_row = (tok0 >= 0) & (((mw >> (8 * (n % 4))) & 0xFF) != 0) + cute.arch.mbarrier_wait(raw_full + r, _phase(gg, self.RS)) + row_off = ( + s_raw + + r * RAW_BUF + + cutlass.Int32(cutlass.select_(is_v, cutlass.Int32(RAW_K), cutlass.Int32(0))) + + n * RAW_LD + ) + w = [_lds4(row_off + 16 * ((c + rot) % 16)) for c in range(16)] + cute.arch.mbarrier_arrive(raw_empty + r) + # a row no verify row selected (not copied) widens to zeros: P = 0 times a stale NaN + # would poison P V + w = [ + [cutlass.Int32(cutlass.select_(live_row, x, cutlass.Int32(0))) for x in wc] + for wc in w + ] + if is_v: + b = gg % self.VS + cute.arch.mbarrier_wait(v_empty + b, _phase(gg, self.VS) ^ 1) + self._widen_row(w, s_v16 + b * F16_SLOT, n, rot) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(v_full + b) + else: + b = gg % self.KS + cute.arch.mbarrier_wait(k_empty + b, _phase(gg, self.KS) ^ 1) + self._widen_row(w, s_k16 + b * F16_SLOT, n, rot) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(k_full + b) + + @cute.jit + def _widen_row(self, w, dst, n, rot): + """Raw e4m3 row words `w` (16 x 4) -> f16 SW128 row `n` at `dst`: 16-byte chunk ci (dims + 8 ci .. 8 ci + 7) of the row's 64-dim block ci // 8 at 8192 (ci // 8) + 128 n + 16 ((ci % + 8) ^ (n % 8)).""" + for c in cutlass.range_constexpr(16): + for half in cutlass.range_constexpr(2): + a = _e4m3x4(w[c][2 * half]) + b = _e4m3x4(w[c][2 * half + 1]) + ci = 2 * ((c + rot) % 16) + half + _sts4( + dst + 8192 * (ci // 8) + 128 * n + 16 * ((ci % 8) ^ (n % 8)), + a[0], + a[1], + b[0], + b[1], + ) + + # ------------------------------------------------------------------ MMA (warp 4) + @cute.jit + def mma_tiles( + self, + g, + n_loc, + qk_mma, + pv_mma_acc, + qk_thr, + pv_thr, + q_layout, + p_layout, + sK, + sV, + tmem_ptr, + q_full, + k_full, + k_empty, + v_full, + v_empty, + s_full, + p_full, + o_full, + ): + """QK(g), then QK(j) / PV(j - 1) pairs, then the last PV: S/P buffer j % 2 at TMEM column + TMEM_S0 + 64 (j % 2); QK(j) overwrites it only after PV(j - 2) was issued (tcgen05 MMAs of + one thread execute in order).""" + tQ = cute.make_tensor(tmem_ptr + TMEM_Q, q_layout.outer) + tSrQ = qk_thr.make_fragment_A(tQ)[None, None, None, 0] + tSrQ = cute.make_tensor(cute.recast_ptr(tmem_ptr + TMEM_Q, dtype=F16), tSrQ.layout) + tSrK = qk_thr.make_fragment_B(sK) + tOrV = pv_thr.make_fragment_B(sV) + tP = cute.make_tensor(tmem_ptr + TMEM_S0, p_layout.outer) + tOrP = pv_thr.make_fragment_A(tP)[None, None, None, 0] + tOrP = cute.make_tensor(cute.recast_ptr(tmem_ptr + TMEM_S0, dtype=F16), tOrP.layout) + tbase = cutlass.Int32(tmem_ptr.toint()) + qk_op = qk_mma.op + pv_op = pv_mma_acc.op + if n_loc > 0: + cute.arch.mbarrier_wait(q_full, 0) + cute.arch.mbarrier_wait(k_full + g % self.KS, _phase(g, self.KS)) + _tc_fence_after() + self.issue_qk(qk_op, tbase, tSrQ, tSrK, sK, g % 2, g % self.KS) + with cute.arch.elect_one(): + tcgen05.commit(s_full + g % 2) + tcgen05.commit(k_empty + g % self.KS) + for j in cutlass.range(1, n_loc + 1, 1, unroll=1): + gj = g + j + if j < n_loc: + cute.arch.mbarrier_wait(k_full + gj % self.KS, _phase(gj, self.KS)) + _tc_fence_after() + self.issue_qk(qk_op, tbase, tSrQ, tSrK, sK, gj % 2, gj % self.KS) + with cute.arch.elect_one(): + tcgen05.commit(s_full + gj % 2) + tcgen05.commit(k_empty + gj % self.KS) + gp = gj - 1 + cute.arch.mbarrier_wait(v_full + gp % self.VS, _phase(gp, self.VS)) + cute.arch.mbarrier_wait(p_full + gp % 2, _phase(gp, 2)) + _tc_fence_after() + self._issue_pv(pv_op, tbase, tOrP, tOrV, sV, gp) + with cute.arch.elect_one(): + tcgen05.commit(v_empty + gp % self.VS) + tcgen05.commit(o_full) + + @cute.jit + def _issue_pv(self, pv_op, tbase, tOrP, tOrV, sV, gp): + """O += P(gp) V(gp): P from tensor memory over the S it came from, V from its slot.""" + _gemm_ts( + pv_op, + tbase + TMEM_O, + tbase + TMEM_S0 + (gp % 2) * BN, + tOrP, + tOrV[None, None, None, gp % self.VS], + sV[None, None, None, gp % self.VS], + False, + ) + + @cute.jit + def issue_qk(self, qk_op, tbase, tSrQ, tSrK, sK, sbuf, kslot): + """S[sbuf] = Q K(kslot)^T: Q from tensor memory, K from SW128 slot `kslot`.""" + _gemm_ts( + qk_op, + tbase + TMEM_S0 + sbuf * BN, + tbase + TMEM_Q, + tSrQ, + tSrK[None, None, None, kslot], + sK[None, None, None, kslot], + True, + ) + + # ------------------------------------------------------------- softmax + epilogue (warps 0-3) + @cute.jit + def stage_q(self, q_s, live, tview, q_full): + """This MMA row's q (bf16 in raw slot 1) -> f16 into tensor memory (the QK A operand).""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + p_cols: cutlass.Constexpr = BN // 2 + cute.arch.cp_async_wait_group(0) + for c in cutlass.range_constexpr(HEAD_DIM // (2 * p_cols)): + rQ = cute.make_rmem_tensor(tPcP.shape, Float32) + for i in cutlass.range_constexpr(p_cols // 4): + w4 = _lds4(q_s + 16 * (c * (p_cols // 4) + i)) + for e in cutlass.range_constexpr(4): + rQ[4 * i + e] = _bits_f32( + cutlass.Int32(cutlass.select_(live, _bf2_to_f2(w4[e]), cutlass.Int32(0))) + ) + cute.copy( + store_p, + rQ, + cute.make_tensor(tPtP0.iterator + (TMEM_Q - TMEM_S0 + c * p_cols), tPtP0.layout), + ) + cute.arch.fence_view_async_tmem_store() + _tc_fence_before() + cute.arch.mbarrier_arrive(q_full) + + @cute.jit + def softmax_tiles( + self, + g, + n_loc, + row_max, + row_sum, + live, + t_row, + warp, + s_tab, + scale_log2, + tview, + s_full, + p_full, + o_full, + ): + """Per tile: S from tensor memory, the max over the row's own entries, the reference max + moved (and O rescaled) only past RESCALE_TH, P = 2^(s - max) as f16 over the S.""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + rpr: cutlass.Constexpr = self.T + s_bits = s_tab + self.t_bits * 4 + tau = Float32(RESCALE_TH) + neg_inf = -Float32.inf + live_warp = warp < self.live_warps + for kl in cutlass.range(n_loc, unroll=1): + gg = g + kl + buf = gg % 2 + cute.arch.mbarrier_wait(s_full + buf, _phase(gg, 2)) + _tc_fence_after() + if live_warp: + lo = cutlass.Int32(0) + hi = cutlass.Int32(0) + if live: + lo = _lds1(s_bits + (kl * rpr + t_row) * 8) + hi = _lds1(s_bits + (kl * rpr + t_row) * 8 + 4) + rS = cute.make_rmem_tensor(tLcS.shape, Float32) + tLtS_b = cute.make_tensor(tLtS0.iterator + buf * BN, tLtS0.layout) + for c in cutlass.range_constexpr(cute.size(rS, mode=[1])): + cute.copy(load_s, tLtS_b[None, c, 0, 0], rS[None, c, 0, 0]) + cute.arch.fence_view_async_tmem_load() + tile_max = neg_inf + for i in cutlass.range_constexpr(BN): + word = lo if i < 32 else hi + x = rS[i] + x = x if (word & (1 << (i % 32))) != 0 else neg_inf + rS[i] = x + tile_max = cute.arch.fmax(tile_max, x) + new_max = cute.arch.fmax(row_max, tile_max) + use_max = row_max + factor = Float32(1.0) + if row_max == neg_inf: + use_max = new_max + elif (new_max - row_max) * scale_log2 > tau: + use_max = new_max + factor = cute.math.exp2((row_max - new_max) * scale_log2, fastmath=True) + need = factor != Float32(1.0) + if cute.arch.vote_any_sync(need): + # O holds PV(..gg - 1): wait for it, then rescale this warp's rows + cute.arch.mbarrier_wait(o_full, (gg - 1) % 2) + _tc_fence_after() + for c in cutlass.range_constexpr(HEAD_DIM // CHUNK_COLS): + rO = cute.make_rmem_tensor(tLcO.shape, Float32) + cute.copy( + load_o, + cute.make_tensor(tLtO.iterator + c * CHUNK_COLS, tLtO.layout), + rO, + ) + for i in cutlass.range_constexpr(cute.size(rO)): + rO[i] = rO[i] * factor + cute.copy( + store_o, + rO, + cute.make_tensor(tStO.iterator + c * CHUNK_COLS, tStO.layout), + ) + cute.arch.fence_view_async_tmem_store() + row_max = use_max + safe = use_max + if use_max == neg_inf: + safe = Float32(0.0) + neg = -safe * scale_log2 + rP = cute.make_rmem_tensor(tPcP.shape, Float32) + rP16 = cute.make_tensor(cute.recast_ptr(rP.iterator, dtype=F16), rS.layout) + acc0 = (Float32(0.0), Float32(0.0)) + acc1 = (Float32(0.0), Float32(0.0)) + for i in cutlass.range_constexpr(0, BN, 4): + e = cute.make_rmem_tensor((4,), Float32) + e[0], e[1] = cute.arch.fma_packed_f32x2( + (rS[i], rS[i + 1]), (scale_log2, scale_log2), (neg, neg) + ) + e[2], e[3] = cute.arch.fma_packed_f32x2( + (rS[i + 2], rS[i + 3]), (scale_log2, scale_log2), (neg, neg) + ) + for u in cutlass.range_constexpr(4): + rS[i + u] = cute.math.exp2(e[u], fastmath=True) + acc0 = cute.arch.add_packed_f32x2(acc0, (rS[i], rS[i + 1])) + acc1 = cute.arch.add_packed_f32x2(acc1, (rS[i + 2], rS[i + 3])) + row_sum = row_sum * factor + ((acc0[0] + acc0[1]) + (acc1[0] + acc1[1])) + rP16.store(rS.load().to(F16)) + cute.copy(store_p, rP, cute.make_tensor(tPtP0.iterator + buf * BN, tPtP0.layout)) + cute.arch.fence_view_async_tmem_store() + _tc_fence_before() + cute.arch.mbarrier_arrive(p_full + buf) + return row_max, row_sum + + @cute.jit + def epilogue( + self, + row_max, + row_sum, + live, + warp, + m, + r0, + rows, + kvh, + split, + nsplit, + tail_row0, + scale_log2, + out_base, + part_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + tview, + o_final, + st, + s_stage, + ): + """O / l: a whole unit's rows as bf16, a split's as an fp32 partial and its log2-sum-exp for + the merge (rows from `tail_row0` on). Gated whole units read the gate before the last P V + completes, stage bf16(O / l) in the free K slots, then apply the gate in coalesced rows.""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + splits: cutlass.Constexpr = self.splits + tailm: cutlass.Constexpr = self.tail or self.bal + has_whole: cutlass.Constexpr = tailm or splits == 1 + has_split: cutlass.Constexpr = tailm or splits > 1 + gated_whole: cutlass.Constexpr = has_whole and self.gated + rows_m: cutlass.Constexpr = self.rows_m + cpr: cutlass.Constexpr = HEAD_DIM // 8 # 16-byte chunks a row + n_gc: cutlass.Constexpr = -(-(rows_m * cpr) // (32 * SOFT_WARPS)) + n_chunks: cutlass.Constexpr = HEAD_DIM // CHUNK_COLS + neg_inf = -Float32.inf + row = r0 + m // group + h = m % group + head = kvh * group + h + prow = cutlass.Int64(row - tail_row0) + prows = cutlass.Int64(rows - tail_row0) + if const_expr(self.bal): + # a CTA's partial slot holds its unit's T rows + prow = cutlass.Int64(m // group) + prows = cutlass.Int64(self.T) + whole = nsplit == 1 + gregs = [] + if const_expr(gated_whole): + for j in cutlass.range_constexpr(n_gc): + k = st + 32 * SOFT_WARPS * j + gm = cutlass.min(k // cpr, rows_m - 1) + grow = cutlass.min(r0 + gm // group, rows - 1) + ghead = kvh * group + gm % group + gregs.append( + _ldg4( + gate_base + + ( + cutlass.Int64(grow) * gate_stride + + cutlass.Int64(ghead) * gate_hs + + (k % cpr) * 8 + ) + * 2 + ) + ) + cute.arch.mbarrier_wait(o_final, 0) + _tc_fence_after() + if warp < self.live_warps: + inv = Float32(0.0) + if row_sum > Float32(0.0): + inv = Float32(1.0) / row_sum + for c in cutlass.range_constexpr(n_chunks): + rO = cute.make_rmem_tensor(tLcO.shape, Float32) + cute.copy(load_o, cute.make_tensor(tLtO.iterator + c * CHUNK_COLS, tLtO.layout), rO) + for v8 in cutlass.range_constexpr(CHUNK_COLS // 8): + vals = [rO[8 * v8 + i] * inv for i in range(8)] + d0 = c * CHUNK_COLS + 8 * v8 + oaddr = out_base + ((cutlass.Int64(row) * heads + head) * HEAD_DIM + d0) * 2 + paddr = ( + part_base + + (((cutlass.Int64(split) * prows + prow) * heads + head) * HEAD_DIM + d0) + * 4 + ) + packed = [_bf2_pack(vals[2 * e], vals[2 * e + 1]) for e in range(4)] + saddr = s_stage + cutlass.min(m, rows_m - 1) * Q_LD + d0 * 2 + if const_expr(has_whole): + if whole: + if const_expr(gated_whole): + if m < rows_m: + _sts4(saddr, packed[0], packed[1], packed[2], packed[3]) + else: + if live: + _stg4(oaddr, packed[0], packed[1], packed[2], packed[3]) + if const_expr(has_split): + if live & (nsplit > 1): + _stg4f(paddr, vals[0], vals[1], vals[2], vals[3]) + _stg4f(paddr + 16, vals[4], vals[5], vals[6], vals[7]) + if const_expr(has_split): + if live & (nsplit > 1): + lse = neg_inf + if row_sum > Float32(0.0): + lse = row_max * scale_log2 + cute.math.log2(row_sum) + _stg1( + lse_base + ((cutlass.Int64(split) * prows + prow) * heads + head) * 4, + _f32_bits(lse), + ) + if const_expr(gated_whole): + if whole: + cute.arch.barrier(barrier_id=1, number_of_threads=32 * SOFT_WARPS) + for j in cutlass.range_constexpr(n_gc): + k = st + 32 * SOFT_WARPS * j + gm = cutlass.min(k // cpr, rows_m - 1) + grow = r0 + gm // group + ghead = kvh * group + gm % group + a4 = _lds4(s_stage + gm * Q_LD + (k % cpr) * 16) + g4 = gregs[j] + res = [ + _bf2_pack( + _bf2_lo(a4[e]) * _sigmoid_fast(_bf2_lo(g4[e])), + _bf2_hi(a4[e]) * _sigmoid_fast(_bf2_hi(g4[e])), + ) + for e in range(4) + ] + if (k < rows_m * cpr) & (grow < rows): + _stg4( + out_base + + ((cutlass.Int64(grow) * heads + ghead) * HEAD_DIM + (k % cpr) * 8) + * 2, + res[0], + res[1], + res[2], + res[3], + ) + + @cute.jit + def fixup( + self, + tidx, + unit, + kvh, + r0, + rows, + nsplit, + bal_base, + cnt_base, + part_base, + lse_base, + out_base, + gate_base, + gate_stride, + gate_hs, + s_flag, + s_w, + ): + """Balanced mode, a unit of several CTAs: the unit's last CTA to finish (a ticket a unit) + merges its partials in slot order as `_RowMergeKernel` does, applies the gate, stores the + rows and resets the ticket; the result does not depend on which CTA merges.""" + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + rows_m: cutlass.Constexpr = self.rows_m + rpr: cutlass.Constexpr = self.T + d4: cutlass.Constexpr = HEAD_DIM // 4 + n_c: cutlass.Constexpr = rows_m * d4 + per: cutlass.Constexpr = -(-n_c // THREADS) + nsp: cutlass.Constexpr = MAX_BAL_SPLITS + neg_inf = -Float32.inf + # the CTA's partials and log-sum-exps (stored before the barrier) released by thread 0's + # acq_rel ticket; the last CTA's thread 0 acquires the others', the barrier passes them on + cute.arch.sync_threads() + if tidx == 0: + _sts1(s_flag, _atom_add_acqrel_g(cnt_base + cutlass.Int64(unit) * 4, cutlass.Int32(1))) + cute.arch.sync_threads() + if _lds1(s_flag) == nsplit - 1: + if tidx == 0: + _stg1(cnt_base + cutlass.Int64(unit) * 4, cutlass.Int32(0)) + # weights of each (verify row, head) of the unit: slots past `nsplit` weigh 0 + for i in cutlass.range_constexpr(-(-rows_m // THREADS)): + k = tidx + i * THREADS + if k < rows_m: + head = kvh * group + k % group + ls = [] + for s_ in cutlass.range_constexpr(nsp): + slot = bal_base + cutlass.min(cutlass.Int32(s_), nsplit - 1) + lv = _bits_f32( + _ldg1cg( + lse_base + + ((cutlass.Int64(slot) * rpr + k // group) * heads + head) * 4 + ) + ) + ls.append(Float32(cutlass.select_(nsplit > s_, lv, neg_inf))) + mx = _tree_fmax(ls) + ws = [ + Float32( + cutlass.select_(mx > neg_inf, cute.math.exp2(lv - mx), Float32(0.0)) + ) + for lv in ls + ] + for s_ in cutlass.range_constexpr(nsp): + _sts1(s_w + (s_ * rows_m + k) * 4, _f32_bits(ws[s_])) + _sts1(s_w + (nsp * rows_m + k) * 4, _f32_bits(_tree_sum(ws))) + cute.arch.sync_threads() + for j in cutlass.range_constexpr(per): + c = tidx + THREADS * j + k = cutlass.min(c // d4, rows_m - 1) + d = (c % d4) * 4 + t = k // group + head = kvh * group + k % group + row = r0 + t + ps = [ + _ldg4cg( + part_base + + ( + ( + ( + cutlass.Int64( + bal_base + cutlass.min(cutlass.Int32(s_), nsplit - 1) + ) + * rpr + + t + ) + * heads + + head + ) + * HEAD_DIM + + d + ) + * 4 + ) + for s_ in range(nsp) + ] + gw = None + if const_expr(self.gated): + gw = _ldg2( + gate_base + + ( + cutlass.Int64(cutlass.min(row, rows - 1)) * gate_stride + + cutlass.Int64(head) * gate_hs + + d + ) + * 2 + ) + ws = [_bits_f32(_lds1(s_w + (s_ * rows_m + k) * 4)) for s_ in range(nsp)] + den = _bits_f32(_lds1(s_w + (nsp * rows_m + k) * 4)) + res = [] + for e in cutlass.range_constexpr(4): + acc = _tree_sum([ws[s_] * _bits_f32(ps[s_][e]) for s_ in range(nsp)]) + merged = Float32(cutlass.select_(den > Float32(0.0), acc / den, Float32(0.0))) + if const_expr(self.gated): + if const_expr(e % 2 == 0): + gx = _bf2_lo(gw[e // 2]) + else: + gx = _bf2_hi(gw[e // 2]) + merged = _round_bf16(merged) * _sigmoid_fast(gx) + res.append(merged) + if (c < n_c) & (row < rows): + _stg2( + out_base + ((cutlass.Int64(row) * heads + head) * HEAD_DIM + d) * 2, + _bf2_pack(res[0], res[1]), + _bf2_pack(res[2], res[3]), + ) + + +#: Threads of a merge CTA (four values a thread at a time). +MERGE_THREADS = 256 +#: Merge CTAs a row, each a contiguous share of the row's heads x head_dim values. +MERGE_PARTS = 8 + + +class _RowMergeKernel: + """The split partials of a row range merged as the per-row kernel's merge does + (log2-sum-exp weights, fp32), the gate applied to the bf16-rounded result: `parts` CTAs a + row, every load of a thread issued before the first use.""" + + def __init__(self, *, heads, splits, gated, pdl, parts): + self.heads, self.splits, self.gated, self.pdl = heads, splits, gated, pdl + self.parts = parts + + @cute.jit + def __call__( + self, + gate_t: cute.Tensor, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + out_ptr: cute.Pointer, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + self.kernel( + part_ptr.toint(), + lse_ptr.toint(), + out_ptr.toint(), + gate_t.iterator.toint(), + rows, + gate_stride, + gate_hs, + ).launch( + grid=[rows * self.parts, 1, 1], + block=[MERGE_THREADS, 1, 1], + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + part_base: cutlass.Int64, + lse_base: cutlass.Int64, + out_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + heads: cutlass.Constexpr = self.heads + sp: cutlass.Constexpr = self.splits + parts: cutlass.Constexpr = self.parts + n_all: cutlass.Constexpr = heads * HEAD_DIM // 4 + n_c: cutlass.Constexpr = -(-n_all // parts) + per: cutlass.Constexpr = -(-n_c // MERGE_THREADS) + neg_inf = -Float32.inf + row = bidx // parts + c0 = (bidx % parts) * n_c + r64 = cutlass.Int64(row) + cs = [ + cutlass.min(c0 + cutlass.min(tidx + MERGE_THREADS * j, n_c - 1), n_all - 1) + for j in range(per) + ] + gws = [] + if const_expr(self.gated): + # the gate is an input of the attention call, ready before the split kernel ran + for c in cs: + head = c // (HEAD_DIM // 4) + d = (c % (HEAD_DIM // 4)) * 4 + gws.append( + _ldg2(gate_base + (r64 * gate_stride + cutlass.Int64(head) * gate_hs + d) * 2) + ) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + lss = [] + pss = [] + for j in cutlass.range_constexpr(per): + head = cs[j] // (HEAD_DIM // 4) + lss.append( + [ + _bits_f32( + _ldg1(lse_base + ((cutlass.Int64(s_) * rows + r64) * heads + head) * 4) + ) + for s_ in range(sp) + ] + ) + pss.append( + [ + _ldg4( + part_base + + ((cutlass.Int64(s_) * rows + r64) * (heads * HEAD_DIM) + 4 * cs[j]) * 4 + ) + for s_ in range(sp) + ] + ) + for j in cutlass.range_constexpr(per): + ls = lss[j] + mx = _tree_fmax(ls) + ws = [ + Float32(cutlass.select_(mx > neg_inf, cute.math.exp2(lv - mx), Float32(0.0))) + for lv in ls + ] + den = _tree_sum(ws) + res = [] + for e in cutlass.range_constexpr(4): + acc = _tree_sum([ws[s_] * _bits_f32(pss[j][s_][e]) for s_ in range(sp)]) + merged = Float32(0.0) + if den > Float32(0.0): + merged = acc / den + if const_expr(self.gated): + if const_expr(e % 2 == 0): + gx = _bf2_lo(gws[j][e // 2]) + else: + gx = _bf2_hi(gws[j][e // 2]) + merged = _round_bf16(merged) * _sigmoid_fast(gx) + res.append(merged) + if (tidx + MERGE_THREADS * j < n_c) & (c0 + tidx + MERGE_THREADS * j < n_all): + _stg2( + out_base + (r64 * (heads * HEAD_DIM) + 4 * cs[j]) * 2, + _bf2_pack(res[0], res[1]), + _bf2_pack(res[2], res[3]), + ) + + +#: Balanced launch: most CTAs of one unit (the last CTA's merge reads this many partials). +MAX_BAL_SPLITS = 4 +#: The plan kernel's threads a CTA, and tokens read a four-column quad (1: the quad's first token, +#: which names the group in the expansion layout, where a quad's valid tokens are a prefix). +PLAN_THREADS, PLAN_QUAD = 512, 1 +#: Most bitmap words (32 four-token groups each) of a plan CTA: a 262144-token block table. +MAX_PLAN_WORDS = 4096 + + +class _BalPlanKernel: + """Balanced launch, step 1, one CTA a request group: the group's union tiles (distinct + four-token groups its valid rows select, / `GPT`, at least 1) to `est`, for + `_GroupedKernel._bal_schedule`.""" + + def __init__(self, *, rpr, width, words, num_requests, pdl): + self.T, self.width, self.words = rpr, width, words + self.num_requests, self.pdl = num_requests, pdl + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + req_ptr: cute.Pointer, + est_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + n_groups: cutlass.Int32, + stream, + ): + @cute.struct + class SharedStorage: + red: cute.struct.MemRange[Int32, 32] + bm: cute.struct.Align[cute.struct.MemRange[Int32, self.words], 16] + + self.shared_storage = SharedStorage + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel(sel_t.iterator.toint(), req, est_ptr.toint(), rows, sel_stride).launch( + grid=[n_groups, 1, 1], + block=[PLAN_THREADS, 1, 1], + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + sel_base: cutlass.Int64, + mReq: cute.Tensor, + est_base: cutlass.Int64, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = cute.arch.lane_idx() + rpr: cutlass.Constexpr = self.T + width: cutlass.Constexpr = self.width + words: cutlass.Constexpr = self.words + n_q: cutlass.Constexpr = -(-width // 4) + n_i: cutlass.Constexpr = -(-n_q // PLAN_THREADS) + nwarps: cutlass.Constexpr = PLAN_THREADS // 32 + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + s_red = cutlass.Int32(storage.red.data_ptr().toint()) + s_bm = cutlass.Int32(storage.bm.data_ptr().toint()) + r0 = bidx * rpr + for i in cutlass.range_constexpr(-(-words // PLAN_THREADS)): + idx = tidx + i * PLAN_THREADS + if idx < words: + _sts1(s_bm + idx * 4, cutlass.Int32(0)) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + # the first valid token of a quad names its four-token group + nu: cutlass.Constexpr = PLAN_QUAD + toks = [] + oks = [] + for t in cutlass.range_constexpr(rpr): + row = r0 + t + xr = cutlass.Int64(mReq[cutlass.min(row, rows - 1)]) + oks.append((row < rows) & (xr >= 0) & (xr < self.num_requests)) + row_sel = sel_base + cutlass.Int64(cutlass.min(row, rows - 1)) * sel_stride * 4 + for i in cutlass.range_constexpr(n_i): + for u in cutlass.range_constexpr(nu): + colc = cutlass.min(4 * (tidx + i * PLAN_THREADS) + u, width - 1) + toks.append(_ldg1(row_sel + cutlass.Int64(colc * 4))) + cute.arch.sync_threads() + for t in cutlass.range_constexpr(rpr): + for i in cutlass.range_constexpr(n_i): + q4 = tidx + i * PLAN_THREADS + gid = cutlass.Int32(-1) + for u in cutlass.range_constexpr(nu): + tok = toks[(t * n_i + i) * nu + u] + ok = oks[t] & (4 * q4 + u < width) & (tok >= 0) + gid = cutlass.Int32(cutlass.select_(ok & (gid < 0), tok >> 2, gid)) + if (gid >= 0) & (gid < words * 32): + _red_or_shared(s_bm + (gid >> 5) * 4, cutlass.Int32(1) << (gid & 31)) + cute.arch.sync_threads() + cnt = cutlass.Int32(0) + for i in cutlass.range_constexpr(-(-words // PLAN_THREADS)): + idx = tidx + i * PLAN_THREADS + if idx < words: + cnt = cnt + _popc(_lds1(s_bm + idx * 4)) + cnt = cute.arch.warp_reduction_sum(cnt) + if lane == 0: + _sts1(s_red + warp * 4, cnt) + cute.arch.sync_threads() + if tidx == 0: + n = cutlass.Int32(0) + for w in cutlass.range_constexpr(nwarps): + n = n + _lds1(s_red + w * 4) + _stg1( + est_base + cutlass.Int64(bidx) * 4, + cutlass.max((n + GPT - 1) // GPT, cutlass.Int32(1)), + ) + + +# --- launch --- + +_COMPILED: dict = {} +#: Geometries whose every reachable launch plan is compiled. +_PLANNED: set = set() + + +@functools.lru_cache(maxsize=None) +def _num_sms(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def plan_splits(n_groups: int, kv_heads: int, width: int, num_sms: int) -> int: + """Splits so the grid fills the GPU once (one CTA an SM): `num_sms // units` CTAs a unit, and + three splits from half to three quarters of a wave while they fit in two waves + (3 * units <= 2 * num_sms). Outputs of different splits differ in the last bits.""" + est_tiles = -(-int(width * 1.25) // BN) + units = n_groups * kv_heads + band = num_sms // 2 < units <= (num_sms * 3) // 4 + if band and 3 * units <= 2 * num_sms: + return min(3, est_tiles) + return max(1, min(MAX_PLAN_SPLITS, est_tiles, num_sms // units)) + + +def plan_launch(n_groups: int, kv_heads: int, width: int, num_sms: int): + """(splits, n_whole) of a launch: n_whole None = every unit (group, KV head) takes `splits` + CTAs (one wave: `plan_splits`); else units below n_whole take one CTA and the units of the + last, partial wave (whole groups) `splits` CTAs each, merged afterwards.""" + units = n_groups * kv_heads + if units <= num_sms: + return plan_splits(n_groups, kv_heads, width, num_sms), None + waves = -(-units // num_sms) + tail = units - (waves - 1) * num_sms + tail = -(-tail // kv_heads) * kv_heads + tail_splits = min(MAX_TAIL_SPLITS, num_sms // tail) + if tail_splits < 2: + return 1, None + return tail_splits, units - tail + + +def reachable_plans(max_groups: int, kv_heads: int, width: int, num_sms: int) -> list: + """Every `(splits, tail)` pair `plan_launch` returns for `MIN_GROUPS..max_groups` groups.""" + plans = set() + for n_groups in range(MIN_GROUPS, max(MIN_GROUPS, max_groups) + 1): + splits, n_whole = plan_launch(n_groups, kv_heads, width, num_sms) + plans.add((max(1, min(MAX_SPLITS, int(splits))), n_whole is not None)) + return sorted(plans) + + +#: Launches from this many units (request groups x KV heads) to one wave take the balanced plan +#: (`_BalPlanKernel`, then the grouped kernel's balanced mode) instead of `plan_launch`'s. A unit's +#: split count depends on the batch, so a row's last bits depend on the other requests' unions. +BALANCED_MIN_UNITS = 88 + + +def takes_balanced(units: int, num_sms: int) -> bool: + """Whether a launch of `units` (group, KV head) units takes the balanced plan: from + `BALANCED_MIN_UNITS` units to one wave.""" + return BALANCED_MIN_UNITS <= units <= num_sms + + +def _plan_words(bt_width: int, tpb: int) -> int: + """Bitmap words (32 four-token groups each) of a plan CTA over a block table.""" + return -(-(bt_width * tpb // 4) // 32) + + +#: The balanced launch's persistent int32 words a device (zeros once; the kernels leave every +#: ticket at zero): one launch at a time on a device. +_BAL_BUFS: dict = {} + + +def _bal_buffers(device: torch.device, n: int) -> torch.Tensor: + """`[8 n + 16]` int32 on `device`: each CTA's plan entry (4 words a CTA, words 0..4n), the + units' merge tickets (4n..5n), the groups' union tiles (5n + 4 .. 6n + 4).""" + key = (device.index if device.index is not None else torch.cuda.current_device(), n) + buf = _BAL_BUFS.get(key) + if buf is None: + buf = torch.zeros(8 * n + 16, dtype=torch.int32, device=device) + _BAL_BUFS[key] = buf + return buf + + +def _compile_plan(pkey: tuple, plan_kw: dict, plan_args: tuple): + """The balanced launch's plan kernel of `pkey`, compiled once.""" + fn = _COMPILED.get(pkey) + if fn is None: + fn = cute.compile( + _BalPlanKernel(**plan_kw), *plan_args, options="--opt-level 2 --enable-tvm-ffi" + ) + _COMPILED[pkey] = fn + return fn + + +def takes_grouped(q, k_cache, selected_tokens, rows_per_request) -> bool: + """Whether `sparse_gqa_grouped` serves this call (on shapes only, so a per-capture constant): + an e4m3 pool on SM103, 2..4 rows a request, `MIN_GROUPS` groups, a shared-memory plan.""" + if q.dim() != 3 or k_cache.dim() != 4 or k_cache.dtype != torch.float8_e4m3fn: + return False + rows, num_q_heads, head_dim = q.shape + rpr = int(rows_per_request) + if rpr < 2 or rpr > MAX_RPR or -(-rows // rpr) < MIN_GROUPS or head_dim != HEAD_DIM: + return False + if k_cache.shape[3] != HEAD_DIM or num_q_heads % k_cache.shape[1]: + return False + if k_cache.stride(3) != 1 or k_cache.stride(2) != HEAD_DIM: + return False + if torch.cuda.get_device_capability(q.device) != (10, 3): + return False + group = num_q_heads // k_cache.shape[1] + return _layout(rpr, group, selected_tokens.shape[1], 1) is not None + + +def _compile_grouped(key: tuple, kern_kw: dict, compile_args: tuple): + """The grouped kernel of `key` (its `splits` and launch mode -- False, True for the tail + mode, "bal" for the balanced one -- at positions 5 and 6), compiled once.""" + fn = _COMPILED.get(key) + if fn is None: + kern = _GroupedKernel(splits=key[5], tail=key[6] is True, bal=key[6] == "bal", **kern_kw) + fn = cute.compile(kern, *compile_args, options="--opt-level 2 --enable-tvm-ffi") + _COMPILED[key] = fn + return fn + + +def _compile_merge(heads: int, splits: int, gated: bool, pdl: bool, merge_args: tuple, gdim: int): + """The row merge of `splits` partials, compiled once per gate tensor rank `gdim` (the + trunk's `[rows, Hq * D]` gate, the draft layer's `[rows, Hq, D]` view, q when ungated).""" + mkey = ("rowmerge", heads, splits, gated, pdl, MERGE_PARTS, gdim) + mfn = _COMPILED.get(mkey) + if mfn is None: + mfn = cute.compile( + _RowMergeKernel(heads=heads, splits=splits, gated=gated, pdl=pdl, parts=MERGE_PARTS), + *merge_args, + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[mkey] = mfn + return mfn + + +def sparse_gqa_grouped( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """The grouped kernel (and the row merge with more than one split) over an e4m3 pool; see + `QsaSparseGqaGroupedCute` for the contract.""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads, tpb, dim = k_cache.shape + rpr = int(rows_per_request) + assert head_dim == HEAD_DIM and dim == HEAD_DIM and tpb == tokens_per_block + assert k_cache.dtype == torch.float8_e4m3fn and v_cache.dtype == k_cache.dtype + assert q.dtype == torch.bfloat16 and q.stride(2) == 1 + assert (q.stride(0) * 2) % 16 == 0 and (q.stride(1) * 2) % 16 == 0 and q.data_ptr() % 16 == 0 + assert k_cache.stride() == v_cache.stride() and k_cache.stride(3) == 1 + assert k_cache.stride(2) == HEAD_DIM, "a page's token rows contiguous (a group: one bulk copy)" + assert v_cache.shape == k_cache.shape + assert k_cache.data_ptr() % 16 == 0 and v_cache.data_ptr() % 16 == 0 + assert all(s % 16 == 0 for s in k_cache.stride()[:3]), "16-byte aligned K/V rows" + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1], "row-contiguous block table" + assert selected_tokens.dtype == torch.int32 and selected_tokens.stride(1) == 1 + assert request_indices.is_contiguous() and request_indices.dtype in (torch.int32, torch.int64) + group = num_q_heads // num_kv_heads + width = selected_tokens.shape[1] + out = torch.empty((rows, num_q_heads, head_dim), dtype=torch.bfloat16, device=q.device) + if rows == 0: + return out + n_groups = -(-rows // rpr) + units = n_groups * num_kv_heads + n_whole = None + n_sms = _num_sms(q.device.index or 0) + words = _plan_words(block_table.shape[1], tpb) + bal_fits = ( + words <= MAX_PLAN_WORDS and units <= n_sms <= THREADS and _layout(rpr, group, width, 1) + ) + bal = bool(bal_fits and takes_balanced(units, n_sms)) + if bal: + splits = 1 + else: + splits, n_whole = plan_launch(n_groups, num_kv_heads, width, n_sms) + n_splits = max(1, min(MAX_SPLITS, int(splits))) + tail = n_whole is not None + assert _layout(rpr, group, width, 1 if (tail or bal) else n_splits) is not None, ( + "no shared-memory plan for this geometry: see `takes_grouped`" + ) + use_pdl = bool(get_env_enable_pdl()) + gated = output_gate is not None + gate_hs = head_dim + if gated: + assert output_gate.dtype == torch.bfloat16 and output_gate.stride(-1) == 1 + if output_gate.dim() == 3: + assert output_gate.shape == q.shape + gate_hs = output_gate.stride(1) + else: + assert output_gate.shape == (rows, num_q_heads * head_dim) + assert (output_gate.stride(0) * 2) % 16 == 0 and (gate_hs * 2) % 16 == 0 + assert output_gate.data_ptr() % 16 == 0, "16-byte aligned gate rows" + req64 = request_indices.dtype == torch.int64 + bbuf = _bal_buffers(q.device, n_sms) + if bal: + n_whole, tail_row0, n_ctas = 0, rows, n_sms + elif tail: + tail_row0 = (n_whole // num_kv_heads) * rpr + n_ctas = n_whole + (units - n_whole) * n_splits + elif n_splits > 1: + n_whole, tail_row0, n_ctas = 0, 0, units * n_splits + else: + n_whole, tail_row0, n_ctas = units, rows, units + merged = rows - tail_row0 + key = ( + rpr, + group, + num_kv_heads, + width, + tpb, + n_splits, + "bal" if bal else tail, + float(softmax_scale), + num_pages, + tuple(block_table.shape), + tuple(k_cache.stride()[:3]), + tuple(q.stride()[:2]), + req64, + gated, + use_pdl, + ) + if bal: + # a partial slot a CTA: its unit's T rows + part = torch.empty( + (n_ctas, rpr, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_ctas, rpr, num_q_heads), dtype=torch.float32, device=q.device) + elif merged > 0: + part = torch.empty( + (n_splits, merged, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_splits, merged, num_q_heads), dtype=torch.float32, device=q.device) + else: + part, lse = out, out + gate_t = output_gate if gated else q + gate_stride = output_gate.stride(0) if gated else 0 + compile_args = ( + # read with scalar loads: a row slice of the selection needs no copy + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(q, cutlass.BFloat16), + _ptr(k_cache, cutlass.Uint8), + _ptr(v_cache, cutlass.Uint8), + _ptr(block_table, cutlass.Int32, 4), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(out, cutlass.BFloat16), + _ptr(part, cutlass.Float32), + _ptr(lse, cutlass.Float32), + _ptr(gate_t, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cutlass.Int32(n_whole), + cutlass.Int32(tail_row0), + cutlass.Int32(n_ctas), + _ptr(bbuf, cutlass.Int32, 16), + _ptr(bbuf, cutlass.Int32, 4), + _ptr(bbuf, cutlass.Int32, 4), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + tab_addr = bbuf.data_ptr() + cnt_addr = tab_addr + 4 * (4 * n_sms) + est_addr = tab_addr + 4 * (5 * n_sms + 4) + plan_kw = dict( + rpr=rpr, width=width, words=words, num_requests=block_table.shape[0], pdl=use_pdl + ) + pkey = ("balplan", rpr, width, words, block_table.shape[0], req64, use_pdl) + plan_args = ( + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(bbuf, cutlass.Int32, 4), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(n_groups), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + merge_args = ( + from_dlpack(gate_t, assumed_align=16).mark_layout_dynamic(leading_dim=gate_t.dim() - 1), + _ptr(part, cutlass.Float32), + _ptr(lse, cutlass.Float32, 4), + _ptr(out, cutlass.BFloat16, 8), + cutlass.Int32(merged), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + kern_kw = dict( + rpr=rpr, + group=group, + kv_heads=num_kv_heads, + width=width, + tpb=tpb, + scale=float(softmax_scale), + num_pages=num_pages, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + k_strides=tuple(k_cache.stride()[:3]), + q_strides=tuple(q.stride()[:2]), + req64=req64, + gated=gated, + pdl=use_pdl, + bal_ctas=n_sms, + ) + # A launch plan depends on the call's row count: the first call of a geometry compiles every + # plan its row counts can reach, so no serving step compiles one. + if key[:5] + key[7:] not in _PLANNED: + _PLANNED.add(key[:5] + key[7:]) + max_groups = block_table.shape[0] + for plan_splits_, plan_tail in reachable_plans( + max_groups, num_kv_heads, width, _num_sms(q.device.index or 0) + ): + if _layout(rpr, group, width, 1 if plan_tail else plan_splits_) is None: + continue + _compile_grouped(key[:5] + (plan_splits_, plan_tail) + key[7:], kern_kw, compile_args) + if plan_tail or plan_splits_ > 1: + _compile_merge(num_q_heads, plan_splits_, gated, use_pdl, merge_args, gate_t.dim()) + if bal_fits: + _compile_grouped(key[:5] + (1, "bal") + key[7:], kern_kw, compile_args) + _compile_plan(pkey, plan_kw, plan_args) + fn = _compile_grouped(key, kern_kw, compile_args) + if bal: + pfn = _compile_plan(pkey, plan_kw, plan_args) + pfn( + selected_tokens, + request_indices.data_ptr(), + est_addr, + rows, + selected_tokens.stride(0), + n_groups, + ) + fn( + selected_tokens, + q.data_ptr(), + k_cache.data_ptr(), + v_cache.data_ptr(), + block_table.data_ptr(), + request_indices.data_ptr(), + out.data_ptr(), + part.data_ptr(), + lse.data_ptr(), + gate_t.data_ptr(), + rows, + selected_tokens.stride(0), + gate_stride, + gate_hs, + n_whole, + tail_row0, + n_ctas, + tab_addr, + cnt_addr, + est_addr, + ) + if bal or merged <= 0: + return out + mfn = _compile_merge(num_q_heads, n_splits, gated, use_pdl, merge_args, gate_t.dim()) + gate_tail = gate_t[tail_row0:] if gated else gate_t + mfn( + gate_tail, + part.data_ptr(), + lse.data_ptr(), + out[tail_row0:].data_ptr(), + merged, + gate_stride, + gate_hs, + ) + return out + + +# --- catalog entry --- + +#: The checkpoint's QSA geometry: 32-token pages, 2051 selected tokens a row, 256 head dims, a +#: `max_seq_len` 262144 table. +_QSA = dict(tokens_per_block=32, compress_ratio=4, token_topk=2048, head_dim=256, pages=8196) +_TEP2 = dict( + _QSA, q_heads=12, kv_heads=1, slots=128, slot_pages=24, layer=11, kv=torch.float8_e4m3fn +) +_DEP4 = dict( + _QSA, q_heads=24, kv_heads=2, slots=128, slot_pages=26, layer=11, kv=torch.float8_e4m3fn +) +_TEP2_ROW = dict(gate="row", row_heads=29) +_DEP4_ROW = dict(gate="row", row_heads=55) + + +def _grouped(member: dict, requests: int, rows: int = 3, last=60000, step=900, **kw) -> dict: + """A verify step of `requests` requests of `rows` rows each, request r's rows ending at + `last - step * r`; `overlap` spreads a request's rows' block scores (None: independent rows).""" + segments = tuple((last + 1 - rows - step * r, rows) for r in range(requests)) + return dict(member, segments=segments, rows_per_request=rows, **{"overlap": 0.05, **kw}) + + +class QsaSparseGqaGroupedCute(OpWrapper): + """`qsa_paged_sparse_gqa` for decode rows that come `rows_per_request` consecutive rows a + request (an MTP verify step) over an e4m3 paged pool: `[rows, Hq, D]` bf16. With `output_gate` + (`[rows, Hq * D]`, or `[rows, Hq, D]` with any head stride; bf16, unit inner stride) the + result is `bf16(attn) * sigmoid(gate)` rounded once more to bf16. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ht_tep2 MTP-2 verify: 128 x 3 = 384 rows, one CTA a group (gate fused into its " + "store), captured", + spec=_grouped(_TEP2, 128, step=400, capture=True, **_TEP2_ROW), + ), + Cell( + why="ht_tep2 MTP-2 verify: 160 x 3 = 480 rows, one CTA a group past the first wave, " + "the last wave's groups split 4 ways and merged, gated", + spec=_grouped(dict(_TEP2, slots=160), 160, step=300, **_TEP2_ROW), + ), + Cell( + why="8 requests x 3 rows deep in long sequences (196k..167k tokens): the fewest " + "groups the kernel takes, 18 splits, gated", + spec=_grouped(_TEP2, 8, last=196607, step=4099, **_TEP2_ROW), + ), + Cell( + why="16 x 3 rows whose selections are independent (union ~3x a row's), ungated", + spec=_grouped(_TEP2, 16, overlap=None), + ), + Cell( + why="ht_dep4 MTP-3 verify: 8 x 4 = 32 rows, 24 q / 2 KV heads over the e4m3 trunk " + "(26 role pages a slot), gated, captured", + spec=_grouped(_DEP4, 8, rows=4, capture=True, **_DEP4_ROW), + ), + Cell( + why="draft layer: 16 x 3 rows, the gate half of the interleaved q projection (head " + "stride 512)", + spec=_grouped(_TEP2, 16, gate="interleaved"), + ), + Cell( + why="groups spanning requests (3, 3, 1, 2, 3 x 6, 2 rows: misaligned and partial " + "groups), short sequences, one foreign row, a row slice of a wider selection, " + "ungated", + spec=dict( + _TEP2, + segments=((77, 3), (2047, 3), (9000, 1), (30000, 2)) + + tuple((40000 + 1000 * r, 3) for r in range(6)) + + ((5, 2),), + rows_per_request=3, + overlap=0.05, + row_offset=1, + foreign=True, + ), + ), + ) + + note = """ + Source: this target's CuTe DSL kernels (`_GroupedKernel`, `_RowMergeKernel`, `_BalPlanKernel`), + `cute.compile`d per (T, splits, launch mode); every reachable plan of a geometry is compiled on + its first call. Called by `modeling._sparse_gqa` for verify / draft-extend rows over an e4m3 + pool with `rows_per_request` > 1 where `serves` (an e4m3 pool on SM103, 2..4 rows a request, + at least `MIN_GROUPS` groups, T x q heads a KV head <= 128); the per-row + `qsa_sparse_gqa_splitk_cute` otherwise. Graph-capturable (the row count is a runtime value). + + Numerics: K/V widened exactly to f16, q rounded to f16, f16 MMAs with fp32 accumulation, P + rounded to f16, the softmax offset moved only past 2^8; within the decode op's band of the + fp32 reference, not bitwise the per-row kernel. Deterministic for one batch. + + Preconditions: as `qsa_sparse_gqa_splitk_cute`, plus expanded selections: each four-column + quad of a row names one four-token group at its own offsets (column % 4 == token % 4), no + group twice in a row. A NaN in a token one row selects reaches the group's other rows of that + request (P = 0 times NaN). A group's pool row `page * tokens_per_block + slot` is 32-bit + (pools below 2^26 pages). + + Not certified: the three-split band of `plan_splits` (77..87 units on 152 SMs), T = 2. + """ + + serves = staticmethod(takes_grouped) + + def __call__( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return sparse_gqa_grouped( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + rows_per_request, + output_gate=output_gate, + ) + + def reference( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """The per-row entry's reference (fp32 sparse GQA over every resident selected token, + rounded once; gated in fp32 over that rounding and rounded again). `rows_per_request` + is ignored: the grouping changes which loads serve a row, not what it attends.""" + return qsa_sparse_gqa_splitk_cute.reference( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + output_gate, + ) + + +qsa_sparse_gqa_grouped_cute = QsaSparseGqaGroupedCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py new file mode 100644 index 000000000000..e0b6167e5b41 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py @@ -0,0 +1,1720 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA for context batches: several query rows per program, K/V read once per tile. + +A plan launch lists, per group of consecutive rows of one request, the union of the rows' +resident selected tokens as cache slots and re-indexes each row's selection onto it; a persistent +row-tile pass walks each list in 64-slot tiles, loading each tile's K/V once for every row and head +of the group. Groups whose rows span requests, or whose union costs more than the rows' own +columns, go to a lone-row pass over each row's own columns. A group is sized to whole 8-row +swizzle atoms (10 rows of 12 heads, 20 of 6). +""" + +from __future__ import annotations + +import math + +import cutlass +import cutlass.cute as cute +import torch +import triton +import triton.language as tl +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( + qsa_paged_sparse_gqa as _per_row_entry, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + BLOCK_M, + BLOCK_N, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + eligible as _cute_eligible, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + num_ctas as _rowtile_ctas, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + rowtile_attention_cute as _rowtile_attention_cute, +) +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + +_INTEGER_DTYPES = (torch.int32, torch.int64) +# The per-row kernel's scan tile: the prefill bound is rounded up to it, as that kernel does. +_BOUND_TILE = 64 +# Programs of the lone-row pass per KV head; each walks the lone list with this stride. +_LONE_PROGRAMS = 256 # capacity: fixed: a launch width; the lone list may be any length +# A one-request group takes the row-tile pass when its list's 64-slot tiles, each costing this +# many quarters of a per-row tile (measured: about half), cost no more than its rows' own +# columns; at 2 every one-request group qualifies, as a union never holds more tiles. +_SHARED_TILE_COST = 2 +#: The O staging's swizzle atom in MMA rows: a group's rows x heads come in whole atoms. +_ROW_ATOM = 8 +_LONE_WARPS = 2 +_LONE_STAGES = 2 + + +@dsl_user_op +def _red_or_shared(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.or.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,l", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +# -------------------------------------------------------------------------------------- +# The token plan (CuTe DSL): one CTA a group ORs its rows' readable selected tokens into a +# shared-memory bitmap over the table's tokens, keeping each row's highest token. A one-request +# group masks each word to resident pages, takes list positions as a block-wide prefix of the +# words' popcounts and decides shared vs lone (`_SHARED_TILE_COST`); a shared group writes each +# listed token's cache slot and each row's re-indexed bits (rank = prefix + popcount below it). +# The list is the union's resident tokens in token order. +# -------------------------------------------------------------------------------------- + +_PLAN_THREADS = 512 +#: CTAs an SM the plan's registers are held to, so three groups' latency chains overlap. +_PLAN_MIN_BLOCKS = 3 +#: Shared scratch words of the plan CTA: rows' highest tokens at 0, rows' columns read at +#: `_MISC_COLS`, the lone list base at `_MISC_LONE`, the scan's warp totals at `_MISC_SCAN`. +_MISC_COLS = 32 +_MISC_LONE = 63 +_MISC_SCAN = 64 + + +@dsl_user_op +def _cu_popc64(x, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(x).ir_value(loc=loc, ip=ip)], + "popc.b64 $0, $1;", + "=r,l", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _cu_ctz64(x, *, loc=None, ip=None): + """Trailing zeros of a nonzero 64-bit word.""" + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(x).ir_value(loc=loc, ip=ip)], + "{ .reg .b64 t; brev.b64 t, $1; clz.b64 $0, t; }", + "=r,l", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _max_shared(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.max.s32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ld_shared_b64(addr, *, loc=None, ip=None): + return cutlass.Int64( + llvm.inline_asm( + T.i64(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip)], + "ld.shared.b64 $0, [$1];", + "=l,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _st_shared_b64(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ st.shared.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,l", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ld_shared_b32(addr, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _st_shared_b32(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +class _GroupPlanKernel: + """The token plan pass, one CTA a group (see the section comment).""" + + def __init__(self, topk, block_topk, compress_ratio, tokens_per_block, table_width, + num_words, rows_per_tile, packed_words, req64, qp64): # fmt: skip + self.topk, self.block_topk, self.cr = topk, block_topk, compress_ratio + self.tpb, self.width, self.num_words = tokens_per_block, table_width, num_words + self.rpt, self.pw = rows_per_tile, packed_words + self.req64, self.qp64 = req64, qp64 + # shared memory: union words | word prefixes | rows' packed words | row scratch + self.o_pref = num_words * 8 + self.o_pack = self.o_pref + ((num_words * 4 + 15) // 16) * 16 + self.o_misc = self.o_pack + rows_per_tile * packed_words * 8 + self.smem = self.o_misc + 96 * 4 + 64 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + qp_ptr: cute.Pointer, + req_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + info_ptr: cute.Pointer, + slots_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + plan_ptr: cute.Pointer, + lone_ptr: cute.Pointer, + lone_count_ptr: cute.Pointer, + rows: cutlass.Int32, + num_groups: cutlass.Int32, + sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, + slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, + num_cache_pages: cutlass.Int32, + stream, + ): + self.kernel( + sel_t.iterator.toint(), + qp_ptr.toint(), + req_ptr.toint(), + bt_ptr.toint(), + info_ptr.toint(), + slots_ptr.toint(), + packed_ptr.toint(), + plan_ptr.toint(), + lone_ptr.toint(), + lone_count_ptr.toint(), + rows, + sel_stride, + bt_stride, + slots_stride, + num_requests, + num_cache_pages, + ).launch( + grid=[num_groups, 1, 1], + block=[_PLAN_THREADS, 1, 1], + smem=self.smem, + stream=stream, + min_blocks_per_mp=_PLAN_MIN_BLOCKS, + ) + + @cute.kernel + def kernel(self, sel_base: cutlass.Int64, qp_base: cutlass.Int64, req_base: cutlass.Int64, + bt_base: cutlass.Int64, info_base: cutlass.Int64, slots_base: cutlass.Int64, + packed_base: cutlass.Int64, plan_base: cutlass.Int64, lone_base: cutlass.Int64, + lone_count_base: cutlass.Int64, rows: cutlass.Int32, sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, num_cache_pages: cutlass.Int32): # fmt: skip + tidx, _, _ = cute.arch.thread_idx() + group, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + warp = tidx // 32 + nthreads: cutlass.Constexpr = _PLAN_THREADS + nwarps: cutlass.Constexpr = _PLAN_THREADS // 32 + rpt: cutlass.Constexpr = self.rpt + nw: cutlass.Constexpr = self.num_words + tpb: cutlass.Constexpr = self.tpb + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem,), stride=(1,)), byte_alignment=128 + ) + s_u = cutlass.Int32(buf.iterator.toint()) + s_pref = s_u + self.o_pref + s_pack = s_u + self.o_pack + s_misc = s_u + self.o_misc + r0 = group * rpt + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < nw: + _st_shared_b64(s_u + w * 8, cutlass.Int64(0)) + if tidx < rpt: + _st_shared_b32(s_misc + tidx * 4, cutlass.Int32(-1)) + # each row's request and columns read (every thread, rpt rows) + reqs = [] + cols = [] + own_tiles = cutlass.Int32(0) + n_live = cutlass.Int32(0) + lowest = cutlass.Int32(2147483647) + highest_req = cutlass.Int32(-1) + for t in cutlass.range_constexpr(rpt): + row = r0 + t + live = row < rows + srow = cutlass.min(row, rows - 1) + if const_expr(self.req64): + rq = cutlass.Int32(_ldg_i64(req_base + cutlass.Int64(srow) * 8)) + else: + rq = _ldg_i32(req_base + cutlass.Int64(srow) * 4) + if const_expr(self.qp64): + qp = cutlass.Int32(_ldg_i64(qp_base + cutlass.Int64(srow) * 8)) + else: + qp = _ldg_i32(qp_base + cutlass.Int64(srow) * 4) + visible = qp + 1 + sb = cutlass.min(visible // self.cr, cutlass.Int32(self.block_topk)) + valid = sb * self.cr + visible % self.cr + bounded = cutlass.min(valid, cutlass.Int32(self.topk)) + cr_ = cutlass.min((bounded + _BOUND_TILE - 1) // _BOUND_TILE * _BOUND_TILE, + cutlass.Int32(self.topk)) # fmt: skip + cr_ = cutlass.max(cr_, cutlass.Int32(0)) + cr_ = cutlass.Int32(cutlass.select_(live, cr_, cutlass.Int32(0))) + reqs.append(rq) + cols.append(cr_) + if tidx == t: + _st_shared_b32(s_misc + (_MISC_COLS + t) * 4, cr_) + own_tiles = own_tiles + (cr_ + BLOCK_N - 1) // BLOCK_N + n_live = n_live + cutlass.Int32( + cutlass.select_(live, cutlass.Int32(1), cutlass.Int32(0)) + ) + lowest = cutlass.Int32(cutlass.select_(live, cutlass.min(lowest, rq), lowest)) + highest_req = cutlass.Int32( + cutlass.select_(live, cutlass.max(highest_req, rq), highest_req) + ) + one_request = (lowest == highest_req) & (lowest >= 0) & (lowest < num_requests) + cute.arch.sync_threads() + # walk 1: the rows' readable tokens into the union bitmap, each row's highest token. + # Items are (row, four-column quad) pairs, item = i * nthreads + tidx, kept in registers + # for walk 2. + nq: cutlass.Constexpr = -(-self.topk // 4) + n_items: cutlass.Constexpr = rpt * nq + n_rounds: cutlass.Constexpr = -(-n_items // nthreads) + toks = [] + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + q = item % nq + ncol = _ld_shared_b32(s_misc + (_MISC_COLS + t) * 4) + ncol = cutlass.Int32(cutlass.select_(item < n_items, ncol, cutlass.Int32(0))) + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + for k in cutlass.range_constexpr(4): + c = q * 4 + k + tok = cutlass.Int32(-1) + if c < ncol: + tok = _ldg_i32(row_sel + cutlass.Int64(c) * 4) + ok = (tok >= 0) & (tok // tpb < self.width) + toks.append(cutlass.Int32(cutlass.select_(ok, tok, cutlass.Int32(-1)))) + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + cur_w = cutlass.Int32(-1) + cur_m = cutlass.Int64(0) + hi_q = cutlass.Int32(-1) + for k in cutlass.range_constexpr(4): + tok = toks[i * 4 + k] + if tok >= 0: + w = tok >> 6 + if w != cur_w: + if cur_w >= 0: + _red_or_shared(s_u + cur_w * 8, cur_m) + cur_w = w + cur_m = cutlass.Int64(0) + cur_m = cur_m | (cutlass.Int64(1) << cutlass.Int64(tok & 63)) + hi_q = cutlass.max(hi_q, tok) + if cur_w >= 0: + _red_or_shared(s_u + cur_w * 8, cur_m) + # the row's highest token: a warp's 32 items span at most two rows (nq > 32) + t_first = cute.arch.shuffle_sync(t, 0) + same = t == t_first + hi_a = cutlass.Int32(cutlass.select_(same, hi_q, cutlass.Int32(-1))) + hi_b = cutlass.Int32(cutlass.select_(same, cutlass.Int32(-1), hi_q)) + for o in (1, 2, 4, 8, 16): + hi_a = cutlass.max(hi_a, cute.arch.shuffle_sync_bfly(hi_a, offset=o)) + hi_b = cutlass.max(hi_b, cute.arch.shuffle_sync_bfly(hi_b, offset=o)) + if lane == 0: + if hi_a >= 0: + _max_shared(s_misc + t_first * 4, hi_a) + t_last = cute.arch.shuffle_sync(t, 31) + if lane == 31: + if (t_last != t_first) & (hi_b >= 0): + _max_shared(s_misc + t_last * 4, hi_b) + cute.arch.sync_threads() + # row_info, the group's highest token + grp_hi = cutlass.Int32(-1) + for t in cutlass.range_constexpr(rpt): + h = _ld_shared_b32(s_misc + t * 4) + grp_hi = cutlass.max(grp_hi, h) + if tidx == t: + if r0 + t < rows: + _stg_i32(info_base + cutlass.Int64(r0 + t) * 8, h) + _stg_i32(info_base + cutlass.Int64(r0 + t) * 8 + 4, cols[t]) + words_needed = cutlass.Int32( + cutlass.select_(one_request, (grp_hi + 64) // 64, cutlass.Int32(0)) + ) + words_needed = cutlass.min(words_needed, cutlass.Int32(nw)) + # resident tokens: a word's bits on a page outside the table or the pool are dropped + bt_row = bt_base + cutlass.Int64(cutlass.max(lowest, 0)) * bt_stride * 4 + cnts = [] + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + cnt = cutlass.Int32(0) + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + if u != 0: + first = w * 64 + for k in cutlass.range_constexpr(_word_pages(tpb)): + lp = first // tpb + k + lo_b = cutlass.max(lp * tpb - first, 0) + hi_b = cutlass.min((lp + 1) * tpb - first, 64) + if lo_b < hi_b: + ok = lp < self.width + pg = cutlass.Int32(-1) + if ok: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + ok = ok & (pg >= 0) & (pg < num_cache_pages) + span = _bits_range64(lo_b, hi_b) + u = cutlass.Int64(cutlass.select_(ok, u, u & ~span)) + _st_shared_b64(s_u + w * 8, u) + cnt = _cu_popc64(u) + cnts.append(cnt) + # list positions: block-wide exclusive prefix of the words' counts in word order; thread + # tidx owns words tidx + i * nthreads, so each round is scanned and the rounds chained. + base = cutlass.Int32(0) + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + incl = cnts[i] + for o in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=o) + if lane >= o: + incl = incl + up + if lane == 31: + _st_shared_b32(s_misc + (_MISC_SCAN + warp) * 4, incl) + cute.arch.sync_threads() + wv = cutlass.Int32(0) + tot = cutlass.Int32(0) + if lane < nwarps: + wv = _ld_shared_b32(s_misc + (_MISC_SCAN + lane) * 4) + tot = wv + before = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, wv, cutlass.Int32(0))) + ) + total = cute.arch.warp_reduction_sum(tot) + w = tidx + i * nthreads + if w < nw: + _st_shared_b32(s_pref + w * 4, base + before + incl - cnts[i]) + base = base + total + cute.arch.sync_threads() + listed = base + tiles = (listed + BLOCK_N - 1) // BLOCK_N + shared = one_request & (tiles * _SHARED_TILE_COST <= own_tiles * 4) + shared_i = cutlass.Int32(cutlass.select_(shared, cutlass.Int32(1), cutlass.Int32(0))) + if tidx == 0: + _stg_i32( + plan_base + cutlass.Int64(group) * 8, + cutlass.Int32(cutlass.select_(shared, tiles, cutlass.Int32(-1))), + ) + _stg_i32(plan_base + cutlass.Int64(group) * 8 + 4, listed) + if shared_i == 0: + lb = _atom_add_g32(lone_count_base, n_live) + _st_shared_b32(s_misc + _MISC_LONE * 4, lb) + if shared_i == 0: + cute.arch.sync_threads() + if tidx < rpt: + if r0 + tidx < rows: + lb = _ld_shared_b32(s_misc + _MISC_LONE * 4) + _stg_i32(lone_base + cutlass.Int64(lb + tidx) * 4, r0 + tidx) + if (shared_i != 0) & (listed > 0): + pw_used = (listed + 63) // 64 + # the rows' packed words, zeroed + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + _st_shared_b64(s_pack + i * 8, cutlass.Int64(0)) + # slots: one thread a word, its set bits in token order + slot_row = slots_base + cutlass.Int64(group) * slots_stride * 4 + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + j = _ld_shared_b32(s_pref + w * 4) + pg_lp = cutlass.Int32(-1) + pg = cutlass.Int32(0) + while u != 0: + b = _cu_ctz64(u) + tok = w * 64 + b + lp = tok // tpb + if lp != pg_lp: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + pg_lp = lp + _stg_i32(slot_row + cutlass.Int64(j) * 4, pg * tpb + tok % tpb) + j = j + 1 + u = u & (u - 1) + cute.arch.sync_threads() + # walk 2 (walk 1's registers): each row's readable resident tokens -> its bit at the + # token's list position; a quad's ranks follow by popcount from its word's prefix. + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tok0 = cutlass.max( + cutlass.max(toks[i * 4], toks[i * 4 + 1]), + cutlass.max(toks[i * 4 + 2], toks[i * 4 + 3]), + ) + if tok0 >= 0: + w = tok0 >> 6 + u = _ld_shared_b64(s_u + w * 8) + pre = _ld_shared_b32(s_pref + w * 4) + cur_w = cutlass.Int32(-1) + cur_m = cutlass.Int64(0) + for k in cutlass.range_constexpr(4): + tok = toks[i * 4 + k] + if tok >= 0: + uw = u + pw_pre = pre + if (tok >> 6) != w: + uw = _ld_shared_b64(s_u + (tok >> 6) * 8) + pw_pre = _ld_shared_b32(s_pref + (tok >> 6) * 4) + bit = cutlass.Int64(1) << cutlass.Int64(tok & 63) + if (uw & bit) != 0: + jj = pw_pre + _cu_popc64(uw & (bit - 1)) + pw_i = jj >> 6 + if pw_i != cur_w: + if cur_w >= 0: + _red_or_shared(s_pack + (t * pw_used + cur_w) * 8, cur_m) + cur_w = pw_i + cur_m = cutlass.Int64(0) + cur_m = cur_m | (cutlass.Int64(1) << cutlass.Int64(jj & 63)) + if cur_w >= 0: + _red_or_shared(s_pack + (t * pw_used + cur_w) * 8, cur_m) + cute.arch.sync_threads() + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + t = i // pw_used + k = i % pw_used + if r0 + t < rows: + _stg_i64(packed_base + (cutlass.Int64(r0 + t) * self.pw + k) * 8, + _ld_shared_b64(s_pack + i * 8)) # fmt: skip + + +def _word_pages(tpb: int) -> int: + """Pages an aligned 64-token word spans at most.""" + if 64 % tpb == 0: + return 64 // tpb + if tpb % 64 == 0: + return 1 + return -(-64 // tpb) + 1 + + +def _bits_range64(lo, hi): + """int64 with bits lo .. hi - 1 set (0 <= lo < hi <= 64).""" + ones = cutlass.Int64(-1) + below_hi = cutlass.Int64( + cutlass.select_(hi >= 64, ones, (cutlass.Int64(1) << cutlass.Int64(hi)) - 1) + ) + below_lo = (cutlass.Int64(1) << cutlass.Int64(lo)) - 1 + return below_hi & ~below_lo + + +@dsl_user_op +def _ldg_i32(addr, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b32 $0, [$1];", + "=r,l", has_side_effects=True, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +@dsl_user_op +def _ldg_i64(addr, *, loc=None, ip=None): + return cutlass.Int64( + llvm.inline_asm( + T.i64(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b64 $0, [$1];", + "=l,l", has_side_effects=True, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +@dsl_user_op +def _stg_i32(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ st.global.b32 [$1], $2; mov.u32 $0, 0; }", "=r,l,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _stg_i64(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ st.global.b64 [$1], $2; mov.u32 $0, 0; }", "=r,l,l", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _atom_add_g32(addr, val, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "atom.global.add.u32 $0, [$1], $2;", "=r,l,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +_GROUP_PLAN: dict = {} + + +def _group_plan(selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, num_words, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages): # fmt: skip + """Launch the token plan pass (one CTA a group).""" + rows, final_topk = selected_tokens.shape + block_topk = (final_topk - compress_ratio + 1) // compress_ratio + req64 = request_indices.dtype == torch.int64 + qp64 = query_positions.dtype == torch.int64 + query_positions = query_positions.contiguous() + key = (final_topk, compress_ratio, tokens_per_block, table_width, rows_per_tile, + packed_words, req64, qp64) # fmt: skip + gm = cute.AddressSpace.gmem + ptrs = ( + query_positions.data_ptr(), request_indices.data_ptr(), block_table.data_ptr(), + row_info.data_ptr(), slots.data_ptr(), packed_bits.data_ptr(), group_plan.data_ptr(), + lone_rows.data_ptr(), lone_count.data_ptr(), + ) # fmt: skip + scalars = (rows, num_groups, selected_tokens.stride(0), block_table.stride(0), slots.stride(0), + block_table.shape[0], num_cache_pages) # fmt: skip + fn = _GROUP_PLAN.get(key) + if fn is None: + i32, i64 = cutlass.Int32, cutlass.Int64 + fn = cute.compile( + _GroupPlanKernel( + final_topk, + block_topk, + compress_ratio, + tokens_per_block, + table_width, + num_words, + rows_per_tile, + packed_words, + req64, + qp64, + ), # fmt: skip + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + make_ptr(i64 if qp64 else i32, ptrs[0], gm, assumed_align=4), + make_ptr(i64 if req64 else i32, ptrs[1], gm, assumed_align=4), + make_ptr(i32, ptrs[2], gm, assumed_align=4), + make_ptr(i32, ptrs[3], gm, assumed_align=8), + make_ptr(i32, ptrs[4], gm, assumed_align=4), + make_ptr(i64, ptrs[5], gm, assumed_align=8), + make_ptr(i32, ptrs[6], gm, assumed_align=8), + make_ptr(i32, ptrs[7], gm, assumed_align=4), + make_ptr(i32, ptrs[8], gm, assumed_align=4), + *(cutlass.Int32(x) for x in scalars), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GROUP_PLAN[key] = fn + fn(selected_tokens, *ptrs, *scalars) + + +# -------------------------------------------------------------------------------------- +# The block plan (CuTe DSL): the token plan's outputs, bit for bit, from the selection's 4-token +# blocks. Every token a row selects below `T0 = 4 * ((p + 1) // 4)` (`p` the group's lowest +# position) lies in a complete block: an aligned column quad below `T0` sets one bit of a block +# bitmap; tokens at or past `T0` (at most `rows_per_tile + 3`) go to one 64-token local word. A +# group naming a token below `T0` outside such a quad goes to the lone pass. Its shared-memory ORs +# are 32-bit (`red.shared.or.b64` is a compare-and-swap loop on SM100). +# -------------------------------------------------------------------------------------- + +_BP_THREADS = 512 +#: CTAs an SM the block plan's registers are held to. +_BP_MIN_BLOCKS = 4 +#: Shared scratch words of the block plan CTA (int32 slots of its misc area). +_BP_HI = 0 # rows' highest tokens (rows_per_tile) +_BP_COLS = 32 # rows' columns read +_BP_SCAN = 64 # the scan's warp totals (16) +_BP_LONE = 80 # the lone list base +_BP_FLAG = 81 # a token below T0 outside an aligned block quad +_BP_NSEG = 82 # the group's segments (runs of one request's rows), 0 for a lone group +_BP_SPLIT = 83 # the second segment's first row +_BP_NLIVE = 84 +_BP_OWN = 85 # the rows' own 64-column tiles +_BP_REQ0 = 86 # each segment's request +_BP_REQ1 = 87 +_BP_T00 = 88 # each segment's T0 +_BP_T01 = 89 +_BP_LOCN = 90 # the segment's resident local tokens +_BP_LOC = 92 # the segment's local word (int64 at words 92-93) +_BP_MISC_WORDS = 96 + + +def _span_pages(tokens: int, tpb: int, aligned: bool) -> int: + """Pages a run of `tokens` tokens spans at most (`aligned`: it starts on a multiple of + `tokens`).""" + if aligned and tokens % tpb == 0: + return tokens // tpb + if aligned and tpb % tokens == 0: + return 1 + return -(-tokens // tpb) + 1 + + +@dsl_user_op +def _stg_v4_i32(addr, a, b, c, d, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)] + + [cutlass.Int32(x).ir_value(loc=loc, ip=ip) for x in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", "=r,l,r,r,r,r", + has_side_effects=True, is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _ldg_nc_i32(addr, *, loc=None, ip=None): + """A global int32 load the compiler may schedule freely (no side effects).""" + return cutlass.Int32( + llvm.inline_asm( + T.i32(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b32 $0, [$1];", + "=r,l", has_side_effects=False, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +def _word_page_ids(bt_row, lp0, tpb, width): + """The physical pages of a block word's logical pages `lp0 ..` (clamped into the table; the + caller drops pages past it), loaded together.""" + out = [] + for k in range(_span_pages(256, tpb, True)): # trace-time unrolled + lp = cutlass.min(lp0 + k, width - 1) + out.append(_ldg_nc_i32(bt_row + cutlass.Int64(lp) * 4)) + return out + + +@dsl_user_op +def _red_or_shared32(addr, val, *, loc=None, ip=None): + """A native shared-memory OR (`red.shared.or.b64` is a compare-and-swap loop on SM100).""" + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.or.b32 [$1], $2; mov.u32 $0, 0; }", "=r,r,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +class _BlockPlanKernel: + """The block plan pass, one CTA a group (see the section comment).""" + + def __init__(self, topk, block_topk, tokens_per_block, table_width, block_words, + rows_per_tile, packed_words, req64, qp64): # fmt: skip + self.topk, self.block_topk = topk, block_topk + self.tpb, self.width, self.nbw = tokens_per_block, table_width, block_words + self.rpt, self.pw = rows_per_tile, packed_words + self.req64, self.qp64 = req64, qp64 + # shared memory: union block words | word prefixes | rows' packed words | misc + self.o_pref = block_words * 8 + self.o_pack = self.o_pref + ((block_words * 4 + 15) // 16) * 16 + self.o_misc = self.o_pack + rows_per_tile * packed_words * 8 + self.smem = self.o_misc + _BP_MISC_WORDS * 4 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + qp_ptr: cute.Pointer, + req_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + info_ptr: cute.Pointer, + slots_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + plan_ptr: cute.Pointer, + lone_ptr: cute.Pointer, + lone_count_ptr: cute.Pointer, + rows: cutlass.Int32, + num_groups: cutlass.Int32, + sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, + slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, + num_cache_pages: cutlass.Int32, + stream, + ): + self.kernel( + sel_t.iterator.toint(), qp_ptr.toint(), req_ptr.toint(), bt_ptr.toint(), + info_ptr.toint(), slots_ptr.toint(), packed_ptr.toint(), plan_ptr.toint(), + lone_ptr.toint(), lone_count_ptr.toint(), rows, sel_stride, bt_stride, slots_stride, + num_requests, num_cache_pages, + ).launch( + grid=[num_groups, 1, 1], + block=[_BP_THREADS, 1, 1], + smem=self.smem, + stream=stream, + min_blocks_per_mp=_BP_MIN_BLOCKS, + ) # fmt: skip + + def _quad(self, sel_base, r0, rows, sel_stride, s_misc, item, ra, rb): + """Walk item `item`'s four columns (a clamped column, masked after): each readable + token, -1 elsewhere (also for an item past the rows or outside rows [ra, rb)).""" + nq = -(-self.topk // 4) + t = cutlass.min(item // nq, self.rpt - 1) + q = item % nq + ncol = _ld_shared_b32(s_misc + (_BP_COLS + t) * 4) + in_seg = (item < self.rpt * nq) & (t >= ra) & (t < rb) + ncol = cutlass.Int32(cutlass.select_(in_seg, ncol, cutlass.Int32(0))) + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + tk = [] + for k in range(4): # trace-time unrolled + c = q * 4 + k + cc = cutlass.Int32(cutlass.select_(c < ncol, c, cutlass.Int32(0))) + tok = _ldg_nc_i32(row_sel + cutlass.Int64(cc) * 4) + ok = (c < ncol) & (tok >= 0) & (tok // self.tpb < self.width) + tk.append(cutlass.Int32(cutlass.select_(ok, tok, cutlass.Int32(-1)))) + return tk + + @cute.kernel + def kernel(self, sel_base: cutlass.Int64, qp_base: cutlass.Int64, req_base: cutlass.Int64, + bt_base: cutlass.Int64, info_base: cutlass.Int64, slots_base: cutlass.Int64, + packed_base: cutlass.Int64, plan_base: cutlass.Int64, lone_base: cutlass.Int64, + lone_count_base: cutlass.Int64, rows: cutlass.Int32, sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, num_cache_pages: cutlass.Int32): # fmt: skip + tidx, _, _ = cute.arch.thread_idx() + group, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + warp = tidx // 32 + nthreads: cutlass.Constexpr = _BP_THREADS + nwarps: cutlass.Constexpr = _BP_THREADS // 32 + rpt: cutlass.Constexpr = self.rpt + nbw: cutlass.Constexpr = self.nbw + tpb: cutlass.Constexpr = self.tpb + pw: cutlass.Constexpr = self.pw + big = cutlass.Int32(2147483647) + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem,), stride=(1,)), byte_alignment=128 + ) + s_u = cutlass.Int32(buf.iterator.toint()) + s_pref = s_u + self.o_pref + s_pack = s_u + self.o_pack + s_misc = s_u + self.o_misc + r0 = group * rpt + # the rows (warp 0, lane t holding row t): columns read, request and position; the + # group's segment (one request's rows; any other group is lone), live rows, own tiles, T0 + if warp == 0: + row = r0 + lane + live = (lane < rpt) & (row < rows) + srow = cutlass.min(row, rows - 1) + if const_expr(self.req64): + rq = cutlass.Int32(_ldg_i64(req_base + cutlass.Int64(srow) * 8)) + else: + rq = _ldg_i32(req_base + cutlass.Int64(srow) * 4) + if const_expr(self.qp64): + qp = cutlass.Int32(_ldg_i64(qp_base + cutlass.Int64(srow) * 8)) + else: + qp = _ldg_i32(qp_base + cutlass.Int64(srow) * 4) + visible = qp + 1 + sb = cutlass.min(visible // 4, cutlass.Int32(self.block_topk)) + valid = sb * 4 + visible % 4 + bounded = cutlass.min(valid, cutlass.Int32(self.topk)) + cr_ = cutlass.min((bounded + _BOUND_TILE - 1) // _BOUND_TILE * _BOUND_TILE, + cutlass.Int32(self.topk)) # fmt: skip + cr_ = cutlass.max(cr_, cutlass.Int32(0)) + cr_ = cutlass.Int32(cutlass.select_(live, cr_, cutlass.Int32(0))) + if lane < rpt: + _st_shared_b32(s_misc + (_BP_COLS + lane) * 4, cr_) + _st_shared_b32(s_misc + (_BP_HI + lane) * 4, cutlass.Int32(-1)) + q0 = cute.arch.shuffle_sync(rq, 0) # row r0 is always live + other = live & (rq != q0) + split = cutlass.Int32(cutlass.select_(other, lane, big)) + q1_lo = cutlass.Int32(cutlass.select_(other, rq, big)) + q1_hi = cutlass.Int32(cutlass.select_(other, rq, cutlass.Int32(-1))) + own = (cr_ + BLOCK_N - 1) // BLOCK_N + nl = cutlass.Int32(cutlass.select_(live, cutlass.Int32(1), cutlass.Int32(0))) + for o in (1, 2, 4, 8, 16): + split = cutlass.min(split, cute.arch.shuffle_sync_bfly(split, offset=o)) + q1_lo = cutlass.min(q1_lo, cute.arch.shuffle_sync_bfly(q1_lo, offset=o)) + q1_hi = cutlass.max(q1_hi, cute.arch.shuffle_sync_bfly(q1_hi, offset=o)) + own = own + cute.arch.shuffle_sync_bfly(own, offset=o) + nl = nl + cute.arch.shuffle_sync_bfly(nl, offset=o) + # rows of request q0 past the split break the run (three runs): lone + late0 = live & (rq == q0) & (lane > split) + seg1 = lane >= split + p0 = cutlass.Int32(cutlass.select_(live & (seg1 == False), qp, big)) # noqa: E712 + p1 = cutlass.Int32(cutlass.select_(live & seg1, qp, big)) + late = cutlass.Int32(cutlass.select_(late0, cutlass.Int32(1), cutlass.Int32(0))) + for o in (1, 2, 4, 8, 16): + p0 = cutlass.min(p0, cute.arch.shuffle_sync_bfly(p0, offset=o)) + p1 = cutlass.min(p1, cute.arch.shuffle_sync_bfly(p1, offset=o)) + late = late | cute.arch.shuffle_sync_bfly(late, offset=o) + if lane == 0: + ok0 = (q0 >= 0) & (q0 < num_requests) & (p0 >= 0) + ok1 = (q1_lo == q1_hi) & (q1_lo >= 0) & (q1_lo < num_requests) & (p1 >= 0) + ok1 = ok1 & (late == 0) + nseg = cutlass.Int32(0) + if split == big: + nseg = cutlass.Int32(cutlass.select_(ok0, cutlass.Int32(1), cutlass.Int32(0))) + _st_shared_b32(s_misc + _BP_NSEG * 4, nseg) + _st_shared_b32(s_misc + _BP_SPLIT * 4, cutlass.min(split, cutlass.Int32(rpt))) + _st_shared_b32(s_misc + _BP_REQ0 * 4, cutlass.max(q0, 0)) + _st_shared_b32(s_misc + _BP_REQ1 * 4, cutlass.max(cutlass.min(q1_lo, big - 1), 0)) + _st_shared_b32(s_misc + _BP_NLIVE * 4, nl) + _st_shared_b32(s_misc + _BP_OWN * 4, own) + _st_shared_b32(s_misc + _BP_T00 * 4, + cutlass.Int32(cutlass.select_(ok0, (p0 + 1) // 4 * 4, cutlass.Int32(0)))) # fmt: skip + _st_shared_b32(s_misc + _BP_T01 * 4, + cutlass.Int32(cutlass.select_(ok1, (p1 + 1) // 4 * 4, cutlass.Int32(0)))) # fmt: skip + _st_shared_b32(s_misc + _BP_FLAG * 4, cutlass.Int32(0)) + cute.arch.sync_threads() + nseg = _ld_shared_b32(s_misc + _BP_NSEG * 4) + split = _ld_shared_b32(s_misc + _BP_SPLIT * 4) + n_live = _ld_shared_b32(s_misc + _BP_NLIVE * 4) + own_tiles = _ld_shared_b32(s_misc + _BP_OWN * 4) + # the rows' packed words (a fixed `packed_words` stride a row), zeroed + for i in cutlass.range_constexpr((rpt * pw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < rpt * pw: + _st_shared_b64(s_pack + w * 8, cutlass.Int64(0)) + nq: cutlass.Constexpr = -(-self.topk // 4) + n_items: cutlass.Constexpr = rpt * nq + n_rounds: cutlass.Constexpr = -(-n_items // nthreads) + listed = cutlass.Int32(0) + for sg in cutlass.range_constexpr(1): + # a lone group (no segment) still walks its rows once: row_info's highest tokens + if sg < cutlass.max(nseg, 1): + # this segment: rows [ra, rb) of request `req`, list positions from `seg_base` + ra = cutlass.Int32(0) if sg == 0 else split + rb = cutlass.Int32(cutlass.select_(nseg == 2, split, cutlass.Int32(rpt))) + if sg == 1: + rb = cutlass.Int32(rpt) + req = _ld_shared_b32(s_misc + (_BP_REQ0 + sg) * 4) + t_lo = _ld_shared_b32(s_misc + (_BP_T00 + sg) * 4) + seg_base = listed + bt_row = bt_base + cutlass.Int64(req) * bt_stride * 4 + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < nbw: + _st_shared_b64(s_u + w * 8, cutlass.Int64(0)) + if tidx == 0: + _st_shared_b64(s_misc + _BP_LOC * 4, cutlass.Int64(0)) + cute.arch.sync_threads() + # walk 1: items are (row, four-column quad) pairs, item = i * nthreads + tidx; a + # block quad below T0 sets its block bit, any other quad its tokens' local bits + # (rolled: unrolled it spills). + for i in cutlass.range(0, n_rounds, unroll=1): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tk = self._quad(sel_base, r0, rows, sel_stride, s_misc, item, ra, rb) + b0 = tk[0] + is_blk = (b0 >= 0) & (b0 < t_lo) & ((b0 & 3) == 0) + is_blk = is_blk & (tk[1] == b0 + 1) & (tk[2] == b0 + 2) & (tk[3] == b0 + 3) + hi_q = cutlass.max(cutlass.max(tk[0], tk[1]), cutlass.max(tk[2], tk[3])) + if is_blk: + blk = b0 >> 2 + _red_or_shared32(s_u + (blk >> 5) * 4, cutlass.Int32(1) << (blk & 31)) + else: + if hi_q >= 0: + lm = cutlass.Int64(0) + bad = cutlass.Int32(0) + for k in cutlass.range_constexpr(4): + d = tk[k] - t_lo + inr = (d >= 0) & (d < 64) + live_k = tk[k] >= 0 + lm = lm | cutlass.Int64( + cutlass.select_( + live_k & inr, + cutlass.Int64(1) << cutlass.Int64(d & 63), + cutlass.Int64(0), + ) + ) + bad = bad | cutlass.Int32( + cutlass.select_( + live_k & ((d < 0) | (d >= 64)), + cutlass.Int32(1), + cutlass.Int32(0), + ) + ) + lm_lo = cutlass.Int32(lm & cutlass.Int64(0xFFFFFFFF)) + lm_hi = cutlass.Int32(lm >> cutlass.Int64(32)) + if lm_lo != 0: + _red_or_shared32(s_misc + _BP_LOC * 4, lm_lo) + if lm_hi != 0: + _red_or_shared32(s_misc + _BP_LOC * 4 + 4, lm_hi) + if bad != 0: + _st_shared_b32(s_misc + _BP_FLAG * 4, cutlass.Int32(1)) + # the row's highest token: a warp's 32 items span at most two rows (nq > 32) + t_first = cute.arch.shuffle_sync(t, 0) + same = t == t_first + hi_a = cutlass.Int32(cutlass.select_(same, hi_q, cutlass.Int32(-1))) + hi_b = cutlass.Int32(cutlass.select_(same, cutlass.Int32(-1), hi_q)) + for o in (1, 2, 4, 8, 16): + hi_a = cutlass.max(hi_a, cute.arch.shuffle_sync_bfly(hi_a, offset=o)) + hi_b = cutlass.max(hi_b, cute.arch.shuffle_sync_bfly(hi_b, offset=o)) + if lane == 0: + if hi_a >= 0: + _max_shared(s_misc + (_BP_HI + t_first) * 4, hi_a) + t_last = cute.arch.shuffle_sync(t, 31) + if lane == 31: + if (t_last != t_first) & (hi_b >= 0): + _max_shared(s_misc + (_BP_HI + t_last) * 4, hi_b) + cute.arch.sync_threads() + use = (_ld_shared_b32(s_misc + _BP_FLAG * 4) == 0) & (nseg > 0) + words_needed = cutlass.Int32( + cutlass.select_(use, (t_lo // 4 + 63) // 64, cutlass.Int32(0)) + ) + words_needed = cutlass.min(words_needed, cutlass.Int32(nbw)) + # resident blocks: a word's bits on a page outside the table or the pool dropped + cnts = [] + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + cnt = cutlass.Int32(0) + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + if u != 0: + first = w * 256 + pgs = _word_page_ids(bt_row, first // tpb, tpb, self.width) + for k in cutlass.range_constexpr(_span_pages(256, tpb, True)): + lp = first // tpb + k + lo_b = cutlass.max(lp * tpb - first, 0) // 4 + hi_b = cutlass.min((lp + 1) * tpb - first, 256) // 4 + if lo_b < hi_b: + span = _bits_range64(lo_b, hi_b) + pg = pgs[k] + ok = (lp < self.width) & (pg >= 0) & (pg < num_cache_pages) + u = cutlass.Int64(cutlass.select_(ok, u, u & ~span)) + _st_shared_b64(s_u + w * 8, u) + cnt = _cu_popc64(u) + cnts.append(cnt) + # the local word's resident tokens + if tidx == nthreads - 1: + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + if use & (lw != 0): + for k in cutlass.range_constexpr(_span_pages(64, tpb, False)): + lp = t_lo // tpb + k + lo_b = cutlass.max(lp * tpb - t_lo, 0) + hi_b = cutlass.min((lp + 1) * tpb - t_lo, 64) + if lo_b < hi_b: + span = _bits_range64(lo_b, hi_b) + if (lw & span) != 0: + ok = lp < self.width + pg = cutlass.Int32(-1) + if ok: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + ok = ok & (pg >= 0) & (pg < num_cache_pages) + lw = cutlass.Int64(cutlass.select_(ok, lw, lw & ~span)) + lw = cutlass.Int64(cutlass.select_(use, lw, cutlass.Int64(0))) + _st_shared_b64(s_misc + _BP_LOC * 4, lw) + _st_shared_b32(s_misc + _BP_LOCN * 4, _cu_popc64(lw)) + cute.arch.sync_threads() + # list positions of the blocks: exclusive prefix of the words' counts, round by round + base = cutlass.Int32(0) + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + if i * nthreads < words_needed: + incl = cnts[i] + for o in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=o) + if lane >= o: + incl = incl + up + if lane == 31: + _st_shared_b32(s_misc + (_BP_SCAN + warp) * 4, incl) + cute.arch.sync_threads() + wv = cutlass.Int32(0) + tot = cutlass.Int32(0) + if lane < nwarps: + wv = _ld_shared_b32(s_misc + (_BP_SCAN + lane) * 4) + tot = wv + before = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, wv, cutlass.Int32(0))) + ) + total = cute.arch.warp_reduction_sum(tot) + w = tidx + i * nthreads + if w < nbw: + _st_shared_b32(s_pref + w * 4, base + before + incl - cnts[i]) + base = base + total + cute.arch.sync_threads() + local_base = seg_base + base * 4 + listed = local_base + _ld_shared_b32(s_misc + _BP_LOCN * 4) + if use & (listed > seg_base): + # slots: a thread a block word, four (one 16-byte store) a listed block + slot_row = slots_base + cutlass.Int64(group) * slots_stride * 4 + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + j = seg_base + _ld_shared_b32(s_pref + w * 4) * 4 + lp0 = (w * 256) // tpb + pgs = _word_page_ids(bt_row, lp0, tpb, self.width) + while u != 0: + b = _cu_ctz64(u) + tok = (w * 64 + b) * 4 + which = tok // tpb - lp0 + pg = pgs[0] + for k in cutlass.range_constexpr(1, len(pgs)): + pg = cutlass.Int32(cutlass.select_(which == k, pgs[k], pg)) + s0 = pg * tpb + tok % tpb + if (seg_base & 3) == 0: + _stg_v4_i32(slot_row + cutlass.Int64(j) * 4, + s0, s0 + 1, s0 + 2, s0 + 3) # fmt: skip + else: + # a second segment's blocks may sit off a 16-byte boundary + _stg_i32(slot_row + cutlass.Int64(j) * 4, s0) + _stg_i32(slot_row + cutlass.Int64(j + 1) * 4, s0 + 1) + _stg_i32(slot_row + cutlass.Int64(j + 2) * 4, s0 + 2) + _stg_i32(slot_row + cutlass.Int64(j + 3) * 4, s0 + 3) + j = j + 4 + u = u & (u - 1) + if tidx == nthreads - 1: + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + j = local_base + while lw != 0: + b = _cu_ctz64(lw) + tok = t_lo + b + pg = _ldg_i32(bt_row + cutlass.Int64(tok // tpb) * 4) + _stg_i32(slot_row + cutlass.Int64(j) * 4, pg * tpb + tok % tpb) + j = j + 1 + lw = lw & (lw - 1) + # walk 2 (quads re-read): a listed block's four bits, a local token's one bit + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + for i in cutlass.range(0, n_rounds, unroll=1): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tk = self._quad(sel_base, r0, rows, sel_stride, s_misc, item, ra, rb) + b0 = tk[0] + is_blk = (b0 >= 0) & (b0 < t_lo) & ((b0 & 3) == 0) + is_blk = is_blk & (tk[1] == b0 + 1) & (tk[2] == b0 + 2) & (tk[3] == b0 + 3) + row_pack = s_pack + t * (pw * 8) + if is_blk: + blk = b0 >> 2 + w = blk >> 6 + bit = cutlass.Int64(1) << cutlass.Int64(blk & 63) + uw = _ld_shared_b64(s_u + w * 8) + if (uw & bit) != 0: + k = _ld_shared_b32(s_pref + w * 4) + _cu_popc64(uw & (bit - 1)) + pos = seg_base + k * 4 + _red_or_shared32(row_pack + (pos >> 5) * 4, + cutlass.Int32(15) << (pos & 31)) # fmt: skip + else: + for k in cutlass.range_constexpr(4): + d = tk[k] - t_lo + if (tk[k] >= 0) & (d >= 0) & (d < 64): + bit = cutlass.Int64(1) << cutlass.Int64(d) + if (lw & bit) != 0: + jj = local_base + _cu_popc64(lw & (bit - 1)) + _red_or_shared32(row_pack + (jj >> 5) * 4, + cutlass.Int32(1) << (jj & 31)) # fmt: skip + cute.arch.sync_threads() + use = (nseg > 0) & (_ld_shared_b32(s_misc + _BP_FLAG * 4) == 0) + listed = cutlass.Int32(cutlass.select_(use, listed, cutlass.Int32(0))) + if tidx < rpt: + if r0 + tidx < rows: + _stg_i32(info_base + cutlass.Int64(r0 + tidx) * 8, + _ld_shared_b32(s_misc + (_BP_HI + tidx) * 4)) # fmt: skip + _stg_i32(info_base + cutlass.Int64(r0 + tidx) * 8 + 4, + _ld_shared_b32(s_misc + (_BP_COLS + tidx) * 4)) # fmt: skip + tiles = (listed + BLOCK_N - 1) // BLOCK_N + shared = use & (tiles * _SHARED_TILE_COST <= own_tiles * 4) + shared_i = cutlass.Int32(cutlass.select_(shared, cutlass.Int32(1), cutlass.Int32(0))) + if tidx == 0: + _stg_i32( + plan_base + cutlass.Int64(group) * 8, + cutlass.Int32(cutlass.select_(shared, tiles, cutlass.Int32(-1))), + ) + _stg_i32(plan_base + cutlass.Int64(group) * 8 + 4, listed) + if shared_i == 0: + lb = _atom_add_g32(lone_count_base, n_live) + _st_shared_b32(s_misc + _BP_LONE * 4, lb) + if shared_i == 0: + cute.arch.sync_threads() + if tidx < rpt: + if r0 + tidx < rows: + lb = _ld_shared_b32(s_misc + _BP_LONE * 4) + _stg_i32(lone_base + cutlass.Int64(lb + tidx) * 4, r0 + tidx) + if (shared_i != 0) & (listed > 0): + pw_used = (listed + 63) // 64 + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + t = i // pw_used + k = i % pw_used + if r0 + t < rows: + _stg_i64(packed_base + (cutlass.Int64(r0 + t) * pw + k) * 8, + _ld_shared_b64(s_pack + (t * pw + k) * 8)) # fmt: skip + + +_BLOCK_PLAN: dict = {} + + +def block_plan_serves(compress_ratio: int, tokens_per_block: int, rows_per_tile: int) -> bool: + """Whether `_block_plan` serves a launch: blocks of 4 tokens, pages of whole blocks, a + group's rows in one warp.""" + return compress_ratio == 4 and tokens_per_block % 4 == 0 and rows_per_tile <= 32 + + +def _block_plan(selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages): # fmt: skip + """Launch the block plan pass (one CTA a group); the fused plan's outputs.""" + rows, final_topk = selected_tokens.shape + block_topk = (final_topk - compress_ratio + 1) // compress_ratio + block_words = triton.cdiv(triton.cdiv(table_width * tokens_per_block, 4), 64) + req64 = request_indices.dtype == torch.int64 + qp64 = query_positions.dtype == torch.int64 + query_positions = query_positions.contiguous() + key = (final_topk, tokens_per_block, table_width, rows_per_tile, packed_words, req64, qp64) + gm = cute.AddressSpace.gmem + ptrs = ( + query_positions.data_ptr(), request_indices.data_ptr(), block_table.data_ptr(), + row_info.data_ptr(), slots.data_ptr(), packed_bits.data_ptr(), group_plan.data_ptr(), + lone_rows.data_ptr(), lone_count.data_ptr(), + ) # fmt: skip + scalars = (rows, num_groups, selected_tokens.stride(0), block_table.stride(0), slots.stride(0), + block_table.shape[0], num_cache_pages) # fmt: skip + fn = _BLOCK_PLAN.get(key) + if fn is None: + i32, i64 = cutlass.Int32, cutlass.Int64 + fn = cute.compile( + _BlockPlanKernel( + final_topk, + block_topk, + tokens_per_block, + table_width, + block_words, + rows_per_tile, + packed_words, + req64, + qp64, + ), # fmt: skip + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + make_ptr(i64 if qp64 else i32, ptrs[0], gm, assumed_align=4), + make_ptr(i64 if req64 else i32, ptrs[1], gm, assumed_align=4), + make_ptr(i32, ptrs[2], gm, assumed_align=4), + make_ptr(i32, ptrs[3], gm, assumed_align=8), + make_ptr(i32, ptrs[4], gm, assumed_align=16), + make_ptr(i64, ptrs[5], gm, assumed_align=8), + make_ptr(i32, ptrs[6], gm, assumed_align=8), + make_ptr(i32, ptrs[7], gm, assumed_align=4), + make_ptr(i32, ptrs[8], gm, assumed_align=4), + *(cutlass.Int32(x) for x in scalars), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _BLOCK_PLAN[key] = fn + fn(selected_tokens, *ptrs, *scalars) + + +@triton.jit(do_not_specialize=["rows", "NUM_CACHE_PAGES", "NUM_REQUESTS"]) +def _rowtile_lone_kernel( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, + lone_rows, lone_count, output, softmax_scale, rows, + q_stride_row, q_stride_head: tl.constexpr, q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, v_stride_dim: tl.constexpr, + block_table_stride_request: tl.constexpr, block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + NUM_CACHE_PAGES, NUM_REQUESTS, LONE_PROGRAMS, + PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_H: tl.constexpr, BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, USE_PDL: tl.constexpr, + LONE_STAGES: tl.constexpr, +): # fmt: skip + """Rows of groups the row-tile pass does not serve: each over its own columns. q's row stride + is a runtime argument, so q in place on the projection row and a compact row share a compile.""" + if USE_PDL: + # Independent of the row-tile pass (disjoint rows): let it start now. + tl.extra.cuda.gdc_launch_dependents() + kv_head = tl.program_id(1) + for slot in range(tl.program_id(0), tl.load(lone_count), LONE_PROGRAMS): + _rowtile_lone_row( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, output, + softmax_scale, tl.load(lone_rows + slot).to(tl.int64), kv_head, + q_stride_row, q_stride_head, q_stride_dim, + k_stride_page, k_stride_head, k_stride_token, k_stride_dim, + v_stride_page, v_stride_head, v_stride_token, v_stride_dim, + block_table_stride_request, block_table_stride_page, + selected_stride_row, selected_stride_token, + output_stride_row, output_stride_head, output_stride_dim, + NUM_CACHE_PAGES, NUM_REQUESTS, PAGE_TABLE_WIDTH, GROUP_SIZE, TOKENS_PER_BLOCK, + BLOCK_H, BLOCK_N, HEAD_DIM, LONE_STAGES, + ) # fmt: skip + + +@triton.jit +def _rowtile_lone_row( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, output, + softmax_scale, row, kv_head, + q_stride_row, q_stride_head, q_stride_dim, + k_stride_page, k_stride_head, k_stride_token, k_stride_dim, + v_stride_page, v_stride_head, v_stride_token, v_stride_dim, + block_table_stride_request, block_table_stride_page, + selected_stride_row, selected_stride_token, + output_stride_row, output_stride_head, output_stride_dim, + NUM_CACHE_PAGES, NUM_REQUESTS, PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, TOKENS_PER_BLOCK: tl.constexpr, BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, HEAD_DIM: tl.constexpr, LONE_STAGES: tl.constexpr, +): # fmt: skip + request = tl.load(request_indices + row).to(tl.int64) + valid_request = (request >= 0) & (request < NUM_REQUESTS) + safe_request = tl.minimum(tl.maximum(request, 0), NUM_REQUESTS - 1) + page_base = block_table + safe_request * block_table_stride_request + columns_read = tl.load(row_info + row * 2 + 1) + head_offsets = tl.arange(0, BLOCK_H) + dim_offsets = tl.arange(0, HEAD_DIM) + query_heads = kv_head * GROUP_SIZE + head_offsets + head_live = head_offsets < GROUP_SIZE + q_tile = tl.load( + q + row * q_stride_row + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=head_live[:, None], + other=0.0, + ) # fmt: skip + accumulator = tl.zeros([BLOCK_H, HEAD_DIM], tl.float32) + running_max = tl.full([BLOCK_H], -1.0e30, tl.float32) + running_sum = tl.zeros([BLOCK_H], tl.float32) + token_offsets = tl.arange(0, BLOCK_N) + for tile in tl.range(0, tl.cdiv(columns_read, BLOCK_N), num_stages=LONE_STAGES): + columns = tile * BLOCK_N + token_offsets + tokens = tl.load( + selected_tokens + row * selected_stride_row + columns * selected_stride_token, + mask=columns < columns_read, + other=-1, + ).to(tl.int32) + safe_tokens = tl.maximum(tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + usable = valid_request & (tokens >= 0) & (logical_pages < PAGE_TABLE_WIDTH) + physical_pages = tl.load( + page_base + tl.minimum(logical_pages, PAGE_TABLE_WIDTH - 1) * block_table_stride_page, + mask=usable, + other=-1, + ).to(tl.int64) + usable = usable & (physical_pages >= 0) & (physical_pages < NUM_CACHE_PAGES) + safe_pages = tl.maximum(physical_pages, 0) + keys = tl.load( + k_cache + safe_pages[None, :] * k_stride_page + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + dim_offsets[:, None] * k_stride_dim, + mask=usable[None, :], + other=0.0, + ).to(q_tile.dtype) # fmt: skip + values = tl.load( + v_cache + safe_pages[:, None] * v_stride_page + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + dim_offsets[None, :] * v_stride_dim, + mask=usable[:, None], + other=0.0, + ).to(q_tile.dtype) # fmt: skip + scores = tl.dot(q_tile, keys) * (softmax_scale * 1.4426950408889634) + scores = tl.where(usable[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where(usable[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0) + accumulator = tl.dot( + probabilities.to(q_tile.dtype), values, accumulator * correction[:, None] + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + normalized = tl.where( + running_sum[:, None] > 0, accumulator / tl.maximum(running_sum[:, None], 1.0e-20), 0.0 + ) + tl.store( + output + row * output_stride_row + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized.to(output.dtype.element_ty), + mask=head_live[:, None], + ) # fmt: skip + + +def _check_inputs(q, k_cache, v_cache, block_table, selected_tokens, request_indices, + tokens_per_block, softmax_scale, query_positions, compress_ratio): # fmt: skip + if q.ndim != 3: + raise ValueError("QSA queries must have shape [rows, query_heads, head_dim]") + if k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA K/V caches must have matching HND paged shapes") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA sparse GQA requires a two-dimensional int32 block table") + if selected_tokens.ndim != 2 or selected_tokens.dtype != torch.int32: + raise ValueError("QSA selected tokens must be a two-dimensional int32 tensor") + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or selected_tokens.shape[1] <= 0: + raise ValueError("QSA selected-token rows must match non-empty query rows") + if request_indices.shape != (rows,) or request_indices.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA request indices must have one integer entry per row") + if not request_indices.is_contiguous(): + raise ValueError("QSA request indices must be contiguous") + if tokens_per_block <= 0 or k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA cache token dimension must match positive tokens_per_block") + if k_cache.shape[3] != head_dim or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA query and K/V cache head geometry is incompatible") + if q.dtype != torch.bfloat16: + raise ValueError("QSA queries must use BF16") + if k_cache.dtype != torch.bfloat16: + raise ValueError("QSA K/V cache must use BF16") + if selected_tokens.stride(1) != 1: + raise ValueError("QSA selected tokens must have unit column stride") + num_kv_heads = k_cache.shape[1] + if num_kv_heads <= 0 or num_q_heads <= 0 or num_q_heads % num_kv_heads: + raise ValueError("QSA query heads must be divisible by non-zero KV heads") + if k_cache.shape[0] <= 0 or block_table.shape[0] <= 0 or block_table.shape[1] <= 0: + raise ValueError("QSA sparse GQA requires non-empty cache and block-table geometry") + if k_cache.shape[0] * tokens_per_block >= 2**31: + raise ValueError("QSA row-tile cache slots must fit int32") + if not math.isfinite(softmax_scale) or softmax_scale <= 0: + raise ValueError("QSA sparse GQA softmax scale must be finite and positive") + if query_positions is None or compress_ratio is None: + raise ValueError("the row-tile kernel needs the prefill pair (positions, compress ratio)") + if query_positions.shape != (rows,) or query_positions.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA query positions must be one integer entry per row") + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + token_topk = selected_tokens.shape[1] - compress_ratio + 1 + if token_topk <= 0 or token_topk % compress_ratio: + raise ValueError("QSA selected-token width is incompatible with its compression ratio") + devices = {t.device for t in (q, k_cache, v_cache, block_table, selected_tokens, + request_indices, query_positions)} # fmt: skip + if len(devices) != 1 or next(iter(devices)).type != "cuda": + raise ValueError("QSA row-tile sparse GQA needs every tensor on one CUDA device") + if num_q_heads // num_kv_heads > BLOCK_M: + raise ValueError("a row's query heads per KV head must fit the 128-row tile") + if not _cute_eligible(q, k_cache, v_cache): + raise ValueError( + "the row-tile pass needs SM103, head_dim 256, unit-stride heads (q's heads packed) " + "and 16-byte-aligned rows" + ) + + +def rows_per_group(group_size: int) -> int: + """Rows of one request a row-tile item serves: the most whose `rows x group_size` MMA + rows fill whole 8-row swizzle atoms of the 128-row tile (10 at 12 heads, 20 at 6).""" + rows = BLOCK_M // group_size + while rows > 1 and rows * group_size % _ROW_ATOM: + rows -= 1 + return rows + + +def plan_launch(block_table, selected_tokens, request_indices, query_positions, + tokens_per_block, compress_ratio, rows_per_tile, num_cache_pages): # fmt: skip + """The row-tile plan of a context launch: `(row_info, slots, packed_bits, group_plan, + lone_rows, counters)` (`counters[0]` the lone list's length, `counters[1]` the row-tile + pass's ticket, zeroed), from one launch: on the selection's blocks wherever + `block_plan_serves`, else on its tokens. Both give the same values.""" + rows = selected_tokens.shape[0] + device = selected_tokens.device + table_width = block_table.shape[1] + num_words = triton.cdiv(table_width * tokens_per_block, 64) + num_groups = triton.cdiv(rows, rows_per_tile) + # A group lists at most its rows' columns: the bound on its slots and re-indexed bits. + listed_cap = min(num_words * 64, rows_per_tile * selected_tokens.shape[1]) + packed_words = triton.cdiv(listed_cap, 64) + slots = torch.empty((num_groups, packed_words * 64), dtype=torch.int32, device=device) + packed_bits = torch.zeros((rows, packed_words), dtype=torch.int64, device=device) + group_plan = torch.empty((num_groups, 2), dtype=torch.int32, device=device) + lone_rows = torch.empty((rows,), dtype=torch.int32, device=device) + counters = torch.zeros((2,), dtype=torch.int32, device=device) + lone_count = counters[:1] + row_info = torch.empty((rows, 2), dtype=torch.int32, device=device) + if block_plan_serves(compress_ratio, tokens_per_block, rows_per_tile): + _block_plan( + selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages, + ) # fmt: skip + else: + _group_plan( + selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, num_words, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages, + ) # fmt: skip + return row_info, slots, packed_bits, group_plan, lone_rows, counters + + +def qsa_rowtile_sparse_gqa( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, +) -> torch.Tensor: + """Plan pass (`plan_launch`), lone-row pass, row-tile pass (`qsa_rowtile_attention_cute`); + returns `[rows, Hq, D]` contiguous.""" + _check_inputs(q, k_cache, v_cache, block_table, selected_tokens, request_indices, + tokens_per_block, softmax_scale, query_positions, compress_ratio) # fmt: skip + rows, num_q_heads, head_dim = q.shape + num_kv_heads = k_cache.shape[1] + group_size = num_q_heads // num_kv_heads + output = torch.empty((rows, num_q_heads, head_dim), dtype=q.dtype, device=q.device) + if rows == 0: + return output + table_width = block_table.shape[1] + rows_per_tile = rows_per_group(group_size) + kv_strides = ( + k_cache.stride(0), k_cache.stride(1), k_cache.stride(2), k_cache.stride(3), + v_cache.stride(0), v_cache.stride(1), v_cache.stride(2), v_cache.stride(3), + ) # fmt: skip + q_strides = (q.stride(0), q.stride(1), q.stride(2)) + common = (*q_strides, *kv_strides, block_table.stride(0), block_table.stride(1)) + geometry = dict( + NUM_CACHE_PAGES=k_cache.shape[0], NUM_REQUESTS=block_table.shape[0], + PAGE_TABLE_WIDTH=table_width, GROUP_SIZE=group_size, + TOKENS_PER_BLOCK=tokens_per_block, HEAD_DIM=head_dim, + ) # fmt: skip + out_strides = (output.stride(0), output.stride(1), output.stride(2)) + num_groups = triton.cdiv(rows, rows_per_tile) + row_info, slots, packed_bits, group_plan, lone_rows, counters = plan_launch( + block_table, selected_tokens, request_indices, query_positions, tokens_per_block, + compress_ratio, rows_per_tile, k_cache.shape[0], + ) # fmt: skip + lone_count = counters[:1] + use_pdl = get_env_enable_pdl() + _rowtile_lone_kernel[(min(rows, _LONE_PROGRAMS), num_kv_heads)]( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, + lone_rows, lone_count, output, softmax_scale, rows, + *common, selected_tokens.stride(0), selected_tokens.stride(1), *out_strides, + BLOCK_H=max(16, triton.next_power_of_2(group_size)), BLOCK_N=64, + LONE_PROGRAMS=min(rows, _LONE_PROGRAMS), USE_PDL=use_pdl, LONE_STAGES=_LONE_STAGES, + num_warps=_LONE_WARPS, **geometry, + ) # fmt: skip + _rowtile_attention_cute[(_rowtile_ctas(num_groups * num_kv_heads, q.device),)]( + q, k_cache, v_cache, slots, packed_bits, group_plan, output, counters[1:], softmax_scale, + rows, + group_size=group_size, rows_per_tile=rows_per_tile, tokens_per_block=tokens_per_block, + use_pdl=use_pdl, + ) # fmt: skip + return output + + +_CHECKPOINT = dict( + q_heads=24, + kv_heads=2, + head_dim=256, + tokens_per_block=32, + compress_ratio=4, + token_topk=2048, + cache_dtype=torch.bfloat16, + q_dtype=torch.bfloat16, + # q is the first heads of the packed [T, Hq + 2 Hkv, 256] QKV buffer. + q_layout="qkv_slice", + request_dtype=torch.int32, + # The engine's table width at max_seq_len 262144, -1 past each sequence. + pages=8196, +) + +#: The trunk: 24 role pages a slot, layer 11's K and V read, pages straddling 2^31 elements. +_TRUNK = dict(cache_layout="trunk", slot_pages=24, layer=11, offset_edge=2**31) +#: Attention DP with one-model MTP: the draft layer's planes (layer 12) join the trunk's slot. +_TRUNK_MTP_DP = dict(cache_layout="trunk", slot_pages=26, layer=12, offset_edge=2**31) +#: tp4 / tep4 rank: 6 q heads and one KV head. +_TP4 = dict(q_heads=6, kv_heads=1, slots=128, **_TRUNK) +#: tp2 / tep2 rank: 12 q heads and one KV head. +_TP2 = dict(q_heads=12, kv_heads=1, slots=128, **_TRUNK) +#: dep4 rank over the bf16 trunk, or over the kv_fp8 pages widened into a bf16 scratch. +_DEP4 = dict(q_heads=24, kv_heads=2, slots=32, **_TRUNK) +_DEP4_SCRATCH = dict(q_heads=24, kv_heads=2, slots=32, cache_layout="scratch") + + +def _spec(segments, member, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CHECKPOINT, **member, "segments": tuple(segments), **overrides} + + +class QsaSparseGqaRowtile(OpWrapper): + """Sparse GQA over the selected tokens of an HND paged K/V pool, for a + launch that carries the prefill pair (`query_positions`, `compress_ratio`). + + For row `r`, query head `h`, KV head `h // (Hq // Hkv)`, over the columns + `j < bound(r)` of `selected_tokens[r]` that name a resident token:: + + t = selected_tokens[r, j] # logical position + page = block_table[request_indices[r], t // tokens_per_block] + s_j = (q[r, h] . k_cache[page, kv, t % tpb]) * softmax_scale + out[r, h] = sum_j softmax(s)_j * v_cache[page, kv, t % tpb] + + `bound(r)` is the per-row kernel's prefill bound -- `min(visible // cr, + block_topk) * cr + visible % cr` columns with `visible = query_positions[r] + + 1`, rounded up to 64 -- so a row reads exactly the columns + `qsa_paged_sparse_gqa` reads on a context launch. Negative indices, pages + past the table, unmapped or out-of-pool pages and foreign requests + contribute nothing; a row left with nothing returns zeros. Returns a + fresh, contiguous `[rows, Hq, D]` tensor of `q.dtype`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="tp4/tep4 2049 rows from 9000: a last group of 9 rows, 6 q / 1 kv heads", + spec=_spec([(9000, 2049)], _TP4), + ), + Cell( + why="tp4/tep4 deep chunk: 8192 rows from 188416 ranking one score vector " + "(a sparse union), 6 q / 1 kv heads, trunk pool straddling 2^31", + spec=_spec([(188416, 8192)], _TP4, shared_scores=True), + ), + Cell( + why="dep4 kv_fp8 deep chunk: 8160 rows from 188416 over the widened bf16 " + "scratch, 24 q / 2 kv heads (10-row groups), [32, 8196] table", + spec=_spec([(188416, 8160)], _DEP4_SCRATCH, shared_scores=True), + ), + Cell( + why="dep4 core: three context requests (0+3000, 40000+4000, 120000+1160) in one " + "launch, 24 q / 2 kv heads, trunk pool straddling 2^31", + spec=_spec([(0, 3000), (40000, 4000), (120000, 1160)], _DEP4), + ), + Cell( + why="dep4 MTP draft layer: 2100 rows from 50000 over layer 12 of a 26-role-page " + "trunk (the draft layer in the trunk's manager), 24 q / 2 kv heads", + spec=_spec([(50000, 2100)], {**_DEP4, **_TRUNK_MTP_DP}, shared_scores=True), + ), + Cell( + why="tp2/tep2 deep chunk: 8192 rows from 120000, 12 q / 1 kv heads (10-row " + "groups), [128, 8196] table, trunk pool straddling 2^31", + spec=_spec([(120000, 8192)], _TP2, shared_scores=True), + ), + ) + + note = """ + Source: this target's plan (CuTe DSL `_block_plan` / `_group_plan`), lone-row (Triton) and + row-tile (`qsa_rowtile_attention_cute`, tcgen05) passes: SM103, bf16 q and K/V, D 256. It + takes `qsa_paged_sparse_gqa`'s arguments with the prefill pair required. + + Preconditions (not checkable without reading the contents): + - within the bound, a row names each token at most once: the row-tile pass attends to + the set of tokens a row names, the per-row kernel to every column. QSA's top-k and + expansion never repeat one. Rows served by the lone-row pass keep per-column semantics. + - as for the per-row kernel: each row is prefix-packed with exactly the bound's count of + valid entries; entries past the 64-rounded bound are ignored. + + Work grows with the union of a group's selections; below a few hundred rows the per-row + kernel is faster. Compiled once per (group size, rows per tile, tokens per block, PDL). + Calls allocate their plan scratch, so CUDA-graph capture is not certified (the prefill path + runs eager). Deterministic. + + Not certified: int64 request indices or positions; rows whose selection resolves no token + (all padding, a foreign request, unmapped pages); `rows == 0`. + """ + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, + ) -> torch.Tensor: + return qsa_rowtile_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + ) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, + ) -> torch.Tensor: + """`qsa_paged_sparse_gqa.reference`: fp32 sparse GQA over every resident selected token, + rounded once (the prefill pair is ignored: the bound reads exactly the valid entries).""" + return _per_row_entry.reference( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + ) + + +qsa_sparse_gqa_rowtile = QsaSparseGqaRowtile() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py new file mode 100644 index 000000000000..e85cf8190595 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py @@ -0,0 +1,1519 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA for decode rows over a bf16 or e4m3 paged pool: CuTe DSL split-K + merge. + +The split-K pipeline and merge of the family's `qsa_paged_sparse_gqa_nvfp4_decode`, reading K +and V as bf16 or exactly widened e4m3 -> f16 instead of NVFP4. + +One CTA (eight warps) serves one (row, KV head, split): it reads q before its PDL wait, +resolves the split's selected tokens to pool rows, streams 64-token K/V tiles through a +cp.async ring, and runs S = q K^T and O += P V on `mma.sync.m16n8k16` with an online +softmax in the log2 domain. The head dimension is permuted identically in q and K (lane +quad c of k-step pair m holds dims 32m + 8c .. 32m + 8c + 7). e4m3 V rows are widened into +an f16 tile over their own ring slot once every raw read of the slot is done, so three +slots fit two CTAs an SM. With one split the CTA stores (and gates) the output; otherwise +the merge kernel combines fp32 partials by their log2-sum-exp and applies the gate. +""" + +from __future__ import annotations + +import math +from typing import Optional + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( + qsa_paged_sparse_gqa as _composition, +) +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + +TILE = 64 +WARPS = 8 +THREADS = WARPS * 32 +HEAD_DIM = 256 +#: Bytes of a raw K row in shared memory, padded so a fragment read is bank-conflict free. +K_LD = {2: 576, 1: 288} +#: Bytes of a raw e4m3 V row (unpadded: the widening pass rotates its reads instead). +V8_LD = 256 +#: Bytes of a V row of the MMA tile (256 + 8 elements: conflict-free ldmatrix). +SV_LD = 528 +#: Bytes of a P row (64 + 8 elements). +SP_LD = 144 +#: Tiles in flight at most (cp.async ring). +STAGES = 3 +LOG2E = 1.4426950408889634 +#: Output elements a merge CTA covers per head. +CHUNK = 128 +MAX_SPLITS = 64 + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _unpack(res, n, ty, loc, ip): + return [ + ty(llvm.extractvalue(T.i32() if ty is cutlass.Int32 else T.f32(), res, [i], loc=loc, ip=ip)) + for i in range(n) + ] + + +@dsl_user_op +def _cp16z(saddr, gaddr, nbytes, *, loc=None, ip=None): + """cp.async of 16 bytes, zero-filled past `nbytes` (0 or 16): no branch for a dead token.""" + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + ], + "{ cp.async.cg.shared.global [$1], [$2], 16, $3; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts4(saddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)] + + [_ir(v, cutlass.Int32, loc, ip) for v in (a, b, c, d)], + "{ st.shared.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts1(saddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds1(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds4(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldg4(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldsm4(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldsm4t(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (f16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([T.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _ex2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "ex2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_to_f2(w, *, loc=None, ip=None): + """bf16x2 -> f16x2 (through f32, round to nearest).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; .reg .f32 fl, fh; mov.b32 {lo, hi}, $1; " + "cvt.f32.bf16 fl, lo; cvt.f32.bf16 fh, hi; cvt.rn.f16x2.f32 $0, fh, fl; }", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _f2_pack(lo, hi, *, loc=None, ip=None): + """Two f32 -> f16x2 (lo in the low half).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.f16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_pack(lo, hi, *, loc=None, ip=None): + """Two f32 -> bf16x2 (lo in the low half).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.bf16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg1(gaddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.global.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg2f(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Float32, loc, ip), + _ir(b, cutlass.Float32, loc, ip), + ], + "{ st.global.v2.f32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg1(gaddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_lo(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_hi(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _f32_bits(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(T.i32(), [_ir(x, cutlass.Float32, loc, ip)], "mov.b32 $0, $1;", "=r,f", loc, ip) + ) + + +@dsl_user_op +def _bits_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(T.f32(), [_ir(x, cutlass.Int32, loc, ip)], "mov.b32 $0, $1;", "=f,r", loc, ip) + ) + + +@dsl_user_op +def _round_bf16(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $1; cvt.f32.bf16 $0, h; }", + "=f,f", + loc, + ip, + ) + ) + + +def _sigmoid(x): + """1 / (1 + exp(-x)) as the bf16 merge computes it.""" + return 1.0 / (1.0 + cute.math.exp(-cutlass.Float32(x))) + + +@dsl_user_op +def _e4m3x4(word, *, loc=None, ip=None): + """Four e4m3 bytes -> two f16x2 (exact), the lower byte of each pair in the low half.""" + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(word, cutlass.Int32, loc, ip)], + "{ .reg .b16 l, h; mov.b32 {l, h}, $2; " + "cvt.rn.f16x2.e4m3x2 $0, l; cvt.rn.f16x2.e4m3x2 $1, h; }", + "=r,=r,r", + loc, + ip, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds2(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v2.b32 {$0, $1}, [$2];", + "=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma_bf16(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (bf16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([T.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +class _SplitKernel: + def __init__( + self, + *, + elem, + group, + kv_heads, + width, + tpb, + splits, + tps, + scale, + num_pages, + bt_width, + num_requests, + kv_strides, + q_strides, + gated, + pdl, + ): + assert elem in (1, 2) and group <= 16 and tpb & (tpb - 1) == 0 + self.elem = elem # bytes an element of the pool: 2 (bf16) or 1 (e4m3) + self.group, self.kvh, self.width, self.tpb = group, kv_heads, width, tpb + self.splits, self.tps = splits, tps + self.n_tiles = -(-width // TILE) + self.scale = scale * LOG2E + self.num_pages, self.bt_width, self.num_requests = num_pages, bt_width, num_requests + self.kvs, self.qs = kv_strides, q_strides + # Two CTAs an SM (at most 128 registers a thread). + self.gated, self.pdl, self.min_blocks = gated, pdl, 2 + self.heads = group * kv_heads + self.k_ld = K_LD[elem] + self.stages = max(1, min(STAGES, tps)) + # shared memory (bytes): token table | ring of tile slots | P | cross-warp exchange. + # A slot holds the tile's K and V rows: bf16 V straight in its MMA tile; e4m3 V raw, + # widened into an MMA tile over the whole slot once its raw K and V have been read. + self.o_tok = 0 + self.tok_bytes = -(-tps * TILE * 4 // 128) * 128 + self.raw_k = TILE * self.k_ld + self.raw_v = TILE * (V8_LD if elem == 1 else SV_LD) + self.raw_buf = self.raw_k + self.raw_v + if elem == 1: + self.raw_buf = max(self.raw_buf, TILE * SV_LD) + self.o_raw = self.o_tok + self.tok_bytes + self.o_sp = self.o_raw + self.stages * self.raw_buf + self.o_red = self.o_sp + 16 * SP_LD + self.smem = self.o_red + 2 * WARPS * 16 * 4 + 128 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + q_ptr: cute.Pointer, + k_ptr: cute.Pointer, + v_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + gate_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + sel = cute.make_tensor( + sel_t.iterator, cute.make_layout((rows, self.width), stride=(sel_stride, 1)) + ) + bt = cute.make_tensor( + bt_ptr, cute.make_layout((self.num_requests, self.bt_width), stride=(self.bt_width, 1)) + ) + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel( + sel, + q_ptr.toint(), + k_ptr.toint(), + v_ptr.toint(), + bt, + req, + out_ptr.toint(), + lse_ptr.toint(), + gate_ptr.toint(), + rows, + gate_stride, + gate_hs, + ).launch( + grid=[rows * self.kvh * self.splits, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=self.min_blocks, + ) + + @cute.jit + def _issue(self, j, buf, tidx, kvh, s_tok, s_raw, k_base, v_base): + """cp.async of tile `j`'s K and V rows into ring slot `buf`, committed as one group. + + The four threads of a token read 64 contiguous bytes an instruction; a token that does + not resolve gets zeros (src-size 0). A bf16 V row lands in its MMA tile swizzled: + 16-byte chunk 8 q + k at 8 q + (k ^ 2 q), the map `ldmatrix.trans` reads through.""" + tpb: cutlass.Constexpr = self.tpb + lt: cutlass.Constexpr = tpb.bit_length() - 1 + elem: cutlass.Constexpr = self.elem + k_ld: cutlass.Constexpr = self.k_ld + ks0, ks1, ks2 = self.kvs + rk = s_raw + buf * self.raw_buf + rv = rk + self.raw_k + tt = tidx // 4 + q4 = tidx % 4 + ent = _lds1(s_tok + (j * TILE + tt) * 4) + ok = ent >= 0 + e = cutlass.max(ent, 0) + pg = cutlass.Int64(e >> lt) + sl = e & (tpb - 1) + n = cutlass.Int32(cutlass.select_(ok, cutlass.Int32(16), cutlass.Int32(0))) + off = (pg * ks0 + cutlass.Int64(kvh * ks1 + sl * ks2)) * elem + if const_expr(elem == 1): + for i in cutlass.range_constexpr(4): + b = 64 * i + 16 * q4 + _cp16z(rk + tt * k_ld + b, k_base + off + b, n) + _cp16z(rv + tt * V8_LD + b, v_base + off + b, n) + else: + for i in cutlass.range_constexpr(8): + b = 64 * i + 16 * q4 + # chunk 4 i + q4 of the row, at its swizzled place + swz = 8 * (i // 2) + ((4 * (i % 2) + q4) ^ (2 * (i // 2))) + _cp16z(rk + tt * k_ld + b, k_base + off + b, n) + _cp16z(rv + tt * SV_LD + 16 * swz, v_base + off + b, n) + cute.arch.cp_async_commit_group() + + @cute.kernel + def kernel( + self, + mSel: cute.Tensor, + q_base: cutlass.Int64, + k_base: cutlass.Int64, + v_base: cutlass.Int64, + mBT: cute.Tensor, + mReq: cute.Tensor, + out_base: cutlass.Int64, + lse_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + g = lane // 4 + c = lane % 4 + elem: cutlass.Constexpr = self.elem + k_ld: cutlass.Constexpr = self.k_ld + group: cutlass.Constexpr = self.group + kvh_n: cutlass.Constexpr = self.kvh + splits: cutlass.Constexpr = self.splits + tps: cutlass.Constexpr = self.tps + n_tiles_all: cutlass.Constexpr = self.n_tiles + width: cutlass.Constexpr = self.width + tpb: cutlass.Constexpr = self.tpb + stages: cutlass.Constexpr = self.stages + row = bidx // (kvh_n * splits) + kvh = (bidx // splits) % kvh_n + split = bidx % splits + t0 = split * tps + n_t = cutlass.min(tps, n_tiles_all - t0) + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_tok = sbase + self.o_tok + s_raw = sbase + self.o_raw + s_p = sbase + self.o_sp + s_red = sbase + self.o_red + + if const_expr(self.pdl): + # The merge waits before it reads anything this grid writes. + cute.arch.griddepcontrol_launch_dependents() + + # ---- q of this KV head, before the wait: rows g and g + 8 of the m16 tile as + # permuted A fragments. Thread (g, c) holds dims 32m + 8c .. 32m + 8c + 7 of heads + # g and g + 8, m = 0..7; a head past the group holds zeros and stores nothing. + qa = cute.make_rmem_tensor(cute.make_layout((8, 4), stride=(4, 1)), cutlass.Int32) + qb = cute.make_rmem_tensor(cute.make_layout((8, 4), stride=(4, 1)), cutlass.Int32) + qs0, qs1 = self.qs + q_row = q_base + (cutlass.Int64(row) * qs0 + cutlass.Int64(kvh * group) * qs1) * 2 + for m in cutlass.range_constexpr(8): + off = cutlass.Int64((32 * m + 8 * c) * 2) + hac = cutlass.min(g, group - 1) + w4 = _ldg4(q_row + cutlass.Int64(hac) * (qs1 * 2) + off) + hb = g + 8 + hbc = cutlass.min(hb, group - 1) + w4b = _ldg4(q_row + cutlass.Int64(hbc) * (qs1 * 2) + off) + for e in cutlass.range_constexpr(4): + wa = w4[e] + wb = w4b[e] + if const_expr(elem == 1): + wa = _bf2_to_f2(wa) + wb = _bf2_to_f2(wb) + qa[m, e] = cutlass.Int32(cutlass.select_(g < group, wa, cutlass.Int32(0))) + qb[m, e] = cutlass.Int32(cutlass.select_(hb < group, wb, cutlass.Int32(0))) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + + # ---- the token table of this split: page * tpb + slot, or -1 --------------------- + rq = cutlass.Int64(mReq[row]) + req_ok = (rq >= 0) & (rq < self.num_requests) + safe_req = cutlass.Int32( + cutlass.min(cutlass.max(rq, cutlass.Int64(0)), cutlass.Int64(self.num_requests - 1)) + ) + for i in cutlass.range_constexpr(-(-(tps * TILE) // THREADS)): + cl = tidx + i * THREADS + col = t0 * TILE + cl + colc = cutlass.min(col, width - 1) + tok = mSel[row, colc] + ok = req_ok & (col < width) & (tok >= 0) + safe_tok = cutlass.max(tok, 0) + lp = safe_tok // tpb + ok = ok & (lp < self.bt_width) + page = mBT[safe_req, cutlass.min(lp, self.bt_width - 1)] + ok = ok & (page >= 0) & (page < self.num_pages) + ent = cutlass.Int32(cutlass.select_(ok, page * tpb + safe_tok % tpb, cutlass.Int32(-1))) + if cl < tps * TILE: + _sts1(s_tok + cl * 4, ent) + cute.arch.sync_threads() + + # Tiles 0 .. stages - 2 in flight (tile 0 alone when one tile is the whole split). + for s in cutlass.range_constexpr(max(stages - 1, 1)): + if s < n_t: + self._issue(s, s, tidx, kvh, s_tok, s_raw, k_base, v_base) + + neg_inf = cutlass.Float32(-math.inf) + st = cute.make_rmem_tensor(cute.make_layout((4,), stride=(1,)), cutlass.Float32) + st[0] = neg_inf # running max, head g + st[1] = neg_inf # running max, head g + 8 + st[2] = cutlass.Float32(0.0) # this thread's share of the running sum, head g + st[3] = cutlass.Float32(0.0) # ... head g + 8 (reduced over the CTA at the end) + acc = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Float32) + acc.fill(0.0) + score_scale = cutlass.Float32(self.scale) + + # loop-invariant per-thread offsets + vt = tidx // 4 + vq = tidx % 4 + v_dst = vt * SV_LD + 128 * vq + ktok = warp * 8 + g + k_src = ktok * k_ld + (8 * c if elem == 1 else 16 * c) + ta = warp * 8 + 2 * c + p_st_g = s_p + g * SP_LD + ta * 2 + p_st_h = p_st_g + 8 * SP_LD + p_ld = s_p + ((lane % 8) + 8 * ((lane // 8) % 2)) * SP_LD + 16 * (lane // 16) + v_ld_row = ((lane % 8) + 8 * ((lane // 8) % 2)) * SV_LD + ci0 = 4 * warp + lane // 16 + ci1 = ci0 + 2 + v_ld0 = v_ld_row + 16 * ((ci0 & ~7) | ((ci0 & 7) ^ (2 * ((ci0 >> 3) & 3)))) + v_ld1 = v_ld_row + 16 * ((ci1 & ~7) | ((ci1 & 7) ^ (2 * ((ci1 >> 3) & 3)))) + red_g = s_red + g * WARPS * 4 + red_h = s_red + (g + 8) * WARPS * 4 + + for j in cutlass.range(n_t): + buf = j % stages + if const_expr(stages > 2): + if j + 1 < n_t: + cute.arch.cp_async_wait_group(stages - 2) + else: + cute.arch.cp_async_wait_group(0) + else: + cute.arch.cp_async_wait_group(0) + # [A] tile j landed; every warp is past tile j - 1, so its ring slot is free + cute.arch.sync_threads() + if const_expr(stages > 1): + if j + stages - 1 < n_t: + self._issue( + j + stages - 1, + (j + stages - 1) % stages, + tidx, + kvh, + s_tok, + s_raw, + k_base, + v_base, + ) + rk = s_raw + buf * self.raw_buf + rv = rk + self.raw_k + # P V reads V from its MMA tile: where bf16 rows landed, or the e4m3 tile widened + # over the slot. + sv = rk if elem == 1 else rv + + # -- S = q K^T for this warp's eight tokens (two accumulator chains) + zero = cutlass.Float32(0.0) + sa = [zero, zero, zero, zero] + sb = [zero, zero, zero, zero] + kb = [] + for m in cutlass.range_constexpr(8): + if const_expr(elem == 1): + w2 = _lds2(rk + k_src + 32 * m) + lo0, hi0 = _e4m3x4(w2[0]) + lo1, hi1 = _e4m3x4(w2[1]) + kb.append([lo0, hi0, lo1, hi1]) + else: + kb.append(_lds4(rk + k_src + 64 * m)) + for m in cutlass.range_constexpr(8): + b = kb[m] + a_lo = (qa[m, 0], qb[m, 0], qa[m, 1], qb[m, 1]) + a_hi = (qa[m, 2], qb[m, 2], qa[m, 3], qb[m, 3]) + if const_expr(m % 2 == 0): + sa = self._mma(a_lo, (b[0], b[1]), sa) + sa = self._mma(a_hi, (b[2], b[3]), sa) + else: + sb = self._mma(a_lo, (b[0], b[1]), sb) + sb = self._mma(a_hi, (b[2], b[3]), sb) + + # mask + scale; tokens 8w + 2c, 8w + 2c + 1 + ea = _lds1(s_tok + (j * TILE + ta) * 4) + eb = _lds1(s_tok + (j * TILE + ta + 1) * 4) + s0 = cutlass.Float32(cutlass.select_(ea >= 0, (sa[0] + sb[0]) * score_scale, neg_inf)) + s1 = cutlass.Float32(cutlass.select_(eb >= 0, (sa[1] + sb[1]) * score_scale, neg_inf)) + s2 = cutlass.Float32(cutlass.select_(ea >= 0, (sa[2] + sb[2]) * score_scale, neg_inf)) + s3 = cutlass.Float32(cutlass.select_(eb >= 0, (sa[3] + sb[3]) * score_scale, neg_inf)) + mg = cute.arch.fmax(s0, s1) + mh = cute.arch.fmax(s2, s3) + mg = cute.arch.fmax(mg, cute.arch.shuffle_sync_bfly(mg, offset=1)) + mh = cute.arch.fmax(mh, cute.arch.shuffle_sync_bfly(mh, offset=1)) + mg = cute.arch.fmax(mg, cute.arch.shuffle_sync_bfly(mg, offset=2)) + mh = cute.arch.fmax(mh, cute.arch.shuffle_sync_bfly(mh, offset=2)) + if c == 0: + _sts1(red_g + warp * 4, _f32_bits(mg)) + _sts1(red_h + warp * 4, _f32_bits(mh)) + cute.arch.sync_threads() # [B] maxes; every warp is past its K reads + vw = [] + if const_expr(elem == 1): + # The raw e4m3 V rows into registers. Thread: token tidx // 4, dims 64q .. + # 64q + 63 (q = tidx % 4), its four 16-byte reads rotated by q // 2 and by the + # token's parity, so a quarter-warp's reads hit eight banks. + for i in cutlass.range_constexpr(4): + ii = (i + vq // 2 + 2 * (vt % 2)) % 4 + vw.append((ii, _lds4(rv + vt * V8_LD + 64 * vq + 16 * ii))) + rg0 = _lds4(red_g) + rg1 = _lds4(red_g + 16) + rh0 = _lds4(red_h) + rh1 = _lds4(red_h + 16) + tmg = st[0] + tmh = st[1] + for e in cutlass.range_constexpr(4): + tmg = cute.arch.fmax(tmg, cute.arch.fmax(_bits_f32(rg0[e]), _bits_f32(rg1[e]))) + tmh = cute.arch.fmax(tmh, cute.arch.fmax(_bits_f32(rh0[e]), _bits_f32(rh1[e]))) + safe_g = cutlass.Float32(cutlass.select_(tmg == neg_inf, cutlass.Float32(0.0), tmg)) + safe_h = cutlass.Float32(cutlass.select_(tmh == neg_inf, cutlass.Float32(0.0), tmh)) + corr_g = _ex2(st[0] - safe_g) + corr_h = _ex2(st[1] - safe_h) + p0 = _ex2(s0 - safe_g) + p1 = _ex2(s1 - safe_g) + p2 = _ex2(s2 - safe_h) + p3 = _ex2(s3 - safe_h) + if const_expr(elem == 1): + _sts1(p_st_g, _f2_pack(p0, p1)) + _sts1(p_st_h, _f2_pack(p2, p3)) + else: + _sts1(p_st_g, _bf2_pack(p0, p1)) + _sts1(p_st_h, _bf2_pack(p2, p3)) + st[0] = tmg + st[1] = tmh + st[2] = st[2] * corr_g + (p0 + p1) + st[3] = st[3] * corr_h + (p2 + p3) + cute.arch.sync_threads() # [C] P; every raw read of this slot is done + if const_expr(elem == 1): + # -- the e4m3 V tile widened (exact) into the f16 MMA tile over the slot + for ii, w in vw: + a0, a1 = _e4m3x4(w[0]) + a2, a3 = _e4m3x4(w[1]) + b0, b1 = _e4m3x4(w[2]) + b2, b3 = _e4m3x4(w[3]) + _sts4(rk + v_dst + 16 * ((2 * ii) ^ (2 * vq)), a0, a1, a2, a3) + _sts4(rk + v_dst + 16 * ((2 * ii + 1) ^ (2 * vq)), b0, b1, b2, b3) + cute.arch.sync_threads() # [D] the V tile + + # -- O += P V: this warp's dims 32w .. 32w + 31 (n-tiles of 8) + if cute.arch.vote_any_sync((corr_g != 1.0) | (corr_h != 1.0)): + for nt in cutlass.range_constexpr(4): + acc[nt, 0] = acc[nt, 0] * corr_g + acc[nt, 1] = acc[nt, 1] * corr_g + acc[nt, 2] = acc[nt, 2] * corr_h + acc[nt, 3] = acc[nt, 3] * corr_h + for kk in cutlass.range_constexpr(4): + a = _ldsm4(p_ld + 32 * kk) + b0 = _ldsm4t(sv + v_ld0 + 16 * kk * SV_LD) + b1 = _ldsm4t(sv + v_ld1 + 16 * kk * SV_LD) + r0 = self._mma(a, (b0[0], b0[1]), (acc[0, 0], acc[0, 1], acc[0, 2], acc[0, 3])) + r1 = self._mma(a, (b0[2], b0[3]), (acc[1, 0], acc[1, 1], acc[1, 2], acc[1, 3])) + r2 = self._mma(a, (b1[0], b1[1]), (acc[2, 0], acc[2, 1], acc[2, 2], acc[2, 3])) + r3 = self._mma(a, (b1[2], b1[3]), (acc[3, 0], acc[3, 1], acc[3, 2], acc[3, 3])) + for e in cutlass.range_constexpr(4): + acc[0, e] = r0[e] + acc[1, e] = r1[e] + acc[2, e] = r2[e] + acc[3, e] = r3[e] + + # ---- the running sums over the quad and the warps ----------------------------------- + lsg = st[2] + lsh = st[3] + lsg = lsg + cute.arch.shuffle_sync_bfly(lsg, offset=1) + lsh = lsh + cute.arch.shuffle_sync_bfly(lsh, offset=1) + lsg = lsg + cute.arch.shuffle_sync_bfly(lsg, offset=2) + lsh = lsh + cute.arch.shuffle_sync_bfly(lsh, offset=2) + cute.arch.sync_threads() + if c == 0: + _sts1(red_g + warp * 4, _f32_bits(lsg)) + _sts1(red_h + warp * 4, _f32_bits(lsh)) + cute.arch.sync_threads() + ug0 = _lds4(red_g) + ug1 = _lds4(red_g + 16) + uh0 = _lds4(red_h) + uh1 = _lds4(red_h + 16) + tsg = cutlass.Float32(0.0) + tsh = cutlass.Float32(0.0) + for e in cutlass.range_constexpr(4): + tsg = tsg + _bits_f32(ug0[e]) + tsh = tsh + _bits_f32(uh0[e]) + for e in cutlass.range_constexpr(4): + tsg = tsg + _bits_f32(ug1[e]) + tsh = tsh + _bits_f32(uh1[e]) + self._epilogue( + acc, + st, + tsg, + tsh, + row, + kvh, + split, + warp, + g, + c, + rows, + out_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + ) + + def _mma(self, a, b, c): + """The tile MMA in the pool's type: f16 over an e4m3 pool, bf16 over a bf16 one.""" + return _mma(a, b, c) if self.elem == 1 else _mma_bf16(a, b, c) + + @cute.jit + def _epilogue( + self, + acc, + st, + lg, + lh, + row, + kvh, + split, + warp, + g, + c, + rows, + out_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + ): + """Heads g and g + 8's normalized outputs: the bf16 result (gated when the gate rides + this launch) with one split, else the fp32 partial and its log2-sum-exp.""" + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + splits: cutlass.Constexpr = self.splits + neg_inf = cutlass.Float32(-math.inf) + ig = cutlass.Float32(0.0) + ih = cutlass.Float32(0.0) + if lg > 0.0: + ig = 1.0 / lg + if lh > 0.0: + ih = 1.0 / lh + hg = kvh * group + g + hh = kvh * group + g + 8 + okg = g < group + okh = (g + 8) < group + for nt in cutlass.range_constexpr(4): + d = 32 * warp + 8 * nt + 2 * c + og0 = acc[nt, 0] * ig + og1 = acc[nt, 1] * ig + oh0 = acc[nt, 2] * ih + oh1 = acc[nt, 3] * ih + if const_expr(splits == 1): + ag = out_base + (cutlass.Int64(row) * heads + hg) * (HEAD_DIM * 2) + d * 2 + ah = out_base + (cutlass.Int64(row) * heads + hh) * (HEAD_DIM * 2) + d * 2 + if okg: + if const_expr(self.gated): + gw = _ldg1( + gate_base + (cutlass.Int64(row) * gate_stride + hg * gate_hs + d) * 2 + ) + og0 = _round_bf16(og0) * _sigmoid(_bf2_lo(gw)) + og1 = _round_bf16(og1) * _sigmoid(_bf2_hi(gw)) + _stg1(ag, _bf2_pack(og0, og1)) + if okh: + if const_expr(self.gated): + gw2 = _ldg1( + gate_base + (cutlass.Int64(row) * gate_stride + hh * gate_hs + d) * 2 + ) + oh0 = _round_bf16(oh0) * _sigmoid(_bf2_lo(gw2)) + oh1 = _round_bf16(oh1) * _sigmoid(_bf2_hi(gw2)) + _stg1(ah, _bf2_pack(oh0, oh1)) + else: + pg_ = ( + out_base + + ((cutlass.Int64(split) * rows + row) * heads + hg) * (HEAD_DIM * 4) + + d * 4 + ) + ph_ = ( + out_base + + ((cutlass.Int64(split) * rows + row) * heads + hh) * (HEAD_DIM * 4) + + d * 4 + ) + if okg: + _stg2f(pg_, og0, og1) + if okh: + _stg2f(ph_, oh0, oh1) + if const_expr(splits > 1): + if warp == 0: + if c == 0: + if okg: + lse_g = neg_inf + if lg > 0.0: + lse_g = st[0] + cute.math.log2(lg) + _stg1( + lse_base + ((cutlass.Int64(split) * rows + row) * heads + hg) * 4, + _f32_bits(lse_g), + ) + if okh: + lse_h = neg_inf + if lh > 0.0: + lse_h = st[1] + cute.math.log2(lh) + _stg1( + lse_base + ((cutlass.Int64(split) * rows + row) * heads + hh) * 4, + _f32_bits(lse_h), + ) + + +# --------------------------------------------------------------------------------------- +# merge kernel (splits > 1) +# --------------------------------------------------------------------------------------- + + +class _MergeKernel: + def __init__(self, *, heads, splits, gated, pdl): + assert HEAD_DIM % CHUNK == 0 and 1 < splits <= MAX_SPLITS + self.heads, self.splits, self.gated, self.pdl = heads, splits, gated, pdl + self.chunks = HEAD_DIM // CHUNK + # at least four warps: the final pass needs a thread per element of the chunk + self.warps = max(4, min(8, splits)) + self.spw = -(-splits // self.warps) + + @cute.jit + def __call__( + self, + gate_t: cute.Tensor, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + out_ptr: cute.Pointer, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + lse = cute.make_tensor( + lse_ptr, + cute.make_layout( + (self.splits, rows, self.heads), stride=(rows * self.heads, self.heads, 1) + ), + ) + out = cute.make_tensor( + out_ptr, + cute.make_layout( + (rows, self.heads, HEAD_DIM), stride=(self.heads * HEAD_DIM, HEAD_DIM, 1) + ), + ) + self.kernel( + gate_t.iterator.toint(), part_ptr.toint(), lse, out, rows, gate_stride, gate_hs + ).launch( + grid=[rows * self.heads * self.chunks, 1, 1], + block=[self.warps * 32, 1, 1], + smem=self.warps * CHUNK * 4 + 64, + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + gate_base: cutlass.Int64, + part_base: cutlass.Int64, + mLse, + mOut, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + per_row: cutlass.Constexpr = self.heads * self.chunks + spw: cutlass.Constexpr = self.spw + row = bidx // per_row + head = (bidx % per_row) // self.chunks + c0 = (bidx % self.chunks) * CHUNK + red = cutlass.utils.SmemAllocator().allocate_tensor( + cutlass.Float32, + cute.make_layout((self.warps, CHUNK), stride=(CHUNK, 1)), + byte_alignment=16, + ) + gval = cutlass.Float32(0.0) + if const_expr(self.gated): + if tidx < CHUNK: + gw = _ldg1( + gate_base + + (cutlass.Int64(row) * gate_stride + head * gate_hs + c0 + (tidx & ~1)) * 2 + ) + gval = cutlass.Float32(cutlass.select_((tidx & 1) == 0, _bf2_lo(gw), _bf2_hi(gw))) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + neg_inf = cutlass.Float32(-math.inf) + l0 = neg_inf + l1 = neg_inf + if lane < self.splits: + l0 = mLse[lane, row, head] + if lane + 32 < self.splits: + l1 = mLse[lane + 32, row, head] + p = cute.make_rmem_tensor(cute.make_layout((spw, 4), stride=(4, 1)), cutlass.Float32) + for jj in cutlass.range_constexpr(spw): + sidx = cutlass.min(warp_i * spw + jj, self.splits - 1) + w4 = _ldg4( + part_base + + ((cutlass.Int64(sidx) * rows + row) * self.heads + head) * (HEAD_DIM * 4) + + (c0 + 4 * lane) * 4 + ) + for e in cutlass.range_constexpr(4): + p[jj, e] = _bits_f32(w4[e]) + mx = cute.arch.warp_reduction_max(cute.arch.fmax(l0, l1)) + w0 = cutlass.Float32(0.0) + w1 = cutlass.Float32(0.0) + if mx > neg_inf: + w0 = cute.math.exp2(l0 - mx) + w1 = cute.math.exp2(l1 - mx) + den = cute.arch.warp_reduction_sum(w0 + w1) + acc = cute.make_rmem_tensor(cute.make_layout((4,), stride=(1,)), cutlass.Float32) + acc.fill(0.0) + for jj in cutlass.range_constexpr(spw): + sidx = warp_i * spw + jj + wa = cute.arch.shuffle_sync(w0, sidx % 32) + wb = cute.arch.shuffle_sync(w1, sidx % 32) + w = cutlass.Float32(cutlass.select_(sidx < 32, wa, wb)) + w = cutlass.Float32(cutlass.select_(sidx < self.splits, w, cutlass.Float32(0.0))) + for e in cutlass.range_constexpr(4): + acc[e] = w * p[jj, e] + acc[e] + for e in cutlass.range_constexpr(4): + red[warp_i, 4 * lane + e] = acc[e] + cute.arch.sync_threads() + if tidx < CHUNK: + tot = cutlass.Float32(0.0) + for w_ in cutlass.range_constexpr(self.warps): + tot = tot + red[w_, tidx] + merged = cutlass.Float32(0.0) + if den > 0.0: + merged = tot / den + if const_expr(self.gated): + att = merged.to(cutlass.BFloat16).to(cutlass.Float32) + merged = att * (1.0 / (1.0 + cute.math.exp(-gval))) + mOut[row, head, c0 + tidx] = merged.to(cutlass.BFloat16) + + +# --------------------------------------------------------------------------------------- +# launch +# --------------------------------------------------------------------------------------- + +_COMPILED: dict = {} + +#: Splits by (rows x KV heads), per pool element size, measured at the routed geometries +#: (`MAX_ROWS`): one tile a CTA while the grid is under about a wave, then fewer, longer splits; +#: one split past the last bound. +_SPLIT_TABLE = { + 2: ((8, 33), (16, 9)), + 1: ((4, 33), (8, 11), (32, 9), (64, 4), (256, 2)), +} +#: Past the table's last bound at one KV head over e4m3 (a tp2 / tep2 rank): two or three +#: shorter splits a row keep more latency-bound tile chains in flight than one split. +_WIDE_SPLITS_ONE_KV = {1: ((320, 2), (1024, 3))} +#: At one KV head over e4m3, from 65 pairs these replace the table's 2: split counts whose CTAs +#: (rows x splits, two an SM) fill whole waves. +_MID_SPLITS_ONE_KV = {1: ((96, 6), (160, 2), (256, 3))} + +_POOL_ELEM = {torch.bfloat16: 2, torch.float8_e4m3fn: 1} + + +def plan_splits(rows: int, kv_heads: int, width: int, elem: int) -> tuple[int, int]: + """(splits, tiles per split) for this many (row, KV head) pairs over a pool of `elem`-byte + elements.""" + n_tiles = -(-width // TILE) + pairs = rows * kv_heads + want = 1 + table = _SPLIT_TABLE[elem] + if kv_heads == 1: + mid = _MID_SPLITS_ONE_KV.get(elem, ()) + lead = tuple(e for e in table if not mid or e[0] <= 64) + rest = tuple(e for e in table if mid and e[0] > 256) + table = lead + mid + rest + _WIDE_SPLITS_ONE_KV.get(elem, ()) + for bound, sp in table: + if pairs <= bound: + want = sp + break + want = max(1, min(n_tiles, want, MAX_SPLITS)) + tps = -(-n_tiles // want) + return -(-n_tiles // tps), tps + + +def _ptr(t: torch.Tensor, dtype, align=16): + return make_ptr(dtype, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + +def sparse_gqa_decode( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """`QsaSparseGqaSplitkCute.__call__`: split pass, then the merge when there are splits.""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads, tpb, dim = k_cache.shape + elem = _POOL_ELEM[k_cache.dtype] + assert head_dim == HEAD_DIM and dim == HEAD_DIM and tpb == tokens_per_block + assert q.dtype == torch.bfloat16 and q.stride(2) == 1 + assert v_cache.dtype == k_cache.dtype and v_cache.shape == k_cache.shape + assert k_cache.stride() == v_cache.stride() and k_cache.stride(3) == 1 + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1], "row-contiguous block table" + assert selected_tokens.dtype == torch.int32 and selected_tokens.stride(1) == 1 + assert request_indices.is_contiguous() and request_indices.dtype in (torch.int32, torch.int64) + group = num_q_heads // num_kv_heads + width = selected_tokens.shape[1] + out = torch.empty((rows, num_q_heads, head_dim), dtype=torch.bfloat16, device=q.device) + if rows == 0: + return out + n_splits, tps = plan_splits(rows, num_kv_heads, width, elem) + use_pdl = bool(get_env_enable_pdl()) + gated = output_gate is not None + gate_hs = head_dim + if gated: + assert output_gate.dtype == torch.bfloat16 and output_gate.stride(-1) == 1 + if output_gate.dim() == 3: + assert output_gate.shape == q.shape + gate_hs = output_gate.stride(1) + else: + assert output_gate.shape == (rows, num_q_heads * head_dim) + req64 = request_indices.dtype == torch.int64 + key = ( + elem, + group, + num_kv_heads, + width, + tpb, + n_splits, + tps, + float(softmax_scale), + num_pages, + tuple(block_table.shape), + tuple(k_cache.stride()[:3]), + tuple(q.stride()[:2]), + req64, + gated and n_splits == 1, + use_pdl, + ) + if n_splits > 1: + part = torch.empty( + (n_splits, rows, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_splits, rows, num_q_heads), dtype=torch.float32, device=q.device) + o_t, l_t = part, lse + else: + o_t, l_t = out, out + gate_t = output_gate if gated else q + gate_stride = output_gate.stride(0) if gated else 0 + fn = _COMPILED.get(key) + if fn is None: + kern = _SplitKernel( + elem=elem, + group=group, + kv_heads=num_kv_heads, + width=width, + tpb=tpb, + splits=n_splits, + tps=tps, + scale=float(softmax_scale), + num_pages=num_pages, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + kv_strides=tuple(k_cache.stride()[:3]), + q_strides=tuple(q.stride()[:2]), + gated=gated and n_splits == 1, + pdl=use_pdl, + ) + fn = cute.compile( + kern, + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(q, cutlass.BFloat16), + _ptr(k_cache, cutlass.Uint8), + _ptr(v_cache, cutlass.Uint8), + _ptr(block_table, cutlass.Int32, 4), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(o_t, cutlass.Float32), + _ptr(l_t, cutlass.Float32), + _ptr(gate_t, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + fn( + selected_tokens, + q.data_ptr(), + k_cache.data_ptr(), + v_cache.data_ptr(), + block_table.data_ptr(), + request_indices.data_ptr(), + o_t.data_ptr(), + l_t.data_ptr(), + gate_t.data_ptr(), + rows, + selected_tokens.stride(0), + gate_stride, + gate_hs, + ) + if n_splits == 1: + return out + mkey = ("merge", num_q_heads, n_splits, gated, gate_t.dim(), use_pdl) + mfn = _COMPILED.get(mkey) + if mfn is None: + mfn = cute.compile( + _MergeKernel(heads=num_q_heads, splits=n_splits, gated=gated, pdl=use_pdl), + from_dlpack(gate_t, assumed_align=16).mark_layout_dynamic(leading_dim=gate_t.dim() - 1), + _ptr(part, cutlass.Float32), + _ptr(lse, cutlass.Float32, 4), + _ptr(out, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[mkey] = mfn + mfn(gate_t, part.data_ptr(), lse.data_ptr(), out.data_ptr(), rows, gate_stride, gate_hs) + return out + + +def serves(q: torch.Tensor, k_cache: torch.Tensor, selected_tokens: torch.Tensor) -> bool: + """Whether the target sends this decode call (no prefill pair) here: a certified head + geometry and pool dtype, at most `MAX_ROWS` rows of it.""" + if q.dim() != 3 or k_cache.dim() != 4 or q.dtype != torch.bfloat16: + return False + rows, num_q_heads, head_dim = q.shape + cap = MAX_ROWS.get((num_q_heads, k_cache.shape[1], k_cache.dtype), 0) + return ( + 0 < rows <= cap + and head_dim == HEAD_DIM + and k_cache.shape[3] == HEAD_DIM + and selected_tokens.shape[1] > TILE + ) + + +#: Rows the target sends here per (q heads, KV heads, pool dtype), up to the row counts measured +#: faster than the composition: a tp4 / tep4 rank over bf16, a dep4 rank and a tp2 / tep2 rank +#: under kv_fp8. Anything else keeps `qsa_paged_sparse_gqa` (24 / 2 over bf16 measured slower: +#: a bf16 tile slot holds 70 KB, one CTA an SM). +MAX_ROWS = { + (6, 1, torch.bfloat16): 16, + (24, 2, torch.float8_e4m3fn): 128, + (12, 1, torch.float8_e4m3fn): 512, +} + +#: QSA geometry: 32-token pages, 2048 + 4 - 1 selected tokens a row, a `max_seq_len` 262144 table. +_QSA = dict(tokens_per_block=32, compress_ratio=4, token_topk=2048, head_dim=256, pages=8196) +#: The engine's trunk: a slot holds every attention layer's K and V role pages (two more under +#: attention DP with the draft layer); the calls read layer 11's. +_TEP4 = dict(_QSA, q_heads=6, kv_heads=1, slots=4, slot_pages=24, layer=11, kv=torch.bfloat16) +_DEP4 = dict( + _QSA, q_heads=24, kv_heads=2, slots=128, slot_pages=26, layer=11, kv=torch.float8_e4m3fn +) +_TEP2 = dict( + _QSA, q_heads=12, kv_heads=1, slots=128, slot_pages=24, layer=11, kv=torch.float8_e4m3fn +) +#: The element offset the cells' pools straddle in the layer's K and V planes. +_OFFSET_EDGE = 2**31 + + +def _verify(member: dict, requests: int, rows: int = 4, last=196607, step=4099, **kw) -> dict: + """A verify step of `requests` requests of `rows` rows each, deep in a long sequence: + request r's rows end at `last - step * r`.""" + segments = tuple((last + 1 - rows - step * r, rows) for r in range(requests)) + return dict(member, segments=segments, **kw) + + +#: The trunk's decode site: the gate is the fused input projection row's gate columns. +_TEP4_ROW = dict(gate="row", row_heads=17) +_DEP4_ROW = dict(gate="row", row_heads=55) +_TEP2_ROW = dict(gate="row", row_heads=29) +#: The draft layer's site: the gate half of its interleaved `[T, Hq, 2 x 256]` q projection. +_DRAFT = dict(gate="interleaved") + + +class QsaSparseGqaSplitkCute(OpWrapper): + """`qsa_paged_sparse_gqa` for decode rows (no prefill pair) over a bf16 or e4m3 paged + pool: sparse GQA over each row's explicitly selected tokens, `[rows, Hq, D]` bf16. With + `output_gate` (`[rows, Hq * D]`, or `[rows, Hq, D]` with any head stride; bf16, unit inner + stride) the result is `bf16(attn) * sigmoid(gate)` rounded once more to bf16, what + `fused_sigmoid_mul_inplace` makes of the ungated call's output. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ll_tep4 MTP3 verify: 1 request x 4 rows at position 196607, 6 q / 1 KV head " + "over the bf16 trunk, 33 splits + gated merge, captured", + spec=_verify(_TEP4, 1, splits=33, capture=True, **_TEP4_ROW), + ), + Cell( + why="tep4 largest captured batch: 4 requests x 4 rows = 16, 6 / 1 heads, bf16, " + "gated, captured", + spec=_verify(_TEP4, 4, capture=True, **_TEP4_ROW), + ), + Cell( + why="tep4 mixed step's generation rows: 3 rows off a wider selection (unaligned " + "slice), short sequences, holes, one foreign row, ungated", + spec=dict( + _TEP4, + segments=((77, 1), (2047, 1), (9000, 1)), + row_offset=1, + holes=True, + foreign=True, + ), + ), + Cell( + why="ht_dep4 b4 verify: 4 x 4 = 16 rows, e4m3, gated, captured", + spec=_verify(_DEP4, 4, capture=True, **_DEP4_ROW), + ), + Cell( + why="ht_dep4 draft layer: 8 x 4 = 32 rows, e4m3, interleaved gate", + spec=_verify(_DEP4, 8, **_DRAFT), + ), + Cell( + why="dep4 b1 verify: 1 x 4 rows, e4m3, gated", + spec=_verify(_DEP4, 1, **_DEP4_ROW), + ), + Cell( + why="dep4 b32 verify: 32 x 4 = 128 rows (the routed cap), e4m3, gated, captured", + spec=_verify(_DEP4, 32, last=60000, step=1700, capture=True, **_DEP4_ROW), + ), + Cell( + why="ht_tep2 verify: 4 requests x 4 rows = 16, 12 q / 1 KV head over the e4m3 " + "trunk, [128, 8196] table, gated, captured", + spec=_verify(_TEP2, 4, capture=True, **_TEP2_ROW), + ), + Cell( + why="tep2 b1 verify: 1 x 4 rows, e4m3, gated", + spec=_verify(_TEP2, 1, **_TEP2_ROW), + ), + Cell( + why="ht_tep2 MTP-2 verify: 160 x 3 = 480 rows (the 160-request graph, three splits " + "a row), e4m3, gated", + spec=_verify( + dict(_TEP2, slots=160), 160, rows=3, last=30000, step=150, splits=3, **_TEP2_ROW + ), + ), + Cell( + why="ht_tep2 draft-extend step's continuing rows: 128 x 1, the gate a contiguous " + "[rows, Hq * D] (rows gathered), e4m3", + spec=_verify(_TEP2, 128, rows=1, last=30000, step=150, gate="contiguous"), + ), + Cell( + why="dep4 mixed step's generation rows: 7 rows off a wider selection (unaligned " + "slice), short sequences, holes, one foreign row, e4m3, ungated", + spec=dict( + _DEP4, + segments=((77, 1), (2047, 1), (5000, 1), (9000, 1), (30000, 1), (1, 1), (4, 1)), + row_offset=1, + holes=True, + foreign=True, + ), + ), + ) + + note = """ + Source: CuTe DSL split pass and merge (TVM-FFI), JIT-compiled per key (pool dtype, heads, + selection width, split plan, pool page count, table shape, plane and q strides, gate fused + or not); the engine's warmup runs each captured batch eagerly, so no compile lands inside a + capture. Routed by `serves` / `MAX_ROWS` from the target's `_sparse_gqa`: the decode + step's trunk layers and the MTP draft layer (gated), and a mixed step's generation rows + (ungated). + + Preconditions: + - PDL: the split pass reads q, and the merge the gate, before their grid-dependency + wait, so both must be complete when the grid ahead releases this one (a launch + without PDL releases at its end; `qsa_topk_expand_decode` only after its own wait). + - a pool below 2^26 pages at 32 tokens a page: the token table's `page * tpb + slot` is + 32-bit (page indices are widened to 64 bits before the plane stride product). + + Numerics: bf16 pool as the Triton composition (bf16 MMAs, fp32 accumulation, bf16 P); e4m3 + pool with K/V widened exactly to f16, q rounded to f16 and f16 P. The merge runs in split + order, so the output differs from the composition's by an ulp on some elements. + Deterministic and graph-capturable (the row count is a runtime value). + + Not certified: other head geometries, head dims or page sizes (refused at compile: at most + 16 q heads a KV head, a power-of-two page); fp16 q or pools; NVFP4 pools + (`qsa_paged_sparse_gqa_nvfp4_decode`); selections of 64 tokens or fewer; int64 request + indices; the prefill pair; the one-split launch (no routed shape reaches it); at one KV + head over e4m3, the 4- and 6-split plans (33..96 rows). + """ + + serves = staticmethod(serves) + + def __call__( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return sparse_gqa_decode( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + output_gate=output_gate, + ) + + def reference( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """fp32 sparse GQA over every resident selected token, rounded once (the + composition entry's reference); with `output_gate`, then gated in fp32 over that + rounding and rounded again.""" + out = _composition.reference( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + ) + if output_gate is None: + return out + gate = torch.sigmoid(output_gate.float().reshape(out.shape)) + return (out.float() * gate).to(out.dtype) + + +qsa_sparse_gqa_splitk_cute = QsaSparseGqaSplitkCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py new file mode 100644 index 000000000000..3207059e0382 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A captured verify step's input prologue in one launch: the overlap scheduler's accepted-count +correction of the mRoPE position ids and the KV lengths, and the QSA row mapping over the +corrected lengths. + +It replaces what `DecoderInputPreparer.preprocess_inputs` does on a generation-only verify step +(the position and KV-length corrections and `on_update_kv_lens()`'s row mapping); +`SpecStepMetadataPrologue` is the engine hook that calls it. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Largest batch the single program covers (requests, rows). +MAX_SEQS = 64 +MAX_ROWS = 512 +#: Requests a step of the row-mapping loop covers once the `[rows, requests]` comparison matrix +#: would pass CHUNK_MIN_ELEMS elements (a single larger int64 matrix spills). Same output bits. +SEQ_CHUNK = 16 +CHUNK_MIN_ELEMS = 8192 + + +@triton.jit +def _spec_step_metadata_kernel( + positions, + stride_coord, + pos_offsets, + num_tokens, + kv_lens, + kv_offsets, + seq_lens, + request_indices, + logical_positions, + sequence_lengths, + visible_blocks, + num_seqs, + pos_out, + COMPRESS_RATIO: tl.constexpr, + BLOCK_ROWS: tl.constexpr, + BLOCK_SEQS: tl.constexpr, + HAS_POS_OUT: tl.constexpr, + SEQ_CHUNK_: tl.constexpr, +): + rows = tl.arange(0, BLOCK_ROWS) + valid = rows < num_tokens + seqs = tl.arange(0, BLOCK_SEQS) + sv = seqs < num_seqs + # Every load before any store (the stores overwrite positions and lengths it reads). + off = tl.load(pos_offsets + rows, mask=valid, other=0) + kv_new = tl.load(kv_lens + seqs, mask=sv, other=0) + tl.load( + kv_offsets + seqs, mask=sv, other=0 + ) + # A row's request is the number of requests ending at or before it, its start the sum of + # their lengths (`_qsa_generation_token_mapping_kernel`'s arithmetic). + request = tl.zeros([BLOCK_ROWS], tl.int32) + request_start = tl.zeros([BLOCK_ROWS], tl.int64) + carry = tl.zeros([1], tl.int64) + rows64 = rows.to(tl.int64) + for s0 in tl.static_range(0, BLOCK_SEQS, SEQ_CHUNK_): + cseq = s0 + tl.arange(0, SEQ_CHUNK_) + csv = cseq < num_seqs + lengths = tl.load(seq_lens + cseq, mask=csv, other=0).to(tl.int64) + ends = tl.cumsum(lengths, axis=0) + carry + ended = (ends[None, :] <= rows64[:, None]) & csv[None, :] + request += tl.sum(ended.to(tl.int32), axis=1) + request_start += tl.sum(tl.where(ended, lengths[None, :], 0), axis=1) + carry += tl.sum(lengths, axis=0) + kv_len = tl.load(kv_lens + request, mask=valid, other=0) + tl.load( + kv_offsets + request, mask=valid, other=0 + ) + seq_len = tl.load(seq_lens + request, mask=valid, other=0) + p0 = tl.load(positions + rows, mask=valid, other=0) + p1 = tl.load(positions + stride_coord + rows, mask=valid, other=0) + p2 = tl.load(positions + 2 * stride_coord + rows, mask=valid, other=0) + tl.debug_barrier() + tl.store(positions + rows, p0 + off, mask=valid) + tl.store(positions + stride_coord + rows, p1 + off, mask=valid) + tl.store(positions + 2 * stride_coord + rows, p2 + off, mask=valid) + if HAS_POS_OUT: + tl.store(pos_out + rows, p0 + off, mask=valid) + tl.store(pos_out + num_tokens + rows, p1 + off, mask=valid) + tl.store(pos_out + 2 * num_tokens + rows, p2 + off, mask=valid) + tl.store(kv_lens + seqs, kv_new, mask=sv) + logical = kv_len.to(tl.int64) - seq_len.to(tl.int64) + rows.to(tl.int64) - request_start + visible_num = logical + 1 + visible = visible_num // COMPRESS_RATIO + visible = tl.where( + (visible_num % COMPRESS_RATIO != 0) & (visible_num < 0), visible - 1, visible + ) + tl.store(request_indices + rows, request, mask=valid) + tl.store(logical_positions + rows, logical, mask=valid) + tl.store(sequence_lengths + rows, kv_len, mask=valid) + tl.store(visible_blocks + rows, visible.to(tl.int32), mask=valid) + + +def _cells(): + return tuple( + Cell(why=f"{n} request(s) x {p} verify rows", spec=dict(n=n, p=p, capture=capture)) + for n, p, capture in ( + (1, 3, True), + (8, 4, False), + (40, 3, True), + (56, 3, True), + (64, 3, False), + ) + ) + + +class QsaSpecStepMetadata(OpWrapper): + """`qsa_spec_step_metadata(positions, pos_offsets, kv_lens, kv_offsets, seq_lens, mapping, + compress_ratio)`, in place: + + - `positions` `[C, 1, T]` int32 (the step's mRoPE positions, unit token stride, any + coordinate stride): `positions[:, :, :T] += pos_offsets[:T]`; + - `kv_lens` `[N]` int32: `kv_lens += kv_offsets[:N]`; + - `mapping = (request_indices, logical_positions, sequence_lengths, visible_blocks)` + (int32, int64, int32, int32, at least `T` long): every row's request, logical position, + KV length and visible-block count over the corrected lengths, with each request's + `seq_lens[i]` rows consecutive -- bitwise the composed corrections followed by + `triton_qsa_generation_token_mapping`. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: one Triton program (`_spec_step_metadata_kernel`), up to 64 requests and 512 rows; + integer arithmetic only. + """ + + def __call__( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ): + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + block_rows = max(16, triton.next_power_of_2(num_tokens)) + block_seqs = max(16, triton.next_power_of_2(num_seqs)) + chunked = block_rows * block_seqs > CHUNK_MIN_ELEMS + _spec_step_metadata_kernel[(1,)]( + positions, + positions.stride(0), + pos_offsets, + num_tokens, + kv_lens, + kv_offsets, + seq_lens, + *mapping, + num_seqs, + pos_out if pos_out is not None else positions, + COMPRESS_RATIO=compress_ratio, + BLOCK_ROWS=block_rows, + BLOCK_SEQS=block_seqs, + HAS_POS_OUT=pos_out is not None, + SEQ_CHUNK_=SEQ_CHUNK if chunked else block_seqs, + num_warps=8 if chunked else 4, + ) + + def reference( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ): + """The composition, in place: the two corrections, then the row mapping in torch.""" + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + positions += pos_offsets[:num_tokens] + if pos_out is not None: + pos_out.copy_(positions.reshape(positions.shape[0], num_tokens)) + kv_lens += kv_offsets[:num_seqs] + request_indices, logical_positions, sequence_lengths, visible_blocks = mapping + lens = seq_lens[:num_seqs].long() + req = torch.repeat_interleave(torch.arange(num_seqs, device=kv_lens.device), lens)[ + :num_tokens + ] + starts = torch.cumsum(lens, 0) - lens + rows = torch.arange(num_tokens, device=kv_lens.device) + kv = kv_lens[req].long() + logical = kv - lens[req] + rows - starts[req] + request_indices[:num_tokens] = req.to(request_indices.dtype) + logical_positions[:num_tokens] = logical.to(logical_positions.dtype) + sequence_lengths[:num_tokens] = kv.to(sequence_lengths.dtype) + visible_blocks[:num_tokens] = torch.div( + logical + 1, compress_ratio, rounding_mode="floor" + ).to(visible_blocks.dtype) + + def is_valid( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ) -> None: + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + assert ( + positions.dim() == 3 and positions.shape[:2] == (3, 1) and positions.stride(2) == 1 + ), ( + tuple(positions.shape), + positions.stride(), + ) + assert positions.dtype == pos_offsets.dtype == torch.int32 + assert kv_lens.dtype == kv_offsets.dtype == seq_lens.dtype == torch.int32 + assert kv_lens.dim() == 1 and kv_lens.is_contiguous() + assert 0 < num_seqs <= MAX_SEQS and 0 < num_tokens <= MAX_ROWS, (num_seqs, num_tokens) + assert pos_offsets.numel() >= num_tokens and kv_offsets.numel() >= num_seqs + assert seq_lens.numel() >= num_seqs + dtypes = (torch.int32, torch.int64, torch.int32, torch.int32) + for t, dt in zip(mapping, dtypes): + assert t.dtype == dt and t.numel() >= num_tokens and t.is_contiguous(), ( + t.dtype, + t.numel(), + ) + + +qsa_spec_step_metadata = QsaSpecStepMetadata() + + +class SpecStepMetadataPrologue: + """The engine's decode-prologue hook (`RuntimeHooks.decode_prologue`): on a generation-only + verify step with the overlap scheduler, runs `qsa_spec_step_metadata` plus + `on_update_kv_lens()`'s host statements and returns True; anything else returns False and + keeps the native code.""" + + def __call__( + self, preparer, inputs, *, enable_spec_decode: bool, runtime_draft_len: int + ) -> bool: + from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAAttentionMetadata + + md = inputs.get("attn_metadata") + if md is not None: + # A packet of an earlier step never reaches this step's forward. + md.spec_step_positions = None + if not enable_spec_decode or preparer._config.disable_overlap_scheduler: + return False + if not isinstance(md, QSAAttentionMetadata) or md.kv_cache_manager is None: + return False + if not md.qsa_has_local_layers or preparer.mapping.has_cp_helix(): + return False + num_seqs, num_tokens = md.num_seqs, md.num_tokens + if md.num_contexts != 0 or md.num_chunked_ctx_requests != 0 or md.num_ctx_tokens != 0: + return False + # the native mapping's generation path (not its one-row-a-request decode path) + if num_tokens == num_seqs or not (0 < num_seqs <= MAX_SEQS) or num_tokens > MAX_ROWS: + return False + pos = inputs["position_ids"] + if ( + pos.ndim != 3 + or tuple(pos.shape[:2]) != (3, 1) + or pos.stride(2) != 1 + or pos.dtype != torch.int32 + ): + return False + if inputs["input_ids"].shape[0] != num_tokens or pos.shape[2] != num_tokens: + return False + buffers = preparer._buffers + pos_off, kv_off = ( + buffers.previous_pos_id_offsets_cuda, + buffers.previous_kv_lens_offsets_cuda, + ) + kv_lens = md.kv_lens_cuda[:num_seqs] + # the mapping reads the runtime view: it must be these lengths + runtime = getattr(md, "kv_lens_cuda_runtime", None) + if runtime is None or runtime.data_ptr() != kv_lens.data_ptr(): + return False + if ( + pos_off.dtype != torch.int32 + or kv_off.dtype != torch.int32 + or kv_lens.dtype != torch.int32 + ): + return False + seq_lens = md.seq_lens_cuda[:num_seqs] + if seq_lens.dtype != torch.int32: + return False + mapping = ( + md.qsa_req_idx_per_token, + md.qsa_logical_positions, + md.qsa_sequence_lengths, + md.qsa_visible_blocks, + ) + if (mapping[0].dtype, mapping[1].dtype, mapping[2].dtype, mapping[3].dtype) != ( + torch.int32, + torch.int64, + torch.int32, + torch.int32, + ): + return False + # The corrected positions also as the contiguous `[C, T]` the decode forward reads + # (`take_step_positions`). + pos3 = torch.empty((pos.shape[0], num_tokens), dtype=torch.int32, device=pos.device) + qsa_spec_step_metadata( + pos, pos_off, kv_lens, kv_off, seq_lens, mapping, md.sparse_metadata_params.compress_ratio, + pos_out=pos3, + ) # fmt: skip + md.spec_step_positions = (pos, pos3) + # `on_update_kv_lens()`'s host statements (the QSA override adds only the mapping). + super(QSAAttentionMetadata, md).on_update_kv_lens() + return True + + +def take_step_positions(md, position_ids): + """The contiguous int32 `[C, T]` positions `SpecStepMetadataPrologue` wrote for exactly this + `position_ids` tensor in this step (consumed once), or None.""" + packet = getattr(md, "spec_step_positions", None) + if packet is None: + return None + md.spec_step_positions = None + return packet[1] if packet[0] is position_ids else None diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py new file mode 100644 index 000000000000..bc0da743883e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py @@ -0,0 +1,753 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA's decode block top-k and token expansion in one CuTe DSL launch, a cluster of CTAs a row. + +Each CTA holds a slice of the row's window in registers; radix passes (12, 11, 8 bits, top down) +narrow to the threshold class, every CTA adding its histogram into every CTA's copy through +distributed shared memory so the cluster narrows in step. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + +#: Consecutive scores a thread loads at once (one 16-byte load): a group. +KPT = 4 +#: The radix passes: (first bit, bits). 31 magnitude bits, top down. +_PASSES = ((19, 12), (8, 11), (0, 8)) +_HIST = 1 << 12 +#: Most rows a launch serves (148 CTAs in clusters of 4); more take the family's one-CTA kernel. +MAX_ROWS = 37 +#: Narrowest table this entry serves: narrower ones take the family's one-CTA kernel. +MIN_WIDTH = 4096 + KPT +#: Widest table (8192 + 128 pages): keeps a CTA's keys below 2^16, the width of its packed counts. +MAX_WIDTH = (8192 + 128) * 32 // 4 +#: CTA size bounds (the scans give each thread at most `_HIST / _MIN_THREADS` bins). +_MAX_THREADS = 1024 +_MIN_THREADS = 128 + + +@dsl_user_op +def _red_add_cluster(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + None, + [ + cutlass.Int32(addr).ir_value(loc=loc, ip=ip), + cutlass.Int32(val).ir_value(loc=loc, ip=ip), + ], + "red.relaxed.cluster.shared::cluster.add.u32 [$0], $1;", + "r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _st_cluster(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + None, + [ + cutlass.Int32(addr).ir_value(loc=loc, ip=ip), + cutlass.Int32(val).ir_value(loc=loc, ip=ip), + ], + "st.shared::cluster.u32 [$0], $1;", + "r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _dsmem(ptr, rank): + """The cluster-shared address of `ptr`'s location in CTA `rank` of the cluster.""" + return cutlass.Int32(cute.arch.map_dsmem_ptr(ptr, rank).toint()) + + +def cluster_size(rows: int) -> int: + """CTAs a row: 8 up to 12 rows (16 clusters of 8 overflow GB300's GPCs in one wave), else 4.""" + return 8 if rows <= 12 else 4 + + +def geometry(width: int, cluster: int) -> tuple[int, int]: + """`(threads, groups)` of a CTA: its `ceil(width / 4 / cluster)` groups over the fewest groups + a thread 1024 threads allow (a group slot costs more than a wider CTA).""" + q4 = -(-(-(-width // KPT)) // cluster) + groups = max(1, -(-q4 // _MAX_THREADS)) + threads = max(_MIN_THREADS, -(-(-(-q4 // groups)) // 32) * 32) + assert threads <= _MAX_THREADS, (width, cluster, threads) + return threads, groups + + +class _TopkExpandClusterKernel: + def __init__(self, *, width, key_stride, block_topk, compress, token_topk, pdl, cluster): + self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( + width, + key_stride, + block_topk, + compress, + token_topk, + pdl, + ) + self.C = cluster + self.threads, self.groups = geometry(width, cluster) + self.warps = self.threads // 32 + self.final = token_topk + compress - 1 + + def smem_bytes(self) -> int: + words = _HIST + sum(1 << bits for _, bits in _PASSES) + 32 + 32 + 32 * self.groups + 16 + return words * 4 + 8 * 128 + + @cute.jit + def __call__( + self, + keys_t: cute.Tensor, + end_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + len_ptr: cute.Pointer, + topk_ptr: cute.Pointer, + sel_ptr: cute.Pointer, + rows: cutlass.Int32, + stream, + ): + # Keys arrive as a tensor (the TVM-FFI entry takes its stream from one); the layout is + # rebuilt with the static width and row stride the 16-byte loads need. + mKeys = cute.make_tensor( + keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) + ) + per_row = cute.make_layout((rows,), stride=(1,)) + mEnd = cute.make_tensor(end_ptr, per_row) + mPos = cute.make_tensor(pos_ptr, per_row) + mLen = cute.make_tensor(len_ptr, per_row) + mTopk = cute.make_tensor(topk_ptr, cute.make_layout((rows, self.k), stride=(self.k, 1))) + mSel = cute.make_tensor( + sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) + ) + ld_keys = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT + ) + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys).launch( + grid=[rows * self.C, 1, 1], + block=[self.threads, 1, 1], + cluster=[self.C, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + ) + + @cute.jit + def _exclusive_scan(self, own, warp, lane, wsum): + """`own`'s exclusive prefix over the CTA's threads in thread order (one barrier).""" + incl = own + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + wsum[warp] = incl + cute.arch.sync_threads() + above = cutlass.Int32(0) + if lane < warp: + above = wsum[lane] + return cute.arch.warp_reduction_sum(above) + incl - own + + @cute.jit + def _narrow( + self, hist, tidx, warp, lane, wsum, state, shift: cutlass.Constexpr, bits: cutlass.Constexpr + ): + """The cluster histogram of a pass read from the top bin down: the bin holding the + `need`-th key of the class becomes the class. Every CTA reads the same counts.""" + nbins: cutlass.Constexpr = 1 << bits + per: cutlass.Constexpr = -(-nbins // self.threads) + prefix = state[0] + mask = state[1] + need = state[2] + h = cute.make_rmem_tensor(cute.make_layout((per,), stride=(1,)), cutlass.Int32) + own = cutlass.Int32(0) + for j in cutlass.range_constexpr(per): + b = nbins - 1 - (tidx * per + j) + hv = cutlass.Int32(0) + if b >= 0: + hv = hist[b] + h[j] = hv + own = own + hv + before = self._exclusive_scan(own, warp, lane, wsum) + if before < need: + if need <= before + own: + run = before + for j in cutlass.range_constexpr(per): + if run < need: + if need <= run + h[j]: + b = nbins - 1 - (tidx * per + j) + rem = need - run + state[0] = prefix | (b << shift) + state[1] = mask | ((nbins - 1) << shift) + state[2] = rem + state[3] = h[j] + state[4] = cutlass.Int32( + cutlass.select_((h[j] == rem) | (shift == 0), 1, 0) + ) + run = run + h[j] + cute.arch.sync_threads() + + @cute.jit + def _pass( + self, + keys, + tidx, + warp, + lane, + local, + merged, + wsum, + state, + shift: cutlass.Constexpr, + bits: cutlass.Constexpr, + q4, + ): + """One radix pass unless narrowing is done: count this CTA's class keys, add the counts + into every CTA's `merged`, cluster barrier, scan. Slot j holds keys only while + `j * threads < q4`.""" + nbins: cutlass.Constexpr = 1 << bits + if state[4] == 0: + prefix = state[0] + mask = state[1] + for j in cutlass.range_constexpr(self.groups): + if j * self.threads < q4: + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + if (kv & mask) == prefix: + cute.arch.atomic_add( + local.iterator + ((kv >> shift) & (nbins - 1)), + cutlass.Int32(1), + scope="cta", + ) + cute.arch.sync_threads() + for b in cutlass.range_constexpr(-(-nbins // self.threads)): + i = tidx + b * self.threads + if i < nbins: + v = local[i] + if v != 0: + local[i] = cutlass.Int32(0) + for r in cutlass.range_constexpr(self.C): + _red_add_cluster(_dsmem(merged.iterator + i, r), v) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + self._narrow(merged, tidx, warp, lane, wsum, state, shift, bits) + + @cute.kernel + def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + rank = cute.arch.block_idx_in_cluster() + C: cutlass.Constexpr = self.C + nt: cutlass.Constexpr = self.threads + nw: cutlass.Constexpr = self.warps + G: cutlass.Constexpr = self.groups + k: cutlass.Constexpr = self.k + cr: cutlass.Constexpr = self.cr + row = bidx // C + smem = cutlass.utils.SmemAllocator() + local = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((_HIST,), stride=(1,)), byte_alignment=128 + ) + merged0 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[0][1],), stride=(1,)), byte_alignment=128 + ) + merged1 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[1][1],), stride=(1,)), byte_alignment=128 + ) + merged2 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[2][1],), stride=(1,)), byte_alignment=128 + ) + ex = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + wsum = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + wtot = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32 * G,), stride=(1,)), byte_alignment=128 + ) + state = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((16,), stride=(1,)), byte_alignment=128 + ) + + # 0. The histograms zeroed and the row's host-filled metadata read before the wait; + # the arrive releases the zeroed counts before any peer adds into them. + for b in cutlass.range_constexpr(-(-_HIST // nt)): + if tidx + b * nt < _HIST: + local[tidx + b * nt] = cutlass.Int32(0) + merged0[tidx + b * nt] = cutlass.Int32(0) + for b in cutlass.range_constexpr(-(-(1 << _PASSES[1][1]) // nt)): + if tidx + b * nt < (1 << _PASSES[1][1]): + merged1[tidx + b * nt] = cutlass.Int32(0) + for b in cutlass.range_constexpr(-(-(1 << _PASSES[2][1]) // nt)): + if tidx + b * nt < (1 << _PASSES[2][1]): + merged2[tidx + b * nt] = cutlass.Int32(0) + n = mEnd[row] + seqlen = mLen[row] + visible = cutlass.Int32(mPos[row]) + 1 + if tidx == 0: + state[0] = cutlass.Int32(0) # the class's fixed bits + state[1] = cutlass.Int32(0) # their mask + state[2] = cutlass.min(n, cutlass.Int32(k)) # still needed from the class + state[3] = n # keys in the class + state[4] = cutlass.Int32(cutlass.select_(n <= k, 1, 0)) # narrowing done + cute.arch.cluster_arrive() + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + # 1. This CTA's slice of the window: groups [g_lo, g_hi) of `KPT` consecutive keys, + # group `g_lo + j * nt + tidx` in slot j of thread tidx; -1 past the window. + n4 = (n + (KPT - 1)) // KPT + q4 = (n4 + (C - 1)) // C + g_lo = rank * q4 + g_hi = cutlass.min(g_lo + q4, n4) + gK = cute.logical_divide(mKeys, (None, KPT)) + keys = cute.make_rmem_tensor(cute.make_layout((KPT, G), stride=(1, KPT)), cutlass.Int32) + keys.fill(-1) + for j in cutlass.range_constexpr(G): + gl = g_lo + j * nt + tidx + if gl < g_hi: + cute.copy(ld_keys, gK[row, (None, gl)], keys[None, j]) + for c in cutlass.range_constexpr(KPT): + v = cutlass.max(keys[c, j], cutlass.Int32(0)) + if (gl >= g_hi) | (gl * KPT + c >= n): + v = cutlass.Int32(-1) + keys[c, j] = v + cute.arch.cluster_wait() + + self._pass( + keys, tidx, warp, lane, local, merged0, wsum, state, _PASSES[0][0], _PASSES[0][1], q4 + ) + self._pass( + keys, tidx, warp, lane, local, merged1, wsum, state, _PASSES[1][0], _PASSES[1][1], q4 + ) + self._pass( + keys, tidx, warp, lane, local, merged2, wsum, state, _PASSES[2][0], _PASSES[2][1], q4 + ) + prefix = state[0] + mask = state[1] + need = state[2] + + # 3. Block order within the CTA (slot j of every thread, then slot j + 1): each + # (thread, slot)'s counts above and in the class, packed (above + class << 16), and their + # exclusive prefix. Slots past `q4` hold no keys in any thread (uniform skip). + pre = cute.make_rmem_tensor(cute.make_layout((G,), stride=(1,)), cutlass.Int32) + pre.fill(0) + for j in cutlass.range_constexpr(G): + if j * nt < q4: + a = cutlass.Int32(0) + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + mk = kv & mask + if mk > prefix: + a = a + 1 + if mk == prefix: + a = a + (1 << 16) + incl = a + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + wtot[j * 32 + warp] = incl + pre[j] = incl - a + cute.arch.sync_threads() + run = cutlass.Int32(0) + for j in cutlass.range_constexpr(G): + if j * nt < q4: + v = cutlass.Int32(0) + if lane < nw: + v = wtot[j * 32 + lane] + above = cutlass.Int32(0) + if lane < warp: + above = v + pre[j] = pre[j] + run + cute.arch.warp_reduction_sum(above) + run = run + cute.arch.warp_reduction_sum(v) + + # 4. Every CTA's totals into every CTA (lane r of warp 0 writes CTA r's slot `rank`). + if tidx < C: + _st_cluster(_dsmem(ex.iterator + rank, tidx), run) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + # The class's `need` are its highest blocks, so later CTAs fill first. `base`: selected + # count of the CTAs before this one; `skip`: this CTA's lowest class keys left out. + base = cutlass.Int32(0) + skip = cutlass.Int32(0) + later = cutlass.Int32(0) + for rr in cutlass.range_constexpr(C): + r = C - 1 - rr + tot = ex[r] + e_r = tot >> 16 + take = cutlass.max(cutlass.min(need - later, e_r), cutlass.Int32(0)) + later = later + e_r + if r < rank: + base = base + (tot & 0xFFFF) + take + if r == rank: + skip = e_r - take + + # 5. Each selected block at its position, with its tokens below the sequence length. + for j in cutlass.range_constexpr(G): + gl = g_lo + j * nt + tidx + ab = pre[j] & 0xFFFF + cb = pre[j] >> 16 + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + mk = kv & mask + sel = cutlass.Int32(0) + pos = cutlass.Int32(0) + if mk > prefix: + sel = cutlass.Int32(1) + pos = base + ab + cutlass.max(cb - skip, cutlass.Int32(0)) + ab = ab + 1 + if mk == prefix: + if cb >= skip: + sel = cutlass.Int32(1) + pos = base + ab + cb - skip + cb = cb + 1 + if sel == 1: + col = gl * KPT + c + mTopk[row, pos] = col + for tt in cutlass.range_constexpr(cr): + tok = col * cr + tt + val = cutlass.Int32(-1) + if tok < seqlen: + val = tok + mSel[row, pos * cr + tt] = val + + # 6. Over the cluster's threads: the block list's -1 padding, then the causal tail and -1. + total = cutlass.min(n, cutlass.Int32(k)) + for b in cutlass.range_constexpr(-(-k // (C * nt))): + slot = (b * C + rank) * nt + tidx + if slot < k: + if slot >= total: + mTopk[row, slot] = cutlass.Int32(-1) + tail_start = (visible // cr) * cr + tail_count = visible - tail_start + body = cutlass.min(total * cr, cutlass.Int32(self.token_topk)) + for b in cutlass.range_constexpr(-(-self.final // (C * nt))): + col = (b * C + rank) * nt + tidx + if col < self.final: + if col >= body: + off = col - body + val = cutlass.Int32(-1) + if off < cr - 1: + if off < tail_count: + if tail_start + off < seqlen: + val = tail_start + off + mSel[row, col] = val + + +#: Compiled launches; the row count is a runtime argument. +_COMPILED: dict = {} +_INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} + + +def launch( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, +): + """Writes `topk_indices`; returns the int32 token selection (`out` when given).""" + rows, width = scores.shape + block_topk = topk_indices.shape[1] + sel = out + if sel is None: + sel = torch.empty( + (rows, token_topk + compress_ratio - 1), dtype=torch.int32, device=scores.device + ) + if rows == 0: + return sel + pdl = bool(get_env_enable_pdl()) + keys = scores.view(torch.int32) + cluster = cluster_size(rows) + key = ( + width, + scores.stride(0), + block_topk, + compress_ratio, + token_topk, + query_positions.dtype, + sequence_lengths.dtype, + pdl, + cluster, + ) + ptrs = ( + row_ends.data_ptr(), + query_positions.data_ptr(), + sequence_lengths.data_ptr(), + topk_indices.data_ptr(), + sel.data_ptr(), + ) + fn = _COMPILED.get(key) + if fn is None: + gm = cute.AddressSpace.gmem + kernel = _TopkExpandClusterKernel( + width=width, + key_stride=scores.stride(0), + block_topk=block_topk, + compress=compress_ratio, + token_topk=token_topk, + pdl=pdl, + cluster=cluster, + ) + fn = _COMPILED[key] = cute.compile( + kernel, + from_dlpack(keys, assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.Int32, ptrs[0], gm, assumed_align=4), + make_ptr( + _INDEX_TYPES[query_positions.dtype], + ptrs[1], + gm, + assumed_align=query_positions.element_size(), + ), + make_ptr( + _INDEX_TYPES[sequence_lengths.dtype], + ptrs[2], + gm, + assumed_align=sequence_lengths.element_size(), + ), + make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + cutlass.Int32(rows), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + fn(keys, *ptrs, rows) + return sel + + +def supports(scores) -> bool: + """Whether this entry serves a decode call over this score table (see `is_valid`).""" + rows, width = scores.shape + return 0 < rows <= MAX_ROWS and width % KPT == 0 and MIN_WIDTH <= width <= MAX_WIDTH + + +#: Score-table widths: 262144 tokens with / without MTP's extra pages, and 131072 tokens. +_TUNE_WIDTH = 65568 +_TUNE_WIDTH_NO_MTP = 65536 +_HALF_WIDTH = 32800 +#: Decode windows of 0, 1, 2 and 9 blocks, k - 1, k, k + 1, the whole table, and between. +_EDGES_16 = ( + 2, + 3, + 11, + 35, + 2047, + 2051, + 2055, + 16380, + 16388, + 99999, + 180224, + 262143, + 50000, + 150001, + 220000, + 7, +) + + +class QsaTopkExpandClusterDecode(OpWrapper): + """`qsa_topk_select` (zero row starts), `qsa_canonical_selection` and + `qsa_expand_block_indices` in one launch, for decode-sized calls over a wide score table. + + Per row: the `block_topk` largest of `scores[row, :row_ends[row]]` (ties to the higher + block) into `topk_indices[row]` in block order, -1 padded, in place. Returns the token + selection (int32 `[rows, token_topk + compress_ratio - 1]`, into `out` when given): selected + groups' tokens below `sequence_lengths[row]`, then the causal tail, then -1. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ll_tep4 draft step: 1 row at ~96k tokens, 65568 wide, relu-sum, graph", + spec=dict(rows=1, pos=(96003,), relu_sum=True, width=_TUNE_WIDTH, graph=True), + ), + Cell( + why="37 rows (MAX_ROWS, 148 CTAs in clusters of 4): 9 requests x 4 verify rows " + "spread to ~200k tokens and a draft row", + spec=dict( + rows=37, + requests=tuple((1000 + 24000 * r, 4) for r in range(9)) + ((131072, 1),), + relu_sum=True, + width=_TUNE_WIDTH, + ), + ), + Cell( + why="4 rows of bf16-resolution log-normal scores: exact ties at the threshold, all " + "three passes, the tie rule across a cluster's CTAs", + spec=dict(rows=4, pos=(100000, 30000, 4095, 2047), lognormal=True, width=_TUNE_WIDTH), + ), + Cell( + why="2 relu-sparse rows (~1.4% positive, negative zeros among the zeros): the zero " + "class straddles the budget across CTAs", + spec=dict(rows=2, pos=(262143, 120000), relu_sparse=True, width=_TUNE_WIDTH_NO_MTP), + ), + Cell( + why="3 rows of equal scores (windows 25000, 513 and 512 blocks): the tie class is " + "the window, the highest blocks taken", + spec=dict(rows=3, pos=(100000, 2051, 2047), equal=True, width=_TUNE_WIDTH), + ), + Cell( + why="20 rows (clusters of 4) at the edges of k and the table, relu-sum, 65536 wide", + spec=dict( + rows=20, + pos=_EDGES_16 + (262140, 4, 2048, 131071), + relu_sum=True, + width=_TUNE_WIDTH_NO_MTP, + ), + ), + Cell( + why="a 131072-token engine's 32800-wide table: 4 verify rows up to its last position", + spec=dict( + rows=4, requests=((131068, 4),), relu_sum=True, width=_HALF_WIDTH, graph=True + ), + ), + ) + + note = """ + Source: this target's CuTe DSL kernel, `cute.compile`d (`--enable-tvm-ffi`) on first call + per table width and row stride, k, compress ratio, token budget, index dtypes, PDL and + cluster size. + + Preconditions: zero row starts; `scores` fp32 `[rows, W]`, unit column stride, 16-byte + aligned rows, `W % 4 == 0`; `topk_indices` int32 `[rows, K]` contiguous; `row_ends` int32 + (each at most `W`), positions and lengths int32 or int64, each a contiguous `[rows]`; the + call within `supports` (1..37 rows, 4100 <= W <= 66560). Ties go to the higher block; a + negative zero ranks as zero. Limits: rows x row stride below 2^31, positions read as Int32. + + Not certified: NaN scores inside the window (they rank above every finite score). + """ + + def __call__( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + return launch( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + def reference( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + """`(topk_indices_after, selection)` via a stable descending sort per row; ignores `out`.""" + rows, _ = scores.shape + k = topk_indices.shape[1] + device = scores.device + blocks = torch.full((rows, k), -1, dtype=torch.int64, device=device) + for r in range(rows): + n = int(row_ends[r]) + s = scores[r, :n].float().clamp(min=0.0) + # A stable sort of the reversed row puts the higher block first among equals. + order = (n - 1) - torch.sort(torch.flip(s, [0]), descending=True, stable=True).indices[ + :k + ] + chosen = torch.sort(order).values + blocks[r, : chosen.numel()] = chosen + final = token_topk + compress_ratio - 1 + lengths = sequence_lengths.long() + offsets = torch.arange(compress_ratio, device=device) + expanded = (blocks[:, :, None] * compress_ratio + offsets).reshape(rows, token_topk) + chosen = (blocks >= 0).repeat_interleave(compress_ratio, dim=1) + keep = chosen & (expanded < lengths[:, None]) + sel = torch.full((rows, final), -1, dtype=torch.int64, device=device) + sel[:, :token_topk] = torch.where(keep, expanded, -1) + first_free = (blocks >= 0).sum(dim=1) * compress_ratio + visible = query_positions.long() + 1 + tail_start = visible // compress_ratio * compress_ratio + row = torch.arange(rows, device=device) + for offset in range(compress_ratio - 1): + column = first_free + offset + token = tail_start + offset + live = (offset < visible - tail_start) & (token < lengths) & (column < final) + sel[row[live], column[live]] = token[live] + return blocks.to(torch.int32), sel.to(torch.int32) + + def is_valid( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + assert scores.dtype == torch.float32 and scores.stride(1) == 1, ( + "fp32 scores, unit column stride" + ) + assert scores.shape[1] % KPT == 0 and scores.shape[1] <= MAX_WIDTH, "W % 4 == 0, W <= 66560" + assert scores.stride(0) % KPT == 0 and scores.data_ptr() % 16 == 0, "16-byte aligned rows" + assert topk_indices.dtype == torch.int32 and topk_indices.is_contiguous(), ( + "int32 topk_indices" + ) + rows = scores.shape[0] + assert row_ends.dtype == torch.int32 and row_ends.shape == (rows,), "int32 row_ends [rows]" + for t in (query_positions, sequence_lengths): + assert t.dtype in _INDEX_TYPES and t.shape == (rows,), "int32/int64 [rows]" + assert all(t.is_contiguous() for t in (row_ends, query_positions, sequence_lengths)), ( + "contiguous row_ends, positions and lengths" + ) + assert token_topk == topk_indices.shape[1] * compress_ratio, ( + "token_topk = k * compress_ratio" + ) + assert out is None or ( + out.dtype == torch.int32 + and out.is_contiguous() + and out.shape == (scores.shape[0], token_topk + compress_ratio - 1) + ), "out: contiguous int32 [rows, token_topk + compress_ratio - 1]" + + +qsa_topk_expand_cluster = QsaTopkExpandClusterDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py new file mode 100644 index 000000000000..26d53f11ee6c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py @@ -0,0 +1,609 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA's block top-k and its expansion for a context chunk's rows over a wide score table. + +Same contract and bits as the family's `qsa_topk_expand_decode` (the `block_topk` largest of +each row's visible scores, ties to the higher block, in block order, -1 padded, then the token +expansion with the causal tail). One CTA a row reads the window twice from global, never staged: +a per-warp bound `lo` with at least `block_topk` keys at or above it, a candidate list of those +keys, radix passes that find the threshold's bin, and bitmaps whose scan orders the selected +blocks. Scores are non-negative fp32 read as int32 bit patterns; -0.0 ranks as zero. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute.runtime import from_dlpack + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + +#: Consecutive scores a thread holds per slab (one 16-byte load). +KPT = 4 +#: Threads a row (a CTA). +THREADS = 512 +#: Slabs whose loads are issued together. +UNROLL = 4 +#: Candidate list entries: about 4 % of a window survives `lo`, so ~100k-block windows fit. +CAPC = 3072 +_HIST = 2048 +_I32_MAX = 0x7FFFFFFF +#: CTAs an SM the register allocation is held to. +MIN_BLOCKS = 3 +#: Score columns this kernel serves: wider than one family CTA's registers, up to a +#: 262144-token table with MTP's extra pages. +MIN_WIDTH = 4096 + KPT +MAX_WIDTH = 1 << 17 + + +class _TopkRowsKernel: + def __init__(self, *, width, key_stride, block_topk, compress, token_topk, pdl): + self.nt = THREADS + self.nw = THREADS // 32 + self.slab = KPT * THREADS + self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( + width, + key_stride, + block_topk, + compress, + token_topk, + pdl, + ) + self.final = token_topk + compress - 1 + self.words = -(-width // 32) + self.wpt = -(-self.words // THREADS) + self.per_warp = -(-block_topk // self.nw) + # Values a lane keeps for the bound: each warp needs `per_warp` of them at or above + # its value, from at most 32 * vpl. + self.vpl = max(2, -(-(3 * self.per_warp) // 64)) + self.union_words = max(2 * self.words, _HIST) + assert self.per_warp <= 32 * self.vpl, "too few lane values for the per-warp bound" + + def smem_bytes(self) -> int: + return (self.union_words + 2 * CAPC + 64 + 32 + 16) * 4 + 6 * 128 + + @cute.jit + def __call__( + self, + keys_t: cute.Tensor, + end_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + len_ptr: cute.Pointer, + topk_ptr: cute.Pointer, + sel_ptr: cute.Pointer, + rows: cutlass.Int32, + stream, + ): + mKeys = cute.make_tensor( + keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) + ) + per_row = cute.make_layout((rows,), stride=(1,)) + mEnd = cute.make_tensor(end_ptr, per_row) + mPos = cute.make_tensor(pos_ptr, per_row) + mLen = cute.make_tensor(len_ptr, per_row) + mTopk = cute.make_tensor(topk_ptr, cute.make_layout((rows, self.k), stride=(self.k, 1))) + mSel = cute.make_tensor( + sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) + ) + ld_keys = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT + ) + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys).launch( + grid=[rows, 1, 1], + block=[self.nt, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=MIN_BLOCKS, + ) + + @cute.jit + def _load(self, ld, gK, row, s, tidx, n, kr): + """Slab `s`'s four keys of thread `tidx` into `kr` (a negative as zero, -1 past the + window).""" + col4 = s * self.nt + tidx + e0 = col4 * KPT + kr.fill(-1) + if e0 < n: + cute.copy(ld, gK[row, (None, col4)], kr) + for c in cutlass.range_constexpr(KPT): + b = cutlass.max(kr[c], cutlass.Int32(0)) + if e0 + c >= n: + b = cutlass.Int32(-1) + kr[c] = b + + @cute.jit + def _insert(self, mv, b): + """`b` into the thread's descending `vpl` largest (`mv[0]` the largest).""" + for j in cutlass.range_constexpr(self.vpl - 1, 0, -1): + mv[j] = cutlass.max(mv[j], cutlass.min(mv[j - 1], b)) + mv[0] = cutlass.max(mv[0], b) + + @cute.jit + def _load_full(self, ld, gK, row, s, tidx, kr): + """`_load` of a slab wholly inside the window (no bounds).""" + cute.copy(ld, gK[row, (None, s * self.nt + tidx)], kr) + for c in cutlass.range_constexpr(KPT): + kr[c] = cutlass.max(kr[c], cutlass.Int32(0)) + + @cute.jit + def _compact(self, kr, s0, tidx, lane, lo, state, ckey, cidx): + """The UNROLL slabs' keys at or above `lo` appended to the list (a warp scan of the + counts, one shared atomic a warp).""" + cnt = cutlass.Int32(0) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + if kr[u][c] >= lo: + cnt = cnt + 1 + incl = cnt + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + base = cutlass.Int32(0) + if lane == 31: + if incl > 0: + base = cute.arch.atomic_add(state.iterator + 5, incl, scope="cta") + base = cute.arch.shuffle_sync(base, offset=31) + pos = base + incl - cnt + if cnt > 0: + for u in cutlass.range_constexpr(UNROLL): + e0 = ((s0 + u) * self.nt + tidx) * KPT + for c in cutlass.range_constexpr(KPT): + if kr[u][c] >= lo: + if pos < CAPC: + ckey[pos] = kr[u][c] + cidx[pos] = e0 + c + pos = pos + 1 + + @cute.jit + def _excl_scan(self, v, warp, lane, ws, half): + """Block-wide exclusive prefix of `v` and the total (one barrier; `ws[half..half+15]`).""" + incl = v + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + ws[half + warp] = incl + cute.arch.sync_threads() + above = cutlass.Int32(0) + if lane < warp: + above = ws[half + lane] + total = cutlass.Int32(0) + if lane < self.nw: + total = ws[half + lane] + pre = cute.arch.warp_reduction_sum(above) + incl - v + total = cute.arch.warp_reduction_sum(total) + return pre, total + + @cute.jit + def _narrow(self, tidx, warp, lane, hist, ws, state, prefix, mask, need, shift): + """One pass's histogram read (and cleared) from the top bin down: the bin holding the + `need`-th key becomes the threshold's (prefix, mask, need left, keys in the bin); the + passes stop once that bin holds exactly the keys still needed or no bits are left.""" + bpt: cutlass.Constexpr = _HIST // self.nt + h = cute.make_rmem_tensor(cute.make_layout((bpt,), stride=(1,)), cutlass.Int32) + h.fill(0) + own = cutlass.Int32(0) + for j in cutlass.range_constexpr(bpt): + h[j] = hist[_HIST - 1 - bpt * tidx - j] + hist[_HIST - 1 - bpt * tidx - j] = cutlass.Int32(0) + own = own + h[j] + before, _ = self._excl_scan(own, warp, lane, ws, 0) + if before < need: + if need <= before + own: + run = before + for j in cutlass.range_constexpr(bpt): + if run < need: + if need <= run + h[j]: + digit = cutlass.Int32(_HIST - 1 - bpt * tidx - j) + state[0] = prefix | (digit << shift) + state[1] = mask | (cutlass.Int32(_HIST - 1) << shift) + state[2] = need - run + state[3] = h[j] + state[4] = cutlass.Int32( + cutlass.select_((h[j] == need - run) | (shift == 0), 1, 0) + ) + state[6] = cutlass.max(shift - 11, cutlass.Int32(0)) + run = run + h[j] + cute.arch.sync_threads() + + @cute.kernel + def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): + tidx, _, _ = cute.arch.thread_idx() + row, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + nt: cutlass.Constexpr = self.nt + k: cutlass.Constexpr = self.k + cr: cutlass.Constexpr = self.cr + words: cutlass.Constexpr = self.words + wpt: cutlass.Constexpr = self.wpt + smem = cutlass.utils.SmemAllocator() + # The radix histogram and the two bitmaps share one region: the passes clear every bin + # before the bitmaps are set. + region = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((self.union_words,), stride=(1,)), byte_alignment=128 + ) + hist = cute.make_tensor(region.iterator, cute.make_layout((_HIST,), stride=(1,))) + selw = cute.make_tensor(region.iterator, cute.make_layout((words,), stride=(1,))) + tiew = cute.make_tensor(region.iterator + words, cute.make_layout((words,), stride=(1,))) + ckey = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((CAPC,), stride=(1,)), byte_alignment=128 + ) + cidx = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((CAPC,), stride=(1,)), byte_alignment=128 + ) + ws = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((64,), stride=(1,)), byte_alignment=128 + ) + wv = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + state = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((16,), stride=(1,)), byte_alignment=128 + ) + + # 0. Shared state and the row's host-filled metadata, before the wait. + for b in cutlass.range_constexpr(-(-self.union_words // nt)): + if tidx + b * nt < self.union_words: + region[tidx + b * nt] = cutlass.Int32(0) + if tidx == 0: + # A window of at most k blocks takes every block (mask 0, prefix 0, nothing skipped). + state[0] = cutlass.Int32(0) + state[1] = cutlass.Int32(0) + state[2] = cutlass.Int32(0) + state[3] = cutlass.Int32(0) + state[4] = cutlass.Int32(1) + state[5] = cutlass.Int32(0) # candidates compacted + state[6] = cutlass.Int32(0) + n = mEnd[row] + seqlen = mLen[row] + visible = cutlass.Int32(mPos[row]) + 1 + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + gK = cute.logical_divide(mKeys, (None, KPT)) + slab: cutlass.Constexpr = self.slab + nslabs = (n + (slab - 1)) // slab + # Slabs in whole UNROLL groups wholly inside the window (no per-key bounds). + nfull = (n // (slab * UNROLL)) * UNROLL + kr = [ + cute.make_rmem_tensor(cute.make_layout((KPT,), stride=(1,)), cutlass.Int32) + for _ in range(UNROLL) + ] + + # 1. Each thread's `vpl` largest keys; the per-warp bound; `lo`. + lo = cutlass.Int32(0) + hb = cutlass.Int32(-1) + if n > k: + mv = cute.make_rmem_tensor(cute.make_layout((self.vpl,), stride=(1,)), cutlass.Int32) + mv.fill(-1) + for s0 in cutlass.range(0, nfull, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load_full(ld_keys, gK, row, s0 + u, tidx, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + self._insert(mv, kr[u][c]) + for s0 in cutlass.range(nfull, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + self._insert(mv, kr[u][c]) + m1 = mv[0] + have_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(self.vpl): + have_t = have_t + cutlass.Int32(cutlass.select_(mv[j] >= 0, 1, 0)) + have = cute.arch.warp_redux_sync(have_t, "add") + v = cutlass.Int32(-1) + if have >= self.per_warp: + t = cutlass.Int32(0) + # The top 16 bits (exponent and 7 mantissa bits): a key at or below the + # exact per-warp value is still a valid bound, within 1/128 of it. + for bit in cutlass.range_constexpr(30, 14, -1): + c = t | cutlass.Int32(1 << bit) + cnt_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(self.vpl): + cnt_t = cnt_t + cutlass.Int32(cutlass.select_(mv[j] >= c, 1, 0)) + cnt = cute.arch.warp_redux_sync(cnt_t, "add") + if cnt >= self.per_warp: + t = c + v = t + wmax = cute.arch.warp_redux_sync(m1, "max") + if lane == 0: + wv[warp] = v + wv[16 + warp] = wmax + cute.arch.sync_threads() + bound = cutlass.Int32(_I32_MAX) + gmax = cutlass.Int32(-1) + if lane < self.nw: + bound = wv[lane] + gmax = wv[16 + lane] + bound = cute.arch.warp_reduction(bound, lambda x, y: cutlass.min(x, y)) + gmax = cute.arch.warp_reduction_max(gmax) + lo = cutlass.max(bound, cutlass.Int32(0)) + # Every candidate shares the bits above the highest bit where `gmax` and `lo` + # differ; the radix passes walk down from there, 11 bits at a time. + diff = gmax ^ lo + if diff != 0: + hb = 31 - cutlass.Int32(cute.arch.clz(diff)) + low_bits = (cutlass.Int64(1) << cutlass.Int64(hb + 1)) - 1 + mask0 = cutlass.Int32(cutlass.Int64(_I32_MAX) & (cutlass.Int64(-1) ^ low_bits)) + if tidx == 0: + state[0] = gmax & mask0 # threshold prefix + state[1] = mask0 # its mask + state[2] = cutlass.Int32(k) # still needed at the threshold + state[3] = cutlass.Int32(0) # keys in the threshold's bin + state[4] = cutlass.Int32(cutlass.select_(hb >= 0, 0, 1)) # narrowing done + state[6] = cutlass.max(hb - 10, cutlass.Int32(0)) # the next pass's shift + + # 2. The candidates (keys at or above `lo`), unordered, with their blocks. + for s0 in cutlass.range(0, nfull, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load_full(ld_keys, gK, row, s0 + u, tidx, kr[u]) + self._compact(kr, s0, tidx, lane, lo, state, ckey, cidx) + for s0 in cutlass.range(nfull, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + self._compact(kr, s0, tidx, lane, lo, state, ckey, cidx) + cute.arch.sync_threads() + m = state[5] + listed = m <= CAPC + + # 3. The threshold's bin, narrowed by radix passes over the list (or the window) + # until it holds exactly the keys still needed or every bit is fixed. + for _p in cutlass.range_constexpr(3): + if state[4] == 0: + prefix = state[0] + mask = state[1] + need = state[2] + shift = state[6] + if listed: + for i in cutlass.range(tidx, m, nt, unroll=1): + key = ckey[i] + if (key & mask) == prefix: + cute.arch.atomic_add( + hist.iterator + ((key >> shift) & (_HIST - 1)), + cutlass.Int32(1), + scope="cta", + ) + else: + for s0 in cutlass.range(0, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + key = kr[u][c] + if key >= 0: + if (key & mask) == prefix: + cute.arch.atomic_add( + hist.iterator + ((key >> shift) & (_HIST - 1)), + cutlass.Int32(1), + scope="cta", + ) + cute.arch.sync_threads() + self._narrow(tidx, warp, lane, hist, ws, state, prefix, mask, need, shift) + prefix = state[0] + mask = state[1] + need = state[2] + at_thr = state[3] + if n > k: + if hb < 0: + # Every candidate equals the largest: the bin is all of them. + at_thr = m + + # 4. Bitmaps of the blocks above the threshold's bin and in it. + if listed: + for i in cutlass.range(tidx, m, nt, unroll=1): + mk = ckey[i] & mask + e = cidx[i] + bit = cutlass.Int32(1) << (e & 31) + if mk > prefix: + cute.arch.atomic_or(selw.iterator + (e >> 5), bit, scope="cta") + elif mk == prefix: + cute.arch.atomic_or(tiew.iterator + (e >> 5), bit, scope="cta") + else: + for s0 in cutlass.range(0, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + e0 = ((s0 + u) * nt + tidx) * KPT + for c in cutlass.range_constexpr(KPT): + key = kr[u][c] + e = e0 + c + bit = cutlass.Int32(1) << (e & 31) + if key >= 0: + mk = key & mask + if mk > prefix: + cute.arch.atomic_or(selw.iterator + (e >> 5), bit, scope="cta") + elif mk == prefix: + cute.arch.atomic_or(tiew.iterator + (e >> 5), bit, scope="cta") + cute.arch.sync_threads() + + # 5. Block order: each thread folds its `wpt` words' kept tie blocks (the lowest `skip` + # dropped) into the selected words; a scan gives each word its first output position. + skip = cutlass.max(at_thr - need, cutlass.Int32(0)) + w0 = tidx * wpt + own_q = cutlass.Int32(0) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + own_q = own_q + cute.arch.popc(tiew[w0 + j]) + tied_before, _ = self._excl_scan(own_q, warp, lane, ws, 0) + fw = cute.make_rmem_tensor(cute.make_layout((wpt,), stride=(1,)), cutlass.Int32) + fw.fill(0) + own_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + tw = tiew[w0 + j] + nq = cute.arch.popc(tw) + drop = cutlass.min(cutlass.max(skip - tied_before, cutlass.Int32(0)), nq) + # clear the lowest `drop` set bits of the word's tie class + for _d in cutlass.range(drop, unroll=1): + tw = tw & (tw - 1) + tied_before = tied_before + nq + f = selw[w0 + j] | tw + fw[j] = f + own_t = own_t + cute.arch.popc(f) + first, _ = self._excl_scan(own_t, warp, lane, ws, 32) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + selw[w0 + j] = fw[j] + tiew[w0 + j] = first + first = first + cute.arch.popc(fw[j]) + cute.arch.sync_threads() + # Output position `o` (one a thread a round): its word by binary search over the words' + # first positions, its block by the in-word rank of the set bit. + total = n + if n > k: + total = cutlass.Int32(k) + for c in cutlass.range_constexpr(-(-k // nt)): + o = tidx + c * nt + if o < total: + lo_w = cutlass.Int32(0) + hi_w = cutlass.Int32(words - 1) + while lo_w < hi_w: + mid = (lo_w + hi_w + 1) >> 1 + if tiew[mid] <= o: + lo_w = mid + else: + hi_w = mid - 1 + x = selw[lo_w] + r = o - tiew[lo_w] + bit = cutlass.Int32(0) + for h in (16, 8, 4, 2, 1): + cnt_h = cute.arch.popc((x >> bit) & cutlass.Int32((1 << h) - 1)) + if r >= cnt_h: + r = r - cnt_h + bit = bit + h + e = lo_w * 32 + bit + mTopk[row, o] = e + for jj in cutlass.range_constexpr(cr): + tok = e * cr + jj + val = cutlass.Int32(-1) + if tok < seqlen: + val = tok + mSel[row, o * cr + jj] = val + + # 6. Padding of the block list; the causal tail after the selected tokens, then -1. + for c in cutlass.range_constexpr(-(-k // nt)): + slot = tidx + c * nt + if slot < k: + if slot >= total: + mTopk[row, slot] = cutlass.Int32(-1) + tail_start = (visible // cr) * cr + tail_count = visible - tail_start + body = total * cr + if body > self.token_topk: + body = cutlass.Int32(self.token_topk) + for c in cutlass.range_constexpr(-(-self.final // nt)): + col = tidx + c * nt + if col < self.final: + if col >= body: + off = col - body + val = cutlass.Int32(-1) + if off < cr - 1: + if off < tail_count: + if tail_start + off < seqlen: + val = tail_start + off + mSel[row, col] = val + + +_COMPILED: dict = {} +_INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} + + +def supports(scores, topk_indices) -> bool: + """Whether this kernel serves a call over this score table and `block_topk`.""" + width = scores.shape[1] + return ( + scores.dtype == torch.float32 + and scores.stride(1) == 1 + and width % KPT == 0 + and scores.stride(0) % KPT == 0 + and MIN_WIDTH <= width <= MAX_WIDTH + and -(-topk_indices.shape[1] // (THREADS // 32)) <= 96 + ) + + +def qsa_topk_rows( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, +): + """`qsa_topk_expand_decode`'s contract, bit for bit, for wide tables (`supports`): writes + `topk_indices` and returns the token selection `[rows, token_topk + compress_ratio - 1]` + int32 (`out` when given).""" + rows, width = scores.shape + block_topk = topk_indices.shape[1] + sel = out + if sel is None: + sel = torch.empty( + (rows, token_topk + compress_ratio - 1), dtype=torch.int32, device=scores.device + ) + if rows == 0: + return sel + pdl = bool(get_env_enable_pdl()) + keys = scores.view(torch.int32) + key = ( + width, + scores.stride(0), + block_topk, + compress_ratio, + token_topk, + query_positions.dtype, + sequence_lengths.dtype, + pdl, + ) + ptrs = ( + row_ends.data_ptr(), + query_positions.data_ptr(), + sequence_lengths.data_ptr(), + topk_indices.data_ptr(), + sel.data_ptr(), + ) + fn = _COMPILED.get(key) + if fn is None: + gm = cute.AddressSpace.gmem + kernel = _TopkRowsKernel( + width=width, + key_stride=scores.stride(0), + block_topk=block_topk, + compress=compress_ratio, + token_topk=token_topk, + pdl=pdl, + ) + fn = _COMPILED[key] = cute.compile( + kernel, + from_dlpack(keys, assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.Int32, ptrs[0], gm, assumed_align=4), + make_ptr( + _INDEX_TYPES[query_positions.dtype], + ptrs[1], + gm, + assumed_align=query_positions.element_size(), + ), + make_ptr( + _INDEX_TYPES[sequence_lengths.dtype], + ptrs[2], + gm, + assumed_align=sequence_lengths.element_size(), + ), + make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + cutlass.Int32(rows), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + fn(keys, *ptrs, rows) + return sel diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py new file mode 100644 index 000000000000..182ddcdc831b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py @@ -0,0 +1,1222 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This target's copies of the QSA Triton kernels for sparse GQA and the K/V append +(`tensorrt_llm._torch.attention.backends.sparse.qsa.kernels`), compiled once per head geometry. + +Extents that follow the engine or the step (pool pages, block-table shape and row stride, row +count) are `do_not_specialize` and the metadata pointers `do_not_specialize_on_alignment`; +geometry, strides and launch tiles stay constexpr. Outputs are bitwise equal to the shared +launchers': the bodies differ only in unsigned range compares under `tl.assume` (decode and +store kernels) and row-major split-K partials in the 64-split bucket. +""" + +from __future__ import annotations + +import math +import os + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + compile_only, + first_time, +) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber +from tensorrt_llm._utils import get_sm_version + +_INTEGER_DTYPES = (torch.int32, torch.int64) +# Scores use base-2 exponentials for the online softmax (the shared kernels' constant). +_LOG2_E = tl.constexpr(1.4426950408889634) + + +def _require_same_cuda_device(name: str, *tensors: torch.Tensor) -> None: + if not tensors or not all(tensor.is_cuda for tensor in tensors): + raise ValueError(f"{name} requires CUDA tensors") + device = tensors[0].device + if any(tensor.device != device for tensor in tensors[1:]): + raise ValueError(f"{name} tensors must be on the same CUDA device") + + +def _pdl_enabled(rows: int) -> bool: + """The shared launcher's PDL rule: two- to four-row speculative graphs only.""" + return rows <= 4 and os.environ.get("TRTLLM_ENABLE_PDL", "1") == "1" and get_sm_version() >= 90 + + +#: Warps of a K/V append program (a store is a move: the warp count cannot change the result). +_KV_STORE_WARPS = 4 + + +def splitk_launch_config(rows: int, num_kv_heads: int, selected_width: int) -> tuple[int, int, int]: + """(BLOCK_N, splits, warps) by `rows * num_kv_heads`: the shared launcher's five buckets, + kept as they are so every launch reduces in the shared kernel's order.""" + base_programs = rows * num_kv_heads + if base_programs <= 4: + block_n, target_splits, num_warps = 16, 64, 4 + elif base_programs < 32: + block_n, target_splits, num_warps = 16, 32, 4 + elif base_programs <= 256: + block_n, target_splits, num_warps = 64, 8, 2 + elif base_programs <= 512: + block_n, target_splits, num_warps = 64, 4, 2 + else: + block_n, target_splits, num_warps = 64, 1, 2 + num_tiles = triton.cdiv(selected_width, block_n) + max_useful_splits = 1 << (num_tiles.bit_length() - 1) + return block_n, min(max_useful_splits, target_splits), num_warps + + +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=["request_indices", "logical_positions", "block_table"], +) +def _qsa_kv_store_stable_kernel( + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + num_rows, + num_pages, + num_requests, + page_table_width, + k_stride_row: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_row: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_dim: tl.constexpr, + k_cache_stride_page: tl.constexpr, + k_cache_stride_head: tl.constexpr, + k_cache_stride_token: tl.constexpr, + k_cache_stride_dim: tl.constexpr, + v_cache_stride_page: tl.constexpr, + v_cache_stride_head: tl.constexpr, + v_cache_stride_token: tl.constexpr, + v_cache_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_DIM: tl.constexpr, +): + # Extents are positive and below 2^31 (the launcher refuses empty ones), so each + # `0 <= x < extent` test is one unsigned compare. + tl.assume(num_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + row = tl.program_id(0) + head = tl.program_id(1) + dims = tl.arange(0, BLOCK_DIM) + valid = (row < num_rows) & (head < NUM_KV_HEADS) & (dims < HEAD_DIM) + + request = tl.load(request_indices + row).to(tl.int32) + logical = tl.load(logical_positions + row).to(tl.int32) + page_column = logical // TOKENS_PER_BLOCK + token_in_page = logical % TOKENS_PER_BLOCK + # `request` in [0, num_requests), `logical >= 0` and `page_column < page_table_width`. + valid_page_lookup = (request.to(tl.uint32) < num_requests.to(tl.uint32)) & ( + logical.to(tl.uint32) < (page_table_width * TOKENS_PER_BLOCK).to(tl.uint32) + ) + page32 = tl.load( + block_table + request * block_table_stride_request + page_column * block_table_stride_page, + mask=valid_page_lookup, + other=-1, + ) + valid &= page32.to(tl.uint32) < num_pages.to(tl.uint32) + page = page32.to(tl.int64) + + k_values = tl.load( + k + row * k_stride_row + head * k_stride_head + dims * k_stride_dim, + mask=valid, + other=0.0, + ) + v_values = tl.load( + v + row * v_stride_row + head * v_stride_head + dims * v_stride_dim, + mask=valid, + other=0.0, + ) + tl.store( + k_cache + + page * k_cache_stride_page + + head * k_cache_stride_head + + token_in_page * k_cache_stride_token + + dims * k_cache_stride_dim, + k_values, + mask=valid, + ) + tl.store( + v_cache + + page * v_cache_stride_page + + head * v_cache_stride_head + + token_in_page * v_cache_stride_token + + dims * v_cache_stride_dim, + v_values, + mask=valid, + ) + + +def qsa_paged_kv_store( + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, +) -> None: + """`triton_qsa_paged_kv_store`'s contract and checks, on `_qsa_kv_store_stable_kernel`.""" + if k.shape != v.shape or k.ndim != 3: + raise ValueError("QSA K and V inputs must have matching [tokens, heads, dim] shapes") + if k_cache.shape != v_cache.shape or k_cache.ndim != 4: + raise ValueError("QSA K and V caches must have matching [pages, heads, tokens, dim] shapes") + rows, num_kv_heads, head_dim = k.shape + if num_kv_heads <= 0 or head_dim <= 0: + raise ValueError("QSA K/V store requires non-empty head geometry") + if k_cache.shape[1] != num_kv_heads or k_cache.shape[3] != head_dim: + raise ValueError("QSA K/V inputs and caches have incompatible head geometry") + if k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA K/V cache token dimension does not match tokens_per_block") + if tokens_per_block <= 0: + raise ValueError("QSA tokens_per_block must be positive") + if k.dtype != v.dtype or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA K/V pairs must use matching dtypes") + if k.dtype not in (torch.bfloat16, torch.float16): + raise ValueError("QSA K/V inputs must use BF16 or FP16") + if k_cache.dtype not in (torch.bfloat16, torch.float16, torch.float8_e4m3fn): + raise ValueError("QSA K/V cache must use BF16, FP16, or FP8 E4M3") + if request_indices.shape != (rows,) or logical_positions.shape != (rows,): + raise ValueError("QSA K/V store metadata must have one entry per input row") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA K/V store requires a two-dimensional int32 block table") + if ( + request_indices.dtype not in _INTEGER_DTYPES + or logical_positions.dtype not in _INTEGER_DTYPES + ): + raise ValueError("QSA K/V store request metadata must use integer storage") + _require_same_cuda_device( + "QSA paged K/V store", + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + ) + if rows == 0: + return + if k_cache.shape[0] == 0 or block_table.shape[0] == 0 or block_table.shape[1] == 0: + raise ValueError("QSA K/V store requires non-empty cache and block-table geometry") + + block_dim = triton.next_power_of_2(head_dim) + _qsa_kv_store_stable_kernel[(rows, num_kv_heads)]( + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + rows, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + TOKENS_PER_BLOCK=tokens_per_block, + BLOCK_DIM=block_dim, + num_warps=_KV_STORE_WARPS, + ) + + +@triton.jit( + do_not_specialize=[ + "num_cache_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=[ + "block_table", + "selected_tokens", + "request_indices", + "query_positions", + ], +) +def _qsa_prefill_pair_stable_kernel( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, + num_cache_pages, + num_requests, + page_table_width, + q_stride_row: tl.constexpr, + q_stride_head: tl.constexpr, + q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, + v_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, + selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + GROUP_SIZE: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + TOPK: tl.constexpr, + BLOCK_TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, +): + row = tl.program_id(0) + kv_head = tl.program_id(1) + tl.assume(num_cache_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + request = tl.load(request_indices + row).to(tl.int64) + # Signed range tests here, as in the shared kernel (the unsigned form is slower over e4m3). + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + query_heads = kv_head * GROUP_SIZE + head_offsets + query_values = tl.load( + q + + row * q_stride_row + + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=(head_offsets < GROUP_SIZE)[:, None], + other=0.0, + ) + + running_max = tl.full([BLOCK_M], -float("inf"), tl.float32) + running_sum = tl.zeros([BLOCK_M], tl.float32) + accumulator = tl.zeros([BLOCK_M, HEAD_DIM], tl.float32) + token_offsets = tl.arange(0, BLOCK_N) + visible_tokens = tl.load(query_positions + row) + 1 + selected_blocks = tl.minimum(visible_tokens // COMPRESS_RATIO, BLOCK_TOPK) + valid_tokens = selected_blocks * COMPRESS_RATIO + visible_tokens % COMPRESS_RATIO + valid_tiles = tl.cdiv(tl.minimum(valid_tokens, TOPK), BLOCK_N) + + for tile in tl.range(0, valid_tiles, num_stages=2): + selected_columns = tile * BLOCK_N + token_offsets + logical_tokens = tl.load( + selected_tokens + row * selected_stride_row + selected_columns * selected_stride_token, + mask=selected_columns < TOPK, + other=-1, + ) + safe_tokens = tl.maximum(logical_tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + valid = ( + valid_request + & (selected_columns < TOPK) + & (logical_tokens >= 0) + & (logical_pages < page_table_width) + ) + pages32 = tl.load( + block_table + + safe_request * block_table_stride_request + + tl.minimum(logical_pages, page_table_width - 1) * block_table_stride_page, + mask=valid, + other=-1, + ) + physical_pages = pages32.to(tl.int64) + valid &= (physical_pages >= 0) & (physical_pages < num_cache_pages) + safe_pages = tl.maximum(physical_pages, 0) + + keys = tl.load( + k_cache + + safe_pages[None, :] * k_stride_page + + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + + dim_offsets[:, None] * k_stride_dim, + mask=valid[None, :], + other=0.0, + ) + values = tl.load( + v_cache + + safe_pages[:, None] * v_stride_page + + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + + dim_offsets[None, :] * v_stride_dim, + mask=valid[:, None], + other=0.0, + ) + keys = keys.to(query_values.dtype) + values = values.to(query_values.dtype) + scores = tl.dot(query_values, keys) * softmax_scale * _LOG2_E + scores = tl.where(valid[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where( + valid[None, :], + tl.math.exp2(scores - next_max[:, None]), + 0.0, + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + accumulator * correction[:, None], + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + + normalized = tl.where( + running_sum[:, None] > 0, + accumulator / tl.maximum(running_sum[:, None], 1.0e-20), + 0.0, + ) + tl.store( + output + + row * output_stride_row + + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized, + mask=(head_offsets < GROUP_SIZE)[:, None], + ) + + +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_cache_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=["block_table", "selected_tokens", "request_indices"], +) +def _qsa_splitk_stable_kernel( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + num_rows, + num_cache_pages, + num_requests, + page_table_width, + q_stride_row: tl.constexpr, + q_stride_head: tl.constexpr, + q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, + v_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, + selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + GROUP_SIZE: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + TOPK: tl.constexpr, + NUM_SPLITS: tl.constexpr, + NUM_TILES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, + USE_PDL: tl.constexpr, + ROW_MAJOR: tl.constexpr, +): + row = tl.program_id(0) + kv_head = tl.program_id(1) + split = tl.program_id(2) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.assume(num_cache_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + request = tl.load(request_indices + row).to(tl.int64) + valid_request = request.to(tl.uint64) < num_requests.to(tl.uint64) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + # A selected token is resident-addressable when `0 <= t` and `t // TOKENS_PER_BLOCK < + # page_table_width`: one unsigned compare against the table's token span. + token_span = (page_table_width * TOKENS_PER_BLOCK).to(tl.uint32) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + token_offsets = tl.arange(0, BLOCK_N) + query_heads = kv_head * GROUP_SIZE + head_offsets + query_values = tl.load( + q + + row * q_stride_row + + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=(head_offsets < GROUP_SIZE)[:, None], + other=0.0, + ) + + running_max = tl.full([BLOCK_M], -float("inf"), tl.float32) + running_sum = tl.zeros([BLOCK_M], tl.float32) + accumulator = tl.zeros([BLOCK_M, HEAD_DIM], tl.float32) + split_tile_start = split * NUM_TILES // NUM_SPLITS + split_tile_end = (split + 1) * NUM_TILES // NUM_SPLITS + + for tile in range(split_tile_start, split_tile_end): + selected_columns = tile * BLOCK_N + token_offsets + logical_tokens = tl.load( + selected_tokens + row * selected_stride_row + selected_columns * selected_stride_token, + mask=selected_columns < TOPK, + other=-1, + ) + safe_tokens = tl.maximum(logical_tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + valid = ( + valid_request & (selected_columns < TOPK) & (logical_tokens.to(tl.uint32) < token_span) + ) + pages32 = tl.load( + block_table + + safe_request * block_table_stride_request + + tl.minimum(logical_pages, page_table_width - 1) * block_table_stride_page, + mask=valid, + other=-1, + ) + valid &= pages32.to(tl.uint32) < num_cache_pages.to(tl.uint32) + safe_pages = tl.maximum(pages32.to(tl.int64), 0) + + keys = tl.load( + k_cache + + safe_pages[None, :] * k_stride_page + + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + + dim_offsets[:, None] * k_stride_dim, + mask=valid[None, :], + other=0.0, + ).to(query_values.dtype) + values = tl.load( + v_cache + + safe_pages[:, None] * v_stride_page + + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + + dim_offsets[None, :] * v_stride_dim, + mask=valid[:, None], + other=0.0, + ).to(query_values.dtype) + scores = tl.dot(query_values, keys) + scores *= softmax_scale * _LOG2_E + scores = tl.where(valid[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where( + valid[None, :], + tl.math.exp2(scores - next_max[:, None]), + 0.0, + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + accumulator * correction[:, None], + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + + has_values = running_sum > 0 + normalized = tl.where( + has_values[:, None], + accumulator / tl.maximum(running_sum[:, None], 1.0e-20), + 0.0, + ) + output_mask = (head_offsets < GROUP_SIZE)[:, None] + if NUM_SPLITS == 1: + tl.store( + output + + row * output_stride_row + + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized, + mask=output_mask, + ) + else: + partial_lse_values = tl.where( + has_values, + running_max + tl.math.log2(tl.maximum(running_sum, 1.0e-20)), + -float("inf"), + ) + # Partial slot of (split, row): row-major in the 64-split bucket, split-major (the shared + # layout) otherwise. Same values, merged in the same order. + if ROW_MAJOR: + slot = row * NUM_SPLITS + split + else: + slot = split * num_rows + row + tl.store( + partial_output + + (slot * NUM_QUERY_HEADS + query_heads[:, None]) * HEAD_DIM + + dim_offsets[None, :], + normalized, + mask=output_mask, + ) + tl.store( + partial_lse + slot * NUM_QUERY_HEADS + query_heads, + partial_lse_values, + mask=head_offsets < GROUP_SIZE, + ) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit(do_not_specialize=["num_rows"]) +def _qsa_merge_stable_kernel( + partial_output, + partial_lse, + output, + output_gate, + num_rows, + gate_stride_row: tl.constexpr, + gate_stride_head: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + BLOCK_SPLITS: tl.constexpr, + HEAD_DIM: tl.constexpr, + USE_PDL: tl.constexpr, + FUSE_OUTPUT_GATE: tl.constexpr, + ROW_MAJOR: tl.constexpr, +): + row = tl.program_id(0) + head = tl.program_id(1) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + split_offsets = tl.arange(0, BLOCK_SPLITS) + dim_offsets = tl.arange(0, HEAD_DIM) + split_mask = split_offsets < NUM_SPLITS + if ROW_MAJOR: + slots = row * NUM_SPLITS + split_offsets + else: + slots = split_offsets * num_rows + row + lse = tl.load( + partial_lse + slots * NUM_QUERY_HEADS + head, + mask=split_mask, + other=-float("inf"), + ) + max_lse = tl.max(lse, axis=0) + has_values = max_lse > -float("inf") + weights = tl.math.exp2(tl.where(split_mask & has_values, lse - max_lse, -float("inf"))) + denominator = tl.sum(weights, axis=0) + partials = tl.load( + partial_output + + (slots[:, None] * NUM_QUERY_HEADS + head) * HEAD_DIM + + dim_offsets[None, :], + mask=split_mask[:, None], + other=0.0, + ) + merged = tl.sum(partials * weights[:, None], axis=0) + merged = tl.where(denominator > 0, merged / denominator, 0.0) + if FUSE_OUTPUT_GATE: + # The merge-then-gate contract: the attention result is rounded to its output dtype + # before the fp32 gate. + attention = merged.to(output.dtype.element_ty).to(tl.float32) + gate = tl.load( + output_gate + row * gate_stride_row + head * gate_stride_head + dim_offsets + ).to(tl.float32) + merged = attention * tl.sigmoid(gate) + tl.store( + output + + row * output_stride_row + + head * output_stride_head + + dim_offsets * output_stride_dim, + merged, + ) + + +def can_fuse_splitk_output_gate( + q: torch.Tensor, + k_cache: torch.Tensor, + selected_tokens: torch.Tensor, + output_gate: torch.Tensor, +) -> bool: + """Whether the split-K merge can consume an attention output gate (the shared rule).""" + if q.ndim != 3 or selected_tokens.ndim != 2 or selected_tokens.shape[1] <= 0: + return False + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or k_cache.ndim != 4: + return False + if not output_gate.is_cuda or output_gate.device != q.device or output_gate.dtype != q.dtype: + return False + if output_gate.stride(-1) != 1: + return False + if output_gate.ndim == 3: + if output_gate.shape != q.shape: + return False + elif output_gate.ndim == 2: + if output_gate.shape != (rows, num_q_heads * head_dim): + return False + else: + return False + _, num_splits, _ = splitk_launch_config(rows, k_cache.shape[1], selected_tokens.shape[1]) + return num_splits > 1 + + +def qsa_paged_sparse_gqa( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, +) -> torch.Tensor: + """`triton_qsa_paged_sparse_gqa`'s contract, checks and branch choice, on these kernels.""" + if q.ndim != 3: + raise ValueError("QSA queries must have shape [rows, query_heads, head_dim]") + if k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA K/V caches must have matching HND paged shapes") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA sparse GQA requires a two-dimensional int32 block table") + if selected_tokens.ndim != 2 or selected_tokens.dtype != torch.int32: + raise ValueError("QSA selected tokens must be a two-dimensional int32 tensor") + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or selected_tokens.shape[1] <= 0: + raise ValueError("QSA selected-token rows must match non-empty query rows") + if request_indices.shape != (rows,) or request_indices.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA request indices must have one integer entry per row") + if not request_indices.is_contiguous(): + raise ValueError("QSA request indices must be contiguous") + if tokens_per_block <= 0 or k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA cache token dimension must match positive tokens_per_block") + if k_cache.shape[3] != head_dim or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA query and K/V cache head geometry is incompatible") + if q.dtype not in (torch.bfloat16, torch.float16): + raise ValueError("QSA queries must use BF16 or FP16") + if k_cache.dtype not in (torch.bfloat16, torch.float16, torch.float8_e4m3fn): + raise ValueError("QSA K/V cache must use BF16, FP16, or FP8 E4M3") + if head_dim <= 0 or head_dim & (head_dim - 1): + raise ValueError("QSA sparse GQA requires a power-of-two head dimension") + num_kv_heads = k_cache.shape[1] + if num_kv_heads <= 0 or num_q_heads <= 0 or num_q_heads % num_kv_heads: + raise ValueError("QSA query heads must be divisible by non-zero KV heads") + if k_cache.shape[0] <= 0 or block_table.shape[0] <= 0 or block_table.shape[1] <= 0: + raise ValueError("QSA sparse GQA requires non-empty cache and block-table geometry") + if not math.isfinite(softmax_scale) or softmax_scale <= 0: + raise ValueError("QSA sparse GQA softmax scale must be finite and positive") + _require_same_cuda_device( + "QSA paged sparse GQA", + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + ) + group_size = num_q_heads // num_kv_heads + block_m = max(16, triton.next_power_of_2(group_size)) + # Contiguous at every row count: the output strides are kernel constexprs, and `empty_like` + # would keep a one-row q's (dense) strides and compile a variant of its own. + output = torch.empty(q.shape, dtype=q.dtype, device=q.device) + if rows == 0: + return output + if output_gate is not None and not can_fuse_splitk_output_gate( + q, k_cache, selected_tokens, output_gate + ): + raise ValueError("QSA output-gate fusion requires a compatible split-K merge") + + if (query_positions is None) != (compress_ratio is None): + raise ValueError("QSA causal-prefix bounds require positions and compression ratio") + if query_positions is not None: + if query_positions.shape != (rows,): + raise ValueError("QSA query positions must match sparse-attention rows") + if query_positions.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA query positions must use integer storage") + _require_same_cuda_device("QSA causal-prefix bounds", q, query_positions) + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + + base_programs = rows * num_kv_heads + if query_positions is not None and base_programs > 512: + final_topk = selected_tokens.shape[1] + token_topk = final_topk - compress_ratio + 1 + if token_topk <= 0 or token_topk % compress_ratio: + raise ValueError("QSA selected-token width is incompatible with its compression ratio") + _qsa_prefill_pair_stable_kernel[(rows, num_kv_heads)]( + *_pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions.contiguous(), + output, + softmax_scale, + ), + **_pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio), + ) + return output + + block_n, num_splits, num_warps = splitk_launch_config( + rows, num_kv_heads, selected_tokens.shape[1] + ) + use_pdl = num_splits > 1 and _pdl_enabled(rows) + # Row-major partials in the 64-split bucket (`splitk_launch_config`: rows x KV heads <= 4). + row_major = rows * num_kv_heads <= 4 + if num_splits == 1: + partial_output = output + partial_lse = output + else: + lead = (rows, num_splits) if row_major else (num_splits, rows) + partial_output = torch.empty( + (*lead, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + partial_lse = torch.empty((*lead, num_q_heads), dtype=torch.float32, device=q.device) + gate = output if output_gate is None else output_gate + _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + output, + gate, + output_gate, + softmax_scale, + group_size, + block_m, + tokens_per_block, + None if query_positions is None else (compress_ratio, query_positions.dtype), + ) + _qsa_splitk_stable_kernel[(rows, num_kv_heads, num_splits)]( + *_splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, + ), + **_splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, + ), + ) + if num_splits == 1: + return output + _qsa_merge_stable_kernel[(rows, num_q_heads)]( + *_merge_args(partial_output, partial_lse, output, gate, output_gate, rows, head_dim), + **_merge_constexprs(num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate), + ) + return output + + +def _pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, +) -> tuple: + return ( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + selected_tokens.stride(0), + selected_tokens.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio) -> dict: + final_topk = selected_tokens.shape[1] + token_topk = final_topk - compress_ratio + 1 + return dict( + GROUP_SIZE=group_size, + TOKENS_PER_BLOCK=k_cache.shape[2], + TOPK=final_topk, + BLOCK_TOPK=token_topk // compress_ratio, + COMPRESS_RATIO=compress_ratio, + BLOCK_M=block_m, + BLOCK_N=64, + HEAD_DIM=q.shape[2], + num_warps=2, + num_stages=2, + ) + + +def _splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, +) -> tuple: + return ( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + selected_tokens.stride(0), + selected_tokens.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, +) -> dict: + return dict( + GROUP_SIZE=group_size, + NUM_QUERY_HEADS=q.shape[1], + TOKENS_PER_BLOCK=tokens_per_block, + TOPK=selected_tokens.shape[1], + NUM_SPLITS=num_splits, + NUM_TILES=triton.cdiv(selected_tokens.shape[1], block_n), + BLOCK_M=block_m, + BLOCK_N=block_n, + HEAD_DIM=q.shape[2], + USE_PDL=use_pdl, + ROW_MAJOR=row_major, + num_warps=num_warps, + num_stages=2, + launch_pdl=use_pdl, + ) + + +def _merge_args(partial_output, partial_lse, output, gate, output_gate, rows, head_dim) -> tuple: + if output_gate is not None and output_gate.ndim == 3: + gate_stride_row = output_gate.stride(0) + gate_stride_head = output_gate.stride(1) + else: + gate_stride_row = gate.stride(0) + gate_stride_head = head_dim + return ( + partial_output, + partial_lse, + output, + gate, + rows, + gate_stride_row, + gate_stride_head, + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _merge_constexprs(num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate) -> dict: + return dict( + NUM_QUERY_HEADS=num_q_heads, + NUM_SPLITS=num_splits, + BLOCK_SPLITS=triton.next_power_of_2(num_splits), + HEAD_DIM=head_dim, + USE_PDL=use_pdl, + FUSE_OUTPUT_GATE=output_gate is not None, + ROW_MAJOR=row_major, + num_warps=2, + num_stages=1, + launch_pdl=use_pdl, + ) + + +#: Row counts reaching every split-K launch variant at one and two KV heads. +_BUCKET_ROWS = (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 128, 129, 256, 257, 512, 513) + + +def decode_bucket_variants(num_kv_heads: int, selected_width: int) -> list[tuple]: + """`(block_n, num_splits, num_warps, use_pdl, row_major)` of every split-K launch, any rows.""" + seen = [] + for rows in _BUCKET_ROWS: + block_n, num_splits, num_warps = splitk_launch_config(rows, num_kv_heads, selected_width) + variant = ( + block_n, + num_splits, + num_warps, + num_splits > 1 and _pdl_enabled(rows), + rows * num_kv_heads <= 4, + ) + if variant not in seen: + seen.append(variant) + return seen + + +def _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + output, + gate, + output_gate, + softmax_scale, + group_size, + block_m, + tokens_per_block, + pair=None, +) -> None: + """Compile (not run) the split-K and merge kernels for every row bucket of this call's + geometry the first time it is seen, so no serving step compiles one.""" + key = ( + q.dtype, + q.shape[1:], + q.stride(), + q.data_ptr() % 16 == 0, + k_cache.dtype, + k_cache.shape[1:], + k_cache.stride(), + v_cache.stride(), + k_cache.data_ptr() % 16 == 0 and v_cache.data_ptr() % 16 == 0, + block_table.stride(1), + selected_tokens.shape[1], + selected_tokens.stride(), + request_indices.dtype, + None if output_gate is None else (output_gate.shape[1:], output_gate.stride()), + tokens_per_block, + pair, + ) + if not first_time("qsa_stable_decode_buckets", key): + return + if output is None: + output = torch.empty(q.shape, dtype=q.dtype, device=q.device) + gate = output + rows = q.shape[0] + num_q_heads, head_dim = q.shape[1], q.shape[2] + scratch = torch.empty(16, dtype=torch.float32, device=q.device) + variants = decode_bucket_variants(k_cache.shape[1], selected_tokens.shape[1]) + splitk, merge = [], [] + for block_n, num_splits, num_warps, use_pdl, row_major in variants: + partials = (output, output) if num_splits == 1 else (scratch, scratch) + splitk.append( + ( + _splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + *partials, + output, + softmax_scale, + rows, + ), + _splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, + ), + ) + ) + if num_splits > 1: + merge.append( + ( + _merge_args(scratch, scratch, output, gate, output_gate, rows, head_dim), + _merge_constexprs( + num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate + ), + ) + ) + for args, constexprs in splitk: + compile_only(_qsa_splitk_stable_kernel, args, **constexprs) + for args, constexprs in merge: + compile_only(_qsa_merge_stable_kernel, args, **constexprs) + if pair is not None: + # A context launch past 512 programs over the same pool (the prefill pair kernel). + compress_ratio, positions_dtype = pair + positions = torch.empty(1, dtype=positions_dtype, device=q.device) + compile_only( + _qsa_prefill_pair_stable_kernel, + _pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + positions, + output, + softmax_scale, + ), + **_pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio), + ) + + +def warm_decode_buckets( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + compress_ratio: int, +) -> None: + """Compile (not run) the split-K composition for every row bucket of a decode call's + geometry, and the prefill pair kernel (int64 positions), for a caller that serves part of + the row range with another kernel.""" + num_q_heads = q.shape[1] + group_size = num_q_heads // k_cache.shape[1] + block_m = max(16, triton.next_power_of_2(group_size)) + _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + None, + None, + None, + softmax_scale, + group_size, + block_m, + tokens_per_block, + (compress_ratio, torch.int64), + ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py new file mode 100644 index 000000000000..bbe60a559c65 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py @@ -0,0 +1,700 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A sparse decode attention layer's work between its input projection and its index +scores, one PDL launch, at one rank's heads over a bf16 or e4m3 K/V pool. + +This target's copy of the NVFP4 default target's entry of the same name, plus an optional row +map for the index query. It fuses both planes' `fused_qk_norm_rope`, `qsa_paged_index_store`, +`qsa_prefill_compress`, the packed-QKV copy and `qsa_paged_kv_store`. Per row: `HPB`-head main +programs (q, k, K pool), one index-query program, one index-key / coordinate / compression +program, one program per v head. + +With PDL, every operand not written since the sublayer's hyper-connection up-gate-mean (page and +slot, rotation factors, norm weights, a group's earlier keys from the index plane) is read +before `griddepcontrol.wait`; after it a program reads only projection rows. The row mapping +and positions are written ahead of that up-gate-mean, which never releases its dependents early. +""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + + +@triton.jit +def _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, D: tl.constexpr, ROT: tl.constexpr, + S1: tl.constexpr, S2: tl.constexpr): # fmt: skip + """A D-wide head's rotation factors at the row's positions (interleaved mRoPE: pair j + reads coordinate 1 when j % 3 == 1 and j < 3 * S1, coordinate 2 when j % 3 == 2 and + j < 3 * S2, else 0): cos, and sin signed for the NeoX halves; identity past ROT.""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + j = d % HALF + p0 = tl.load(positions + token) + p1 = tl.load(positions + num_tokens + token) + p2 = tl.load(positions + 2 * num_tokens + token) + pos = tl.where((j % 3 == 1) & (j < 3 * S1), p1, tl.where((j % 3 == 2) & (j < 3 * S2), p2, p0)) + row = cos_sin + pos.to(tl.int64) * stride_cs + c = tl.load(row + j, mask=rot, other=1.0) + s = tl.load(row + HALF + j, mask=rot, other=0.0) + s = tl.where(d < HALF, -s, s) + return c, s + + +@triton.jit +def _norm_weights( + heads, weight, weight_k, D: tl.constexpr, ROT: tl.constexpr, WEIGHT_SPLIT: tl.constexpr +): + """`1 + w` for a tile's elements and for their rotation partners; heads at or past + WEIGHT_SPLIT (when > 0) take the second weight (the k heads).""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + partner = tl.where(d < HALF, d + HALF, tl.where(rot, d - HALF, d)) + w = tl.load(weight + d).to(tl.float32) + wp = tl.load(weight + partner).to(tl.float32) + if WEIGHT_SPLIT > 0: + is_k = (heads >= WEIGHT_SPLIT)[:, None] + wk = tl.load(weight_k + d).to(tl.float32) + wkp = tl.load(weight_k + partner).to(tl.float32) + w2 = tl.where(is_k, wk[None, :], w[None, :]) + 1.0 + wp2 = tl.where(is_k, wkp[None, :], wp[None, :]) + 1.0 + else: + w2 = w[None, :] + 1.0 + wp2 = wp[None, :] + 1.0 + return w2, wp2 + + +@triton.jit +def _norm_rope_apply(src, dst, heads, head_mask, w2, wp2, c, s, eps, D: tl.constexpr, ROT: tl.constexpr, + dst_head_stride): # fmt: skip + """rms over each head, `* (1 + w)`, rotate; fp32 throughout, one rounding to `dst`'s + dtype, stored at `dst`. Returns the stored (rounded) tile.""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + partner = tl.where(d < HALF, d + HALF, tl.where(rot, d - HALF, d)) + hm = head_mask[:, None] + x = tl.load(src + heads[:, None] * D + d[None, :], mask=hm, other=0.0).to(tl.float32) + xp = tl.load(src + heads[:, None] * D + partner[None, :], mask=hm & rot[None, :], other=0.0).to( + tl.float32 + ) + rstd = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps) + y = x * rstd[:, None] * w2 + yp = xp * rstd[:, None] * wp2 + out = (y * c[None, :] + yp * s[None, :]).to(dst.dtype.element_ty) + tl.store(dst + heads[:, None] * dst_head_stride + d[None, :], out, mask=hm) + return out + + +@triton.jit +def _compress_norm_rope(x, weight, cosine, sine, eps, HEAD_DIM: tl.constexpr, ROTARY_PAIRS: tl.constexpr): # fmt: skip + """QSA's `_qsa_gemma_norm_rope` for one row with its static factors given (`weight` is + `1 + w`; the fp32 table makes its intermediate rounding to the table's dtype a no-op).""" + x_f32 = x.to(tl.float32) + reciprocal_rms = tl.rsqrt(tl.sum(x_f32 * x_f32, axis=1) / HEAD_DIM + eps) + normalized = x_f32 * reciprocal_rms[:, None] * weight[None, :] + rotated, passthrough = tl.split( + tl.permute(tl.reshape(normalized, (1, 2, HEAD_DIM // 2)), (0, 2, 1)) + ) + first, second = tl.split(tl.permute(tl.reshape(rotated, (1, 2, ROTARY_PAIRS)), (0, 2, 1))) + output_first = first * cosine[None, :] - second * sine[None, :] + output_second = second * cosine[None, :] + first * sine[None, :] + rotated = tl.reshape( + tl.permute(tl.join(output_first, output_second), (0, 2, 1)), (1, HEAD_DIM // 2) + ) + return tl.reshape( + tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), + (1, HEAD_DIM), + ).to(tl.bfloat16) + + +@triton.jit(do_not_specialize=["num_tokens", "num_requests", "table_width", "num_pages"]) +def _attn_decode_prologue_kernel( + proj, + stride_proj, + out, + stride_out, + index_q, + stride_iq_token, + stride_iq_head, + positions, + num_tokens, + cos_sin, + stride_cs, + q_weight, + k_weight, + index_q_weight, + index_k_weight, + eps, + k_cache, + v_cache, + s_kc_page, + s_kc_head, + s_kc_tok, + s_vc_page, + s_vc_head, + s_vc_tok, + request_indices, + logical_positions, + block_table, + s_bt_req, + s_bt_col, + num_requests, + table_width, + num_pages, + index_cache, + s_ic_page, + s_ic_slot, + position_cache, + s_pc_page, + s_pc_slot, + s_pc_axis, + coords, + s_c_row, + s_c_axis, + index_q_rows, + HQ: tl.constexpr, + HK: tl.constexpr, + D: tl.constexpr, + IH: tl.constexpr, + ID: tl.constexpr, + IQ_COL: tl.constexpr, + ROT: tl.constexpr, + S1: tl.constexpr, + S2: tl.constexpr, + HPB: tl.constexpr, + MAIN_PROGRAMS: tl.constexpr, + TPB: tl.constexpr, + CR: tl.constexpr, + ROTARY_PAIRS: tl.constexpr, + IN_LAUNCH_GROUPS: tl.constexpr, + IQ_ROW_MAP: tl.constexpr, + USE_PDL: tl.constexpr, +): + token = tl.program_id(0).to(tl.int64) + program = tl.program_id(1) + if USE_PDL: + # The consumer (the index scores) reads nothing of this launch before its own wait. + tl.extra.cuda.gdc_launch_dependents() + row = proj + token * stride_proj + out_row = out + token * stride_out + + # The row's page and slot (shared by the K/V pool, index plane and coordinates); an unmapped + # row writes no cache. Written before the up-gate-mean, so read before the wait. + request = tl.load(request_indices + token).to(tl.int64) + logical = tl.load(logical_positions + token).to(tl.int64) + column = logical // TPB + slot = logical % TPB + live = (request >= 0) & (request < num_requests) & (logical >= 0) & (column < table_width) + page = tl.load(block_table + request * s_bt_req + column * s_bt_col, mask=live, other=-1).to( + tl.int64 + ) + live = live & (page >= 0) & (page < num_pages) + + if program < MAIN_PROGRAMS: + main_heads = program * HPB + tl.arange(0, HPB) + c, s = _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, D, ROT, S1, S2) + w2, wp2 = _norm_weights(main_heads, q_weight, k_weight, D, ROT, HQ) + if USE_PDL: + gdc_wait_with_memory_clobber() + rotated = _norm_rope_apply( + row, out_row, main_heads, main_heads < HQ + HK, w2, wp2, c, s, eps, D, ROT, D + ) + if program * HPB + HPB > HQ: + # The rounded rows the composition's store read back from the packed buffer. + is_k = (main_heads >= HQ) & (main_heads < HQ + HK) & live + k_heads = (main_heads - HQ).to(tl.int64)[:, None] + d = tl.arange(0, D) + tl.store( + k_cache + page * s_kc_page + k_heads * s_kc_head + slot * s_kc_tok + d[None, :], + rotated.to(k_cache.dtype.element_ty), + mask=is_k[:, None], + ) + elif program == MAIN_PROGRAMS: + index_heads = tl.arange(0, IH) + c, s = _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, ID, ROT, 0, 0) + w2, wp2 = _norm_weights(index_heads, index_q_weight, index_q_weight, ID, ROT, 0) + if USE_PDL: + gdc_wait_with_memory_clobber() + iq_token = token + if IQ_ROW_MAP: + # The row's slot in the split index selection's permuted order. + iq_token = tl.load(index_q_rows + token).to(tl.int64) + _norm_rope_apply( + row + IQ_COL, index_q + iq_token * stride_iq_token, index_heads, index_heads < IH, + w2, wp2, c, s, eps, ID, ROT, stride_iq_head, + ) # fmt: skip + elif program >= MAIN_PROGRAMS + 2: + # One v head a program: the V pool, as the K/V append stores it, and the packed buffer. + h = program - MAIN_PROGRAMS - 2 + d = tl.arange(0, D) + if USE_PDL: + gdc_wait_with_memory_clobber() + v = tl.load(row + (HQ + HK + h) * D + d) + tl.store(out_row + (HQ + HK + h) * D + d, v) + if live: + tl.store( + v_cache + page * s_vc_page + h * s_vc_head + slot * s_vc_tok + d, + v.to(v_cache.dtype.element_ty), + ) + else: + # Before the wait: the coordinates and, at a group's last token, the group's earlier raw + # keys, the first token's coordinate 0, its rotation and the key norm weight. + axes = tl.arange(0, 4) + coordinate = tl.load(coords + token * s_c_row + axes * s_c_axis, mask=axes < 3, other=0) + dims = tl.arange(0, ID) + group_end = live & ((logical + 1) % CR == 0) + offsets = tl.arange(0, CR) + members = logical - (CR - 1) + offsets + member_pages = tl.load( + block_table + request * s_bt_req + (tl.maximum(members, 0) // TPB) * s_bt_col, + mask=group_end, + other=0, + ).to(tl.int64) + earlier = group_end & (offsets < CR - 1) + if IN_LAUNCH_GROUPS: + # A request's rows are consecutive at consecutive positions: an earlier member at + # row `token - (CR - 1) + o` of the same request is written by this launch, so its + # raw key (and, as the first member, its coordinates) come from its own row. + member_rows = token - (CR - 1) + offsets + maybe = earlier & (member_rows >= 0) + member_requests = tl.load(request_indices + member_rows, mask=maybe, other=-1) + member_logical = tl.load(logical_positions + member_rows, mask=maybe, other=-1) + in_launch = ( + maybe + & (member_requests.to(tl.int64) == request) + & (member_logical.to(tl.int64) == members) + ) + else: + in_launch = offsets < 0 + group = tl.load( + index_cache + + member_pages[:, None] * s_ic_page + + (tl.maximum(members, 0) % TPB)[:, None] * s_ic_slot + + dims[None, :], + mask=(earlier & ~in_launch)[:, None], + other=0.0, + ) + first = tl.maximum(logical - (CR - 1), 0) + first_page = tl.load( + block_table + request * s_bt_req + (first // TPB) * s_bt_col, mask=group_end, other=0 + ).to(tl.int64) + first_base = position_cache + first_page * s_pc_page + (first % TPB) * s_pc_slot + if IN_LAUNCH_GROUPS: + # A first member written by this launch has its coordinates in its row. + first_in_launch = tl.sum(tl.where(offsets == 0, in_launch, False).to(tl.int32)) > 0 + cached = group_end & ~first_in_launch + pos_t = tl.load(coords + (token - (CR - 1)) * s_c_row, mask=first_in_launch, other=0) + pos_t = tl.where(cached, tl.load(first_base, mask=cached, other=0), pos_t) + else: + pos_t = tl.load(first_base, mask=group_end, other=0) + # The compressed key rotates on the first token's coordinate 0 (QSA's plain rope). + pairs = tl.arange(0, ROTARY_PAIRS) + cs_row = cos_sin + pos_t.to(tl.int64) * stride_cs + cosine = tl.load(cs_row + pairs, mask=group_end, other=0.0) + sine = tl.load(cs_row + ROTARY_PAIRS + pairs, mask=group_end, other=0.0) + key_weight = tl.load(index_k_weight + dims).to(tl.float32) + 1.0 + if USE_PDL: + gdc_wait_with_memory_clobber() + if live: + # The coordinates, as the index store writes them. + tl.store( + position_cache + page * s_pc_page + slot * s_pc_slot + axes * s_pc_axis, + coordinate.to(position_cache.dtype.element_ty), + mask=axes < 3, + ) + # The raw index key -- or, at a group's last token, the group compressed in + # its place (the compression kernel overwrites the same slot). + current_k = tl.load(row + IQ_COL + IH * ID + dims) + stored_k = current_k.to(index_cache.dtype.element_ty) + if (logical + 1) % CR == 0: + if IN_LAUNCH_GROUPS: + launch_k = tl.load( + proj + + member_rows[:, None] * stride_proj + + IQ_COL + + IH * ID + + dims[None, :], + mask=in_launch[:, None], + other=0.0, + ) + group = tl.where( + in_launch[:, None], launch_k.to(index_cache.dtype.element_ty), group + ) + group = tl.where(offsets[:, None] == CR - 1, stored_k[None, :], group) + pooled = (tl.sum(group.to(tl.float32), axis=0) / CR).to(tl.bfloat16) + stored_k = tl.reshape( + _compress_norm_rope( + pooled[None, :], key_weight, cosine, sine, eps, ID, ROTARY_PAIRS + ), + (ID,), + ) + tl.store(index_cache + page * s_ic_page + slot * s_ic_slot + dims, stored_k) + + +def _launch(num_tokens: int) -> Tuple[int, int]: + """Main-plane heads per program and warps per program, by row count.""" + if num_tokens <= 16: + return 1, 2 + if num_tokens <= 64: + return 2, 2 + return 8, 8 + + +#: Rank geometries: tp4 / tep4 6 q heads and 1 KV head; tp2 / tep2 12 and 1; dep4 / dep2 / tp1 +#: 24 and 2 (heads of 256, a 4 x 128 index plane). +_TP4 = dict(geom="tp4") +_TP2 = dict(geom="tp2") +_DEP4 = dict(geom="dep4") + + +class Qwen4ExpAttnDecodePrologue(OpWrapper): + """The main and index planes' Gemma norm + rope, the packed-QKV copy, the paged K/V append, + the index key and coordinate store and the group compression of one sparse decode + attention layer, in one launch. Returns `(out, index_q)`. + + `proj` `[T, W]` (read only): q heads `[0, Hq)`, k heads `[Hq, Hq + Hk)` and v heads + `[Hq + Hk, Hq + 2 Hk)` of `head_dim`, the index query heads at `index_q_col` and the raw + index key right after them. Writes `out` `[T, (Hq + 2 Hk) * head_dim]` (q and k normalized + and rotated, v copied); `index_q` `[T, index_heads, index_head_dim]`; the rotated k and raw + v into `k_cache` / `v_cache` `[pages, Hk, tpb, head_dim]` (bf16 or e4m3); the raw index key + (or, at a group's last token, the compressed group) and the coordinates into the index + plane and the position cache. + + The rotation is NeoX over the first `rotary_dim` dims, factors from `cos_sin` (fp32 + `[P, rotary_dim]`, cos block then sin block). The main plane uses interleaved mRoPE over + `position_ids` `[3, T]` (pair `j` reads coordinate 1 when `j % 3 == 1 and j < 3 * + mrope_section1`, 2 when `j % 3 == 2 and j < 3 * mrope_section2`, else 0); the index plane + and the compressed key (`qsa_prefill_compress`'s, at the group's first token) use + coordinate 0. `rows_per_request` > 1 declares a verify step: each request's rows + consecutive, at consecutive positions. With `index_q_rows` (`[T]` int), row t's index query + goes to `index_q[index_q_rows[t]]`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="tep4 rank, verify 2 requests x 4 rows: a group end on a page's last slot " + "with one member in the plane, one with all three in the launch; zero, -0.0 and " + "outlier blocks in k and v", + spec=dict(_TP4, pos=(4093, 8696), rows_per_request=4, edge=True), + ), + Cell( + why="tep4 rank, MTP-7 verify 2 requests x 8 rows, one up to position 262143 " + "(the table's last row)", + spec=dict(_TP4, pos=(1001, 262136), rows_per_request=8), + ), + Cell( + why="tp4 rank, decode 3 rows: a group end, a mid-group row, an unmapped row", + spec=dict(_TP4, pos=(8699, 1201, 31), unmapped=2), + ), + Cell( + why="tp4 rank, decode 72 rows (eight-head programs: the one main program masks " + "its eighth head)", + spec=dict(_TP4, pos=tuple(1000 + 113 * r for r in range(72))), + ), + Cell( + why="tep4 rank, engine-laid pool: K/V the role planes of layer slot 11 of a " + "24-role page, the index plane strided by 12 layers' pages, rows on pages past " + "2^31 elements of the pool", + spec=dict(_TP4, pos=(900, 1203), rows_per_request=4, engine=True, large=True), + ), + Cell( + why="tep2 rank, 32 requests x 4 MTP-3 verify rows (a generation server's 128-row " + "graph), e4m3 pool", + spec=dict( + _TP2, pos=tuple(3000 + 6007 * r for r in range(32)), rows_per_request=4, fp8=True + ), + ), + Cell( + why="dep4 rank, 128 requests x 4 MTP-3 verify rows (ht_dep4's largest graph " + "batch, eight-head programs), e4m3 pool", + spec=dict( + _DEP4, + pos=tuple(3000 + 1531 * r for r in range(128)), + rows_per_request=4, + fp8=True, + ), # fmt: skip + ), + Cell( + why="dep2 / tp1 rank, decode 32 requests over the core's bf16 pool", + spec=dict(_DEP4, pos=tuple(2000 + 151 * r for r in range(32))), + ), + ) + + note = """ + Source: the Triton kernel above, grid `(T, cdiv(Hq + Hk, HPB) + 2 + Hk)`; PDL when the + engine enables it. Pages, rows and request slots index in int64 (pools past 2^31 elements). + + Preconditions: the index plane one KV head of `index_head_dim` (bf16); the position cache + int32 `[pages, tpb, >=3]`; `compress_ratio` divides `tokens_per_block`; one row per request, + or with `rows_per_request` > 1 each request's rows consecutive at consecutive positions. A + row whose page lookup fails writes no cache; its packed row and index query are still written. + + Numerics: fp32, one rounding at each store; v, raw keys and coordinates bitwise. The rotation + reads the fp32 cos/sin table, not `fused_qk_norm_rope`'s `__sincosf`. + + Not certified: NVFP4 pools (the forward keeps the composition there); non-zero index-plane + mRoPE sections; mRoPE on the compressed key; `T == 0` (returns at once); head geometries other + than 6/1, 12/1 and 24/2 x 256 with a 4 x 128 index plane. + """ + + def __call__( + self, + proj: torch.Tensor, + out: torch.Tensor, + index_q: torch.Tensor, + position_ids: torch.Tensor, + coords: torch.Tensor, + cos_sin: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + index_q_weight: torch.Tensor, + index_k_weight: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + index_heads: int, + index_head_dim: int, + index_q_col: int, + rotary_dim: int, + eps: float, + mrope_section1: int, + mrope_section2: int, + rows_per_request: int = 1, + index_q_rows: torch.Tensor | None = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + num_tokens = proj.shape[0] + if num_tokens == 0: + return out, index_q + hpb, warps = _launch(num_tokens) + main_programs = triton.cdiv(num_heads_q + num_heads_k, hpb) + pdl = get_env_enable_pdl() + _attn_decode_prologue_kernel[(num_tokens, main_programs + 2 + num_heads_k)]( + proj, + proj.stride(0), + out, + out.stride(0), + index_q, + index_q.stride(0), + index_q.stride(1), + position_ids, + num_tokens, + cos_sin, + cos_sin.stride(0), + q_weight, + k_weight, + index_q_weight, + index_k_weight, + eps, + k_cache, + v_cache, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + request_indices, + logical_positions, + block_table, + block_table.stride(0), + block_table.stride(1), + block_table.shape[0], + block_table.shape[1], + k_cache.shape[0], + index_cache, + index_cache.stride(0), + index_cache.stride(1), + position_cache, + position_cache.stride(0), + position_cache.stride(1), + position_cache.stride(2), + coords, + coords.stride(0), + coords.stride(1), + proj if index_q_rows is None else index_q_rows, + HQ=num_heads_q, + HK=num_heads_k, + D=head_dim, + IH=index_heads, + ID=index_head_dim, + IQ_COL=index_q_col, + ROT=rotary_dim, + S1=mrope_section1, + S2=mrope_section2, + HPB=hpb, + MAIN_PROGRAMS=main_programs, + TPB=tokens_per_block, + CR=compress_ratio, + ROTARY_PAIRS=cos_sin.shape[1] // 2, + IN_LAUNCH_GROUPS=rows_per_request > 1, + IQ_ROW_MAP=index_q_rows is not None, + USE_PDL=pdl, + num_warps=warps, + launch_pdl=pdl, + ) + return out, index_q + + def reference( + self, + proj: torch.Tensor, + out: torch.Tensor, + index_q: torch.Tensor, + position_ids: torch.Tensor, + coords: torch.Tensor, + cos_sin: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + index_q_weight: torch.Tensor, + index_k_weight: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + index_heads: int, + index_head_dim: int, + index_q_col: int, + rotary_dim: int, + eps: float, + mrope_section1: int, + mrope_section2: int, + rows_per_request: int = 1, + index_q_rows: torch.Tensor | None = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Plain torch, fp32, writing the given buffers: both planes' norm + rope, then every row's + stores, then the compression of every group a row ends (reading the plane those left).""" + rows, dev = proj.shape[0], proj.device + half = rotary_dim // 2 + pos = position_ids.long() + table = cos_sin.float() + j = torch.arange(half, device=dev) + + def factors(s1: int, s2: int): + coord = torch.zeros(half, dtype=torch.long, device=dev) + coord[(j % 3 == 1) & (j < 3 * s1)] = 1 + coord[(j % 3 == 2) & (j < 3 * s2)] = 2 + p = pos[coord, :].t() # [T, half] + return table[p, j][:, None, :], table[p, half + j][:, None, :] + + def plane(x: torch.Tensor, w: torch.Tensor, cos, sin) -> torch.Tensor: + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + w) + x1, x2 = x[..., :half], x[..., half:rotary_dim] + return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin, x[..., rotary_dim:]], -1) + + main_heads = num_heads_q + num_heads_k + width = main_heads * head_dim + v_width = num_heads_k * head_dim + weights = torch.cat( + [ + q_weight.float().expand(num_heads_q, head_dim), + k_weight.float().expand(num_heads_k, head_dim), + ] + ) + main = plane( + proj[:, :width].float().reshape(rows, main_heads, head_dim), + weights, + *factors(mrope_section1, mrope_section2), + ).to(out.dtype) + out[:, :width] = main.reshape(rows, width) + v = proj[:, width : width + v_width] + out[:, width : width + v_width] = v + iq = proj[:, index_q_col : index_q_col + index_heads * index_head_dim].float() + iq = plane( + iq.reshape(rows, index_heads, index_head_dim), index_q_weight.float(), *factors(0, 0) + ) + iq = iq.to(index_q.dtype) + if index_q_rows is None: + index_q.copy_(iq) + else: + index_q[index_q_rows.long()] = iq + key_col = index_q_col + index_heads * index_head_dim + raw = proj[:, key_col : key_col + index_head_dim] + + requests = request_indices.long().tolist() + logical = logical_positions.long().tolist() + num_requests, table_width = block_table.shape + + def page_of(request: int, position: int): + column = position // tokens_per_block + if not (0 <= request < num_requests and position >= 0 and column < table_width): + return None + page = int(block_table[request, column]) + return page if 0 <= page < k_cache.shape[0] else None + + k_rows = main[:, num_heads_q:] + v_rows = v.reshape(rows, num_heads_k, head_dim) + slots = [page_of(r, p) for r, p in zip(requests, logical)] + for t, page in enumerate(slots): + if page is None: + continue + slot = logical[t] % tokens_per_block + k_cache[page, :, slot] = k_rows[t].to(k_cache.dtype) + v_cache[page, :, slot] = v_rows[t].to(v_cache.dtype) + index_cache[page, slot, 0] = raw[t].to(index_cache.dtype) + position_cache[page, slot, :3] = coords[t].to(position_cache.dtype) + + key_weight = 1.0 + index_k_weight.float() + rot = 2 * (cos_sin.shape[1] // 2) + for t, page in enumerate(slots): + if page is None or (logical[t] + 1) % compress_ratio: + continue + members = [logical[t] - compress_ratio + 1 + o for o in range(compress_ratio)] + keys = torch.stack( + [ + index_cache[page_of(requests[t], m), m % tokens_per_block, 0].float() + for m in members + ] + ) + pooled = (keys.sum(0) / compress_ratio).to(torch.bfloat16).float() + first_page = page_of(requests[t], members[0]) + first = int(position_cache[first_page, members[0] % tokens_per_block, 0]) + x = pooled * torch.rsqrt(pooled.pow(2).mean() + eps) * key_weight + cos, sin = table[first, : rot // 2], table[first, rot // 2 : rot] + x1, x2 = x[: rot // 2], x[rot // 2 : rot] + key = torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin, x[rot:]]) + index_cache[page, logical[t] % tokens_per_block, 0] = key.to(index_cache.dtype) + return out, index_q + + def is_valid(self, proj, out, index_q, position_ids, coords, cos_sin, *args, **kwargs) -> None: + assert proj.stride(1) == 1 and proj.dtype == torch.bfloat16, "bf16 proj, unit column stride" + assert out.stride(1) == 1 and out.dtype == torch.bfloat16, "bf16 out, unit column stride" + assert index_q.stride(2) == 1, "index_q must have a unit dim stride" + assert position_ids.is_contiguous() and position_ids.shape == (3, proj.shape[0]), ( + "contiguous [3, T] positions (the kernel reads coordinate a of row t at a * T + t)" + ) + assert position_ids.dtype == torch.int32, "int32 positions" + assert coords.shape == (proj.shape[0], 3), "coords [T, 3]" + assert cos_sin.dtype == torch.float32 and cos_sin.stride(1) == 1, ( + "an fp32 cos|sin table with a unit column stride" + ) + + +qwen4_exp_attn_decode_prologue = Qwen4ExpAttnDecodePrologue() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py new file mode 100644 index 000000000000..ec4d712847df --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A captured verify step's QSA rollback staging in one launch: every generation row's QSA slot +(`block_table[req, pos // tpb]`, `pos % tpb`), the pre-step contents of that slot in every +attention layer's index-K plane, and in the shared position plane. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: One index-K row: 128 bf16 a token (one 128-wide head). +ROW = 128 +#: A position-plane row: the token's three mRoPE coordinates, int32. +POS_W = 3 + + +# Program (l, r) resolves row r's slot and copies its row of plane l; the program past the last +# index plane copies the position triple and writes the slot. +@triton.jit( + do_not_specialize=["rows"], + do_not_specialize_on_alignment=["req_idx", "logical"], +) +def _begin_kernel( + req_idx, + logical, + block_table, + bt_stride, + ptrs, + page_strides, + position, + pos_s0, + pos_s1, + pages_out, + within_out, + index_out, + position_out, + rows, + TPB: tl.constexpr, + ROW_: tl.constexpr, + PLANES: tl.constexpr, + POS_W_: tl.constexpr, +): + plane = tl.program_id(0) + r = tl.program_id(1) + pos = tl.load(logical + r).to(tl.int64) + req = tl.load(req_idx + r).to(tl.int64) + page = tl.load(block_table + req * bt_stride + pos // TPB).to(tl.int64) + tok = pos % TPB + if plane < PLANES: + base = tl.load(ptrs + plane).to(tl.pointer_type(tl.bfloat16)) + stride = tl.load(page_strides + plane) + offs = tl.arange(0, ROW_) + vals = tl.load(base + page * stride + tok * ROW_ + offs) + tl.store(index_out + (plane.to(tl.int64) * rows + r) * ROW_ + offs, vals) + else: + w = tl.arange(0, 4) + wm = w < POS_W_ + v = tl.load(position + page * pos_s0 + tok * pos_s1 + w, mask=wm) + tl.store(position_out + r * POS_W_ + w, v, mask=wm) + tl.store(pages_out + r, page) + tl.store(within_out + r, tok) + + +def _cells(): + return tuple( + Cell( + why=f"tep4 MTP verify, {n} request(s) x {p} rows over 12 planes", + spec=dict(requests=n, p=p, planes=12, capture=capture), + ) + for n, p, capture in ( + (1, 3, False), + (4, 3, True), + (4, 4, False), + ) + ) + + +class SpecRollbackBegin(OpWrapper): + """`(pages, within, index_all, position_old) = spec_rollback_begin(req_idx, logical, + block_table, tokens_per_block, ptrs, page_strides, position)` for a verify step's rows. + + - `req_idx` `[R]` int32 / int64 and `logical` `[R]` int64: the generation rows' request + index (into `block_table`) and logical position (`qsa_req_idx_per_token`, + `qsa_logical_positions`); + - `block_table` `[S, max_pages]` int32 (unit column stride), `tokens_per_block` its page size; + - `ptrs` / `page_strides`: `plane_table(planes)` of the attention layers' index-K planes; + - `position` `[pages, tpb, 3]` int32 (unit innermost stride): the shared position plane. + + Returns `pages` / `within` `[R]` int64 (`block_table[req, pos // tpb]`, `pos % tpb`), + `index_all` `[L, R, 128]` bf16 (every row's slot of every plane, plane order -- what + `qsa_rollback_gather` returns) and `position_old` `[R, 3]` int32 (`position[pages, within]`). + Nothing but the outputs is written. Run it before any layer of the step writes a plane. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: one Triton kernel (`_begin_kernel`), JIT-compiled per (tokens_per_block, plane + count); grid (planes + 1, R). Copies and integer arithmetic only: bitwise the composed torch + path. + + Preconditions: the dtypes, ranks and strides `is_valid` checks; every slot inside its pool + (positions non-negative and inside the table) is the caller's. + """ + + def __call__( + self, req_idx, logical, block_table, tokens_per_block: int, ptrs, page_strides, position + ): + rows = logical.shape[0] + planes = ptrs.shape[0] + dev = logical.device + pages = torch.empty([rows], dtype=torch.int64, device=dev) + within = torch.empty([rows], dtype=torch.int64, device=dev) + index_all = torch.empty([planes, rows, ROW], dtype=torch.bfloat16, device=dev) + position_old = torch.empty([rows, POS_W], dtype=torch.int32, device=dev) + if rows: + _begin_kernel[(planes + 1, rows)]( + req_idx, + logical, + block_table, + block_table.stride(0), + ptrs, + page_strides, + position, + position.stride(0), + position.stride(1), + pages, + within, + index_all, + position_old, + rows, + TPB=tokens_per_block, + ROW_=ROW, + PLANES=planes, + POS_W_=POS_W, + num_warps=1, + ) + return pages, within, index_all, position_old + + def reference(self, req_idx, logical, block_table, tokens_per_block: int, planes, position): + """The composed torch path; takes the planes themselves where `__call__` takes their + `plane_table` (torch cannot dereference raw addresses).""" + positions = logical.to(torch.long) + pages = block_table[req_idx.to(torch.long), positions // tokens_per_block].to(torch.long) + within = positions % tokens_per_block + index_all = torch.stack([t[pages, within].reshape(-1, ROW) for t in planes]) + return pages, within, index_all, position[pages, within] + + def is_valid( + self, req_idx, logical, block_table, tokens_per_block: int, ptrs, page_strides, position + ) -> None: + rows = logical.shape[0] + assert logical.dtype == torch.int64 and logical.dim() == 1 and rows >= 1, ( + logical.dtype, + logical.shape, + ) + assert req_idx.dtype in (torch.int32, torch.int64) and tuple(req_idx.shape) == (rows,), ( + req_idx.dtype, + req_idx.shape, + ) + assert ( + block_table.dtype == torch.int32 + and block_table.dim() == 2 + and block_table.stride(1) == 1 + ), ( + block_table.dtype, + block_table.stride(), + ) + assert tokens_per_block > 0 + assert ( + ptrs.dtype == page_strides.dtype == torch.int64 and ptrs.shape == page_strides.shape + ), "ptrs / page_strides must be plane_table's int64 tables" + assert ( + position.dtype == torch.int32 + and position.dim() == 3 + and position.shape[2] == POS_W + and position.stride(2) == 1 + ), (position.dtype, position.shape, position.stride()) + + +spec_rollback_begin = SpecRollbackBegin() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py new file mode 100644 index 000000000000..2b309b091c32 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, Tri Dao. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Adapted from flash-attn 2 (https://github.com/Dao-AILab/flash-attention), +# `flash_attn/ops/triton/rotary.py`: `rotary_kernel` and `apply_rotary`. +"""Vision-tower 2-D rotary, in place on the q|k prefix of the packed qkv projection. + +The caller builds the per-token `cos` / `sin` table once per tower forward from `grid_thw`. +The kernel is flash-attn 2's `rotary_kernel` (header above) cut down to one sequence, NeoX +pairing, int64 token offsets and in-place writes; the registered rope ops take 1-D positions only. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: This checkpoint's bf16 tower: 1152 / 16 heads = a 72-wide head, a qkv row of 3 * 16 * 72. +NUM_HEADS = 16 +HEAD_DIM = 72 + +#: Heads x tokens per program. +_BLOCK_H = 4 +_BLOCK_M = 8 + + +@triton.jit +def _rotary_neox_inplace_kernel( + X, + COS, + SIN, + seqlen, + nheads, + seqlen_ro, + stride_x_seqlen, + stride_x_nheads, + stride_x_headdim, + # Constexpr so the compiler sees the mask is constant every 8 elements (128-bit loads). + ROTARY_DIM: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_M: tl.constexpr, +): + ROTARY_DIM_HALF: tl.constexpr = ROTARY_DIM // 2 + pid_head = tl.program_id(axis=0) + pid_m = tl.program_id(axis=1) + if pid_m * BLOCK_M >= seqlen: + return + + rh = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + # Token offsets in int64: rows x row stride passes 2^31 at 621379 rows of a 3456-wide row. + rm64 = rm.to(tl.int64) + rk_half = tl.arange(0, BLOCK_K // 2) + + COS = COS + (rm64[:, None] * ROTARY_DIM_HALF + rk_half[None, :]) + SIN = SIN + (rm64[:, None] * ROTARY_DIM_HALF + rk_half[None, :]) + mask_cs = (rm[:, None] < seqlen_ro) & (rk_half[None, :] < ROTARY_DIM_HALF) + cos = tl.load(COS, mask=mask_cs, other=1.0).to(tl.float32) + sin = tl.load(SIN, mask=mask_cs, other=0.0).to(tl.float32) + + # Pairs (i, i + ROTARY_DIM_HALF), stored back where they were read. + X = X + ( + rh[:, None, None] * stride_x_nheads + + rm64[None, :, None] * stride_x_seqlen + + rk_half[None, None, :] * stride_x_headdim + ) + mask = ( + (rh[:, None, None] < nheads) + & (rm[None, :, None] < seqlen) + & (rk_half[None, None, :] < ROTARY_DIM_HALF) + ) + x0 = tl.load(X, mask=mask, other=0.0).to(tl.float32) + x1 = tl.load(X + ROTARY_DIM_HALF * stride_x_headdim, mask=mask, other=0.0).to(tl.float32) + o0 = x0 * cos - x1 * sin + o1 = x0 * sin + x1 * cos + tl.store(X, o0, mask=mask) + tl.store(X + ROTARY_DIM_HALF * stride_x_headdim, o1, mask=mask) + + +class VisionRope2dPackedQk(OpWrapper): + """Rotate every head of `qk` by its token's angles, in place; return `qk`. + + half = head_dim // 2 + qk[t, h, i] <- x[i] cos[t, i] - x[i + half] sin[t, i] + qk[t, h, i + half] <- x[i] sin[t, i] + x[i + half] cos[t, i] i < half + + In fp32, rounded once on the store. `qk` is the `[T, 2 * heads, head_dim]` q|k prefix of + the packed `[T, 3 * heads, head_dim]` qkv, addressed through its strides (the v heads are + never touched). `cos` / `sin` are fp32 `[T, half]`: the height angles, then the width angles. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: grid -- `grid_thw` rows the test builds the angle table from; grid_dtype -- its + # host dtype; dtype -- the qkv dtype. Heads 16 x 72, merge 2, theta 1e4 throughout. + CELLS: tuple[Cell, ...] = ( + Cell( + why="62x90 with the processor's int64 grid_thw: 5580 rows", + spec=dict(grid=((1, 62, 90),), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + Cell( + why="video (3,8,8) + image (1,16,24), two grid rows: 576 rows", + spec=dict(grid=((3, 8, 8), (1, 16, 24)), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + Cell( + why="video (64,16,32): 32768 rows, the tower's per-call ceiling", + spec=dict(grid=((64, 16, 32),), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + ) + + note = """ + Source: the Triton kernel above. Called eagerly by every vision block; the tower is + replicated, so every member runs these shapes. Bitwise equal to `reference` at every cell. + + Preconditions: `qk` 3-D, any strides; `cos` and `sin` fp32, contiguous, `[T, head_dim // 2]`. + `is_valid` refuses the silent failures: a non-fp32 table (rotates by rounded angles), a + short table (tail tokens unrotated), a narrow or wide table (partial rotation, or the last + k head writes into the first v head), a strided table (wrong angles). + + Not certified: float16 / float32 / float8 q|k; head geometries other than 16 x 72; partial + rotary; calls past 32768 rows (the tower's per-call ceiling); T = 0; CUDA-graph capture; + out-of-place use. + """ + + def __call__(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + seqlen, nheads, _ = qk.shape + rotary_dim = 2 * cos.shape[-1] + grid = (triton.cdiv(nheads, _BLOCK_H), triton.cdiv(seqlen, _BLOCK_M)) + _rotary_neox_inplace_kernel[grid]( + qk, + cos, + sin, + seqlen, + nheads, + cos.shape[0], + qk.stride(0), + qk.stride(1), + qk.stride(2), + ROTARY_DIM=rotary_dim, + BLOCK_K=triton.next_power_of_2(rotary_dim), + BLOCK_H=_BLOCK_H, + BLOCK_M=_BLOCK_M, + ) + return qk + + def reference(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + """Each head's halves as one complex number times `cos + i sin`, in fp32; `qk` is not + mutated.""" + half = qk.shape[-1] // 2 + x = qk.float() + rotor = torch.complex(cos.float(), sin.float()).unsqueeze(-2) + rotated = torch.complex(x[..., :half], x[..., half:]) * rotor + return torch.cat((rotated.real, rotated.imag), dim=-1).to(qk.dtype) + + def is_valid(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> None: + assert cos.dtype == torch.float32 and sin.dtype == torch.float32, ( + f"cos and sin must be fp32; got {cos.dtype} / {sin.dtype} -- the kernel widens a " + "narrower table and rotates by its rounded angles" + ) + half = qk.shape[-1] // 2 + assert cos.shape == (qk.shape[0], half) and sin.shape == cos.shape, ( + f"cos and sin must be [T, head_dim // 2] = [{qk.shape[0]}, {half}]; got " + f"{tuple(cos.shape)} / {tuple(sin.shape)} -- a short table leaves the tail tokens " + "unrotated, a narrow one rotates only a prefix of each head" + ) + assert cos.is_contiguous() and sin.is_contiguous(), ( + "cos and sin must be contiguous; the kernel indexes them as t * half + i and a " + "strided table reads the wrong angles" + ) + + +vision_rope_2d_packed_qk = VisionRope2dPackedQk() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py new file mode 100644 index 000000000000..e0a347d6462c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Two-rank row exchange on the copy engines: each rank pushes a tensor into the peer's receive +buffer over NVLink with `cudaMemcpyAsync` and the pair signals with stream memory operations, so a +GEMM it overlaps keeps every SM. + +Protocol (`exchange`), on the caller's stream, every flag in the memory of the rank that waits on +it, every value a constant (so a CUDA graph replays it): + + 1. write peer.ready = 1 my receive buffer is free (everything before on this stream is done) + 2. wait my.ready == 1; write my.ready = 0 the peer's buffer is free + 3. copy my rows -> the peer's receive buffer (copy engine) + 4. write peer.data = 1 (the write's default memory barrier orders it after the copy) + 5. wait my.data == 1; write my.data = 0 the peer's rows are in my buffer + +A rank's step-1 signal for exchange k is issued after its step 5 of exchange k - 1, which follows +the peer's reset of `ready` for k - 1 (step 2 precedes the peer's step 4); likewise for `data`, so no +signal is lost or seen twice. The result is a view of the receive buffer, valid until the next +exchange's step 1. Both ranks must call `exchange` identically (same calls, order and sizes). +""" + +from __future__ import annotations + +import torch + +#: Bytes kept for the two flags after the data region (one 128-byte line each). +_FLAG_BYTES = 256 + + +class _CudaArray: + """`__cuda_array_interface__` over a raw device allocation (uint8), for `torch.as_tensor`.""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "shape": (nbytes,), + "typestr": "|u1", + "data": (ptr, False), + "version": 3, + "strides": None, + } + + +class CopyEngineExchange: + """`exchange(x)` -> the peer's `x` (a view of this rank's receive buffer, same shape and + dtype). `capacity` bytes a call at most. Built on the attention-TP group of a two-rank + `mapping` (IPC memory inside a node); `create` returns None where that does not hold.""" + + def __init__(self, mapping, capacity: int): + from cuda.bindings import driver, runtime + + from tensorrt_llm._ipc_utils import IpcMemory + + assert mapping.tp_size == 2 + self._driver, self._runtime = driver, runtime + self.capacity = capacity + self._ipc = IpcMemory(mapping, capacity + _FLAG_BYTES, open_ipc=True) + rank = mapping.tp_rank + local, peer = self._ipc.peer_ptrs[rank], self._ipc.peer_ptrs[1 - rank] + assert local and peer, "IPC memory did not open" + self._peer = peer + self._my_ready, self._my_data = local + capacity, local + capacity + 128 + self._peer_ready, self._peer_data = peer + capacity, peer + capacity + 128 + self._recv = torch.as_tensor(_CudaArray(local, capacity), device=torch.device("cuda")) + + @classmethod + def create(cls, mapping, capacity: int): + """The exchange over `mapping`'s attention-TP pair, or None (not two ranks, ranks on + different nodes, or IPC memory unavailable on either rank). A collective of the pair: + both ranks build it or neither.""" + if mapping.tp_size != 2 or mapping.tp_size > mapping.gpus_per_node: + return None + from tensorrt_llm._torch.distributed.communicator import Distributed + + try: + exchange = cls(mapping, capacity) + except Exception: # noqa: BLE001 -- keep NCCL where the copy engines cannot be reached + exchange = None + if not all(Distributed.get(mapping).tp_allgather(exchange is not None)): + return None + return exchange + + def _write(self, stream, addr: int, value: int) -> None: + d = self._driver + (err,) = d.cuStreamWriteValue32( + stream, + d.CUdeviceptr(addr), + value, + d.CUstreamWriteValue_flags.CU_STREAM_WRITE_VALUE_DEFAULT, + ) + assert err == d.CUresult.CUDA_SUCCESS, err + + def _wait(self, stream, addr: int, value: int) -> None: + d = self._driver + (err,) = d.cuStreamWaitValue32( + stream, + d.CUdeviceptr(addr), + value, + d.CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ, + ) + assert err == d.CUresult.CUDA_SUCCESS, err + + def exchange(self, x: torch.Tensor) -> torch.Tensor: + assert x.is_contiguous() and x.is_cuda + nbytes = x.numel() * x.element_size() + assert nbytes <= self.capacity, (nbytes, self.capacity) + stream = self._driver.CUstream(torch.cuda.current_stream().cuda_stream) + r = self._runtime + self._write(stream, self._peer_ready, 1) + self._wait(stream, self._my_ready, 1) + self._write(stream, self._my_ready, 0) + (err,) = r.cudaMemcpyAsync( + self._peer, + x.data_ptr(), + nbytes, + r.cudaMemcpyKind.cudaMemcpyDefault, + r.cudaStream_t(torch.cuda.current_stream().cuda_stream), + ) + assert err == r.cudaError_t.cudaSuccess, err + self._write(stream, self._peer_data, 1) + self._wait(stream, self._my_data, 1) + self._write(stream, self._my_data, 0) + return self._recv[:nbytes].view(x.dtype).view(x.shape) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py new file mode 100644 index 000000000000..62d3e1040a5a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode block's TP sum and the hyper-connection site's injection + grouped norm in one launch. + +`native_ops.hc_allreduce_inject_norm` (`csrc/hcAllreduceInjectNorm.cu`) does the IPC one-shot +Lamport all-reduce of `catalog/comm/allreduce` over the same workspace and protocol (so it +interleaves with the plain all-reduce's calls), then `_hc_inject_norm_launch`'s injection and +norm from the summed row, its sum of squares in the Triton program's reduction tree at +`program_warps` warps. Both outputs are bitwise the all-reduce followed by the Triton kernel. + +Only the norm weights are read before the grid-dependency wait; the residual, injection weights +and shared-expert operands are read after it. +""" + +from __future__ import annotations + +import torch + +from ... import native_ops + +HIDDEN = 2560 +HC_COUNT = 4 + + +def _aligned(t: torch.Tensor) -> bool: + return t.data_ptr() % 16 == 0 and t.stride(-1) == 1 and t.stride(0) % 8 == 0 + + +def fits(hc, partial, inject, shared=None, gate=None) -> bool: + """Whether the native kernel takes these operands (else the caller sums and injects apart).""" + rows = hc.shape[0] + if partial.dim() != 2 or partial.shape != (rows, HIDDEN) or not partial.is_contiguous(): + return False + if hc.shape != (rows, HC_COUNT * HIDDEN) or not _aligned(hc): + return False + if inject is None or inject.shape[0] != rows or inject.shape[1] < HC_COUNT: + return False + if inject.stride(-1) != 1: + return False + if (shared is None) != (gate is None): + return False + if shared is not None and (shared.shape != (rows, HIDDEN) or not _aligned(shared)): + return False + dtypes = {t.dtype for t in (hc, partial, inject, shared, gate) if t is not None} + return dtypes == {torch.bfloat16} + + +def hc_allreduce_inject_norm( + partial: torch.Tensor, + workspace: torch.Tensor, + rank: int, + nranks: int, + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + program_warps: int = 4, +) -> tuple[torch.Tensor, torch.Tensor]: + """`(hc_out, normed)` of the site after a block whose per-rank output is `partial`: bitwise + `_hc_inject_norm_launch(hc, tp_sum(partial), inject, norm, eps, 4, shared, gate, + program_warps)`.""" + hc_out, normed = native_ops.hc_allreduce_inject_norm( + partial, + workspace, + int(rank), + int(nranks), + hc, + inject, + shared, + gate, + norm, + float(eps), + int(program_warps), + # trigger_completion_at_end=False: release the next launch right after this kernel's wait. + False, + ) + return hc_out, normed diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py new file mode 100644 index 000000000000..088dd772cf53 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The gather of a decode step's QSA selection split by request over the TP group, one launch. + +Each rank selects some requests' rows (`_decode_index_selection`); one Triton launch gathers the +int32 selection rows in rank order as a one-shot Lamport exchange over CUDA IPC buffers: each +rank stores its rows into every peer's buffer (NVLink) and its own output, then copies the +peers' rows out of its own buffer as they land. Bitwise the NCCL gather. + +Buffers: `[SLOTS, max_words]` int32 a rank, every word `SENTINEL` (int32 min; tokens are >= -1) +until a peer's store lands. Slot protocol: every program reads the call's slot (`state[0]`) and +the previous call's share (`state[1]`) after its grid-dependency wait and takes a ticket +(`state[2]`, acq_rel) after its stores; the last ticket sets the next slot and share and resets +the ticket. Each call clears the previous call's slot back to `SENTINEL`. A peer stores into +call k's slot only after reading this rank's call k - 1 stores, made after call k - 2 completed +(stream order), so call k's slot (call k - 3's) was cleared by call k - 2. Every rank of the +group must make the same sequence of calls (the split depends only on the row count). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + +SLOTS = 3 +SENTINEL = -(2**31) +_BLOCK = 1024 +_PROGRAMS = 256 + + +class _IpcBuffers: + """This rank's buffer and the peers' opened ones (addresses in rank order); freed and + closed with the object.""" + + def __init__(self, rank: int, local: int, peers: list): + self.rank, self.local, self.peers = rank, local, peers + + def __del__(self): + import sys + + if sys.is_finalizing(): + return + from tensorrt_llm._ipc_utils import cudart + + for q, ptr in enumerate(self.peers): + if ptr and q != self.rank: + cudart.cudaIpcCloseMemHandle(ptr) + if self.local: + cudart.cudaFree(self.local) + + +@dataclass(frozen=True) +class SelectionLamport: + """One rank's view of the group's buffers: `buffers` the IPC allocations (kept alive here), + `state` the `[4]` int32 slot state (`state[0]` the next call's slot, `state[1]` the + previous call's share in words, `state[2]` the ticket counter), `args` the `[8]` int64 the + kernel reads: the peers' buffer addresses at [0, world), this rank's own at `rank`.""" + + rank: int + world: int + max_words: int + buffers: _IpcBuffers + state: torch.Tensor + args: torch.Tensor + + def fits(self, words: int) -> bool: + """Whether a call gathering `words` int32 in all (every rank's share) fits a slot.""" + return 0 < words <= self.max_words + + +def build(rank: int, world: int, max_words: int, allgather) -> SelectionLamport: + """Allocate this rank's buffers and open the peers' (CUDA IPC, one node). `allgather(b)` + gathers one `bytes` a rank in rank order (e.g. `Distributed.get(mapping).tp_allgather`): a + collective, so every rank calls `build` at the same point, before any CUDA graph capture.""" + from tensorrt_llm._ipc_utils import ( + _ipc_mem_handle_from_bytes, + _ipc_mem_handle_to_bytes, + _raise_if_error, + cuda, + cudart, + ) + + assert 2 <= world <= 4, "the kernel addresses 2 to 4 ranks" + nbytes = -(-(SLOTS * max_words * 4) // (1 << 21)) * (1 << 21) + err, local = cudart.cudaMalloc(nbytes) + _raise_if_error(err) + (err,) = cuda.cuMemsetD32(local, SENTINEL & 0xFFFFFFFF, nbytes // 4) + _raise_if_error(err) + err, handle = cudart.cudaIpcGetMemHandle(local) + _raise_if_error(err) + handles = allgather(_ipc_mem_handle_to_bytes(handle)) + assert len(handles) == world, (len(handles), world) + peers = [] + for q, h in enumerate(handles): + if q == rank: + peers.append(int(local)) + continue + err, ptr = cudart.cudaIpcOpenMemHandle( + _ipc_mem_handle_from_bytes(h), cudart.cudaIpcMemLazyEnablePeerAccess + ) + _raise_if_error(err) + peers.append(int(ptr)) + dev = torch.device("cuda", torch.cuda.current_device()) + state = torch.zeros(4, dtype=torch.int32, device=dev) + args = torch.tensor(peers + [0] * (8 - world), dtype=torch.int64, device=dev) + torch.cuda.synchronize() + # every rank's buffer is filled and opened before any rank's first push + allgather(b"") + return SelectionLamport( + rank=rank, + world=world, + max_words=max_words, + buffers=_IpcBuffers(rank, int(local), peers), + state=state, + args=args, + ) + + +@triton.jit +def _dst_words(dst_rows, g, m, width, ROW_MAP: tl.constexpr): + """Output word of gathered word `g`: itself, or with `ROW_MAP` the same column of the row + `dst_rows` names for its row.""" + if ROW_MAP: + # int32 arithmetic (a slot holds < 2^31 words): cheaper than a 64-bit division per word. + g32 = g.to(tl.int32) + row = g32 // width + to = tl.load(dst_rows + row, mask=m, other=0).to(tl.int64) + return to * width + (g32 - row * width) + return g + + +@triton.jit(do_not_specialize=["share", "rank", "slot_words", "width"]) +def _lamport_allgather_kernel( + src, + dst, + args, + state, + share, + rank, + slot_words, + dst_rows, + width, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, + ROW_MAP: tl.constexpr, + USE_PDL: tl.constexpr, +): + """Program p moves words p * BLOCK, (p + P) * BLOCK, ... of each share (`share` words a + rank): pushes this rank's from `src` to every peer's slot and to `dst`, clears the previous + call's peer shares of the previous slot, then copies each peer's share out of this rank's + slot as it lands. With `ROW_MAP`, gathered row `j` (of `width` words) lands in row + `dst_rows[j]` of `dst` instead of row `j`.""" + pid = tl.program_id(0) + nprog = tl.num_programs(0) + if USE_PDL: + # With a memory clobber: no load of the predecessor's output (`src`) or of the state is + # hoisted above the wait. + gdc_wait_with_memory_clobber() + slot = tl.load(state) + prev_share = tl.load(state + 1) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + offs = tl.arange(0, BLOCK) + local = tl.load(args + rank).to(tl.pointer_type(tl.int32)) + slot_off = slot.to(tl.int64) * slot_words + own0 = rank.to(tl.int64) * share + # 1. this rank's share: to every peer's slot (same offset) and to the output + for start in range(pid * BLOCK, share, nprog * BLOCK): + w = start + offs + m = w < share + out = dst + _dst_words(dst_rows, own0 + w, m, width, ROW_MAP) + v = tl.load(src + w, mask=m, other=0) + tl.store(out, v, mask=m) + for q in tl.static_range(WORLD): + if q != rank: + peer = tl.load(args + q).to(tl.pointer_type(tl.int32)) + tl.store(peer + slot_off + own0 + w, v, mask=m) + # 2. the previous call's peer shares of the previous slot back to the marker + prev_off = ((slot + 2) % 3).to(tl.int64) * slot_words + for q in tl.static_range(WORLD): + if q != rank: + q0 = q * prev_share.to(tl.int64) + for start in range(pid * BLOCK, prev_share, nprog * BLOCK): + w = start + offs + tl.store( + local + prev_off + q0 + w, + tl.full([BLOCK], -2147483648, tl.int32), + mask=w < prev_share, + ) + # Every program has read the state (at its start): the last ticket moves it on. + ticket = tl.atomic_add(state + 2, 1, sem="acq_rel", scope="gpu") + if ticket == nprog - 1: + tl.store(state, (slot + 1) % 3) + tl.store(state + 1, share) + tl.store(state + 2, 0) + # 3. each peer's share, once it has landed in this rank's slot, to the output + for q in tl.static_range(WORLD): + if q != rank: + q0 = q * share.to(tl.int64) + for start in range(pid * BLOCK, share, nprog * BLOCK): + w = start + offs + m = w < share + out = dst + _dst_words(dst_rows, q0 + w, m, width, ROW_MAP) + v = tl.load(local + slot_off + q0 + w, mask=m, other=0, volatile=True) + pending = tl.sum(((v == -2147483648) & m).to(tl.int32), axis=0) + while pending > 0: + v = tl.load(local + slot_off + q0 + w, mask=m, other=0, volatile=True) + pending = tl.sum(((v == -2147483648) & m).to(tl.int32), axis=0) + tl.store(out, v, mask=m) + + +def lamport_allgather( + src: torch.Tensor, lam: SelectionLamport, dst_rows: torch.Tensor | None = None +) -> torch.Tensor: + """`src`'s rows from every rank of `lam`'s group concatenated in rank order (the NCCL + `allgather` with equal shares): `src` is this rank's contiguous int32 `[rows, width]` + share. Returns a fresh `[world * rows, width]` int32 tensor. With `dst_rows` (int32 + `[world * rows]`, a permutation, the same on every rank) the gathered row `j` is stored at + row `dst_rows[j]` instead (`out[dst_rows] = gathered`), in the same launch.""" + assert src.dtype == torch.int32 and src.is_contiguous() and src.ndim == 2 + rows, width = src.shape + share = rows * width + assert lam.fits(share * lam.world), (share, lam.world, lam.max_words) + dst = torch.empty([lam.world * rows, width], dtype=torch.int32, device=src.device) + if dst_rows is not None: + assert dst_rows.dtype == torch.int32 and dst_rows.is_contiguous() + assert dst_rows.numel() == lam.world * rows + pdl = get_env_enable_pdl() + _lamport_allgather_kernel[(_PROGRAMS,)]( + src, + dst, + lam.args, + lam.state, + share, + lam.rank, + lam.max_words, + dst if dst_rows is None else dst_rows, + width, + WORLD=lam.world, + BLOCK=_BLOCK, + ROW_MAP=dst_rows is not None, + USE_PDL=pdl, + num_warps=4, + launch_pdl=pdl, + ) + return dst diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py new file mode 100644 index 000000000000..8e241359265a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Sum across the TP group through a multicast NVLink (MNNVL) workspace, nothing fused. + +This target's variant of `catalog/comm/mnnvl_allreduce` (which certifies only the residual + +RMSNorm fusion) for the decode-sized calls of `tp_sum`: the op's plain sum (fusion `NONE`).""" + +from __future__ import annotations + +from typing import Sequence + +import torch + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.catalog.comm.mnnvl_allreduce import ( + mnnvl_workspace_args, + workspace_bytes, +) +from tensorrt_llm.functional import AllReduceFusionOp + +NONE = int(AllReduceFusionOp.NONE) + +HIDDEN = 2560 + +__all__ = ["mnnvl_tp_sum", "mnnvl_workspace_args"] + +_TP_SUM = dict(hidden=HIDDEN, dtype=torch.bfloat16) + + +class MnnvlTpSum(OpWrapper): + """Sum `input` (`[rows, HIDDEN]` bf16) over the TP group; returns a fresh tensor. The + workspace is built once by `mnnvl_workspace_args` (a TP collective) for a row budget the + kernel does not check; `is_valid` does.""" + + ARCHS = frozenset({Arch.SM_100, Arch.SM_103}) + + CELLS: tuple[Cell, ...] = tuple( + Cell(why=f"tp_sum, {why}, tp{world}", spec=dict(world=world, rows=rows, **_TP_SUM)) + for world, rows, why in ( + (4, 1, "one draft row"), + (4, 32, "an MTP-3 verify of 8 requests"), + (2, 4, "an MTP-3 verify of 1 request"), + ) + ) + + def __call__( + self, input: torch.Tensor, comm_buffer: torch.Tensor, buffer_flags: torch.Tensor, world: int + ) -> torch.Tensor: + """`world` is not passed to the op, which reads it from the workspace; it is here so + `is_valid` can size the call against the buffer.""" + return torch.ops.trtllm.mnnvl_fusion_allreduce( + input, None, None, 1e-6, comm_buffer, buffer_flags, False, None, NONE + )[0] + + def reference(self, inputs_in_rank_order: Sequence[torch.Tensor]) -> torch.Tensor: + """The fp32 sum of every rank's `input`, in rank order, rounded once. The documented + collective exception to mirroring `__call__`.""" + total = torch.zeros_like(inputs_in_rank_order[0], dtype=torch.float32) + for x in inputs_in_rank_order: + total += x.float() + return total.to(inputs_in_rank_order[0].dtype) + + def is_valid( + self, input: torch.Tensor, comm_buffer: torch.Tensor, buffer_flags: torch.Tensor, world: int + ) -> None: + hidden = input.shape[-1] + need = workspace_bytes(input.numel() // hidden, hidden, world, input.dtype) + have = comm_buffer.shape[-1] * comm_buffer.element_size() + assert need <= have, (need, have) + + +mnnvl_tp_sum = MnnvlTpSum() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py new file mode 100644 index 000000000000..cbed53aae03e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The attention-DP MoE round trip as an all-to-all (`NVLinkOneSided`), run when +`TRTLLM_MODELING_V2_MOE_ALLTOALL=1` (`modeling.MOE_ALLTOALL_ENV`) in place of the default +all-gather / reduce-scatter. `moe_a2a_dispatch` sends every rank the rows with a top-k expert in +its whole-expert window, with their top-k ids and weights; `moe_a2a_combine` returns each +partial to the row's rank and sums them there. The strategy (`build_strategy`) holds the MNNVL +workspace and the state between a dispatch and its combine: build it once, pass it to every call.""" + +from __future__ import annotations + +from typing import List, NamedTuple, Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +#: The id the strategy writes over a received slot that holds no token: outside every window. +NO_EXPERT = -1 + + +def build_strategy(mapping, max_num_tokens_per_rank: int): + """The stock `NVLinkOneSided` strategy over the attention-DP group (`mapping`'s TP group, + every rank one EP rank), or None where the platform has no MNNVL. Collective: every rank of + the group builds it at the same point.""" + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided + + if not NVLinkOneSided.is_platform_supported(): + return None + return NVLinkOneSided( + mapping, + num_slots=NUM_EXPERTS, + top_k=TOPK, + max_num_tokens_per_rank=max_num_tokens_per_rank, + hidden_size=HIDDEN, + dtype=torch.bfloat16, + ) + + +def _cell(world: int, counts: tuple, why: str) -> Cell: + """One round trip over an attention-DP group of `world` ranks holding `counts` rows.""" + return Cell( + why=why, + spec=dict(world=world, counts=counts, hidden=HIDDEN, topk=TOPK, dtype=torch.bfloat16), + ) + + +_CELLS = ( + _cell(2, (1, 1), "dep2: a decode step at batch 1"), + _cell(2, (5, 300), "dep2: a ragged eager step, one rank holding a context chunk"), + _cell(4, (0, 17, 3, 0), "dep4: a ragged step where two ranks hold no rows"), + _cell(4, (2048,) * 4, "dep4: a prefill chunk of 2048 rows a rank"), +) + + +class Dispatched(NamedTuple): + """What `moe_a2a_dispatch` hands this rank, `[world * max(counts), ...]` each: the received + `rows`, their `topk_ids` (global expert ids; `NO_EXPERT` on a slot that holds no token) and + `topk_weights`. Block `s` holds source rank `s`'s tokens in an order the kernel picks.""" + + rows: torch.Tensor + topk_ids: torch.Tensor + topk_weights: torch.Tensor + + +class MoeA2aDispatch(OpWrapper): + """Send every rank the rows routed into its window; see the module docstring.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _CELLS + + note = """ + Source: `NVLinkOneSided.dispatch` (`torch.ops.trtllm.moe_a2a_dispatch`): each token written + once to every rank one of its top-k experts lives on, into that rank's block of the sender + (slot order inside a block is not fixed), the slots past each block's count stamped + `NO_EXPERT`. Measured on GB300; received rows compared with `reference` as a set. + Not certified: world sizes other than 2 and 4, groups spanning nodes, sm_100, post-quant + dispatch, low-precision combine, EPLB statistics. + """ + + def __call__( + self, + strategy, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + all_rank_num_tokens: List[int], + ) -> Dispatched: + # The weights travel as fp32 (bf16 -> fp32 -> bf16 is exact). + rows, _, ids, weights = strategy.dispatch( + x, None, topk_ids, topk_weights.float(), list(all_rank_num_tokens) + ) + return Dispatched(rows, ids, weights.to(topk_weights.dtype)) + + def reference( + self, + per_rank_inputs: Sequence[Sequence[torch.Tensor]], + all_rank_num_tokens: List[int], + experts_per_rank: int, + rank: int = 0, + ) -> List[tuple]: + """What `rank` receives from every rank's `(x, topk_ids, topk_weights)` (ascending rank + order): per source, its tokens with a top-k id in `rank`'s window, in token order. + Compare as a set: the kernel's order inside a block is its own.""" + lo, hi = rank * experts_per_rank, (rank + 1) * experts_per_rank + out = [] + for x, ids, w in per_rank_inputs: + hit = ((ids >= lo) & (ids < hi)).any(1) + out.append((x[hit], ids[hit], w[hit])) + return out + + def is_valid( + self, + strategy, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + all_rank_num_tokens: List[int], + ) -> None: + assert x.shape[0] == topk_ids.shape[0] == topk_weights.shape[0], ( + x.shape, + topk_ids.shape, + topk_weights.shape, + ) + assert x.shape[0] <= strategy.max_num_tokens_per_rank, ( + f"{x.shape[0]} rows past the workspace's {strategy.max_num_tokens_per_rank}" + ) + assert topk_ids.dtype == torch.int32, topk_ids.dtype + + +class MoeA2aCombine(OpWrapper): + """Return each partial to the rank its row came from and sum a row's partials there. + `partials` is `[world * max(counts), hidden]` in the order `moe_a2a_dispatch` handed the rows + (a slot that holds no token may hold anything); returns this rank's `[rows, hidden]`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _CELLS + + note = """ + Source: `NVLinkOneSided.combine` (`torch.ops.trtllm.moe_a2a_combine`). Must follow the + strategy's `dispatch` (the strategy keeps the slot map between them). Not certified: as + `MoeA2aDispatch`. + """ + + def __call__(self, strategy, partials: torch.Tensor) -> torch.Tensor: + return strategy.combine(partials) + + def reference(self, per_rank_token_partials: Sequence[torch.Tensor]) -> torch.Tensor: + """This rank's rows: the partial every rank computed for each of them (zero where the + row was not sent there), `[rows, hidden]` each in ascending rank order, summed in fp32 + and rounded once.""" + out = torch.zeros_like(per_rank_token_partials[0], dtype=torch.float32) + for partial in per_rank_token_partials: + out += partial.float() + return out.to(per_rank_token_partials[0].dtype) + + def is_valid(self, strategy, partials: torch.Tensor) -> None: + assert partials.dim() == 2 and partials.shape[1] == HIDDEN, partials.shape + + +moe_a2a_dispatch = MoeA2aDispatch() +moe_a2a_combine = MoeA2aCombine() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py new file mode 100644 index 000000000000..a84b7d349b2a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's token embedding in one launch: the rows of the table at the step's ids, +optionally with negative ids read as row 0, written `copies` times side by side (once per +hyper-connection stream). +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +BLOCK = 512 + + +# `ids` is a view of the step's token buffer at any row (a gather index, read by scalar loads). +@triton.jit(do_not_specialize_on_alignment=["ids"]) +def _embed_rows_kernel( + ids, + table, + out, + stride_table, + stride_out, + H: tl.constexpr, + COPIES: tl.constexpr, + CLAMP: tl.constexpr, + BLOCK_: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.program_id(1) * BLOCK_ + tl.arange(0, BLOCK_) + mask = cols < H + token = tl.load(ids + row).to(tl.int64) + if CLAMP: + token = tl.maximum(token, 0) + vals = tl.load(table + token * stride_table + cols, mask=mask) + for c in tl.static_range(COPIES): + tl.store(out + row.to(tl.int64) * stride_out + c * H + cols, vals, mask=mask) + + +class EmbedRows(OpWrapper): + """`out = embed_rows(ids, table, copies, clamp)`: `[T, copies * H]`, block `c` of row `t` + `table[ids[t]]` (`table[max(ids[t], 0)]` with `clamp`); `ids` `[T]` int32 / int64 + contiguous, `table` `[V, H]` with unit column stride. Bitwise the composed `F.embedding` + path. Ids past the table (or negative without `clamp`) read out of bounds: the caller's. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="batch-4 verify: 12 rows over 4 streams", spec=dict(rows=12, copies=4, clamp=False)), + Cell(why="draft decode step: 1 row, clamped", spec=dict(rows=1, copies=1, clamp=True)), + ) # fmt: skip + + note = """ + Source: one Triton kernel (`_embed_rows_kernel`), grid `(T, ceil(H / 512))`, JIT-compiled per + (H, copies, clamp, id dtype); the engine's warmup forward compiles it before capture. Copies + only: bitwise the composed path for every in-range id. + """ + + def __call__( + self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool + ) -> torch.Tensor: + rows = ids.shape[0] + hidden = table.shape[1] + out = torch.empty([rows, copies * hidden], dtype=table.dtype, device=table.device) + if rows: + _embed_rows_kernel[(rows, triton.cdiv(hidden, BLOCK))]( + ids, + table, + out, + table.stride(0), + out.stride(0), + H=hidden, + COPIES=copies, + CLAMP=clamp, + BLOCK_=BLOCK, + num_warps=4, + ) + return out + + def reference( + self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool + ) -> torch.Tensor: + h = torch.nn.functional.embedding(ids.clamp_min(0) if clamp else ids, table) + return torch.cat([h] * copies, -1) + + def is_valid(self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool) -> None: + assert ids.dim() == 1 and ids.is_contiguous() and ids.dtype in (torch.int32, torch.int64), ( + ids.shape, + ids.dtype, + ) + assert table.dim() == 2 and table.stride(1) == 1, (tuple(table.shape), table.stride()) + assert copies >= 1 + + +embed_rows = EmbedRows() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py new file mode 100644 index 000000000000..c071facbdeba --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Decode-shaped bf16 projection: one kernel per call, its source picked per (M, N, K). + +At decode M a dense bf16 projection streams its weight once, and FlashInfer's low-M CuTe-DSL +kernels (`dense_bf16_gemm_direct.run_direct_dense`, the register-prefetch kernel, and +`dense_bf16_gemm_warp_splitk.run_warp_splitk_dense`, cp.async + mma.sync warp split-K) beat +cuBLAS there. A per-(N, K) table by M bucket picks one and its tactic; every other shape and +every M above the table stays on the trtllm cuBLASLt op. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple + +import torch +from flashinfer.gemm.kernels import dense_bf16_gemm_direct as _fi_direct +from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fi_warp + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.modeling_v2.catalog._op import ( + Arch, + Cell, + OpWrapper, + true_fp32_matmul, +) + + +class _Plan(NamedTuple): + """`kind` is "direct", "warp" or "cublas"; `tactic` is the kernel's own tactic tuple.""" + + kind: str + tactic: tuple = () + + +def _d(block_size: int, outputs_per_block: int) -> tuple: + """A direct-kernel choice; rows_per_block is the kernel's default for the runtime M.""" + return ("direct", block_size, outputs_per_block) + + +def _w(output_tile: int, token_tile: int, k_tile: int, stages: int, b_loaders: int) -> tuple: + """A warp split-K choice.""" + return ("warp", output_tile, token_tile, k_tile, stages, b_loaders) + + +#: (N, K) -> ((M upper bound, choice), ...) in ascending M; other shapes and larger M run on +#: cuBLAS. The decode projections of a tp4/tep4 rank (1..16 rows) and a tp2/tep2 and tp1 rank's +#: attention in / out (3..16 rows), each raced on GB300 over the kernel's `autotune_tactics`. +_TABLE: dict[tuple[int, int], tuple[tuple[int, tuple], ...]] = { + # Attention input projection [q|gate 6 x 256 | k | v | index q|k 640 | pad 128] = 4352. + (4352, 2560): ((1, _d(64, 2)), (8, _w(16, 8, 256, 9, 2)), (16, _w(16, 16, 256, 6, 2))), + # GDN input projection 4120, padded to 4128 (`weights.IN_PROJ_ROW_ALIGN`) so the warp split-K's + # 16-wide output tile divides it. + (4128, 2560): ( + (2, _d(64, 2)), + (4, _w(16, 8, 256, 8, 2)), + (8, _w(16, 8, 128, 11, 2)), + (16, _w(16, 16, 256, 6, 2)), + ), + # Attention o and GDN out (1536 in), the draft layer's o too. + (2560, 1536): ((1, _d(192, 2)), (2, _d(64, 2)), (16, _w(16, 8, 128, 11, 1))), + # MoE router (512 experts), trunk and draft layer. + (512, 2560): ((4, _d(64, 1)), (16, _w(16, 8, 256, 10, 1))), + # Shared expert gate|up|gate logit|pad, 1288: the warp kernel cannot tile it. + (1288, 2560): ((4, _d(64, 2)),), + # Shared expert down projection. + (2560, 640): ((4, _w(16, 8, 128, 5, 2)),), + # Draft layer `fc_embedding` (M rows) and `fc_hidden` (4M rows, one per stream). + (2560, 2560): ((1, _d(64, 2)), (4, _w(16, 8, 256, 9, 1)), (16, _w(32, 16, 256, 7, 1))), + # Draft layer q|gate. + (3072, 2560): ((1, _d(64, 2)), (4, _w(16, 8, 256, 9, 1))), + # Draft layer k and v. + (256, 2560): ((4, _d(64, 1)),), + # Draft layer index q|k. + (640, 2560): ((4, _d(64, 1)),), + # tp2/tep2 attention input projection (7424). Its GDN input projection (8240) is not a whole + # number of 32-wide tiles and stays on cuBLAS. + (7424, 2560): ((8, _w(32, 8, 256, 5, 2)), (16, _w(32, 16, 256, 4, 2))), + # tp2/tep2 attention o and GDN out (3072 in). + (2560, 3072): ((8, _w(32, 8, 256, 11, 2)), (16, _w(32, 16, 256, 9, 2))), + # tp1 attention input projection (14080); cuBLAS catches up from 12 rows, and GDN in (16480) + # stays on cuBLAS. + (14080, 2560): ((8, _w(32, 8, 256, 5, 2)),), + # tp1 attention o and GDN out (6144 in). + (2560, 6144): ((8, _w(32, 8, 256, 11, 2)), (16, _w(32, 16, 256, 9, 2))), +} + + +@functools.lru_cache(maxsize=None) +def plan(m: int, n: int, k: int) -> _Plan: + """The kernel and tactic `LowMBf16Mm` launches for an [M, K] x [K, N] product.""" + for bound, choice in _TABLE.get((n, k), ()): + if m <= bound: + if choice[0] == "direct": + t = _fi_direct.DirectTactic( + choice[1], choice[2], _fi_direct.default_tactic(m, n, k).rows_per_block + ) + _fi_direct.validate_tactic(t, m, n, k) + return _Plan("direct", tuple(choice[1:]) + (t.rows_per_block,)) + _fi_warp.validate_tactic(_fi_warp.WarpSplitKTactic(*choice[1:]), m, n, k) + return _Plan("warp", tuple(choice[1:])) + return _Plan("cublas") + + +def _cell(why: str, m: int, n: int, k: int, kind: str, pdl: bool = True) -> Cell: + """`kind` is the branch `plan` must pick for this shape; the test holds it to that.""" + return Cell(why=why, spec=dict(m=m, n=n, k=k, kind=kind, pdl=pdl)) + + +class LowMBf16Mm(OpWrapper): + """`mat_a @ mat_b` in bf16 with fp32 accumulation: one kernel launch per call. + + A drop-in for `catalog/gemm/cublas_mm` without bias or `out_dtype`: `mat_a` dense row-major + `[M, K]`, `mat_b` dense column-major `[K, N]` (a weight's `.t()`), the result a fresh + `[M, N]` bf16 tensor. `plan(M, N, K)` picks the FlashInfer direct kernel (M <= 2, and up to 4 + on narrow outputs), its warp split-K kernel (M 2..16 where 16 divides N) or cuBLAS. + + `pdl` launches the FlashInfer kernels with programmatic dependent launch: each streams its + weight before `griddepcontrol.wait` and reads `mat_a` and writes its output only after it, + so it is safe after any kernel; it changes nothing on the cuBLAS branch. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # One cell per `_TABLE` tactic (PDL on), N=1288, both cuBLAS fallbacks and PDL off for each + # FlashInfer kernel; `kind` pins the branch (the test's plan pins hold the other M buckets). + CELLS: tuple[Cell, ...] = ( + _cell("tep4 attn in M=4 (MTP-3 verify, 1 request): warp split-K", 4, 4352, 2560, "warp"), + _cell("tep4 attn in M=16 (MTP-3, 4 requests): warp 16-token tile", 16, 4352, 2560, "warp"), + _cell("tep4 attn in M=1 (decode, no MTP): direct", 1, 4352, 2560, "direct"), + _cell("tep4 attn in M=24: cuBLAS above the table", 24, 4352, 2560, "cublas"), + _cell("tep4 GDN in (padded) M=8 (MTP-3, 2 requests): warp 128-K", 8, 4128, 2560, "warp"), + _cell("tep4 GDN in (padded) M=1, PDL off: direct", 1, 4128, 2560, "direct", pdl=False), + _cell("tep4 GDN in (padded) M=3, PDL off: warp", 3, 4128, 2560, "warp", pdl=False), + _cell("tep4 o / GDN out M=3 (MTP-2 verify): warp, ragged tile", 3, 2560, 1536, "warp"), + _cell("tep4 o / GDN out M=1: direct 192-thread", 1, 2560, 1536, "direct"), + _cell("dep4 GDN in M=32: cuBLAS (not in the table)", 32, 16480, 2560, "cublas"), + _cell("tep4 router M=4 (MTP-3 verify / draft extend): direct", 4, 512, 2560, "direct"), + _cell("tep4 router M=16 (MTP-3, 4 requests): warp", 16, 512, 2560, "warp"), + _cell("tep4 shared gate|up M=4: direct", 4, 1288, 2560, "direct"), + _cell("tep4 shared down M=4: warp", 4, 2560, 640, "warp"), + _cell("tep4 draft fc_embedding M=4 (draft extend): warp", 4, 2560, 2560, "warp"), + _cell("tep4 draft fc_hidden M=16 (4 rows x 4 streams): warp 32x16", 16, 2560, 2560, "warp"), + _cell("tep2 attn in M=3 (MTP-2 verify): warp 32-wide", 3, 7424, 2560, "warp"), + _cell("tep2 attn in M=16 (4 requests x 4 rows): warp 16-token", 16, 7424, 2560, "warp"), + _cell("tep2 o / GDN out M=12: warp 16-token", 12, 2560, 3072, "warp"), + _cell("tp1 o / GDN out M=1: warp", 1, 2560, 6144, "warp"), + ) + + note = """ + Preconditions: bf16, `mat_a` dense row-major, `mat_b` dense column-major, both 32-byte + aligned (the FlashInfer kernels refuse padded operands; cuBLAS reads them as dense, see + `is_valid`). The FlashInfer kernels are JIT-compiled per (N, K, tactic, pdl; direct also per + rows_per_block) on the first call, which must come before CUDA-graph capture, and cost tens + of microseconds of host time a call: route only captured decode calls here. + + Certified on GB300 within the test's 4 / 2 bf16-ulp band at the cells: M 1, 3, 4, 8, 12, 16 + on the FlashInfer kernels, 24 and 32 on cuBLAS. Not certified: M above 16 on FlashInfer. + """ + + def __call__(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> torch.Tensor: + m, k = mat_a.shape + n = mat_b.shape[1] + p = plan(m, n, k) + if p.kind == "cublas": + return torch.ops.trtllm.cublas_mm(mat_a, mat_b, None, None, 0, None) + out = torch.empty((m, n), dtype=mat_a.dtype, device=mat_a.device) + if p.kind == "direct": + return _fi_direct.run_direct_dense( + mat_a, mat_b, out, pdl, _fi_direct.DirectTactic(*p.tactic) + ) + return _fi_warp.run_warp_splitk_dense( + mat_a, mat_b, out, pdl, _fi_warp.WarpSplitKTactic(*p.tactic) + ) + + def reference(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> torch.Tensor: + """fp32-accumulated, TF32 off; `pdl` is a launch attribute and is ignored.""" + with true_fp32_matmul(): + return (mat_a.float() @ mat_b.float()).to(mat_a.dtype) + + def is_valid(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> None: + # The cuBLAS branch reads a row-padded mat_a or column-padded mat_b as dense: silently a + # different product. The FlashInfer branches raise on those. + assert mat_a.dtype == torch.bfloat16 and mat_b.dtype == torch.bfloat16, "bf16 operands only" + assert mat_a.is_contiguous(), "mat_a must be dense row-major [M, K]" + assert mat_b.stride(0) == 1 and mat_b.stride(1) == mat_b.shape[0], ( + "mat_b must be dense column-major [K, N] (e.g. weight.t())" + ) + + +low_m_bf16_mm = LowMBf16Mm() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py new file mode 100644 index 000000000000..11717c851d94 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py @@ -0,0 +1,1046 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Decode hyper-connection site for 1..16 rows on warp tensor cores: two PDL launches, the +first emitting the bottleneck `low`, the second the up projection, gate and stream mean. +The NVFP4 tp1 target's `qwen4_exp_hc_decode_rows_tc` kernels, bf16 variants only (the routed +experts' output arrives finalized and summed; the mix feeds bf16 blocks). + +first (`_First2`): a (2 K halves x 4 streams) cluster per 24-column chunk forms the injection, +the per-stream sums of squares and `y = bf16(x * (1 + w))` and multiplies `y` by the weight +rows on mma.sync; cluster rank o reduces rows (2o, 2o+1) into `rstd_s = (ss_s / H + eps)^-1/2`, +the logits `z = sum_s rstd_s (P_s,0 + P_s,1) / C`, `low = bf16(silu(z))` (columns < 320) and +the injection weights `2 sigmoid(z)` (columns 320..323). mix (`_Mix2`): the up projection of +`low`, gate x normed, the stream mean. + +`hc_out` is bitwise `qwen4_exp_hc_inject_norm`'s; the rest is bf16 operands on tensor cores +with fp32 accumulation (1-2 bf16 ulp of the float64 composition). +""" + +from __future__ import annotations + +import functools + +import cuda.bindings.driver as cuda_drv +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute import experimental as cute_ext +from cutlass.cute.nvgpu import cpasync, warp +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + _asm, + _mapa, + _sigmoid, +) +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl + +#: This checkpoint's hyper-connection bottleneck width. +RANK = 320 +#: Rows a launch serves (one m16 MMA tile). +MAX_ROWS = 16 +M = 16 +VEC = 8 +PAD = 8 +KH = 2 # K halves +KHW = 1280 +NC = 24 # columns a CTA +NCH = 14 # 14 x 24 = 336 >= 324 +WARPS = 8 +THREADS = WARPS * 32 +KW = KHW // WARPS # K a warp +CL = KH * 4 # cluster size +RPO = M // CL +#: Item iterations a thread whose activation loads issue together (all of them at <= 16 rows). +GROUP_IT = 10 +#: Rows of First2's A stage allocated at least; the footprint leaves an SM room for the mix CTAs. +A_MIN_ROWS = 8 +BLOCK_H = 16 + + +@dsl_user_op +def _st_async_f32(addr, val, mbar, *, loc=None, ip=None): + """4 bytes into a cluster CTA's shared memory, completing on that CTA's mbarrier.""" + _asm( + T.i32(), + [cutlass.Int32(addr), cutlass.Float32(val), cutlass.Int32(mbar)], + "{ st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [$1], $2, [$3]; " + "mov.u32 $0, 0; }", + "=r,r,f,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _mbar_init_tx(mbar, nbytes, *, loc=None, ip=None): + """Init an mbarrier (one arrival), make it visible to the cluster, arrive expecting + `nbytes` of transactions.""" + _asm( + T.i32(), + [cutlass.Int32(mbar), cutlass.Int32(nbytes)], + "{ mbarrier.init.shared::cta.b64 [$1], 1; fence.mbarrier_init.release.cluster; " + "mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _mbar_wait0(mbar, *, loc=None, ip=None): + _asm( + T.i32(), + [cutlass.Int32(mbar)], + "{ .reg .pred p; WAIT: mbarrier.try_wait.parity.shared::cta.b64 p, [$1], 0; " + "@!p bra WAIT; mov.u32 $0, 0; }", + "=r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _bulk_g2s(dst, src, nbytes: int, mbar, *, loc=None, ip=None): + """One non-tensor bulk copy global -> this CTA's shared memory, completing on `mbar`.""" + _asm( + T.i32(), + [cutlass.Int32(dst), cutlass.Int64(src), cutlass.Int32(mbar)], + "{ cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], " + + str(nbytes) + + ", [$3]; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side_effects=True, + ) + + +class _First2: + def __init__(self, *, hidden, rank, n_cols, rows, inject, combine, pdl): + assert hidden == KH * KHW and n_cols <= NC * NCH and rows <= M + assert inject or not combine, "the combine forms the injected block output" + self.hidden, self.rank, self.n_cols, self.rows = hidden, rank, n_cols, rows + self.inject, self.combine, self.pdl = inject, combine, pdl + self.ld = KHW + PAD + self.rld = NC + 2 # pushed slot: NC partials + the sum of squares (+pad) + + def smem_bytes(self): + def r(n): + return -(-n // 128) * 128 + + assert WARPS * M * NC * 4 + M * NC * 4 <= NC * self.ld * 2 + b = r(max(self.rows, A_MIN_ROWS) * self.ld * 2) + r(NC * self.ld * 2) + b += r(self.rows * (KHW // VEC) * 4) + b += r(CL * RPO * self.rld * 4) + r(M * 4) + b += r(KHW * 2) # the norm-weight slice + return b + 256 + 128 + + @cute.jit + def __call__( + self, + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps: cutlass.Float32, + stream: cuda_drv.CUstream, + ): + mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(cutlass.BFloat16, cutlass.Float32, (16, 8, 16)), + cute.make_layout((1, 1, 1)), + ) + g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.BFloat16, num_bits_per_copy=128) + self.kernel( + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps, + mma, + g2s, + ).launch( + grid=[KH, 4, NCH], + block=[THREADS, 1, 1], + cluster=[KH, 4, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps: cutlass.Float32, + mma: cute.TiledMma, + g2s: cute.CopyAtom, + ): + tidx, _, _ = cute.arch.thread_idx() + kh, s, nc = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + rows: cutlass.Constexpr = self.rows + hidden: cutlass.Constexpr = self.hidden + rank: cutlass.Constexpr = self.rank + n_cols: cutlass.Constexpr = self.n_cols + ld: cutlass.Constexpr = self.ld + rld: cutlass.Constexpr = self.rld + nvec: cutlass.Constexpr = KHW // VEC + k0 = kh * KHW + n0 = nc * NC + me = kh + KH * s # cluster rank + + smem = cutlass.utils.SmemAllocator() + # sA holds the R live rows only; the MMA's 16-row A view runs on into sW (rows past R + # are garbage, their outputs discarded: the MMA's rows are independent). + sA = smem.allocate_tensor( + cutlass.BFloat16, + cute.make_layout((max(rows, A_MIN_ROWS), KHW), stride=(ld, 1)), + byte_alignment=128, + ) + sW = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((NC, KHW), stride=(ld, 1)), byte_alignment=128 + ) + sSs = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((rows, nvec), stride=(nvec, 1)), byte_alignment=128 + ) + # The warps' partials and their sum reuse sW once every warp holds its B fragment. + wp_ptr = cute.recast_ptr(sW.iterator, dtype=cutlass.Float32) + sWp = cute.make_tensor(wp_ptr, cute.make_layout((WARPS, M, NC), stride=(M * NC, NC, 1))) + sAcc = cute.make_tensor(wp_ptr + WARPS * M * NC, cute.make_layout((M, NC), stride=(NC, 1))) + sRed = smem.allocate_tensor( + cutlass.Float32, + cute.make_layout((CL, RPO, rld), stride=(RPO * rld, rld, 1)), + byte_alignment=128, + ) + sSsr = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M,), stride=(1,)), byte_alignment=128 + ) + sNw = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((1, KHW), stride=(KHW, 1)), byte_alignment=128 + ) + sBar = smem.allocate_tensor(cutlass.Int64, cute.make_layout((2,)), byte_alignment=8) + rbar = cutlass.Int32(sBar.iterator.toint()) + wbar = rbar + 8 # the weight rows' bulk copies + # This CTA receives rows [me * RPO, me * RPO + RPO) of the cluster's partials from all CL + # CTAs, by st.async on `rbar` (no cluster barrier, no fence, after the pushes). + rows_me = cutlass.max(cutlass.min(cutlass.Int32(RPO), rows - me * RPO), cutlass.Int32(0)) + if tidx == 0: + _mbar_init_tx(rbar, rows_me * (CL * (NC + 1) * 4)) + _mbar_init_tx(wbar, NC * KHW * 2) + cute.arch.cluster_arrive() + + # 1. Before the wait: the norm weight's slice (its own cp.async group, waited on right + # after the PDL wait), then the CTA's 24 weight rows over its K half. + nw_vec = cute.logical_divide(mNorm, (None, VEC)) + snw_vec = cute.logical_divide(sNw, (None, VEC)) + if tidx < KHW // VEC: + cute.copy( + g2s, + nw_vec[0, (None, (s * hidden + k0) // VEC + tidx)], + snw_vec[0, (None, tidx)], + ) + cute.arch.cp_async_commit_group() + # One bulk copy a weight row (2560 B) into its padded shared row, completing on `wbar`. + col0 = (s * hidden + k0) // VEC + if tidx == 0: + w_base = mW.iterator.toint() + cutlass.Int64(col0 * VEC * 2) + sw_base = cutlass.Int32(sW.iterator.toint()) + for n in cutlass.range_constexpr(NC): + _bulk_g2s( + sw_base + n * (ld * 2), + w_base + cutlass.Int64(n0 + n) * (4 * hidden * 2), + KHW * 2, + wbar, + ) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + + # 2. The K half's injection, sums of squares, y (bf16) into sA. A thread issues all + # loads of a group of GROUP_IT items before using the first: one L2 round trip a group. + hc_vec = cute.logical_divide(mHc, (None, VEC)) + b_vec = cute.logical_divide(mBlock, (None, VEC)) + sh_vec = cute.logical_divide(mShared, (None, VEC)) + o_vec = cute.logical_divide(mHcOut, (None, VEC)) + sa_vec = cute.logical_divide(sA, (None, VEC)) + items: cutlass.Constexpr = rows * nvec + n_it: cutlass.Constexpr = -(-items // THREADS) + cute.arch.cp_async_wait_group(0) # the norm slice (the weights land on `wbar`) + cute.arch.sync_threads() + nwr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + outr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + yr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + for g0 in cutlass.range_constexpr(0, n_it, GROUP_IT): + g_n: cutlass.Constexpr = min(GROUP_IT, n_it - g0) + xg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + bg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + shg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + ig = cute.make_rmem_tensor(cute.make_layout((g_n,), stride=(1,)), cutlass.Float32) + zg = cute.make_rmem_tensor(cute.make_layout((g_n,), stride=(1,)), cutlass.Float32) + for k in cutlass.range_constexpr(g_n): + # Past the last item a thread reads the last item's operands (unused). + c = cutlass.min(tidx + (g0 + k) * THREADS, cutlass.Int32(items - 1)) + r = c // nvec + p = c % nvec + cute.autovec_copy(hc_vec[r, (None, col0 + p)], xg[k, None]) + if const_expr(self.inject): + cute.autovec_copy(b_vec[r, (None, k0 // VEC + p)], bg[k, None]) + ig[k] = mInj[r, s].to(cutlass.Float32) + if const_expr(self.combine): + cute.autovec_copy(sh_vec[r, (None, k0 // VEC + p)], shg[k, None]) + zg[k] = mGate[r, 0].to(cutlass.Float32) + for k in cutlass.range_constexpr(g_n): + c = tidx + (g0 + k) * THREADS + if c < items: + r = c // nvec + p = c % nvec + cute.autovec_copy(snw_vec[0, (None, p)], nwr) + ssum = cutlass.Float32(0.0) + if const_expr(self.inject): + g = ig[k] + if const_expr(self.combine): + # `tl.sigmoid`'s instructions, so the combine (and so `hc_out`) + # is bitwise `qwen4_exp_hc_inject_norm`'s. + sg = _sigmoid(zg[k]) + for v in cutlass.range_constexpr(VEC): + bv = bg[k, v].to(cutlass.Float32) + if const_expr(self.combine): + # The MoE block output, rounded once. + bv = ( + cute.math.fma(sg, shg[k, v].to(cutlass.Float32), bv) + .to(cutlass.BFloat16) + .to(cutlass.Float32) + ) + outr[v] = cute.math.fma(bv, g, xg[k, v].to(cutlass.Float32)).to( + cutlass.BFloat16 + ) + if nc == 0: + cute.autovec_copy(outr, o_vec[r, (None, col0 + p)]) + else: + for v in cutlass.range_constexpr(VEC): + outr[v] = xg[k, v] + for v in cutlass.range_constexpr(VEC): + xv = outr[v].to(cutlass.Float32) + ssum = xv * xv + ssum + yr[v] = (xv * (1.0 + nwr[v].to(cutlass.Float32))).to(cutlass.BFloat16) + cute.autovec_copy(yr, sa_vec[r, (None, p)]) + sSs[r, p] = ssum + _mbar_wait0(wbar) + cute.arch.sync_threads() + + # 3. Warp w: rows x its 160-wide K slice x the 24 columns; the warps' partials summed in + # warp order; the K half's sums of squares per row. + # The warp's K slice: 320-byte offsets, so 16-byte aligned (ldmatrix rows). + a_w = cute.make_tensor( + cute.make_ptr( + cutlass.BFloat16, + sA.iterator.toint() + warp_i * (KW * 2), + cute.AddressSpace.smem, + assumed_align=16, + ), + cute.make_layout((M, KW), stride=(ld, 1)), + ) + b_w = cute.make_tensor( + cute.make_ptr( + cutlass.BFloat16, + sW.iterator.toint() + warp_i * (KW * 2), + cute.AddressSpace.smem, + assumed_align=16, + ), + cute.make_layout((NC, KW), stride=(ld, 1)), + ) + thr = mma.get_slice(lane) + tCsA = thr.partition_A(a_w) + tCrA = thr.make_fragment_A(tCsA) + tCsB = thr.partition_B(b_w) + tCrB = thr.make_fragment_B(tCsB) + # Fragments by ldmatrix (4 / 2 8x8 matrices an instruction). + ldsm_a = cute.make_tiled_copy_A( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 4), cutlass.BFloat16), mma + ) + ldsm_b = cute.make_tiled_copy_B( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 2), cutlass.BFloat16), mma + ) + cpa = ldsm_a.get_slice(lane) + cpb = ldsm_b.get_slice(lane) + cute.copy(ldsm_a, cpa.partition_S(a_w), cpa.retile(tCrA)) + cute.copy(ldsm_b, cpb.partition_S(b_w), cpb.retile(tCrB)) + acc = cute.make_rmem_tensor(thr.partition_shape_C((M, NC)), cutlass.Float32) + acc.fill(0.0) + cute.gemm(mma, acc, tCrA, tCrB, acc) + cute.arch.sync_threads() # every warp's fragments are loaded: sW may be reused + co = thr.partition_C(cute.make_identity_tensor((M, NC))) + for j in cutlass.range_constexpr(cute.size(acc)): + rr, nn = co[j] + sWp[warp_i, rr, nn] = acc[j] + # Sums of squares: 16 threads a row (10 vectors each), then a 16-lane butterfly. + segs: cutlass.Constexpr = THREADS // M + per_seg: cutlass.Constexpr = nvec // segs + rq = tidx // segs + sq = tidx % segs + t = cutlass.Float32(0.0) + if rq < rows: + for p in cutlass.range_constexpr(per_seg): + t = t + sSs[rq, sq * per_seg + p] + t = cute.arch.warp_reduction_sum(t, threads_in_group=segs) + if sq == 0: + if rq < rows: + sSsr[rq] = t + cute.arch.sync_threads() + for i in cutlass.range_constexpr(-(-(M * NC) // THREADS)): + e = tidx + i * THREADS + if e < M * NC: + rr = e // NC + nn = e % NC + t = cutlass.Float32(0.0) + for w in cutlass.range_constexpr(WARPS): + t = t + sWp[w, rr, nn] + sAcc[rr, nn] = t + cute.arch.sync_threads() + + # 4. Push rows (2o, 2o+1) (and their sums of squares) to cluster rank o, slot `me`. + cute.arch.cluster_wait() + red_addr = cutlass.Int32(sRed.iterator.toint()) + per: cutlass.Constexpr = RPO * (NC + 1) + for it in cutlass.range_constexpr(-(-(CL * per) // THREADS)): + c = tidx + it * THREADS + if c < CL * per: + o = c // per + rem = c % per + rl = rem // (NC + 1) + nn = rem % (NC + 1) + rr = o * RPO + rl + if rr < rows: + v = cutlass.Float32(0.0) + if nn < NC: + v = sAcc[rr, nn] + else: + v = sSsr[rr] + dst = red_addr + ((me * RPO + rl) * rld + nn) * 4 + _st_async_f32(_mapa(dst, o), v, _mapa(rbar, o)) + _mbar_wait0(rbar) + + # 5. Rank `me` finishes rows (2 me, 2 me + 1): rstd per stream, the logits, low / injection. + for it in cutlass.range_constexpr(-(-(RPO * NC) // THREADS)): + c = tidx + it * THREADS + if c < RPO * NC: + rl = c // NC + nn = c % NC + rr = me * RPO + rl + col = n0 + nn + if rr < rows: + if col < n_cols: + z = cutlass.Float32(0.0) + for st in cutlass.range_constexpr(4): + ssv = sRed[KH * st, rl, NC] + sRed[KH * st + 1, rl, NC] + rs = cute.math.rsqrt(ssv / hidden + eps) + z = (sRed[KH * st, rl, nn] + sRed[KH * st + 1, rl, nn]) * rs + z + z = z * 0.25 + sig = 1.0 / (1.0 + cute.math.exp(-z)) + if col < rank: + mLow[rr, col] = (z * sig).to(cutlass.BFloat16) + else: + mInjOut[rr, col - rank] = (2.0 * sig).to(cutlass.BFloat16) + if nc == 0: + if tidx < RPO * 4: + rl = tidx // 4 + st = tidx % 4 + rr = me * RPO + rl + if rr < rows: + ssv = sRed[KH * st, rl, NC] + sRed[KH * st + 1, rl, NC] + mRstd[rr, st] = cute.math.rsqrt(ssv / hidden + eps) + + +def _dl(t: torch.Tensor, align: int = 16): + return from_dlpack(t, assumed_align=align) + + +@functools.lru_cache(maxsize=None) +def _compiled_first( + hidden, + hc_count, + rank, + n_cols, + n_wrows, + rows, + inject, + combine, + pdl, + hc_stride, + block_stride, + gate_stride, +): + kern = _First2( + hidden=hidden, + rank=rank, + n_cols=n_cols, + rows=rows, + inject=inject, + combine=combine, + pdl=pdl, + ) + dev, bf = "cuda", torch.bfloat16 + width = hidden * hc_count + hc = torch.empty(rows, hc_stride, dtype=bf, device=dev)[:, :width] + block = torch.empty(rows, block_stride, dtype=bf, device=dev)[:, :hidden] + inj = torch.empty(rows, hc_count, dtype=bf, device=dev) + shared = torch.empty(rows, hidden, dtype=bf, device=dev) + gate = torch.empty(rows, gate_stride, dtype=bf, device=dev)[:, :1] + norm = torch.empty(1, width, dtype=bf, device=dev) + w = torch.empty(n_wrows, width, dtype=bf, device=dev) + hc_out = torch.empty(rows, width, dtype=bf, device=dev) + low = torch.empty(rows, rank, dtype=bf, device=dev) + inj_out = torch.empty(rows, hc_count, dtype=bf, device=dev) + rstd = torch.empty(rows, hc_count, dtype=torch.float32, device=dev) + stream = cuda_drv.CUstream(torch.cuda.current_stream().cuda_stream) + return cute_ext.compile( + kern, + _dl(hc), + _dl(block), + _dl(inj, 2), + _dl(shared), + _dl(gate, 2), + _dl(norm), + _dl(w), + _dl(hc_out), + _dl(low), + _dl(inj_out, 2), + _dl(rstd, 4), + cutlass.Float32(1e-6), + stream, + ) + + +def first_rows( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + rank=RANK, +): + """Returns `(hc_out, low [R, rank] bf16, inject_out [R, C] bf16 or None, rstd [R, C] fp32)`; + `n_cols` = rank + C with the injection, rank without (the stack-final mixer).""" + rows, width = hyper_input.shape + hidden = width // hc_count + dev = hyper_input.device + has_inject = block_out is not None + has_shared = shared is not None + hc_out = ( + torch.empty((rows, width), dtype=hyper_input.dtype, device=dev) + if has_inject + else hyper_input + ) + low = torch.empty((rows, rank), dtype=hyper_input.dtype, device=dev) + inj_out = torch.empty((rows, hc_count), dtype=hyper_input.dtype, device=dev) + rstd = torch.empty((rows, hc_count), dtype=torch.float32, device=dev) + weight = first.t() + assert weight.shape[0] >= NC * NCH + blk = block_out if has_inject else hyper_input.new_empty(rows, hidden) + inj = inject if has_inject else hyper_input.new_empty(rows, hc_count) + sh = shared if has_shared else hyper_input.new_empty(rows, hidden) + # The gate logit is a [R, 1] column (the shared expert's gate|up output carries it). + gate = gate_logits.reshape(rows, 1) if has_shared else hyper_input.new_empty(rows, 1) + fn = _compiled_first( + hidden, + hc_count, + rank, + n_cols, + weight.shape[0], + rows, + has_inject, + has_shared, + bool(get_env_enable_pdl()), + hyper_input.stride(0), + blk.stride(0), + gate.stride(0), + ) + stream = cuda_drv.CUstream(torch.cuda.current_stream(dev).cuda_stream) + fn( + _dl(hyper_input), + _dl(blk), + _dl(inj, 2), + _dl(sh), + _dl(gate, 2), + _dl(norm_weight.reshape(1, -1)), + _dl(weight), + _dl(hc_out), + _dl(low), + _dl(inj_out, 2), + _dl(rstd, 4), + cutlass.Float32(eps), + stream, + ) + return hc_out, low, (inj_out if n_cols > rank else None), rstd + + +class _Mix2: + def __init__(self, *, hidden, streams, rank, rows, pdl): + assert streams * BLOCK_H == WARPS * 8 and rows <= M and rank % 16 == 0 + self.hidden, self.streams, self.rank, self.rows, self.pdl = hidden, streams, rank, rows, pdl + self.ld = rank + PAD + + def smem_bytes(self): + def r(n): + return -(-n // 128) * 128 + + nrow = self.streams * BLOCK_H + b = r(nrow * self.ld * 2) + r(M * self.ld * 2) + b += r(M * nrow * 4) + r(M * nrow * 2) + r(M * self.streams * 4) + r(nrow * 4) + return b + 256 + + @cute.jit + def __call__(self, mLow, mRstd, mUp, mHc, mNorm, mMixed, stream: cuda_drv.CUstream): + mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(cutlass.BFloat16, cutlass.Float32, (16, 8, 16)), + cute.make_layout((1, WARPS, 1)), + ) + g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.BFloat16, num_bits_per_copy=128) + self.kernel(mLow, mRstd, mUp, mHc, mNorm, mMixed, mma, g2s).launch( + grid=[self.hidden // BLOCK_H, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=2, + ) + + @cute.kernel + def kernel( + self, + mLow, + mRstd, + mUp, + mHc, + mNorm, + mMixed, + mma: cute.TiledMma, + g2s: cute.CopyAtom, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + hidden: cutlass.Constexpr = self.hidden + streams: cutlass.Constexpr = self.streams + rank: cutlass.Constexpr = self.rank + rows: cutlass.Constexpr = self.rows + ld: cutlass.Constexpr = self.ld + nrow: cutlass.Constexpr = streams * BLOCK_H + i0 = bidx * BLOCK_H + + smem = cutlass.utils.SmemAllocator() + sU = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((nrow, rank), stride=(ld, 1)), byte_alignment=128 + ) + sL = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((M, rank), stride=(ld, 1)), byte_alignment=128 + ) + sV = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M, nrow), stride=(nrow, 1)), byte_alignment=128 + ) + sX = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((M, nrow), stride=(nrow, 1)), byte_alignment=128 + ) + sR = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M, streams), stride=(streams, 1)), byte_alignment=128 + ) + sG = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((nrow,), stride=(1,)), byte_alignment=128 + ) + + u_vec = cute.logical_divide(mUp, (None, VEC)) + su_vec = cute.logical_divide(sU, (None, VEC)) + pieces: cutlass.Constexpr = nrow * (rank // VEC) + for i in cutlass.range_constexpr(-(-pieces // THREADS)): + c = tidx + i * THREADS + if c < pieces: + n = c // (rank // VEC) + p = c % (rank // VEC) + jrow = (n // BLOCK_H) * hidden + i0 + n % BLOCK_H + cute.copy(g2s, u_vec[jrow, (None, p)], su_vec[n, (None, p)]) + cute.arch.cp_async_commit_group() + if tidx < nrow: + sG[tidx] = 1.0 + mNorm[0, (tidx // BLOCK_H) * hidden + i0 + tidx % BLOCK_H].to( + cutlass.Float32 + ) + # Rows of sL past R are never written (MMA rows are independent; discarded). Before the + # wait the up rows land and every thread loads its B fragments into registers. + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.cp_async_wait_group(0) + cute.arch.sync_threads() + thr = mma.get_slice(tidx) + tCsB = thr.partition_B(sU) + tCrB = thr.make_fragment_B(tCsB) + cute.autovec_copy(tCsB, tCrB) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + # low rows (16-byte pieces), rstd, this CTA's residual elements. + l_vec = cute.logical_divide(mLow, (None, VEC)) + sl_vec = cute.logical_divide(sL, (None, VEC)) + lp: cutlass.Constexpr = rows * (rank // VEC) + for i in cutlass.range_constexpr(-(-lp // THREADS)): + c = tidx + i * THREADS + if c < lp: + rr = c // (rank // VEC) + p = c % (rank // VEC) + cute.copy(g2s, l_vec[rr, (None, p)], sl_vec[rr, (None, p)]) + cute.arch.cp_async_commit_group() + if tidx < rows * streams: + sR[tidx // streams, tidx % streams] = mRstd[tidx // streams, tidx % streams] + h_vec = cute.logical_divide(mHc, (None, VEC)) + sx_vec = cute.logical_divide(sX, (None, VEC)) + xp: cutlass.Constexpr = rows * (nrow // VEC) + if tidx < xp: + rr = tidx // (nrow // VEC) + q = tidx % (nrow // VEC) + cute.copy( + g2s, + h_vec[ + rr, + ( + None, + ((q * VEC) // BLOCK_H) * (hidden // VEC) + + (i0 + (q * VEC) % BLOCK_H) // VEC, + ), + ], + sx_vec[rr, (None, q)], + ) + cute.arch.cp_async_commit_group() + cute.arch.cp_async_wait_group(0) + cute.arch.sync_threads() + + tCsA = thr.partition_A(sL) + tCrA = thr.make_fragment_A(tCsA) + ldsm_a = cute.make_tiled_copy_A( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 4), cutlass.BFloat16), mma + ) + cpa = ldsm_a.get_slice(tidx) + cute.copy(ldsm_a, cpa.partition_S(sL), cpa.retile(tCrA)) + acc = cute.make_rmem_tensor(thr.partition_shape_C((M, nrow)), cutlass.Float32) + acc.fill(0.0) + cute.gemm(mma, acc, tCrA, tCrB, acc) + co = thr.partition_C(cute.make_identity_tensor((M, nrow))) + for j in cutlass.range_constexpr(cute.size(acc)): + rr, n = co[j] + if rr < rows: + gate = 1.0 / (1.0 + cute.math.exp(-acc[j])) + x = sX[rr, n].to(cutlass.Float32) + sV[rr, n] = gate * (x * sR[rr, n // BLOCK_H] * sG[n]) + cute.arch.sync_threads() + + for rb in cutlass.range_constexpr(0, rows, WARPS): + r = rb + warp_i + if r < rows: + if lane < BLOCK_H: + tot = cutlass.Float32(0.0) + for t in cutlass.range_constexpr(streams): + tot = tot + sV[r, t * BLOCK_H + lane] + mMixed[r, i0 + lane] = (tot * (1.0 / streams)).to(cutlass.BFloat16) + + +@functools.lru_cache(maxsize=None) +def _compiled_mix(hidden, streams, rank, rows, hc_stride, pdl): + kern = _Mix2(hidden=hidden, streams=streams, rank=rank, rows=rows, pdl=pdl) + dev, bf = "cuda", torch.bfloat16 + width = hidden * streams + low = torch.empty(rows, rank, dtype=bf, device=dev) + rstd = torch.empty(rows, streams, dtype=torch.float32, device=dev) + up = torch.empty(width, rank, dtype=bf, device=dev) + hc = torch.empty(rows, hc_stride, dtype=bf, device=dev)[:, :width] + norm = torch.empty(1, width, dtype=bf, device=dev) + mixed = torch.empty(rows, hidden, dtype=bf, device=dev) + stream = cuda_drv.CUstream(torch.cuda.current_stream().cuda_stream) + return cute_ext.compile( + kern, + _dl(low), + _dl(rstd, 4), + _dl(up), + _dl(hc, 16), + _dl(norm, 2), + _dl(mixed, 2), + stream, + ) + + +def mix_rows(low, rstd, up, hc_out, norm_weight, hc_count): + rows, width = hc_out.shape + hidden = width // hc_count + rank = up.shape[0] + dev, dt = hc_out.device, hc_out.dtype + mixed = torch.empty((rows, hidden), dtype=dt, device=dev) + fn = _compiled_mix(hidden, hc_count, rank, rows, hc_out.stride(0), bool(get_env_enable_pdl())) + stream = cuda_drv.CUstream(torch.cuda.current_stream(dev).cuda_stream) + fn( + _dl(low), + _dl(rstd, 4), + _dl(up.t()), + _dl(hc_out, 16), + _dl(norm_weight.reshape(1, -1), 2), + _dl(mixed, 2), + stream, + ) + return mixed + + +def _reference_first( + hyper_input, block_out, inject, norm_weight, first, hc_count, n_cols, eps, shared, gate_logits +): + """float64: the combine and the injection each rounded once as `qwen4_exp_hc_inject_norm` + does, the grouped RMS, the projection operand `y = x * (1 + w)` rounded once to bf16 (the + MMA's A operand), the logits `z = sum_s rstd_s (y_s @ W_s) / C`, `low = silu(z)` and + `2 sigmoid(z)` each rounded once, rstd in fp32.""" + dt = hyper_input.dtype + rows, width = hyper_input.shape + hidden = width // hc_count + if block_out is None: + hc_out = hyper_input + else: + blk = block_out.double() + if shared is not None: + sg = torch.sigmoid(gate_logits.double().reshape(rows, 1)) + blk = (blk + sg * shared.double()).to(dt).double() + streams = hyper_input.double().unflatten(-1, (hc_count, hidden)) + hc_out = (streams + blk.unsqueeze(-2) * inject.double().unsqueeze(-1)).flatten(-2).to(dt) + x = hc_out.double().unflatten(-1, (hc_count, hidden)) + rstd = torch.rsqrt(x.pow(2).mean(-1) + eps) + y = (x * (1.0 + norm_weight.double().unflatten(-1, (hc_count, hidden)))).to(dt).double() + z = (y * rstd.unsqueeze(-1)).flatten(-2) @ first[:, :n_cols].double() / hc_count + low = torch.nn.functional.silu(z[:, :RANK]).to(dt) + inj = (2.0 * torch.sigmoid(z[:, RANK:])).to(dt) if n_cols > RANK else None + return hc_out, low, inj, rstd.float() + + +class Qwen4ExpHcDecodeRowsFirst(OpWrapper): + """First launch of a 1..16-row decode hyper-connection site: the previous block's injection + (and, with `shared` / `gate_logits`, the MoE block's combine `routed + sigmoid(gate) * shared` + rounded once), the grouped RMS, the first projection over `[down; block_inject]`, the SiLU + bottleneck and the injection weights. Returns `(hc_out, low [R, 320] bf16, inject_out [R, C] + or None, rstd [R, C] fp32)`; `hc_out` *is* `hyper_input` without `block_out`. + + `hc_out` is bitwise `qwen4_exp_hc_inject_norm`'s; `low` and the injection weights are within + one bf16 step of the float64 composition over `y = bf16(x * (1 + w))` (`reference`). + + Fusion boundary: `qwen4_exp_hc_inject_norm` + `qwen4_exp_hc_decode_tc_first`, with the rstd + handed on instead of the `[R, C * H]` normed stream. + """ + + ARCHS = frozenset({Arch.SM_103}) + # spec: rows, site (combine: after a MoE block; inject: after an attention / GDN block or + # the draft front's embedding add; norm: no pending block), cols (324, or 320 for the + # stack-final / draft-head mixer); the checkpoint's geometry, bf16. + CELLS: tuple[Cell, ...] = ( + Cell( + why="1 row after a MoE block: a one-request decode step / draft pass", + spec=dict(rows=1, site="combine", cols=324), + ), + Cell( + why="1 row after an attention / GDN block: a draft pass's MoE site", + spec=dict(rows=1, site="inject", cols=324), + ), + Cell( + why="4 rows, the stack-final mixer after the last MoE block (320 cols)", + spec=dict(rows=4, site="combine", cols=320), + ), + Cell( + why="4 rows, norm only: the stack's first site (324 cols)", + spec=dict(rows=4, site="norm", cols=324), + ), + Cell( + why="16 rows after a MoE block: a 4-request MTP-3 verify step (MAX_ROWS)", + spec=dict(rows=16, site="combine", cols=324), + ), + Cell( + why="16 rows after an attention / GDN block: a 16-request rank graph's draft MoE site", + spec=dict(rows=16, site="inject", cols=324), + ), + Cell( + why="16 rows, norm only, 320 cols (MAX_ROWS)", + spec=dict(rows=16, site="norm", cols=320), + ), + ) + note = """ + Source: `_First2` above (CuTe DSL). Grid (2 K halves, 4 streams, 14 N chunks), cluster + (2, 4, 1); the norm slice and the CTA's 24 weight rows staged before the PDL wait, + dependents released there. The gate logit is read through its row stride (a column of the + shared expert's gate|up output). One compile per row count, variant and operand row + strides (lru_cache, first use: compile before capture). + Not certified: hc_count other than 4, hidden other than 2560, rank other than 320. + """ + + def __call__( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ): + return first_rows( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared, + gate_logits, + ) + + def reference( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ): + """float64 (`_reference_first`).""" + return _reference_first( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared, + gate_logits, + ) + + def is_valid( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ) -> None: + rows, width = hyper_input.shape + assert 1 <= rows <= MAX_ROWS, f"1..{MAX_ROWS} rows" + assert hyper_input.dtype == torch.bfloat16 and hyper_input.stride(1) == 1 + assert hc_count == 4 and width // hc_count == KH * KHW, "hc_count 4, hidden 2560" + assert n_cols in (RANK, RANK + hc_count), "the bottleneck (+ the injection) columns" + assert first.stride(0) == 1 and first.shape[1] >= NC * NCH, ( + "the padded [width, 384] .t() view" + ) + assert (block_out is None) == (inject is None) + assert (shared is None) == (gate_logits is None) + assert shared is None or block_out is not None, "the combine needs the routed rows" + if block_out is not None: + assert block_out.shape == (rows, width // hc_count) and block_out.stride(1) == 1 + assert inject.is_contiguous() and inject.shape == (rows, hc_count) + if shared is not None: + assert shared.is_contiguous() and shared.shape == block_out.shape + assert gate_logits.numel() == rows and gate_logits.stride(-1) == 1 + for t in (hyper_input, block_out, shared, first): + assert t is None or t.data_ptr() % 16 == 0, "16-byte aligned rows" + assert hyper_input.stride(0) % VEC == 0, "16-byte aligned rows" + assert block_out is None or block_out.stride(0) % VEC == 0, "16-byte aligned rows" + + +class Qwen4ExpHcDecodeRowsMix(OpWrapper): + """Second launch of a 1..16-row decode hyper-connection site: the up projection of `low`, + the sigmoid gate on the normed stream (`hc_out * rstd * (1 + w)`, fp32) and the stream + mean, rounded once. Returns `mixed` `[R, hidden]`. Fusion boundary: + `qwen4_exp_hc_up_gate_mean`, from `hc_out` and its rstd instead of the normed stream. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="1 row: a one-request decode step / draft pass", spec=dict(rows=1)), + Cell(why="16 rows: a 4-request MTP-3 verify step (MAX_ROWS)", spec=dict(rows=16)), + ) + note = """ + Source: `_Mix2` above (CuTe DSL), after `qwen4_exp_hc_decode_rows_first`. The up weight's + 64 rows staged and their B fragments loaded before the PDL wait; dependents released at + the start. + """ + + def __call__(self, low, rstd, up, hc_out, norm_weight, hc_count): + return mix_rows(low, rstd, up, hc_out, norm_weight, hc_count) + + def reference(self, low, rstd, up, hc_out, norm_weight, hc_count): + """float64 mixed, rounded once.""" + hidden = hc_out.shape[1] // hc_count + gate = torch.sigmoid(low.double() @ up.double()) + x = hc_out.double().unflatten(-1, (hc_count, hidden)) * rstd.double().unsqueeze(-1) + normed = x.flatten(-2) * (1.0 + norm_weight.double()) + return (gate * normed).unflatten(-1, (hc_count, hidden)).mean(-2).to(hc_out.dtype) + + def is_valid(self, low, rstd, up, hc_out, norm_weight, hc_count) -> None: + rows = hc_out.shape[0] + assert 1 <= rows <= MAX_ROWS and low.shape == (rows, RANK) and low.is_contiguous() + assert rstd.shape == (rows, hc_count) and rstd.is_contiguous() + assert hc_count == 4 and hc_out.shape[1] == hc_count * KH * KHW, "hc_count 4, hidden 2560" + assert up.shape == (RANK, hc_out.shape[1]) and up.t().is_contiguous(), ( + "up the [R, width] view of a row-major [width, R] weight" + ) + assert hc_out.stride(1) == 1 and hc_out.stride(0) % VEC == 0 + + +qwen4_exp_hc_decode_rows_first = Qwen4ExpHcDecodeRowsFirst() +qwen4_exp_hc_decode_rows_mix = Qwen4ExpHcDecodeRowsMix() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py new file mode 100644 index 000000000000..6bb9b7f5755a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py @@ -0,0 +1,805 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hyper-connection mixer first half at decode row counts (1..640) as one tcgen05 split-K +GEMM: `normed @ [down; block_inject]^T` with the SiLU and injection activations in its +epilogue. The NVFP4 default target's kernel; HC is replicated, so only the rows vary. + +A CTA computes a `[128, BN]` partial over `K / KS` (grid z is the M tile): a TMA warp streams +the normed rows (past `T` zero-filled) and the `n_cols` real weight rows (the padding is never +read), one MMA warp accumulates in tensor memory, four epilogue warps push the fp32 partial +into the shared memory of the cluster CTA that reduces those rows; after one cluster barrier +each CTA sums its rows' `KS` partials in K-slice order (deterministic). PDL: the weight +streams before the grid-dependency wait, and the consumer is released at once. +""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.pipeline as pipeline +import cutlass.utils as utils +import torch +import torch.nn.functional as F +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.custom_pipeline import PipelineTmaUmma, PipelineUmmaAsync +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + +#: Largest row count this entry is certified for: five 128-row MMA tiles. +MAX_ROWS = 640 + +#: Programmatic dependent launch (sm_90+), off with TRTLLM_ENABLE_PDL=0. +_PDL = get_env_enable_pdl() + + +def _asm(ret, args, text, constraints, loc, ip, side_effects=False): + return llvm.inline_asm( + ret, + [a.ir_value(loc=loc, ip=ip) for a in args], + text, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _mapa(addr, rank, *, loc=None, ip=None): + """This CTA's shared-memory address `addr` in cluster CTA `rank`'s window.""" + return cutlass.Int32( + _asm( + T.i32(), + [cutlass.Int32(addr), cutlass.Int32(rank)], + "mapa.shared::cluster.u32 $0, $1, $2;", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sts_v4_f32(addr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [cutlass.Int32(addr)] + [cutlass.Float32(v) for v in (a, b, c, d)], + "{ st.shared.v4.f32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,f,f,f,f", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _bulk_to_peer(dst, src, nbytes, mbar, peer, *, loc=None, ip=None): + """`nbytes` of this CTA's shared memory at `src` into cluster CTA `peer`'s shared + memory at `dst`, completing on `peer`'s mbarrier at `mbar` (`dst` / `mbar` are offsets + in this CTA's window: every CTA has the same layout).""" + _asm( + T.i32(), + [cutlass.Int32(x) for x in (dst, src, nbytes, mbar, peer)], + "{ .reg .b32 rd, rb; mapa.shared::cluster.u32 rd, $1, $5; " + "mapa.shared::cluster.u32 rb, $4, $5; " + "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes " + "[rd], [$2], $3, [rb]; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _bulk_commit_wait_read(*, loc=None, ip=None): + """Commit this thread's bulk copies and wait until they have read their sources.""" + _asm( + T.i32(), + [], + "{ cp.async.bulk.commit_group; cp.async.bulk.wait_group.read 0; mov.u32 $0, 0; }", + "=r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _lds_v4_f32(addr, *, loc=None, ip=None): + """16 bytes of this CTA's shared memory; side-effecting, so it stays behind the + cluster barrier that makes them valid.""" + ty = T.f32() + res = _asm( + llvm.StructType.get_literal([ty] * 4), + [cutlass.Int32(addr)], + "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,r", + loc, + ip, + side_effects=True, + ) + return [cutlass.Float32(llvm.extractvalue(ty, res, [i], loc=loc, ip=ip)) for i in range(4)] + + +@dsl_user_op +def _bf16x2(lo, hi, *, loc=None, ip=None): + """Two fp32 values rounded (RN, as `.to(BFloat16)`) into one bf16x2 word, `lo` low.""" + return cutlass.Int32( + _asm( + T.i32(), + [cutlass.Float32(hi), cutlass.Float32(lo)], + "cvt.rn.bf16x2.f32 $0, $1, $2;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg_v2(addr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [cutlass.Int64(addr), cutlass.Int32(a), cutlass.Int32(b)], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side_effects=True, + ) + + +class HcDecodeTcFirstKernel: + """`Qwen4ExpHcDecodeTcFirst`'s kernel. Compile-time: `bn` (MMA N, a multiple of 16), + `ks` (K split = cluster size), `hc`, `rank`, `n_cols`, `k`, `with_inject`; runtime: the + row count (one grid-z M tile per 128 rows) and the pointers.""" + + def __init__(self, bn, ks, hc, rank, n_cols, k, with_inject, epi_cols=16): + assert bn % epi_cols == 0 and bn % 16 == 0 and 16 <= bn <= 256 + assert k % (64 * ks) == 0, "K must split into 64-wide tiles per CTA" + assert rank % 4 == 0 and hc % 4 == 0, "the reduction writes 4-column groups" + self.bn, self.ks, self.hc, self.rank, self.n_cols, self.k = bn, ks, hc, rank, n_cols, k + self.with_inject = with_inject + self.n_tiles = (n_cols + bn - 1) // bn + self.acc_dtype = cutlass.Float32 + self.io_dtype = cutlass.BFloat16 + self.mma_tiler = (128, bn, 1) + self.epi_cols = epi_cols + self.cta_group = tcgen05.CtaGroup.ONE + # The MMA itself is single-CTA: no multicast; the launch cluster is along K only. + self.cluster_shape_mn = (1, 1) + self.epilogue_warp_id = (0, 1, 2, 3) + self.mma_warp_id = 4 + self.tma_warp_id = 5 + self.threads_per_cta = 32 * 6 + self.epilog_sync_bar_id = 1 + self.tmem_alloc_sync_bar_id = 2 + self.num_acc_stage = 1 + # Accumulator rows of a column tile each cluster CTA reduces (the last may own fewer). + self.rows_per = (128 + ks - 1) // ks + # 4-column groups a CTA reduces, at most, per thread (all six warps). + groups = self.rows_per * (bn // 4) + self.red_iters = (groups + self.threads_per_cta - 1) // self.threads_per_cta + + def _setup(self, a_major, b_major): + tiled_mma = utils.sm100.make_trivial_tiled_mma( + self.io_dtype, a_major, b_major, self.acc_dtype, self.cta_group, self.mma_tiler[:2] + ) + inst_k = cute.size(tiled_mma.shape_mnk, mode=[2]) + self.mma_tiler = (self.mma_tiler[0], self.mma_tiler[1], inst_k * 4) + self.cta_tile_shape_mnk = self.mma_tiler + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout((*self.cluster_shape_mn, 1)), (tiled_mma.thr_id.shape,) + ) + self.epi_tile = (self.mma_tiler[0], self.epi_cols) + a_one = utils.sm100.make_smem_layout_a(tiled_mma, self.mma_tiler, self.io_dtype, 1) + b_one = utils.sm100.make_smem_layout_b(tiled_mma, self.mma_tiler, self.io_dtype, 1) + ab_bytes = cute.size_in_bytes(self.io_dtype, a_one) + cute.size_in_bytes( + self.io_dtype, b_one + ) + smem_capacity = utils.get_smem_capacity_in_bytes() + self.k_tiles = self.k // self.mma_tiler[2] // self.ks + # The receive buffer: for each K slice j (row block j) the `rows_per` accumulator rows + # this CTA reduces, as its peer j pushed them, padded by 16 bytes a row. + self.p_stride = self.bn + 4 + p_bytes = self.ks * self.rows_per * self.p_stride * 4 + self.num_ab_stage = max(2, min(self.k_tiles, (smem_capacity - 2048 - p_bytes) // ab_bytes)) + self.n_pre = min(self.num_ab_stage, self.k_tiles) + self.a_smem_layout_staged = utils.sm100.make_smem_layout_a( + tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage + ) + self.b_smem_layout_staged = utils.sm100.make_smem_layout_b( + tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage + ) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + acc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + self.num_tmem_alloc_cols = utils.get_num_tmem_alloc_cols(acc_fake, arch="sm_100") + return tiled_mma + + @cute.jit + def __call__( + self, + m: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + low_t: cute.Tensor, + inj_ptr: cute.Pointer, + stream: cuda.CUstream, + ): + k, ks = self.k, self.ks + a = cute.make_tensor(a_ptr, cute.make_layout((m, k, 1), stride=(k, 1, 0))) + b = cute.make_tensor(b_ptr, cute.make_layout((self.n_cols, k, 1), stride=(k, 1, 0))) + # `low` arrives as a tensor (the TVM-FFI entry takes its stream from a GPU tensor). + low = cute.make_tensor( + low_t.iterator, cute.make_layout((m, self.rank), stride=(self.rank, 1)) + ) + inj = cute.make_tensor(inj_ptr, cute.make_layout((m, self.hc), stride=(self.hc, 1))) + k_major = utils.LayoutEnum.ROW_MAJOR.mma_major_mode() + tiled_mma = self._setup(k_major, k_major) + + a_op = utils.sm100.cluster_shape_to_tma_atom_A(self.cluster_shape_mn, tiled_mma.thr_id) + a_smem = cute.slice_(self.a_smem_layout_staged, (None, None, None, 0)) + tma_atom_a, tma_a = cute.nvgpu.make_tiled_tma_atom_A( + a_op, a, a_smem, self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape + ) + b_op = utils.sm100.cluster_shape_to_tma_atom_B(self.cluster_shape_mn, tiled_mma.thr_id) + b_smem = cute.slice_(self.b_smem_layout_staged, (None, None, None, 0)) + tma_atom_b, tma_b = cute.nvgpu.make_tiled_tma_atom_B( + b_op, b, b_smem, self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape + ) + self.num_tma_load_bytes = cute.size_in_bytes(self.io_dtype, a_smem) + cute.size_in_bytes( + self.io_dtype, b_smem + ) + self.kernel( + tiled_mma, + tma_atom_a, + tma_a, + tma_atom_b, + tma_b, + low, + inj, + m, + self.cluster_layout_vmnk, + self.a_smem_layout_staged, + self.b_smem_layout_staged, + self.epi_tile, + ).launch( + grid=[ks, self.n_tiles, (m + 127) // 128], + block=[self.threads_per_cta, 1, 1], + cluster=[ks, 1, 1], + stream=stream, + use_pdl=_PDL, + ) + + @cute.kernel + def kernel( + self, + tiled_mma: cute.TiledMma, + tma_atom_a: cute.CopyAtom, + mA_mkl: cute.Tensor, + tma_atom_b: cute.CopyAtom, + mB_nkl: cute.Tensor, + mLow: cute.Tensor, + mInj: cute.Tensor, + m: cutlass.Int32, + cluster_layout_vmnk: cute.Layout, + a_smem_layout_staged: cute.ComposedLayout, + b_smem_layout_staged: cute.ComposedLayout, + epi_tile: cute.Tile, + ): + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + # The dependent may start now: it streams its own weight, then waits for this grid. + cute.arch.griddepcontrol_launch_dependents() + if warp_idx == self.tma_warp_id: + cpasync.prefetch_descriptor(tma_atom_a) + cpasync.prefetch_descriptor(tma_atom_b) + tidx, _, _ = cute.arch.thread_idx() + bidx, bidy, bidz = cute.arch.block_idx() + kr = bidx # K slice == rank in the (KS, 1, 1) cluster + nt = bidy # column tile + mt = bidz # 128-row M tile + + @cute.struct + class SharedStorage: + ab_full_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_ab_stage * 2] + acc_full_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2] + tmem_dealloc_mbar: cutlass.Int64 + recv_full: cutlass.Int64 + tmem_holding_buf: cutlass.Int32 + + smem = utils.SmemAllocator() + storage = smem.allocate(SharedStorage) + # Completes once every K slice's partial of this CTA's rows has landed. + if tidx == 0: + cute.arch.mbarrier_init(storage.recv_full, 1) + cute.arch.mbarrier_init_fence() + ab_producer, ab_consumer = PipelineTmaUmma.create( + barrier_storage=storage.ab_full_mbar_ptr.data_ptr(), + num_stages=self.num_ab_stage, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread), + consumer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, 1), + tx_count=self.num_tma_load_bytes, + cta_layout_vmnk=cluster_layout_vmnk, + ).make_participants() + acc_pipeline = PipelineUmmaAsync.create( + barrier_storage=storage.acc_full_mbar_ptr.data_ptr(), + num_stages=self.num_acc_stage, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread), + consumer_group=pipeline.CooperativeGroup( + pipeline.Agent.Thread, len(self.epilogue_warp_id) + ), + cta_layout_vmnk=cluster_layout_vmnk, + ) + tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=self.tmem_alloc_sync_bar_id, + num_threads=32 * len((self.mma_warp_id, *self.epilogue_warp_id)), + ) + tmem = utils.TmemAllocator( + storage.tmem_holding_buf.ptr, + barrier_for_retrieve=tmem_alloc_barrier, + allocator_warp_id=self.epilogue_warp_id[0], + is_two_cta=False, + two_cta_tmem_dealloc_mbar_ptr=storage.tmem_dealloc_mbar.ptr, + ) + pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True) + + sA = smem.allocate_tensor( + element_type=self.io_dtype, + layout=a_smem_layout_staged.outer, + byte_alignment=128, + swizzle=a_smem_layout_staged.inner, + ) + sB = smem.allocate_tensor( + element_type=self.io_dtype, + layout=b_smem_layout_staged.outer, + byte_alignment=128, + swizzle=b_smem_layout_staged.inner, + ) + sP = smem.allocate_tensor( + element_type=self.acc_dtype, + layout=cute.make_layout(self.ks * self.rows_per * self.p_stride), + byte_alignment=16, + ) + p_base = cutlass.Int32(sP.iterator.toint()) + row0 = mt * 128 + nreal = cutlass.min(m - row0, 128) + + gA_mkl = cute.local_tile( + mA_mkl, cute.slice_(self.mma_tiler, (None, 0, None)), (None, None, None) + ) + gB_nkl = cute.local_tile( + mB_nkl, cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) + ) + k0 = kr * self.k_tiles + + thr_mma = tiled_mma.get_slice(0) + tCgA = thr_mma.partition_A(gA_mkl) + tCgB = thr_mma.partition_B(gB_nkl) + tAsA, tAgA = cpasync.tma_partition( + tma_atom_a, + 0, + cute.make_layout(1), + cute.group_modes(sA, 0, 3), + cute.group_modes(tCgA, 0, 3), + ) + tBsB, tBgB = cpasync.tma_partition( + tma_atom_b, + 0, + cute.make_layout(1), + cute.group_modes(sB, 0, 3), + cute.group_modes(tCgB, 0, 3), + ) + tCrA = tiled_mma.make_fragment_A(sA) + tCrB = tiled_mma.make_fragment_B(sB) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + tCtAcc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + + pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn) + # Start-up cluster barrier (split): a peer pushes partials into this CTA only after + # its wait, i.e. after this CTA initialized `recv_full`. + cute.arch.cluster_arrive_relaxed() + if tidx == 0: + cute.arch.mbarrier_arrive_and_expect_tx( + storage.recv_full, self.ks * self._rows_of(kr, nreal) * self.p_stride * 4 + ) + + if warp_idx == self.tma_warp_id: + tAgA_slice = tAgA[(None, mt, None, 0)] + tBgB_slice = tBgB[(None, nt, None, 0)] + ab_producer.reset() + # The first stages' weight tiles stream before the grid-dependency wait; the + # normed rows, the previous kernel's output, after it. + pre = [] + for kk in cutlass.range_constexpr(self.n_pre): + handle = ab_producer.acquire_and_advance() + cute.copy( + tma_atom_b, + tBgB_slice[(None, k0 + kk)], + tBsB[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + pre.append(handle) + cute.arch.griddepcontrol_wait() + for kk in cutlass.range_constexpr(self.n_pre): + handle = pre[kk] + cute.copy( + tma_atom_a, + tAgA_slice[(None, k0 + kk)], + tAsA[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + for k_tile in cutlass.range(self.n_pre, self.k_tiles, 1, unroll=1): + handle = ab_producer.acquire_and_advance() + cute.copy( + tma_atom_a, + tAgA_slice[(None, k0 + k_tile)], + tAsA[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + cute.copy( + tma_atom_b, + tBgB_slice[(None, k0 + k_tile)], + tBsB[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + ab_producer.tail() + + if warp_idx == self.mma_warp_id: + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + acc_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_acc_stage + ) + tCtAcc = tCtAcc_base[(None, None, None, acc_producer_state.index)] + ab_consumer.reset() + peek = ab_consumer.try_wait() + acc_pipeline.producer_acquire(acc_producer_state) + tiled_mma.set(tcgen05.Field.ACCUMULATE, False) + for k_tile in cutlass.range(0, self.k_tiles, 1, unroll=1): + handle = ab_consumer.wait_and_advance(peek) + num_kblocks = cute.size(tCrA, mode=[2]) + for kb in cutlass.range(num_kblocks, unroll_full=True): + kcrd = (None, None, kb, handle.index) + cute.gemm(tiled_mma, tCtAcc, tCrA[kcrd], tCrB[kcrd], tCtAcc) + tiled_mma.set(tcgen05.Field.ACCUMULATE, True) + handle.release() + peek = cutlass.Boolean(1) + if handle.count + 1 < self.k_tiles: + peek = ab_consumer.try_wait() + acc_pipeline.producer_commit(acc_producer_state) + acc_producer_state.advance() + acc_pipeline.producer_tail(acc_producer_state) + + if warp_idx < self.mma_warp_id: + tmem.allocate(self.num_tmem_alloc_cols) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + acc_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_acc_stage + ) + copy_atom_t2r = utils.sm100.get_tmem_load_op( + self.cta_tile_shape_mnk, + utils.LayoutEnum.ROW_MAJOR, + self.acc_dtype, + self.acc_dtype, + epi_tile, + False, + ) + tAcc_epi = cute.flat_divide(tCtAcc_base[((None, None), 0, 0, None)], epi_tile) + tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + tTR_tAcc_base = thr_copy_t2r.partition_S(tAcc_epi) + tTR_cP = thr_copy_t2r.partition_D( + cute.flat_divide(cute.make_identity_tensor((128, self.bn)), epi_tile) + ) + tTR_rAcc = cute.make_rmem_tensor(tTR_cP[(None, None, None, 0, 0)].shape, self.acc_dtype) + tTR_tAcc = tTR_tAcc_base[(None, None, None, None, None, acc_consumer_state.index)] + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + # 32dp32b: warp w reads TMEM lanes [32w, 32w + 32), so this thread holds accumulator + # row `tidx`. Rows are staged in row order in the operand ring (dead once the + # accumulator is complete), so each owner's rows are one contiguous block. + send_base = cutlass.Int32(sA.iterator.toint()) + acc_pipeline.consumer_wait(acc_consumer_state) + for sub in cutlass.range_constexpr(self.bn // self.epi_cols): + cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, sub)], tTR_rAcc) + if tidx < nreal: + for q in cutlass.range_constexpr(self.epi_cols // 4): + _sts_v4_f32( + send_base + (tidx * self.p_stride + sub * self.epi_cols + 4 * q) * 4, + tTR_rAcc[4 * q], + tTR_rAcc[4 * q + 1], + tTR_rAcc[4 * q + 2], + tTR_rAcc[4 * q + 3], + ) + cute.arch.fence_proxy("async.shared", space="cta") + with cute.arch.elect_one(): + acc_pipeline.consumer_release(acc_consumer_state) + acc_consumer_state.advance() + + cute.arch.barrier( + barrier_id=self.epilog_sync_bar_id, + number_of_threads=32 * len(self.epilogue_warp_id), + ) + tmem.relinquish_alloc_permit() + tmem.free(tmem_ptr) + # Peers' `recv_full` are initialized (the start-up barrier; warp-aligned wait). + cute.arch.cluster_wait() + if tidx == 0: + # One bulk copy per owner (this CTA included): its rows, into its block `kr`. + for o in cutlass.range_constexpr(self.ks): + rows_o = self._rows_of(o, nreal) + if rows_o > 0: + _bulk_to_peer( + p_base + kr * self.rows_per * self.p_stride * 4, + send_base + o * self.rows_per * self.p_stride * 4, + rows_o * self.p_stride * 4, + storage.recv_full.toint(), + o, + ) + else: + cute.arch.cluster_wait() + + # Every K slice's partial of this CTA's rows is local once `recv_full` completes; + # no DSMEM access follows (a CTA may exit once its own copies have read their source). + cute.arch.mbarrier_wait(storage.recv_full, 0) + + # This CTA's rows [kr * rows_per, ...) of the tile's real rows, in 4-column groups, + # all 6 warps; an element's `KS` partials summed in K-slice order (deterministic). + r_lo = kr * self.rows_per + mine = self._rows_of(kr, nreal) + g4 = self.bn // 4 + groups = mine * g4 + inv_hc = cutlass.Float32(1.0 / self.hc) + low_base = mLow.iterator.toint() + inj_base = mInj.iterator.toint() + for it in cutlass.range_constexpr(self.red_iters): + g = tidx + it * self.threads_per_cta + if g < groups: + rl = g // g4 + c = (g - rl * g4) * 4 + src = p_base + (rl * self.p_stride + c) * 4 + parts = [ + _lds_v4_f32(src + j * self.rows_per * self.p_stride * 4) for j in range(self.ks) + ] + n = nt * self.bn + c + row = cutlass.Int64(row0 + r_lo + rl) + lows, injs = [], [] + for e in cutlass.range_constexpr(4): + acc = parts[0][e] + for j in cutlass.range_constexpr(1, self.ks): + acc = acc + parts[j][e] + z = acc * inv_hc + sg = cutlass.Float32(1.0) / (cutlass.Float32(1.0) + cute.math.exp(-z)) + lows.append(z * sg) + injs.append(cutlass.Float32(2.0) * sg) + if n < self.rank: + _stg_v2( + low_base + (row * self.rank + n) * 2, + _bf16x2(lows[0], lows[1]), + _bf16x2(lows[2], lows[3]), + ) + if cutlass.const_expr(self.with_inject): + if n >= self.rank and n < self.rank + self.hc: + _stg_v2( + inj_base + (row * self.hc + n - self.rank) * 2, + _bf16x2(injs[0], injs[1]), + _bf16x2(injs[2], injs[3]), + ) + if tidx == 0: + _bulk_commit_wait_read() + + def _rows_of(self, owner, nreal): + """Real rows of the tile that cluster CTA `owner` reduces (0 past the real rows).""" + lo = owner * self.rows_per + hi = cutlass.min(lo + self.rows_per, 128) + return cutlass.max(cutlass.min(nreal, hi) - lo, 0) + + +_COMPILED: dict = {} + + +def _launcher(bn, ks, hc, rank, n_cols, k, with_inject, ptrs): + """Compile once per static configuration (the row count is a runtime argument); the + entry launches on the caller's current stream. `ptrs`: `(normed, weight, low, inject)`.""" + key = (bn, ks, hc, rank, n_cols, k, with_inject) + fn = _COMPILED.get(key) + if fn is None: + kern = HcDecodeTcFirstKernel(bn, ks, hc, rank, n_cols, k, with_inject) + gmem = cute.AddressSpace.gmem + cptrs = [ + make_ptr(cutlass.BFloat16, ptrs[0], gmem, assumed_align=16), + make_ptr(cutlass.BFloat16, ptrs[1], gmem, assumed_align=16), + from_dlpack(ptrs[2], assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.BFloat16, ptrs[3], gmem, assumed_align=2), + ] + fn = cute.compile( + kern, + MAX_ROWS, + *cptrs, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + +def hc_decode_tc_first(normed, first, rank, hc_count, with_inject, bn, ks): + """Launch with an explicit tile (`bn` columns, `ks`-way K split). `first` is + the `[K, >= n_cols]` transposed view of the row-major weight, as + `cublas_mm(normed, first)` takes it. Returns `(low, inject)`.""" + rows, k = normed.shape + n_cols = rank + (hc_count if with_inject else 0) + dev, dt = normed.device, normed.dtype + low = torch.empty((rows, rank), dtype=dt, device=dev) + inject = torch.empty((rows, hc_count), dtype=dt, device=dev) if with_inject else None + if rows > 0: + ptrs = ( + normed.data_ptr(), + first.data_ptr(), + low, + (inject if with_inject else low).data_ptr(), + ) + _launcher(bn, ks, hc_count, rank, n_cols, k, with_inject, ptrs)(rows, *ptrs) + return low, inject + + +#: Tile `(BN, KS)` by row count: 48 weight rows per CTA (7 tiles cover the 324 real rows) +#: and a `KS`-way K split, one cluster per column tile, so the grid stays about one wave of +#: one CTA an SM (70 to 140 CTAs). +_TILES = ( + (4, (48, 10)), + (128, (48, 16)), + (256, (48, 8)), + (384, (48, 5)), + (512, (48, 4)), + (MAX_ROWS, (48, 4)), +) +#: Every `_TILES` K split divides K into 64-wide tiles per CTA when K divides by this. +_K_ALIGN = 64 * 80 + + +def _tile(rows: int) -> tuple[int, int]: + return next(t for hi, t in _TILES if rows <= hi) + + +def precompile(k: int, rank: int, hc_count: int, device) -> None: + """Compile every `(tile, with_inject)` variant for a `[*, k]` stream without launching, + so no serving step stalls on a compile. The placeholder operands' contents are unread.""" + normed = torch.empty((MAX_ROWS, k), dtype=torch.bfloat16, device=device) + first = torch.empty((rank + hc_count, k), dtype=torch.bfloat16, device=device) + low = torch.empty((MAX_ROWS, rank), dtype=torch.bfloat16, device=device) + ptrs = (normed.data_ptr(), first.data_ptr(), low, low.data_ptr()) + for _, (bn, ks) in _TILES: + for with_inject in (True, False): + n_cols = rank + (hc_count if with_inject else 0) + _launcher(bn, ks, hc_count, rank, n_cols, k, with_inject, ptrs) + + +class Qwen4ExpHcDecodeTcFirst(OpWrapper): + """With `R = rank`, `C = hc_count`, `normed` the `[T, K]` grouped-normed + stream and `first` the `[K, >= R (+ C)]` transposed view of the load-time + `[down; block_inject; 0]` weight, `L = normed @ first` in fp32:: + + low = silu(L[:, :R] / C) # [T, R], normed's dtype + inject = 2 * sigmoid(L[:, R : R + C] / C) # [T, C], only with_inject + + fp32 accumulation (a fixed-order sum of the K-split partials), each activation rounded + once. Returns fresh `(low, inject)`; `inject` is None exactly when `with_inject` is + False. Fusion boundary: `cublas_mm(normed, first, None, fp32)` + `qwen4_exp_hc_lowrank_act`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: rows, inject (True); the checkpoint's geometry (hc 4, rank 320, the padded + # [384, K] weight, bf16). With inject one cell per `_TILES` bucket at partial and full + # M tiles; without, three. + CELLS: tuple[Cell, ...] = ( + Cell( + why="1 row: a one-request decode step without MTP (tp1..dep4), a draft pass", + spec=dict(rows=1), + ), + Cell( + why="16 rows: ll_tep4's 4-request MTP-3 verify step; a dep4 rank's draft pass", + spec=dict(rows=16), + ), + Cell( + why="128 rows: ht_dep4's 32-request rank graph cap (MTP-3 verify), one M tile", + spec=dict(rows=128), + ), + Cell(why="129 rows: a 1-row second M tile", spec=dict(rows=129)), + Cell( + why="288 rows: tep2 gen 96-request MTP-2 verify step, three M tiles", + spec=dict(rows=288), + ), + Cell( + why="480 rows: tep2 gen 160-request MTP-2 verify step, four M tiles", + spec=dict(rows=480), + ), + Cell(why="640 rows: MAX_ROWS, five full M tiles", spec=dict(rows=640)), + Cell( + why="stack-final mixer: 320 cols, no inject, 200 rows (two M tiles)", + spec=dict(rows=200, inject=False), + ), + Cell( + why="stack-final mixer: 320 cols, no inject, 384 rows (tep2 gen b128 verify)", + spec=dict(rows=384, inject=False), + ), + Cell( + why="MTP draft head mixer: 320 cols, no inject, 1 row (ll draft step)", + spec=dict(rows=1, inject=False), + ), + ) + note = """ + Source: `HcDecodeTcFirstKernel` above (CuTe DSL). Preconditions (bf16): `normed` `[T, K]` + contiguous, T <= 640; `first` the `.t()` view of a row-major `[N, K]` weight with + N >= R (+ C); 16-byte aligned; K a multiple of 64 * 80. Only the R (+ C) real weight rows + are read. One CTA an SM (~227 KB shared memory); the partials meet in distributed shared + memory, no global workspace. PDL off with TRTLLM_ENABLE_PDL=0. Graph-capturable (current + stream, no host sync, no persistent state); compiles once per (BN, KS, C, R, N, K, + inject), so call `precompile` before capture. + Certified on GB300 at the cells' rows, graph replay bitwise the eager launch. + Not certified: hc_count other than 4, an unpadded weight (N = R + C), zero rows. + """ + + def __call__( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ): + return hc_decode_tc_first( + normed, first, rank, hc_count, with_inject, *_tile(normed.shape[0]) + ) + + def reference( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ): + """float64 projection and activations, one rounding per output.""" + n_cols = rank + (hc_count if with_inject else 0) + lg = normed.double() @ first[:, :n_cols].double() + low = F.silu(lg[:, :rank] / hc_count).to(normed.dtype) + inject = None + if with_inject: + inject = (2.0 * torch.sigmoid(lg[:, rank:n_cols] / hc_count)).to(normed.dtype) + return low, inject + + def is_valid( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ) -> None: + rows, k = normed.shape + n_cols = rank + (hc_count if with_inject else 0) + assert normed.dtype == first.dtype == torch.bfloat16, "bf16 operands" + assert rows <= MAX_ROWS, f"at most {MAX_ROWS} rows (five MMA tiles)" + assert normed.stride() == (k, 1), "normed contiguous [T, K]" + assert first.shape[0] == k and first.shape[1] >= n_cols, "first [K, >= n_cols]" + assert first.stride() == (1, k), "first must be the .t() view of a row-major [N, K] weight" + assert k % _K_ALIGN == 0, f"K must be a multiple of {_K_ALIGN}" + for t in (normed, first): + assert t.data_ptr() % 16 == 0, "16-byte aligned operands" + + +qwen4_exp_hc_decode_tc_first = Qwen4ExpHcDecodeTcFirst() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py new file mode 100644 index 000000000000..0c11293a3316 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py @@ -0,0 +1,1858 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is copied and modified from cutlass example https://github.com/NVIDIA/cutlass/blob/main/examples/python/CuTeDSL/blackwell/blockwise_gemm/blockwise_gemm.py + +"""The target's copy of the stock CuTe DSL `Sm100BlockwiseGemmKernel` +(`tensorrt_llm/_torch/kernels/blackwell/blockwise_gemm/blockwise_gemm.py`), reduced to the +contiguous grouped mode `moe_prefill_grouped__prefill` runs (1-CTA instructions, fp32 +accumulator): + +- A / SFA / C are M tiles of rows back to back, each tile one group's rows; B / SFB are batched by + group and `tile_group[l]` names tile l's group; tiles from the device scalar `num_tiles[0]` on + are skipped, so a launch over an upper bound of tiles needs no host synchronization; +- with `epi_quant=True` (`wrapper_grouped_quant`), C holds each row's 128-column block as e4m3 + codes and `c_scale` its fp32 scale instead of the accumulator. + +C = (SFA * A) * (SFB * B), fp8 e4m3 A / B, fp32 1x128 SFA and 128x128 SFB. Warp roles: a scheduler +warp, a TMA warp (A / B), a scale warp (SFA / SFB through cp.async), an MMA warp (tcgen05.mma into +TMEM), four acc-update warps (scale each k tile's partial and sum it in registers, then store the +sum back to TMEM) and four epilogue warps (TMEM -> registers -> SMEM -> TMA store). + +Constraints: MMA tiler M 64 / 128 / 256, N 128 (the SFB granularity); cluster M / N powers of 2, +at most 16 CTAs; the contiguous dimension of A / B / C 16-byte aligned. +""" + +import math +from typing import Optional, Tuple, Type, Union + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.pipeline as pipeline +import cutlass.utils as utils +import cutlass.utils.blackwell_helpers as sm100_utils +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait + + +@dsl_user_op +def _div_rn(a, b, *, loc=None, ip=None): + """IEEE fp32 division (div.rn.f32), as Triton's `tl.math.div_rn`.""" + return cutlass.Float32( + llvm.inline_asm( + T.f32(), + [ + cutlass.Float32(a).ir_value(loc=loc, ip=ip), + cutlass.Float32(b).ir_value(loc=loc, ip=ip), + ], + "div.rn.f32 $0, $1, $2;", + "=f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +class Sm100BlockwiseGemmKernel: + """The grouped blockwise FP8 GEMM; see the module docstring for the operand layouts.""" + + def __init__( + self, + mma_tiler_mn: Tuple[int, int], + cluster_shape_mn: Tuple[int, int], + epi_quant: bool = False, + ): + """`epi_quant` (128-column tiles only): store each tile row's 1x128 e4m3 round trip + instead of the accumulator: C holds the codes `acc * (448 / amax)` and + `c_scale[n_tile, row]` the fp32 scale `amax / 448` (amax floored at 1e-10).""" + self.epi_quant = epi_quant + assert not epi_quant or mma_tiler_mn[1] == 128 + self.acc_dtype: Type[cutlass.Numeric] = cutlass.Float32 + self.cluster_shape_mn = cluster_shape_mn + # K is set in _setup_attributes. + self.mma_tiler = (*mma_tiler_mn, 1) + self.cta_group = tcgen05.CtaGroup.ONE + + self.occupancy = 1 + self.acc_update_warp_id = (0, 1, 2, 3) + self.epilog_warp_id = (4, 5, 6, 7) + self.mma_warp_id = 8 + self.tma_warp_id = 9 + self.scale_warp_id = 10 + self.sched_warp_id = 11 + self.threads_per_warp = 32 + self.threads_per_cta = self.threads_per_warp * len( + ( + *self.acc_update_warp_id, + *self.epilog_warp_id, + self.mma_warp_id, + self.tma_warp_id, + self.scale_warp_id, + self.sched_warp_id, + ) + ) + self.threads_wo_sched = self.threads_per_warp * len( + ( + *self.acc_update_warp_id, + *self.epilog_warp_id, + self.mma_warp_id, + self.tma_warp_id, + self.scale_warp_id, + ) + ) + self.num_regs_uniform_warps = 64 + self.num_regs_sched_warps = 64 + self.num_regs_epilogue_warps = 216 + self.num_regs_acc_update_warps = 216 + + self.epilog_sync_barrier = pipeline.NamedBarrier( + barrier_id=1, + num_threads=32 * len(self.epilog_warp_id), + ) + self.tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=2, + num_threads=32 + * len((self.mma_warp_id, *self.epilog_warp_id, *self.acc_update_warp_id)), + ) + self.sched_sync_barrier = pipeline.NamedBarrier( + barrier_id=3, + num_threads=self.threads_per_warp, + ) + self.num_smem_capacity = utils.get_smem_capacity_in_bytes("sm_100") + # TMEM column of the acc-update warps' summed accumulator (the epilogue's source). + self.tmem_final_offset = 384 + + def _setup_attributes(self): + """Tile shapes, stage counts and SMEM layouts that depend on the operands' dtypes and + majorness (set by `__call__`).""" + tiled_mma = sm100_utils.make_trivial_tiled_mma( + self.a_dtype, + self.a_major_mode, + self.b_major_mode, + self.acc_dtype, + self.cta_group, + self.mma_tiler[:2], + ) + + mma_inst_shape_k = cute.size(tiled_mma.shape_mnk, mode=[2]) + mma_inst_tile_k = 4 + self.mma_tiler = ( + self.mma_tiler[0], + self.mma_tiler[1], + mma_inst_shape_k * mma_inst_tile_k, + ) + self.cta_tile_shape_mnk = ( + self.mma_tiler[0] // cute.size(tiled_mma.thr_id.shape), + self.mma_tiler[1], + self.mma_tiler[2], + ) + + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout((*self.cluster_shape_mn, 1)), + (tiled_mma.thr_id.shape,), + ) + + self.scale_granularity_m = 1 + self.scale_granularity_n = 128 + self.scale_granularity_k = 128 + self.scale_m_per_tile = self.cta_tile_shape_mnk[0] // self.scale_granularity_m + self.scale_n_per_tile = self.cta_tile_shape_mnk[1] // self.scale_granularity_n + self.scale_k_per_tile = self.cta_tile_shape_mnk[2] // self.scale_granularity_k + + if self.scale_k_per_tile != 1: + raise ValueError("scale_k_per_tile must be 1") + if self.scale_m_per_tile != self.cta_tile_shape_mnk[0]: + raise ValueError("scale_m_per_tile must be cta_tile_m") + if self.scale_n_per_tile != 1: + raise ValueError("scale_n_per_tile must be 1") + + self.num_mcast_ctas_a = cute.size(self.cluster_layout_vmnk.shape[2]) + self.num_mcast_ctas_b = cute.size(self.cluster_layout_vmnk.shape[1]) + self.is_a_mcast = self.num_mcast_ctas_a > 1 + self.is_b_mcast = self.num_mcast_ctas_b > 1 + + self.epi_tile = sm100_utils.compute_epilogue_tile_shape( + self.cta_tile_shape_mnk, + False, + self.c_layout, + self.c_dtype, + ) + + ( + self.num_acc_stage, + self.num_ab_stage, + self.num_c_stage, + self.num_scale_stage, + self.num_tile_stage, + ) = self._compute_stages( + tiled_mma, + self.mma_tiler, + self.a_dtype, + self.b_dtype, + self.epi_tile, + self.c_dtype, + self.c_layout, + self.sfa_dtype, + self.sfb_dtype, + self.scale_m_per_tile * self.scale_k_per_tile, + self.scale_n_per_tile * self.scale_k_per_tile, + self.num_smem_capacity, + self.occupancy, + ) + + self.a_smem_layout_staged = sm100_utils.make_smem_layout_a( + tiled_mma, + self.mma_tiler, + self.a_dtype, + self.num_ab_stage, + ) + self.b_smem_layout_staged = sm100_utils.make_smem_layout_b( + tiled_mma, + self.mma_tiler, + self.b_dtype, + self.num_ab_stage, + ) + self.c_smem_layout_staged = sm100_utils.make_smem_layout_epi( + self.c_dtype, + self.c_layout, + self.epi_tile, + self.num_c_stage, + ) + self.sfa_smem_layout_staged = cute.make_layout( + ( + (self.scale_granularity_m, self.scale_m_per_tile), + (self.scale_granularity_k, self.scale_k_per_tile), + self.num_scale_stage, + ), + stride=( + (0, self.scale_k_per_tile), + (0, 1), + self.scale_k_per_tile * self.scale_m_per_tile, + ), + ) + self.sfb_smem_layout_staged = cute.make_layout( + ( + (self.scale_granularity_n, self.scale_n_per_tile), + (self.scale_granularity_k, self.scale_k_per_tile), + self.num_scale_stage, + ), + stride=( + (0, self.scale_k_per_tile), + (0, 1), + self.scale_k_per_tile * self.scale_n_per_tile, + ), + ) + self.num_tmem_alloc_cols = 512 + + @cute.jit + def __call__( + self, + a: cute.Tensor, + b: cute.Tensor, + c: cute.Tensor, + sfa: cute.Tensor, + sfb: cute.Tensor, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + tile_group: Optional[cute.Tensor] = None, + num_tiles: Optional[cute.Tensor] = None, + c_scale: Optional[cute.Tensor] = None, + ): + """Build the TMA atoms, grid and shared storage for these operands and launch `kernel` + on `stream`. Called through `wrapper_grouped` / `wrapper_grouped_quant`.""" + self.a_dtype: Type[cutlass.Numeric] = a.element_type + self.b_dtype: Type[cutlass.Numeric] = b.element_type + self.c_dtype: Type[cutlass.Numeric] = c.element_type + self.sfa_dtype: Type[cutlass.Numeric] = sfa.element_type + self.sfb_dtype: Type[cutlass.Numeric] = sfb.element_type + self.a_major_mode = utils.LayoutEnum.from_tensor(a).mma_major_mode() + self.b_major_mode = utils.LayoutEnum.from_tensor(b).mma_major_mode() + self.c_layout = utils.LayoutEnum.from_tensor(c) + + if cutlass.const_expr(self.a_dtype != self.b_dtype): + raise TypeError(f"Type must match: {self.a_dtype} != {self.b_dtype}") + + self._setup_attributes() + + tiled_mma = sm100_utils.make_trivial_tiled_mma( + self.a_dtype, + self.a_major_mode, + self.b_major_mode, + self.acc_dtype, + self.cta_group, + self.mma_tiler[:2], + ) + atom_thr_size = cute.size(tiled_mma.thr_id.shape) + + a_op = self._get_tma_atom_kind(atom_thr_size, self.is_a_mcast) + a_smem_layout = cute.slice_(self.a_smem_layout_staged, (None, None, None, 0)) + tma_atom_a, tma_tensor_a = cute.nvgpu.make_tiled_tma_atom_A( + a_op, + a, + a_smem_layout, + self.mma_tiler, + tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=(cutlass.TFloat32 if a.element_type is cutlass.Float32 else None), + ) + + b_op = self._get_tma_atom_kind(atom_thr_size, self.is_b_mcast) + b_smem_layout = cute.slice_(self.b_smem_layout_staged, (None, None, None, 0)) + tma_atom_b, tma_tensor_b = cute.nvgpu.make_tiled_tma_atom_B( + b_op, + b, + b_smem_layout, + self.mma_tiler, + tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=(cutlass.TFloat32 if b.element_type is cutlass.Float32 else None), + ) + + a_copy_size = cute.size_in_bytes(self.a_dtype, a_smem_layout) + b_copy_size = cute.size_in_bytes(self.b_dtype, b_smem_layout) + self.num_tma_load_bytes = (a_copy_size + b_copy_size) * atom_thr_size + + c_cta_v_layout = cute.composition(cute.make_identity_layout(c.shape), self.epi_tile) + epi_smem_layout = cute.slice_(self.c_smem_layout_staged, (None, None, 0)) + tma_atom_c, tma_tensor_c = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileS2GOp(), + c, + epi_smem_layout, + c_cta_v_layout, + ) + + tensor_sfa = cute.make_tensor( + sfa.iterator, + cute.make_layout( + ( + (self.scale_granularity_m, sfa.shape[0]), + (self.scale_granularity_k, sfa.shape[1]), + sfa.shape[2], + ), + stride=( + (0, sfa.layout.stride[0]), + (0, sfa.layout.stride[1]), + sfa.layout.stride[2], + ), + ), + ) + tensor_sfb = cute.make_tensor( + sfb.iterator, + cute.make_layout( + ( + (self.scale_granularity_n, sfb.shape[0]), + (self.scale_granularity_k, sfb.shape[1]), + sfb.shape[2], + ), + stride=( + (0, sfb.layout.stride[0]), + (0, sfb.layout.stride[1]), + sfb.layout.stride[2], + ), + ), + ) + + self.tile_sched_params, grid = self._compute_grid( + c, self.cta_tile_shape_mnk, self.cluster_shape_mn, max_active_clusters + ) + + self.buffer_align_bytes = 1024 + + c_smem_size = cute.cosize(self.c_smem_layout_staged.outer) + + @cute.struct + class SharedStorage: + sInfo: cute.struct.Align[ + cute.struct.MemRange[cutlass.Int32, 4 * self.num_tile_stage], + 1, + ] + ab_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_ab_stage * 2] + scale_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_scale_stage * 2] + acc_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2] + tile_info_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_tile_stage * 2] + epi_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 1 * 2] + tmem_dealloc_mbar_ptr: cutlass.Int64 + tmem_holding_buf: cutlass.Int32 + sC: cute.struct.Align[ + cute.struct.MemRange[ + self.c_dtype, + c_smem_size, + ], + self.buffer_align_bytes, + ] + sA: cute.struct.Align[ + cute.struct.MemRange[self.a_dtype, cute.cosize(self.a_smem_layout_staged.outer)], + self.buffer_align_bytes, + ] + sB: cute.struct.Align[ + cute.struct.MemRange[self.b_dtype, cute.cosize(self.b_smem_layout_staged.outer)], + self.buffer_align_bytes, + ] + sSFA: cute.struct.Align[ + cute.struct.MemRange[self.sfa_dtype, cute.cosize(self.sfa_smem_layout_staged)], + self.buffer_align_bytes, + ] + sSFB: cute.struct.Align[ + cute.struct.MemRange[self.sfb_dtype, cute.cosize(self.sfb_smem_layout_staged)], + self.buffer_align_bytes, + ] + + self.shared_storage = SharedStorage + + self.kernel( + tiled_mma, + tma_atom_a, + tma_tensor_a, + tma_atom_b, + tma_tensor_b, + tma_atom_c, + tma_tensor_c, + tensor_sfa, + tensor_sfb, + self.cluster_layout_vmnk, + self.a_smem_layout_staged, + self.b_smem_layout_staged, + self.c_smem_layout_staged, + self.sfa_smem_layout_staged, + self.sfb_smem_layout_staged, + self.epi_tile, + self.tile_sched_params, + tile_group, + num_tiles, + c_scale, + ).launch( + grid=grid, + block=[self.threads_per_cta, 1, 1], + cluster=(*self.cluster_shape_mn, 1), + smem=self.shared_storage.size_in_bytes(), + stream=stream, + min_blocks_per_mp=1, + ) + return + + @cute.kernel + def kernel( + self, + tiled_mma: cute.TiledMma, + tma_atom_a: cute.CopyAtom, + mA_mkl: cute.Tensor, + tma_atom_b: cute.CopyAtom, + mB_nkl: cute.Tensor, + tma_atom_c: cute.CopyAtom, + mC_mnl: cute.Tensor, + mSFA_mkl: cute.Tensor, + mSFB_nkl: cute.Tensor, + cluster_layout_vmnk: cute.Layout, + a_smem_layout_staged: cute.ComposedLayout, + b_smem_layout_staged: cute.ComposedLayout, + c_smem_layout_staged: Union[cute.Layout, cute.ComposedLayout, None], + sfa_smem_layout_staged: cute.Layout, + sfb_smem_layout_staged: cute.Layout, + epi_tile: cute.Tile, + tile_sched_params: utils.PersistentTileSchedulerParams, + tile_group: Optional[cute.Tensor], + num_tiles: Optional[cute.Tensor], + mCS: Optional[cute.Tensor], + ): + """The persistent warp-specialized kernel; `mCS` is `c_scale` (`epi_quant` only).""" + warp_idx = cute.arch.warp_idx() + warp_idx = cute.arch.make_warp_uniform(warp_idx) + lane_idx = cute.arch.lane_idx() + + if warp_idx == self.tma_warp_id: + cpasync.prefetch_descriptor(tma_atom_a) + cpasync.prefetch_descriptor(tma_atom_b) + cpasync.prefetch_descriptor(tma_atom_c) + + bidx, bidy, bidz = cute.arch.block_idx() + mma_tile_coord_v = bidx % cute.size(tiled_mma.thr_id.shape) + is_leader_cta = mma_tile_coord_v == 0 + cta_rank_in_cluster = cute.arch.make_warp_uniform(cute.arch.block_idx_in_cluster()) + block_in_cluster_coord_vmnk = cluster_layout_vmnk.get_flat_coord(cta_rank_in_cluster) + tidx, _, _ = cute.arch.thread_idx() + + smem = utils.SmemAllocator() + storage = smem.allocate(self.shared_storage) + + tmem_dealloc_mbar_ptr = storage.tmem_dealloc_mbar_ptr + tmem_holding_buf = storage.tmem_holding_buf + + ab_pipeline_producer_group = pipeline.CooperativeGroup(pipeline.Agent.Thread) + num_tma_producer = self.num_mcast_ctas_a + self.num_mcast_ctas_b - 1 + ab_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, num_tma_producer + ) + ab_pipeline = pipeline.PipelineTmaUmma.create( + barrier_storage=storage.ab_mbar_ptr.data_ptr(), + num_stages=self.num_ab_stage, + producer_group=ab_pipeline_producer_group, + consumer_group=ab_pipeline_consumer_group, + tx_count=self.num_tma_load_bytes, + cta_layout_vmnk=cluster_layout_vmnk, + defer_sync=True, + ) + + scale_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * 1, + ) + scale_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.epilog_warp_id), + ) + scale_pipeline = pipeline.PipelineCpAsync.create( + barrier_storage=storage.scale_mbar_ptr.data_ptr(), + num_stages=self.num_scale_stage, + producer_group=scale_pipeline_producer_group, + consumer_group=scale_pipeline_consumer_group, + defer_sync=True, + ) + + acc_pipeline_producer_group = pipeline.CooperativeGroup(pipeline.Agent.Thread) + num_acc_consumer_threads = len(self.epilog_warp_id) + acc_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, num_acc_consumer_threads + ) + acc_pipeline = pipeline.PipelineUmmaAsync.create( + barrier_storage=storage.acc_mbar_ptr.data_ptr(), + num_stages=self.num_acc_stage, + producer_group=acc_pipeline_producer_group, + consumer_group=acc_pipeline_consumer_group, + cta_layout_vmnk=cluster_layout_vmnk, + defer_sync=True, + ) + + epi_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.acc_update_warp_id), + ) + epi_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.epilog_warp_id), + ) + epi_pipeline = pipeline.PipelineAsync.create( + barrier_storage=storage.epi_mbar_ptr.data_ptr(), + num_stages=1, + producer_group=epi_pipeline_producer_group, + consumer_group=epi_pipeline_consumer_group, + defer_sync=True, + ) + + tile_info_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * 1, + ) + tile_info_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_wo_sched, + ) + tile_info_pipeline = pipeline.PipelineAsync.create( + barrier_storage=storage.tile_info_mbar_ptr.data_ptr(), + num_stages=self.num_tile_stage, + producer_group=tile_info_pipeline_producer_group, + consumer_group=tile_info_pipeline_consumer_group, + defer_sync=True, + ) + + tmem = utils.TmemAllocator( + tmem_holding_buf, + barrier_for_retrieve=self.tmem_alloc_barrier, + allocator_warp_id=self.epilog_warp_id[0], + is_two_cta=False, + two_cta_tmem_dealloc_mbar_ptr=tmem_dealloc_mbar_ptr, + ) + + pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True) + + sC = storage.sC.get_tensor(c_smem_layout_staged.outer, swizzle=c_smem_layout_staged.inner) + sA = storage.sA.get_tensor(a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner) + sB = storage.sB.get_tensor(b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner) + sSFA = storage.sSFA.get_tensor(sfa_smem_layout_staged) + sSFB = storage.sSFB.get_tensor(sfb_smem_layout_staged) + info_layout = cute.make_layout((4, self.num_tile_stage), stride=(1, 4)) + sInfo = storage.sInfo.get_tensor(info_layout) + + a_full_mcast_mask = None + b_full_mcast_mask = None + if cutlass.const_expr(self.is_a_mcast or self.is_b_mcast): + a_full_mcast_mask = cpasync.create_tma_multicast_mask( + cluster_layout_vmnk, block_in_cluster_coord_vmnk, mcast_mode=2 + ) + b_full_mcast_mask = cpasync.create_tma_multicast_mask( + cluster_layout_vmnk, block_in_cluster_coord_vmnk, mcast_mode=1 + ) + + gA_mkl = cute.local_tile( + mA_mkl, cute.slice_(self.mma_tiler, (None, 0, None)), (None, None, None) + ) + gB_nkl = cute.local_tile( + mB_nkl, cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) + ) + gC_mnl = cute.local_tile( + mC_mnl, cute.slice_(self.mma_tiler, (None, None, 0)), (None, None, None) + ) + gSFA_mkl = cute.local_tile( + mSFA_mkl, + cute.slice_(self.cta_tile_shape_mnk, (None, 0, None)), + (None, None, None), + ) + gSFB_nkl = cute.local_tile( + mSFB_nkl, + cute.slice_(self.cta_tile_shape_mnk, (0, None, None)), + (None, None, None), + ) + cSFA_mkl = cute.make_identity_tensor(cute.shape(mSFA_mkl)) + cSFB_nkl = cute.make_identity_tensor(cute.shape(mSFB_nkl)) + cSFA = cute.local_tile( + cSFA_mkl, + cute.slice_(self.cta_tile_shape_mnk, (None, 0, None)), + (None, None, None), + ) + cSFB = cute.local_tile( + cSFB_nkl, + cute.slice_(self.cta_tile_shape_mnk, (0, None, None)), + (None, None, None), + ) + k_tile_cnt = cute.size(gA_mkl, mode=[3]) + + thr_mma = tiled_mma.get_slice(mma_tile_coord_v) + tCgA = thr_mma.partition_A(gA_mkl) + tCgB = thr_mma.partition_B(gB_nkl) + tCgC = thr_mma.partition_C(gC_mnl) + + sSFA_view_as_C_layout = cute.make_layout( + ( + (self.scale_granularity_m, self.scale_m_per_tile), + self.cta_tile_shape_mnk[1], + self.num_scale_stage, + ), + stride=((0, 1), 0, self.scale_m_per_tile), + ) + sSFB_view_as_C_layout = cute.make_layout( + ( + self.cta_tile_shape_mnk[0], + (self.scale_granularity_n, self.scale_n_per_tile), + self.num_scale_stage, + ), + stride=(0, (0, 1), self.scale_n_per_tile), + ) + sSFA_view_as_C = cute.make_tensor(sSFA.iterator, sSFA_view_as_C_layout) + sSFB_view_as_C = cute.make_tensor(sSFB.iterator, sSFB_view_as_C_layout) + + a_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, 0, None, 0)).shape) + tAsA, tAgA = cpasync.tma_partition( + tma_atom_a, + block_in_cluster_coord_vmnk[2], + a_cta_layout, + cute.group_modes(sA, 0, 3), + cute.group_modes(tCgA, 0, 3), + ) + b_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, None, 0, 0)).shape) + tBsB, tBgB = cpasync.tma_partition( + tma_atom_b, + block_in_cluster_coord_vmnk[1], + b_cta_layout, + cute.group_modes(sB, 0, 3), + cute.group_modes(tCgB, 0, 3), + ) + + atom_copy = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp(), + mSFA_mkl.element_type, + num_bits_per_copy=mSFA_mkl.element_type.width, + ) + tiled_copy_sfa = cute.make_tiled_copy_tv( + atom_copy, cute.make_layout((32,)), cute.make_layout((1,)) + ) + tiled_copy_sfb = cute.make_tiled_copy_tv( + atom_copy, cute.make_layout((32,)), cute.make_layout((1,)) + ) + thr_copy_sfa = tiled_copy_sfa.get_slice(lane_idx) + thr_copy_sfb = tiled_copy_sfb.get_slice(lane_idx) + tAgSFA_mkl = thr_copy_sfa.partition_S(gSFA_mkl) + tAsSFA = thr_copy_sfa.partition_D(sSFA) + tAcSFA = thr_copy_sfa.partition_S(cSFA) + tBgSFB_nkl = thr_copy_sfb.partition_S(gSFB_nkl) + tBsSFB = thr_copy_sfb.partition_D(sSFB) + tBcSFB = thr_copy_sfb.partition_S(cSFB) + + tCrA = tiled_mma.make_fragment_A(sA) + tCrB = tiled_mma.make_fragment_B(sB) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + tCtAcc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + + # Cluster wait before tensor memory alloc + pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn) + + # Specialized Schedule warp + if warp_idx == self.sched_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_sched_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + tile_info_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_tile_stage + ) + + sched_valid = cutlass.Int32(work_tile.is_valid_tile) + if work_tile.tile_idx[2] >= num_tiles[0]: + sched_valid = cutlass.Int32(0) + + while sched_valid == 1: + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + + tile_info_pipeline.producer_acquire(tile_info_producer_state) + + cur_tile_coord = work_tile.tile_idx + sched_valid = cutlass.Int32(work_tile.is_valid_tile) + # Tiles are visited in increasing batch order, so the first tile past the + # live ones ends this CTA's schedule. + if cur_tile_coord[2] >= num_tiles[0]: + sched_valid = cutlass.Int32(0) + with cute.arch.elect_one(): + sInfo[(0, tile_info_producer_state.index)] = cur_tile_coord[0] + sInfo[(1, tile_info_producer_state.index)] = cur_tile_coord[1] + sInfo[(2, tile_info_producer_state.index)] = cur_tile_coord[2] + sInfo[(3, tile_info_producer_state.index)] = sched_valid + + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + self.sched_sync_barrier.arrive_and_wait() + tile_info_pipeline.producer_commit(tile_info_producer_state) + tile_info_producer_state.advance() + + tile_info_pipeline.producer_tail(tile_info_producer_state) + + # Specialized TMA load warp + if warp_idx == self.tma_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + ab_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_ab_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + mma_tile_coord_mnl = ( + tile_info[0] // cute.size(tiled_mma.thr_id.shape), + tile_info[1], + tile_info[2], + ) + a_batch = mma_tile_coord_mnl[2] + tAgA_slice = tAgA[(None, mma_tile_coord_mnl[0], None, a_batch)] + b_batch = tile_group[mma_tile_coord_mnl[2]] + tBgB_slice = tBgB[(None, mma_tile_coord_mnl[1], None, b_batch)] + + # Peek (try_wait) AB buffer empty for k_tile = prefetch_k_tile_cnt + ab_producer_state.reset_count() + peek_ab_empty_status = cutlass.Boolean(1) + if ab_producer_state.count < k_tile_cnt: + peek_ab_empty_status = ab_pipeline.producer_try_acquire(ab_producer_state) + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tAgA_k = tAgA_slice[(None, ab_producer_state.count)] + tBgB_k = tBgB_slice[(None, ab_producer_state.count)] + tAsA_pipe = tAsA[(None, ab_producer_state.index)] + tBsB_pipe = tBsB[(None, ab_producer_state.index)] + + tma_bar = ab_pipeline.producer_get_barrier(ab_producer_state) + + ab_pipeline.producer_acquire(ab_producer_state, peek_ab_empty_status) + + # Peek (try_wait) AB buffer empty for k_tile = prefetch_k_tile_cnt + k_tile + 1 + ab_producer_state.advance() + peek_ab_empty_status = cutlass.Boolean(1) + if ab_producer_state.count < k_tile_cnt: + peek_ab_empty_status = ab_pipeline.producer_try_acquire(ab_producer_state) + + cute.copy( + tma_atom_a, + tAgA_k, + tAsA_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=a_full_mcast_mask, + ) + cute.copy( + tma_atom_b, + tBgB_k, + tBsB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=b_full_mcast_mask, + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + ab_pipeline.producer_tail(ab_producer_state) + + # Specialized Scale load warp + if warp_idx == self.scale_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + scale_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_scale_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + sfa_batch = tile_info[2] + sfb_batch = tile_group[tile_info[2]] + tApSFA = cute.make_rmem_tensor( + cute.make_layout( + cute.filter_zeros(cute.slice_(tAsSFA, (None, None, None, 0))).shape + ), + cutlass.Boolean, + ) + tBpSFB = cute.make_rmem_tensor( + cute.make_layout( + cute.filter_zeros(cute.slice_(tBsSFB, (None, None, None, 0))).shape + ), + cutlass.Boolean, + ) + + # Peek (try_wait) SCALE buffer empty + scale_producer_state.reset_count() + peek_scale_empty_status = cutlass.Boolean(1) + if scale_producer_state.count < k_tile_cnt: + peek_scale_empty_status = scale_pipeline.producer_try_acquire( + scale_producer_state + ) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tAsSFA_pipe = cute.filter_zeros( + tAsSFA[(None, None, None, scale_producer_state.index)] + ) + tBsSFB_pipe = cute.filter_zeros( + tBsSFB[(None, None, None, scale_producer_state.index)] + ) + tAgSFA_k = cute.filter_zeros( + tAgSFA_mkl[ + ( + None, + None, + None, + tile_info[0], + scale_producer_state.count, + sfa_batch, + ) + ] + ) + tBgSFB_k = cute.filter_zeros( + tBgSFB_nkl[ + ( + None, + None, + None, + tile_info[1], + scale_producer_state.count, + sfb_batch, + ) + ] + ) + + tAcSFA_compact = cute.filter_zeros( + cute.slice_( + tAcSFA, + ( + None, + None, + None, + tile_info[0], + scale_producer_state.count, + sfa_batch, + ), + ) + ) + tBcSFB_compact = cute.filter_zeros( + cute.slice_( + tBcSFB, + ( + None, + None, + None, + tile_info[1], + scale_producer_state.count, + sfb_batch, + ), + ) + ) + for i in cutlass.range_constexpr(cute.size(tApSFA, mode=[1])): + tApSFA[((0, 0), i, (0, 0))] = cute.elem_less( + tAcSFA_compact[(i)][0], mSFA_mkl.shape[0] + ) + for i in cutlass.range_constexpr(cute.size(tBpSFB, mode=[1])): + tBpSFB[((0, 0), i, (0, 0))] = cute.elem_less( + tBcSFB_compact[(i)][0], mSFB_nkl.shape[0] + ) + + scale_pipeline.producer_acquire(scale_producer_state, peek_scale_empty_status) + + cute.copy(tiled_copy_sfa, tAgSFA_k, tAsSFA_pipe, pred=tApSFA) + cute.copy(tiled_copy_sfb, tBgSFB_k, tBsSFB_pipe, pred=tBpSFB) + + scale_pipeline.producer_commit(scale_producer_state) + + # Peek (try_wait) Scale buffer empty + scale_producer_state.advance() + peek_scale_empty_status = cutlass.Boolean(1) + if scale_producer_state.count < k_tile_cnt: + peek_scale_empty_status = scale_pipeline.producer_try_acquire( + scale_producer_state + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + scale_pipeline.producer_tail(scale_producer_state) + + # Specialized MMA warp + if warp_idx == self.mma_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + ab_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_ab_stage + ) + acc_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_acc_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + # Peek (try_wait) AB buffer full for k_tile = 0 + ab_consumer_state.reset_count() + peek_ab_full_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt and is_leader_cta: + peek_ab_full_status = ab_pipeline.consumer_try_wait(ab_consumer_state) + + # Peek (try_wait) Acc buffer empty for k_tile = 0 + acc_producer_state.reset_count() + peek_acc_empty_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt and is_leader_cta: + peek_acc_empty_status = acc_pipeline.producer_try_acquire(acc_producer_state) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tCtAcc = tCtAcc_base[(None, None, None, acc_producer_state.index)] + + if is_leader_cta: + acc_pipeline.producer_acquire(acc_producer_state, peek_acc_empty_status) + + tiled_mma.set(tcgen05.Field.ACCUMULATE, False) + + if is_leader_cta: + ab_pipeline.consumer_wait(ab_consumer_state, peek_ab_full_status) + + # tCtAcc += tCrA * tCrB + num_kblocks = cute.size(tCrA, mode=[2]) + for kblock_idx in cutlass.range(num_kblocks, unroll_full=True): + kblock_coord = ( + None, + None, + kblock_idx, + ab_consumer_state.index, + ) + + cute.gemm( + tiled_mma, + tCtAcc, + tCrA[kblock_coord], + tCrB[kblock_coord], + tCtAcc, + ) + # Enable accumulate on tCtAcc after first kblock + tiled_mma.set(tcgen05.Field.ACCUMULATE, True) + + ab_pipeline.consumer_release(ab_consumer_state) + + # Peek (try_wait) AB buffer full for k_tile = k_tile + 1 + ab_consumer_state.advance() + peek_ab_full_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt: + if is_leader_cta: + peek_ab_full_status = ab_pipeline.consumer_try_wait(ab_consumer_state) + + if is_leader_cta: + acc_pipeline.producer_commit(acc_producer_state) + + # Peek (try_wait) Acc buffer empty for k_tile = k_tile + 1 + acc_producer_state.advance() + if acc_producer_state.count < k_tile_cnt: + if is_leader_cta: + peek_acc_empty_status = acc_pipeline.producer_try_acquire( + acc_producer_state + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + acc_pipeline.producer_tail(acc_producer_state) + + # Specialized acc update warps + if warp_idx <= self.acc_update_warp_id[-1]: + cute.arch.warpgroup_reg_alloc(self.num_regs_acc_update_warps) + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + tCtAcc_final = cute.make_tensor( + tCtAcc_base.iterator + self.tmem_final_offset, tCtAcc_base.layout + ) + + epi_tidx = tidx % 128 + ( + tiled_copy_t2r, + tiled_copy_r2t, + tTR_tAcc_base, + tTR_rAcc, + tTR_rAcc_final, + tTR_sSFA, + tTR_sSFB, + tRT_rAcc, + tRT_tAcc_base, + ) = self.acc_update_tmem_copy_and_partition( + epi_tidx, + tCtAcc_base, + tCtAcc_final, + tCgC, + sSFA_view_as_C, + sSFB_view_as_C, + epi_tile, + ) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + acc_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_acc_stage + ) + + scale_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_scale_stage + ) + + epi_producer_state = pipeline.make_pipeline_state(pipeline.PipelineUserType.Producer, 1) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + tTR_rAcc_final.fill(0.0) + + tTR_rSFA = cute.make_rmem_tensor( + cute.slice_(tTR_sSFA, (None, None, None, 0, None, 0)).shape, + self.acc_dtype, + ) + tTR_rSFB = cute.make_rmem_tensor( + cute.slice_(tTR_sSFB, (None, None, None, 0, None, 0)).shape, + self.acc_dtype, + ) + + scale_consumer_state.reset_count() + peek_scale_full_status = cutlass.Boolean(1) + if scale_consumer_state.count < k_tile_cnt: + peek_scale_full_status = scale_pipeline.consumer_try_wait(scale_consumer_state) + + acc_consumer_state.reset_count() + peek_acc_full_status = cutlass.Boolean(1) + if acc_consumer_state.count < k_tile_cnt: + peek_acc_full_status = acc_pipeline.consumer_try_wait(acc_consumer_state) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tTR_tAcc = tTR_tAcc_base[ + (None, None, None, None, None, acc_consumer_state.index) + ] + + scale_pipeline.consumer_wait(scale_consumer_state, peek_scale_full_status) + + tTR_sSFA_slice = cute.slice_( + tTR_sSFA, + (None, None, None, 0, None, scale_consumer_state.index), + ) + tTR_sSFB_slice = cute.slice_( + tTR_sSFB, + (None, None, None, 0, None, scale_consumer_state.index), + ) + + scale_atom_copy = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + self.acc_dtype, + num_bits_per_copy=self.acc_dtype.width, + ) + + cute.copy(scale_atom_copy, tTR_sSFA_slice, tTR_rSFA) + cute.copy(scale_atom_copy, tTR_sSFB_slice, tTR_rSFB) + + acc_pipeline.consumer_wait(acc_consumer_state, peek_acc_full_status) + + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + + subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) + for subtile_idx in cutlass.range(subtile_cnt): + tTR_tAcc_mn = tTR_tAcc[(None, None, None, subtile_idx)] + cute.copy(tiled_copy_t2r, tTR_tAcc_mn, tTR_rAcc) + + tTR_rAcc_subtile = tTR_rAcc_final[(None, None, None, subtile_idx)] + tTR_rSFA_subtile = tTR_rSFA[(None, None, None, subtile_idx)] + tTR_rSFB_subtile = tTR_rSFB[(None, None, None, subtile_idx)] + + acc_vec = tTR_rAcc.load() + final_vec = tTR_rAcc_subtile.load() + scale_a = tTR_rSFA_subtile.load() + scale_b = tTR_rSFB_subtile.load() + scale = scale_a * scale_b + final_vec = acc_vec * scale + final_vec + tTR_rAcc_subtile.store(final_vec.to(self.acc_dtype)) + + scale_pipeline.consumer_release(scale_consumer_state) + scale_consumer_state.advance() + + peek_scale_full_status = cutlass.Boolean(1) + if scale_consumer_state.count < k_tile_cnt: + peek_scale_full_status = scale_pipeline.consumer_try_wait( + scale_consumer_state + ) + with cute.arch.elect_one(): + acc_pipeline.consumer_release(acc_consumer_state) + acc_consumer_state.advance() + + peek_acc_full_status = cutlass.Boolean(1) + if acc_consumer_state.count < k_tile_cnt: + peek_acc_full_status = acc_pipeline.consumer_try_wait(acc_consumer_state) + + tRT_tAcc = tRT_tAcc_base[(None, None, None, None, None, 0)] + tRT_tAcc = cute.group_modes(tRT_tAcc, 3, cute.rank(tRT_tAcc)) + + epi_pipeline.producer_acquire(epi_producer_state) + + cute.copy(tiled_copy_r2t, tTR_rAcc_final, tRT_tAcc) + cute.arch.fence_view_async_tmem_store() + + epi_pipeline.producer_commit(epi_producer_state) + epi_producer_state.advance() + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + # Specialized epilogue warps + if warp_idx <= self.epilog_warp_id[-1] and warp_idx >= self.epilog_warp_id[0]: + cute.arch.warpgroup_reg_alloc(self.num_regs_epilogue_warps) + tmem.allocate(self.num_tmem_alloc_cols) + + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base_ = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + tCtAcc_final = cute.make_tensor( + tCtAcc_base_.iterator + self.tmem_final_offset, tCtAcc_base_.layout + ) + + epi_tidx = tidx % 128 + ( + tiled_copy_t2r, + tTR_tAcc_base, + tTR_rAcc, + ) = self.epilog_tmem_copy_and_partition(epi_tidx, tCtAcc_final, tCgC, epi_tile) + + tTR_rC = None + tiled_copy_r2s = None + tRS_rC = None + tRS_sC = None + bSG_sC = None + bSG_gC_partitioned = None + tTR_rC = cute.make_rmem_tensor(tTR_rAcc.shape, self.c_dtype) + tiled_copy_r2s, tRS_rC, tRS_sC = self.epilog_smem_copy_and_partition( + tiled_copy_t2r, tTR_rC, epi_tidx, sC + ) + ( + tma_atom_c, + bSG_sC, + bSG_gC_partitioned, + ) = self.epilog_gmem_copy_and_partition(epi_tidx, tma_atom_c, tCgC, epi_tile, sC) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + epi_consumer_state = pipeline.make_pipeline_state(pipeline.PipelineUserType.Consumer, 1) + + c_pipeline = None + c_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + 32 * len(self.epilog_warp_id), + ) + c_pipeline = pipeline.PipelineTmaStore.create( + num_stages=self.num_c_stage, + producer_group=c_producer_group, + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + num_prev_subtiles = cutlass.Int32(0) + + while is_valid_tile: + mma_tile_coord_mnl = ( + tile_info[0] // cute.size(tiled_mma.thr_id.shape), + tile_info[1], + tile_info[2], + ) + bSG_gC = None + bSG_gC = bSG_gC_partitioned[ + ( + None, + None, + None, + mma_tile_coord_mnl[0], + mma_tile_coord_mnl[1], + mma_tile_coord_mnl[2], + ) + ] + + tTR_tAcc = tTR_tAcc_base[(None, None, None, None, None, epi_consumer_state.index)] + + epi_pipeline.consumer_wait(epi_consumer_state) + + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + bSG_gC = cute.group_modes(bSG_gC, 1, cute.rank(bSG_gC)) + + subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) + q_inv = cutlass.Float32(1.0) + if cutlass.const_expr(self.epi_quant): + # A thread holds one row of the 128-column tile (lane = row of the 32dp TMEM + # loads), so the row's 1x128 amax is thread-local: one pass over the subtiles + # for it, the store pass below scales by 448 / amax. + q_amax = cutlass.Float32(0.0) + for subtile_idx in cutlass.range_constexpr(subtile_cnt): + cute.copy( + tiled_copy_t2r, tTR_tAcc[(None, None, None, subtile_idx)], tTR_rAcc + ) + for i in cutlass.range_constexpr(cute.size(tTR_rAcc)): + q_v = tTR_rAcc[i] + q_amax = cutlass.max( + q_amax, cutlass.max(q_v, cutlass.Float32(0.0) - q_v) + ) + q_amax = cutlass.max(q_amax, cutlass.Float32(1.0e-10)) + q_inv = _div_rn(cutlass.Float32(448.0), q_amax) + mCS[ + ( + mma_tile_coord_mnl[1], + mma_tile_coord_mnl[2] * self.cta_tile_shape_mnk[0] + epi_tidx, + ) + ] = _div_rn(q_amax, cutlass.Float32(448.0)) + for subtile_idx in cutlass.range(subtile_cnt): + tTR_tAcc_mn = tTR_tAcc[(None, None, None, subtile_idx)] + cute.copy(tiled_copy_t2r, tTR_tAcc_mn, tTR_rAcc) + + acc_vec = tiled_copy_r2s.retile(tTR_rAcc).load() + if cutlass.const_expr(self.epi_quant): + acc_vec = (acc_vec * q_inv).to(self.c_dtype) + else: + acc_vec = acc_vec.to(self.c_dtype) + tRS_rC.store(acc_vec) + + num_prev_subtiles = num_prev_subtiles + 1 + c_buffer = num_prev_subtiles % self.num_c_stage + cute.copy( + tiled_copy_r2s, + tRS_rC, + tRS_sC[(None, None, None, c_buffer)], + ) + # Fence and barrier to make sure shared memory store is visible to TMA store + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + self.epilog_sync_barrier.arrive_and_wait() + + if warp_idx == self.epilog_warp_id[0]: + cute.copy( + tma_atom_c, + bSG_sC[(None, c_buffer)], + bSG_gC[(None, subtile_idx)], + ) + # Fence and barrier to make sure shared memory store is visible to TMA store + c_pipeline.producer_commit() + c_pipeline.producer_acquire() + self.epilog_sync_barrier.arrive_and_wait() + + epi_pipeline.consumer_release(epi_consumer_state) + epi_consumer_state.advance() + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + tmem.relinquish_alloc_permit() + self.epilog_sync_barrier.arrive_and_wait() + tmem.free(tmem_ptr) + c_pipeline.producer_tail() + + def acc_update_tmem_copy_and_partition( + self, + tidx: cutlass.Int32, + tAcc: cute.Tensor, + tAcc_final: cute.Tensor, + gC_mnl: cute.Tensor, + sSFA: cute.Tensor, + sSFB: cute.Tensor, + epi_tile: cute.Tile, + ) -> Tuple[ + cute.TiledCopy, + cute.TiledCopy, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + ]: + """TMEM <-> register copies of the acc-update warps: `(tiled_copy_t2r, tiled_copy_r2t, + tTR_tAcc, tTR_rAcc, tTR_rAcc_final, tTR_sSFA, tTR_sSFB, tRT_rAcc_final, tRT_tAcc_final)` -- + each k tile's partial loaded from `tAcc`, the running sum held in registers and stored to + `tAcc_final`, and SFA / SFB partitioned like the partial.""" + tmem_load_atom = None + tmem_store_atom = None + if cutlass.const_expr(self.mma_tiler[0] == 64): + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld16x256bOp(tcgen05.copy.Repetition(8)), + self.acc_dtype, + ) + else: + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), + self.acc_dtype, + ) + if cutlass.const_expr(self.mma_tiler[0] == 64): + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St16x256bOp(tcgen05.copy.Repetition(8)), + self.acc_dtype, + ) + else: + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), + self.acc_dtype, + ) + + tAcc_epi = cute.flat_divide(tAcc[((None, None), 0, 0, None)], epi_tile) + tAcc_final_epi = cute.flat_divide(tAcc_final[((None, None), 0, 0, None)], epi_tile) + + tiled_copy_t2r = tcgen05.make_tmem_copy(tmem_load_atom, tAcc_epi[(None, None, 0, 0, 0)]) + tiled_copy_r2t = tcgen05.make_tmem_copy( + tmem_store_atom, tAcc_final_epi[(None, None, 0, 0, 0)] + ) + + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + thr_copy_r2t = tiled_copy_r2t.get_slice(tidx) + + tTR_tAcc = thr_copy_t2r.partition_S(tAcc_epi) + gC_mnl_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + sSFA_epi = cute.flat_divide(sSFA, epi_tile) + sSFB_epi = cute.flat_divide(sSFB, epi_tile) + tTR_gC = thr_copy_t2r.partition_D(gC_mnl_epi) + tTR_sSFA = thr_copy_t2r.partition_D(sSFA_epi) + tTR_sSFB = thr_copy_t2r.partition_D(sSFB_epi) + tTR_rAcc = cute.make_rmem_tensor( + tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype + ) + tTR_rAcc_final_ = cute.make_rmem_tensor( + tTR_gC[(None, None, None, None, None, 0, 0, 0)].shape, self.acc_dtype + ) + tTR_rAcc_final = cute.group_modes(tTR_rAcc_final_, 3, cute.rank(tTR_rAcc_final_)) + + tRT_gC = thr_copy_r2t.partition_S(gC_mnl_epi) + tRT_tAcc_final = thr_copy_r2t.partition_D(tAcc_final_epi) + tRT_rAcc_final_ = cute.make_rmem_tensor( + tRT_gC[(None, None, None, None, None, 0, 0, 0)].shape, self.acc_dtype + ) + tRT_rAcc_final = cute.group_modes(tRT_rAcc_final_, 3, cute.rank(tRT_rAcc_final_)) + + return ( + tiled_copy_t2r, + tiled_copy_r2t, + tTR_tAcc, + tTR_rAcc, + tTR_rAcc_final, + tTR_sSFA, + tTR_sSFB, + tRT_rAcc_final, + tRT_tAcc_final, + ) + + def epilog_tmem_copy_and_partition( + self, + tidx: cutlass.Int32, + tAcc: cute.Tensor, + gC_mnl: cute.Tensor, + epi_tile: cute.Tile, + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: + """`(tiled_copy_t2r, tTR_tAcc, tTR_rAcc)`: the epilogue warps' TMEM -> register copy over + `epi_tile` subtiles.""" + copy_atom_t2r = sm100_utils.get_tmem_load_op( + self.cta_tile_shape_mnk, + self.c_layout, + self.c_dtype, + self.acc_dtype, + epi_tile, + False, + ) + tAcc_epi = cute.flat_divide( + tAcc[((None, None), 0, 0, None)], + epi_tile, + ) + tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) + + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + tTR_tAcc = thr_copy_t2r.partition_S(tAcc_epi) + + gC_mnl_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + tTR_gC = thr_copy_t2r.partition_D(gC_mnl_epi) + + tTR_rAcc = cute.make_rmem_tensor( + tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype + ) + + return tiled_copy_t2r, tTR_tAcc, tTR_rAcc + + def epilog_smem_copy_and_partition( + self, + tiled_copy_t2r: cute.TiledCopy, + tTR_rC: cute.Tensor, + tidx: cutlass.Int32, + sC: cute.Tensor, + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: + """`(tiled_copy_r2s, tRS_rC, tRS_sC)`: the register -> SMEM copy of a C subtile.""" + copy_atom_r2s = sm100_utils.get_smem_store_op( + self.c_layout, self.c_dtype, self.acc_dtype, tiled_copy_t2r + ) + tiled_copy_r2s = cute.make_tiled_copy_D(copy_atom_r2s, tiled_copy_t2r) + thr_copy_r2s = tiled_copy_r2s.get_slice(tidx) + tRS_sC = thr_copy_r2s.partition_D(sC) + tRS_rC = tiled_copy_r2s.retile(tTR_rC) + return tiled_copy_r2s, tRS_rC, tRS_sC + + def epilog_gmem_copy_and_partition( + self, + tidx: cutlass.Int32, + atom: Union[cute.CopyAtom, cute.TiledCopy], + gC_mnl: cute.Tensor, + epi_tile: cute.Tile, + sC: cute.Tensor, + ) -> Tuple[cute.CopyAtom, cute.Tensor, cute.Tensor]: + """`(tma_atom_c, bSG_sC, bSG_gC)`: the TMA store partition of a C subtile, SMEM to GMEM.""" + gC_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + tma_atom_c = atom + sC_for_tma_partition = cute.group_modes(sC, 0, 2) + gC_for_tma_partition = cute.group_modes(gC_epi, 0, 2) + bSG_sC, bSG_gC = cpasync.tma_partition( + tma_atom_c, + 0, + cute.make_layout(1), + sC_for_tma_partition, + gC_for_tma_partition, + ) + return tma_atom_c, bSG_sC, bSG_gC + + @staticmethod + def _compute_stages( + tiled_mma: cute.TiledMma, + mma_tiler_mnk: Tuple[int, int, int], + a_dtype: Type[cutlass.Numeric], + b_dtype: Type[cutlass.Numeric], + epi_tile: cute.Tile, + c_dtype: Type[cutlass.Numeric], + c_layout: utils.LayoutEnum, + sfa_dtype: Type[cutlass.Numeric], + sfb_dtype: Type[cutlass.Numeric], + sfa_count: int, + sfb_count: int, + num_smem_capacity: int, + occupancy: int, + ) -> Tuple[int, int, int]: + """`(ACC, A/B, C, scale, tile info)` stage counts for the CTA's SMEM at `occupancy` CTAs an + SM.""" + num_acc_stage = 3 if mma_tiler_mnk[0] / tiled_mma.thr_id.shape == 128 else 6 + + num_c_stage = 2 + + num_scale_stage = 10 + + num_tile_stage = 2 + + a_smem_layout_stage_one = sm100_utils.make_smem_layout_a( + tiled_mma, + mma_tiler_mnk, + a_dtype, + 1, # a tmp 1 stage is provided + ) + b_smem_layout_staged_one = sm100_utils.make_smem_layout_b( + tiled_mma, + mma_tiler_mnk, + b_dtype, + 1, # a tmp 1 stage is provided + ) + c_smem_layout_staged_one = sm100_utils.make_smem_layout_epi( + c_dtype, + c_layout, + epi_tile, + 1, + ) + + ab_bytes_per_stage = cute.size_in_bytes( + a_dtype, a_smem_layout_stage_one + ) + cute.size_in_bytes(b_dtype, b_smem_layout_staged_one) + # 1024B alignment + mbar_helpers_bytes = 1024 + c_bytes_per_stage = cute.size_in_bytes(c_dtype, c_smem_layout_staged_one) + c_bytes = c_bytes_per_stage * num_c_stage + sfa_bytes = sfa_count * (sfa_dtype.width // 8) * num_scale_stage + sfb_bytes = sfb_count * (sfb_dtype.width // 8) * num_scale_stage + scale_bytes = math.ceil((sfa_bytes + sfb_bytes) / 1024) * 1024 + + # A/B stages fill what is left of the CTA's SMEM after the barriers, scales and C stages; + # the C stages then take what the A/B stages leave. + num_ab_stage = ( + num_smem_capacity // occupancy - (mbar_helpers_bytes + c_bytes + scale_bytes) + ) // ab_bytes_per_stage + + num_c_stage += ( + num_smem_capacity + - occupancy * ab_bytes_per_stage * num_ab_stage + - occupancy * (mbar_helpers_bytes + c_bytes + scale_bytes) + ) // (occupancy * c_bytes_per_stage) + return num_acc_stage, num_ab_stage, num_c_stage, num_scale_stage, num_tile_stage + + @staticmethod + def _compute_grid( + c: cute.Tensor, + cta_tile_shape_mnk: Tuple[int, int, int], + cluster_shape_mn: Tuple[int, int], + max_active_clusters: cutlass.Constexpr, + ) -> Tuple[utils.PersistentTileSchedulerParams, Tuple[int, int, int]]: + """`(tile_sched_params, grid)` of the persistent tile scheduler over C's tiles.""" + c_shape = cute.slice_(cta_tile_shape_mnk, (None, None, 0)) + gc = cute.zipped_divide(c, tiler=c_shape) + num_ctas_mnl = gc[(0, (None, None, None))].shape + cluster_shape_mnl = (*cluster_shape_mn, 1) + + tile_sched_params = utils.PersistentTileSchedulerParams(num_ctas_mnl, cluster_shape_mnl) + grid = utils.StaticPersistentTileScheduler.get_grid_shape( + tile_sched_params, max_active_clusters + ) + + return tile_sched_params, grid + + @staticmethod + def _get_tma_atom_kind( + atom_sm_cnt: cutlass.Int32, mcast: cutlass.Boolean + ) -> Union[cpasync.CopyBulkTensorTileG2SMulticastOp, cpasync.CopyBulkTensorTileG2SOp]: + """The TMA G2S copy op for `atom_sm_cnt` SMs, multicast or not; ValueError otherwise.""" + if atom_sm_cnt == 2 and mcast: + return cpasync.CopyBulkTensorTileG2SMulticastOp(tcgen05.CtaGroup.TWO) + elif atom_sm_cnt == 2 and not mcast: + return cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.TWO) + elif atom_sm_cnt == 1 and mcast: + return cpasync.CopyBulkTensorTileG2SMulticastOp(tcgen05.CtaGroup.ONE) + elif atom_sm_cnt == 1 and not mcast: + return cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE) + + raise ValueError(f"Invalid atom_sm_cnt: {atom_sm_cnt} and {mcast}") + + @cute.jit + def wrapper_grouped( + self, + tiles: cutlass.Int32, + n: cutlass.Int32, + k: cutlass.Int32, + sf_m: cutlass.Int32, + sf_n: cutlass.Int32, + sf_k: cutlass.Int32, + num_groups: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + c_tensor: cute.Tensor, + tile_group_ptr: cute.Pointer, + num_tiles_ptr: cute.Pointer, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + ): + """The contiguous grouped GEMM over raw pointers: A holds `tiles` M tiles of + rows back to back (`[tiles * M_tile, k]`, k-major), each tile of one group; SFA is M-major + over all of those rows (`[sf_k, sf_m]`, `sf_m >= tiles * M_tile`); B `[num_groups, n, k]` + and SFB `[num_groups, sf_n, sf_k]` are batched by group; C is `(M_tile, n, tiles)` over + `[tiles * M_tile, n]`. `tile_group[l]` (int32 `[tiles]`) names tile l's group; tiles from + `num_tiles[0]` (int32 `[1]`) on are skipped (so `tiles` can be a capture-time upper bound). + Visiting order: every N tile of an M tile, then the next M tile -- a group's tiles back to + back, its B from L2. + """ + mt = self.mma_tiler[0] + a_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((mt, k, tiles), order=(1, 0, 2)), + ) + b_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, num_groups), order=(1, 0, 2)), + ) + sfa_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_layout((mt, sf_k, tiles), stride=(1, sf_m, mt)), + ) + sfb_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout((sf_n, sf_k, num_groups), order=(1, 0, 2)), + ) + tile_group = cute.make_tensor(tile_group_ptr, layout=cute.make_layout((tiles,))) + num_tiles = cute.make_tensor(num_tiles_ptr, layout=cute.make_layout((1,))) + self( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + max_active_clusters, + stream, + tile_group=tile_group, + num_tiles=num_tiles, + ) + + @cute.jit + def wrapper_grouped_quant( + self, + tiles: cutlass.Int32, + n: cutlass.Int32, + k: cutlass.Int32, + sf_m: cutlass.Int32, + sf_n: cutlass.Int32, + sf_k: cutlass.Int32, + num_groups: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + c_ptr: cute.Pointer, + c_scale: cute.Tensor, + tile_group_ptr: cute.Pointer, + num_tiles_ptr: cute.Pointer, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + ): + """`wrapper_grouped` for `epi_quant=True`: C `[tiles * M_tile, n]` e4m3 codes (`c_ptr`), + `c_scale` `[n / 128, >= tiles * M_tile]` fp32 (row-major) the scale of each row's 128-column + block.""" + mt = self.mma_tiler[0] + a_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((mt, k, tiles), order=(1, 0, 2)), + ) + b_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, num_groups), order=(1, 0, 2)), + ) + sfa_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_layout((mt, sf_k, tiles), stride=(1, sf_m, mt)), + ) + sfb_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout((sf_n, sf_k, num_groups), order=(1, 0, 2)), + ) + c_tensor = cute.make_tensor( + c_ptr, + layout=cute.make_ordered_layout((mt, n, tiles), order=(1, 0, 2)), + ) + tile_group = cute.make_tensor(tile_group_ptr, layout=cute.make_layout((tiles,))) + num_tiles = cute.make_tensor(num_tiles_ptr, layout=cute.make_layout((1,))) + self( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + max_active_clusters, + stream, + tile_group=tile_group, + num_tiles=num_tiles, + c_scale=c_scale, + ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py new file mode 100644 index 000000000000..2bf7d75d6e1c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py @@ -0,0 +1,616 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""DeepSeek-style FP8 block-scale mixture-of-experts layer: top-k routed grouped FC1 GEMM + +SwiGLU + FP8 requantization + grouped FC2 GEMM + routing-weighted combine, in one trtllm-gen call. + +This target's copy of the family entry `models/qwen4_exp/ops/moe/fp8_block_scale_moe_runner.py`, +with the logits-routed decode front end (`fp8_block_scale_moe_front`) and pinned decode tactics.""" + +from __future__ import annotations + +from typing import Callable, Optional + +import torch + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.modeling_v2.catalog._op import ( + Arch, + Cell, + OpWrapper, + true_fp32_matmul, +) +from tensorrt_llm._torch.autotuner import AutoTuner +from tensorrt_llm._torch.custom_ops.trtllm_gen_custom_ops import FP8BlockScaleMoERunner + +from ... import native_ops + +#: Block edge on every scale axis: 1x128 activations, 128x128 weights. +BLOCK = 128 + +#: Largest finite e4m3 magnitude. +E4M3_MAX = 448.0 + +#: The activation kernel's floor on a block amax (`kDsActAmaxEpsilon` in +#: trtllmGenKernels/blockScaleMoe/DevKernel.cu). Only an all-zero block reaches it. +ACT_AMAX_FLOOR = 1.0e-10 + + +def dequantize_block_weight(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """One expert's `[M, K]` fp32 weight from e4m3 codes and `[M/128, K/128]` scales.""" + m, k = codes.shape + blocks = codes.float().reshape(m // BLOCK, BLOCK, k // BLOCK, BLOCK) + return (blocks * scales.float().reshape(m // BLOCK, 1, k // BLOCK, 1)).reshape(m, k) + + +def requantize_1x128(x: torch.Tensor) -> torch.Tensor: + """The e4m3 round trip the kernel applies to both intermediates: per row and 128 columns, + `scale = max(amax, 1e-10) / 448` (not a power of two), round-to-nearest-even saturating at 448. + """ + shape = x.shape + blocks = x.reshape(*shape[:-1], shape[-1] // BLOCK, BLOCK) + scale = blocks.abs().amax(dim=-1, keepdim=True).clamp(min=ACT_AMAX_FLOOR) / E4M3_MAX + codes = (blocks / scale).clamp(-E4M3_MAX, E4M3_MAX).to(torch.float8_e4m3fn) + return (codes.float() * scale).reshape(shape) + + +def _geom(hidden, inter, num_experts, top_k, offset=0, local=None, capture=False): + """A cell's MoE geometry (every expert local by default; `capture`: a captured graph).""" + return dict( + hidden=hidden, + inter=inter, + num_experts=num_experts, + top_k=top_k, + local_expert_offset=offset, + local_num_experts=num_experts if local is None else local, + capture=capture, + ) + + +def _whole(capture: bool = False) -> dict: + """The checkpoint's routed experts (H 2560, I 640, 512, top-10), all local (tp1).""" + return _geom(2560, 640, 512, 10, capture=capture) + + +_CKPT = _whole() + + +def _window(offset: int, local: int = 128, capture: bool = False) -> dict: + """One rank's window of the 512 (128 or 256 wide); `num_experts` stays 512.""" + return _geom(2560, 640, 512, 10, offset=offset, local=local, capture=capture) + + +_CUSTOM_OP = "trtllm::fp8_block_scale_moe_runner" + +#: The geometry `DECODE_TACTICS` was measured on: (num_experts, top_k, intermediate, routing). +DECODE_TACTICS_GEOMETRY = (512, 10, 640, 1) + +#: trtllm-gen tactic `(tileN, config index)` per (window width, rows) for logits-routed decode +#: calls (`routing_logits=`, Renormalize). The engine's tuning pass profiles pre-routed shapes +#: only, so the AutoTuner holds nothing for these and would hand them the slower cold fallback +#: `[32, 0]`. Every tactic gives the fallback's bits: the choice moves time only. A call not +#: listed takes the AutoTuner's tactic for the same call pre-routed (`_dispatch`), else the fallback. +DECODE_TACTICS: dict[tuple[int, int], tuple[int, int]] = { + (128, 1): (8, 13), + (128, 2): (8, 13), + (128, 3): (8, 13), + (128, 4): (8, 13), + (128, 6): (8, 12), + (128, 8): (8, 12), +} + + +#: The tactic logits-routed decode calls over a 256-expert window (tp2 / tep2) run from +#: `WIDE_DECODE_MIN_ROWS` rows on. The AutoTuner profiles near-uniform routing, where tileN 8 wins; +#: this checkpoint's skewed routing makes hot experts restream weight tiles at tileN 8, so tileN 32 +#: is faster from 168 rows (GB300, recorded routing). Every tactic writes the same bits. Config 8 +#: pairs the persistent-scheduler FC1 (`u2`) and FC2 cubins. +WIDE_DECODE_TACTIC: tuple[int, int] = (32, 8) +WIDE_DECODE_MIN_ROWS = 168 +WIDE_DECODE_WIDTH = 256 + +#: The same calls below `WIDE_DECODE_MIN_ROWS`, by the row count each tactic starts at: what a +#: full-size tuning call picks. Pinned because a small first tuning call profiles every bucket on +#: a few experts (the tuner repeats its rows' `topk_ids`) and picks tiles slower under real routing. +WIDE_DECODE_SMALL_TACTICS: tuple[tuple[int, tuple[int, int]], ...] = ( + (1, (8, 13)), + (4, (8, 12)), + (8, (8, 8)), +) + + +#: `(tactic, min_rows)` of the 256-wide decode calls; None (tests set it) leaves them to the AutoTuner. +_WIDE_DECODE: Optional[tuple[tuple[int, int], int]] = (WIDE_DECODE_TACTIC, WIDE_DECODE_MIN_ROWS) + + +def _pinned_tactic(geometry: tuple, inputs: list, runner) -> Optional[list]: + """`DECODE_TACTICS`'s tactic for a logits-routed call at its geometry, if listed, or for a + 256-wide window `WIDE_DECODE_TACTIC` from `WIDE_DECODE_MIN_ROWS` rows on and + `WIDE_DECODE_SMALL_TACTICS` below -- either only if valid at the call's rows; else None.""" + experts, top_k, inter, _, local, routing = geometry[:6] + if inputs[0] is None or inputs[9] is not None: + return None + if (experts, top_k, inter, routing) != DECODE_TACTICS_GEOMETRY: + return None + rows = inputs[2].shape[0] + tactic = DECODE_TACTICS.get((local, rows)) + if tactic is None and _WIDE_DECODE is not None and local == WIDE_DECODE_WIDTH: + if rows >= _WIDE_DECODE[1]: + tactic = _WIDE_DECODE[0] + else: + tactic = [t for start, t in WIDE_DECODE_SMALL_TACTICS if rows >= start][-1] + if tactic is None: + return None + hidden = inputs[2].shape[1] + valid = runner.get_runner().get_valid_configs(top_k, None, hidden, inter, local, rows) + if list(tactic) not in [[int(v) for v in t] for t in valid]: + return None + return list(tactic) + + +# The custom op's AutoTuner choice, keyed on everything its lookup reads (geometry, input shapes, +# the profiling cache's size: a lookup made before the tuning pass is not reused after it). +_DISPATCH: dict = {} + + +def _dispatch(geometry: tuple, inputs: list) -> Optional[tuple]: + """`(runner, tactic)` as `torch.ops.trtllm.fp8_block_scale_moe_runner` would choose them, + remembered; None while the AutoTuner tunes or captures (the custom op then runs as is).""" + tuner = AutoTuner.get() + if tuner.is_tuning_mode or tuner._active_capture is not None: + return None + shapes = tuple(None if t is None else tuple(t.shape) for t in inputs) + key = (geometry, shapes, len(tuner.profiling_cache)) + hit = _DISPATCH.get(key) + if hit is not None: + return hit + experts, top_k, inter, offset, local, routing, act, clamp, tune_max, use_dp = geometry + runner = FP8BlockScaleMoERunner( + experts, + top_k, + None, + None, + None, + inter, + offset, + local, + None, + routing, + act, + tune_max_num_tokens=tune_max, + use_dp=use_dp, + gemm1_clamp_limit_value=clamp, + ) + pinned = _pinned_tactic(geometry, inputs, runner) + if pinned is not None: + _DISPATCH[key] = (runner, pinned) + return runner, pinned + lookup = inputs + if inputs[0] is not None and inputs[9] is None: + # Routed from logits, which the tuning pass never profiles: use the tactic held for the + # same call pre-routed (the GEMMs' shapes are the same). + rows, device = inputs[2].shape[0], inputs[2].device + lookup = list(inputs) + lookup[0] = None + lookup[8] = torch.empty((rows, top_k), dtype=torch.bfloat16, device=device) + lookup[9] = torch.empty((rows, top_k), dtype=torch.int32, device=device) + runner, tactic = tuner.choose_one(_CUSTOM_OP, [runner], runner.tuning_config, lookup) + if tactic == -1: + tactic = runner.get_fallback_tactic(inputs[2].shape[1], inputs[2].shape[0]) + _DISPATCH[key] = (runner, tactic) + return runner, tactic + + +def fp8_block_scale_moe_front( + routing_logits: torch.Tensor, + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routing_method_type: int, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, +) -> Optional[Callable[[Optional[torch.Tensor]], torch.Tensor]]: + """The logits-routed runner call through this target's one-launch decode routing + (`native_ops.fp8_block_scale_moe_front`, Renormalize bitwise the runner's) at the tactic the + custom op would choose. Returns a callable taking the bf16 hidden states (then also quantized + into `hidden_states` / `hidden_states_scale`, bitwise `fp8_quantize_1x128`) or None; returns + None while the AutoTuner tunes or captures (the caller makes the plain call).""" + inputs = [ + routing_logits, + None, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + None, + None, + ] + geometry = ( + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + 0, + None, + tune_max_num_tokens, + use_dp, + ) + chosen = _dispatch(geometry, inputs) + if chosen is None: + return None + runner, tactic = chosen + tactic = [int(v) for v in tactic] + if len(tactic) != 2 or tactic[0] <= 0 or tactic[1] < 0: + return None + + def call(hidden_states_bf16: Optional[torch.Tensor]) -> torch.Tensor: + return native_ops.fp8_block_scale_moe_front( + routing_logits, + hidden_states_bf16, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + tactic[0], + tactic[1], + ) + + return call + + +class Fp8BlockScaleMoeRunner(OpWrapper): + """One FP8 block-scale MoE layer over e4m3 activations, entered pre-routed. + + Per token `t` and slot `j`, with `A[t]` the fp32 activations + (`hidden_states[t, k] * hidden_states_scale[k // 128, t]`) and `W_*` the + dequantized 128x128-block weights of local expert `e = topk_ids[t, j] - + local_expert_offset` (slots outside `[0, local_num_experts)` contribute + nothing): + + up, gate = requant(A[t] @ [W_up; W_gate].T).split(I) + gate, up = min(gate, lim), clamp(up, -lim, lim) # gemm1_clamp_limit + out[t] += topk_weights[t, j] * (requant(silu(gate) * up) @ W_down.T) + + accumulated in fp32 and stored bf16, `requant` being `requantize_1x128`. + FC1 rows are `[up ; gate]` (the loader's `[w3; w1]`). The router and top-k, the input + quantization (`fp8_quantize_1x128(x, use_ue8m0=False)`), the shared expert and any + collective are the caller's. + + `hidden_states_scale` is `[H/128, T]`, as `fp8_quantize_1x128` returns it: at `T == H/128` + (20 tokens) its transpose has the same shape, passes every check and is wrong. + + Returns a fresh `[T, H]` bf16 tensor, or an empty `[0]` tensor when `output` is given (the + result is then in `output`). + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + # Each window width (512 / 256 / 128) at its smallest and largest rows, first and last + # offsets, captured where the engine captures, and one non-power-of-two decode batch. + Cell( + why="MTPLayer._moe decode: one token, 10 of 512 experts", spec=dict(tokens=1, **_CKPT) + ), + Cell( + why="20 tokens == H/128: the activation scale buffer is square", + spec=dict(tokens=20, **_CKPT), + ), + Cell( + why="16384 rows: max_num_tokens of the tuned config; past the last tuning bucket", + spec=dict(tokens=16384, **_CKPT), + ), + Cell( + why="tp1 captured decode, 128 rows: the whole 512-expert stack", + spec=dict(tokens=128, **_whole(capture=True)), + ), + Cell( + why="tep2 captured decode, 1 row: 256-expert window 0", + spec=dict(tokens=1, **_window(0, 256, capture=True)), + ), + Cell( + why="dep2 captured decode, 2 x 128 gathered rows: 256-expert window 0", + spec=dict(tokens=256, **_window(0, 256, capture=True)), + ), + Cell( + why="tep2 prefill, a 16384-row chunk (max_num_tokens 16384): 256-expert window 256", + spec=dict(tokens=16384, **_window(256, 256)), + ), + Cell( + why="tep4 captured decode, 1 row: 128-expert window 256", + spec=dict(tokens=1, **_window(256, capture=True)), + ), + Cell( + why="tp4 captured decode, 128 rows: 128-expert window 0", + spec=dict(tokens=128, **_window(0, capture=True)), + ), + Cell( + why="dep4 captured MTP3 verify, 4 x 128 x 4 gathered rows: 128-expert window 384", + spec=dict(tokens=2048, **_window(384, capture=True)), + ), + Cell( + why="tep2/tp2 captured decode, batch 40: 256-expert window 0", + spec=dict(tokens=40, **_window(0, 256, capture=True)), + ), + Cell( + why="tep4/tp4 captured decode, batch 56: 128-expert window 128", + spec=dict(tokens=56, **_window(128, capture=True)), + ), + ) + + note = """ + Source: the custom op `trtllm::fp8_block_scale_moe_runner` + (`_torch/custom_ops/trtllm_gen_custom_ops.py`) over `torch.classes.trtllm.FP8BlockScaleMoERunner` + (`cpp/tensorrt_llm/thop/moe/trtllmGen/fp8BlockScaleMoe.cpp`): trtllm-gen's precompiled + batched-GEMM cubins, nothing JIT-compiled. A warm tuned tactic is bitwise the cold fallback; + the tactic is chosen on the host at capture and fixed in the graph, and the runner's + workspaces come from the graph pool, so a replay is bitwise the eager call. + + Preconditions: `hidden_states` `[T, H]` e4m3, `hidden_states_scale` `[H/128, T]` fp32 (see + the class docstring); plain row-major weights, no shuffle or swizzle: `gemm1_weights` + `[E, 2I, H]` e4m3 rows `[up ; gate]`, `gemm2_weights` `[E, H, I]`, scales `[E, 2I/128, H/128]` + and `[E, H/128, I/128]` fp32; `topk_ids` `[T, top_k]` int32 and `topk_weights` `[T, top_k]` + bf16, used as given. H and I multiples of 128, `top_k <= 32` (the op raises otherwise). Ids + must be distinct per row: a repeated id counts once or twice by the routing kernel the row + count picks. + + Windows: local slot i is global id `local_expert_offset + i`; `num_experts` stays the routing + space (512) and the weights hold the window's experts only; a token with no slot in the + window gets zero. The window's own count passed as `num_experts` silently zeroes or reads out + of bounds; `is_valid` refuses a window starting outside `[0, num_experts)`. + `gemm1_clamp_limit` is live; `routing_method_type`, `n_group` / `topk_group`, + `routed_scaling_factor`, `tune_max_num_tokens` and `use_dp` are inert on the pre-routed path. + The registered fake returns `[T, 2H]` (upstream defect; eager is unaffected). + + Certified on GB300 with near-uniform random routing at the checkpoint's E 512, H 2560, I 640, + top-10, within the test's 16 / 0.75 bf16-ulp gate: the whole stack, 256-wide windows at + offsets 0 / 256 and 128-wide at 0 / 128 / 256 / 384, at the cells' rows (1 to 16384, captured + where the cells say). Not certified: other windows or geometries, rows past 16384 (the target + chunks at 8192 under attention DP and 16384 under TP), captured rows other than the cells', + repeated or out-of-range ids, `gemm1_clamp_limit`, `output=`, skewed routing beyond the + 256-wide logits-routed calls the test holds, multi-rank execution, sm_100. + """ + + def __call__( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> torch.Tensor: + # Outside a tuning pass, a pre-routed (or plain logits-routed) unfused call enters the + # runner directly at the custom op's own choice (`_dispatch`), skipping its dispatch and + # AutoTuner lookup (~0.1 ms of host time a call). + if ( + (routing_logits is None) != (topk_ids is None) + and routing_bias is None + and num_fused_shared_experts is None + and n_group is None + and topk_group is None + and routed_scaling_factor is None + ): + inputs = [ + routing_logits, + None, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + topk_weights, + topk_ids, + ] + geometry = ( + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + act_type, + gemm1_clamp_limit, + tune_max_num_tokens, + use_dp, + ) + chosen = _dispatch(geometry, inputs) + if chosen is not None: + runner, tactic = chosen + result = runner.forward(inputs, tactic=tactic, output=output) + if output is not None: + return torch.empty(0, device=result.device, dtype=result.dtype) + return result + return torch.ops.trtllm.fp8_block_scale_moe_runner( + routing_logits, + routing_bias, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + num_fused_shared_experts, + n_group, + topk_group, + intermediate_size, + local_expert_offset, + local_num_experts, + routed_scaling_factor, + routing_method_type, + topk_weights=topk_weights, + topk_ids=topk_ids, + act_type=act_type, + gemm1_clamp_limit=gemm1_clamp_limit, + output=output, + tune_max_num_tokens=tune_max_num_tokens, + use_dp=use_dp, + ) + + def reference( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> torch.Tensor: + """The layer over bit-identical operands, fp32 GEMMs, bf16 store. Pre-routed only + (`routing_logits` None); the arguments inert on that path are ignored, and the `[T, H]` + result is returned even when `output` is given.""" + assert routing_logits is None and topk_ids is not None and topk_weights is not None, ( + "the reference models the pre-routed path only" + ) + num_tokens, hidden = hidden_states.shape + nb = hidden // BLOCK + act_in = ( + hidden_states.float().reshape(num_tokens, nb, BLOCK) + * hidden_states_scale.t().reshape(num_tokens, nb, 1).float() + ).reshape(num_tokens, hidden) + out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=hidden_states.device) + local = topk_ids.long() - local_expert_offset + live = local[(local >= 0) & (local < local_num_experts)] + with true_fp32_matmul(): + for e in torch.unique(live).tolist(): + tok, slot = (local == e).nonzero(as_tuple=True) + w1 = dequantize_block_weight(gemm1_weights[e], gemm1_weights_scale[e]) + y1 = requantize_1x128(act_in[tok] @ w1.t()) + up, gate = y1[:, :intermediate_size], y1[:, intermediate_size:] + if gemm1_clamp_limit is not None: + gate = gate.clamp(max=gemm1_clamp_limit) + up = up.clamp(-gemm1_clamp_limit, gemm1_clamp_limit) + act = requantize_1x128(torch.nn.functional.silu(gate) * up) + w2 = dequantize_block_weight(gemm2_weights[e], gemm2_weights_scale[e]) + y = act @ w2.t() + out.index_add_(0, tok, y * topk_weights[tok, slot].float().unsqueeze(1)) + return out.to(torch.bfloat16) + + def is_valid( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> None: + """Inputs the op accepts and answers wrongly.""" + # The kernel takes raw data pointers: a strided view is read as dense. + for name, tensor in ( + ("routing_logits", routing_logits), + ("routing_bias", routing_bias), + ("hidden_states", hidden_states), + ("hidden_states_scale", hidden_states_scale), + ("gemm1_weights", gemm1_weights), + ("gemm1_weights_scale", gemm1_weights_scale), + ("gemm2_weights", gemm2_weights), + ("gemm2_weights_scale", gemm2_weights_scale), + ("topk_weights", topk_weights), + ("topk_ids", topk_ids), + ("output", output), + ): + if tensor is not None: + assert tensor.is_contiguous(), ( + f"{name} must be contiguous; a strided view is read as if dense " + "and silently produces wrong results" + ) + # The Python runner overwrites the activation with SwiGlu. + assert act_type == 0, ( + f"act_type={act_type} is silently ignored -- this op always computes " + "SwiGLU. Use a different runner for a non-gated activation" + ) + # Once topk_ids carry the routing, nothing reads the bias. + if routing_bias is not None: + assert topk_ids is None, ( + "routing_bias is ignored when topk_ids/topk_weights are given " + "(routing has already happened); fold it into the router logits" + ) + # `num_experts` is the routing space, not the window: a window starting at or past it + # holds no routable id (silent zeros, or an out-of-bounds read at 16 rows). + assert 0 <= local_expert_offset < num_experts, ( + f"local_expert_offset={local_expert_offset} is outside the routing space " + f"[0, num_experts={num_experts}); num_experts is every routed expert " + "(512 for a 128-expert window of 512), not the window's count" + ) + + +fp8_block_scale_moe_runner = Fp8BlockScaleMoeRunner() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py new file mode 100644 index 000000000000..9057d2fac40f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's MoE front end in one Triton launch: the router's bf16 logits and the 1x128 e4m3 +quantization of the same rows (bitwise `fp8_quantize_1x128`); the runner routes from the logits +itself (`RoutingMethodType::Renormalize`). +""" + +from __future__ import annotations + +import functools + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import ( + Arch, + Cell, + OpWrapper, + true_fp32_matmul, +) +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + +#: Columns per activation scale. +QUANT_BLOCK = 128 +#: Largest finite e4m3 magnitude. +E4M3_MAX = 448.0 +#: `fp8_quantize_1x128`'s amax floor, compared in bf16: bfloat16(1e-10). +AMAX_FLOOR = float(torch.tensor(1e-10, dtype=torch.bfloat16)) +#: Largest row count the entry takes (tep4 MTP-3 verify of 4 requests). +MAX_ROWS = 16 +#: The checkpoint's router: 512 experts over hidden 2560. +EXPERTS = 512 +HIDDEN = 2560 + + +@triton.jit +def _scales(amax): + """`(448 / amax, 1 / (448 / amax))` as IEEE divisions, bitwise the CUDA kernel's.""" + quant = tl.math.div_rn(tl.full(amax.shape, 448.0, tl.float32), amax) + dequant = tl.math.div_rn(tl.full(amax.shape, 1.0, tl.float32), quant) + return quant, dequant + + +@triton.jit +def _moe_front_kernel( + x_ptr, + w_ptr, + logits_ptr, + xq_ptr, + xsf_ptr, + rows, + ROWS_PAD: tl.constexpr, + HIDDEN_: tl.constexpr, + CHUNK: tl.constexpr, + EXPERTS_: tl.constexpr, + QBLOCK: tl.constexpr, + AMAX_MIN: tl.constexpr, + USE_PDL: tl.constexpr, +): + # One expert a program: the hidden width as five CHUNK-wide chunks, the weight row's `[1, CHUNK]` + # tiles broadcast over the rows' `[ROWS_PAD, CHUNK]` tiles, reduced once per row at the end. + tl.static_assert(HIDDEN_ == 5 * CHUNK) + pid = tl.program_id(0) + router_program = pid < EXPERTS_ + offs_k = tl.arange(0, CHUNK)[None, :] + offs_r = tl.arange(0, ROWS_PAD)[:, None] + # Nothing in the step writes the router weight, so its row is in flight before the PDL wait. + wmask = (offs_k < CHUNK) & router_program + w_row = w_ptr + pid * HIDDEN_ + offs_k + w0 = tl.load(w_row, mask=wmask, other=0.0) + w1 = tl.load(w_row + CHUNK, mask=wmask, other=0.0) + w2 = tl.load(w_row + 2 * CHUNK, mask=wmask, other=0.0) + w3 = tl.load(w_row + 3 * CHUNK, mask=wmask, other=0.0) + w4 = tl.load(w_row + 4 * CHUNK, mask=wmask, other=0.0) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + xmask = (offs_r < rows) & router_program + x_rows = x_ptr + offs_r * HIDDEN_ + offs_k + acc = tl.load(x_rows, mask=xmask, other=0.0).to(tl.float32) * w0.to(tl.float32) + acc += tl.load(x_rows + CHUNK, mask=xmask, other=0.0).to(tl.float32) * w1.to(tl.float32) + acc += tl.load(x_rows + 2 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w2.to(tl.float32) + acc += tl.load(x_rows + 3 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w3.to(tl.float32) + acc += tl.load(x_rows + 4 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w4.to(tl.float32) + dot = tl.sum(acc, axis=1) + r = tl.arange(0, ROWS_PAD) + tl.store(logits_ptr + r * EXPERTS_ + pid, dot.to(tl.bfloat16), mask=(r < rows) & router_program) + # One 1x128 block of the activation per program, `fp8_quantize_1x128`'s arithmetic. + nblocks: tl.constexpr = HIDDEN_ // QBLOCK + if pid < rows * nblocks: + row = pid // nblocks + blk = pid - row * nblocks + offs_q = blk * QBLOCK + tl.arange(0, QBLOCK) + xb = tl.load(x_ptr + row * HIDDEN_ + offs_q).to(tl.float32) + # The block amax on every lane (the scales are formed lane-wise, the store is one). + amax = tl.zeros([QBLOCK], tl.float32) + tl.max(tl.abs(xb), axis=0) + quant, dequant = _scales(tl.maximum(amax, AMAX_MIN)) + tl.store(xq_ptr + row * HIDDEN_ + offs_q, (xb * quant).to(tl.float8e4nv)) + lane = tl.arange(0, QBLOCK) + tl.store(xsf_ptr + blk * rows + row + lane * 0, dequant, mask=lane == 0) + + +@functools.cache +def _pdl(device_index: int) -> bool: + return get_env_enable_pdl() and torch.cuda.get_device_capability(device_index)[0] >= 9 + + +#: Warps a program runs with, by padded row count (fastest of 2 / 4 / 8 on GB300). +_NUM_WARPS = {1: 4, 2: 4, 4: 4, 8: 4, 16: 2} + + +def _launch(x, router_rows, logits, xq, xsf, num_warps: int, pdl: bool) -> None: + rows = x.shape[0] + # One program per expert, and at least one per 1x128 block of the activation. + grid = (max(EXPERTS, rows * (HIDDEN // QUANT_BLOCK)),) + _moe_front_kernel[grid]( + x, + router_rows, + logits, + xq, + xsf, + rows, + ROWS_PAD=triton.next_power_of_2(rows), + HIDDEN_=HIDDEN, + CHUNK=HIDDEN // 5, + EXPERTS_=EXPERTS, + QBLOCK=QUANT_BLOCK, + AMAX_MIN=AMAX_FLOOR, + USE_PDL=pdl, + num_warps=num_warps, + **(dict(launch_pdl=True) if pdl else {}), + ) + + +def _router_cells(): + rows = (1, 2, 3, 6, 16) + eager = tuple( + Cell( + why=f"decode MoE front end at {r} rows (tep4 / tp1 LL verify and draft steps)", + spec=dict(rows=r, capture=False), + ) + for r in rows + ) + captured = tuple( + Cell( + why=f"captured decode MoE front end at {r} rows, replayed over two steps", + spec=dict(rows=r, capture=True), + ) + for r in (1, 12) + ) + return eager + captured + + +class MoeFrontRouterQuant(OpWrapper): + """`(logits, xq, xsf) = moe_front_router_quant(x, router_rows)` over a decode step's rows. + + `x` is the MoE site's bf16 `[T, 2560]` mixer output, `router_rows` the router weight as + the checkpoint stores it, `[512, 2560]` bf16 row-major (the target's `router` operand is + its `.t()` view; pass `router.t()`). Returns: + + - `logits` `[T, 512]` bf16: `x @ router_rows.T`, a SIMT dot accumulated in fp32 in the + kernel's own order and rounded once (the router projection's value, not its bits); + - `xq` `[T, 2560]` e4m3 and `xsf` `[20, T]` fp32: bitwise `fp8_quantize_1x128(x, + use_ue8m0=False)`, the scales transposed and compact (the layout the FP8 block-scale runner + reads). + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _router_cells() + + note = """ + Source: one Triton kernel (`_moe_front_kernel`), JIT-compiled per padded row count + (1, 2, 4, 8, 16); the engine's warmup forward compiles it before capture. Stateless; runs on + the ambient stream; PDL per `TRTLLM_ENABLE_PDL` on sm_90+. Preconditions, refused by + `is_valid`: `x` contiguous bf16 `[T, 2560]`, `1 <= T <= MAX_ROWS`; `router_rows` the + contiguous `[512, 2560]` checkpoint layout, not the target's `.t()` view. + """ + + def __call__( + self, x: torch.Tensor, router_rows: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + rows = x.shape[0] + logits = torch.empty([rows, EXPERTS], dtype=torch.bfloat16, device=x.device) + xq = torch.empty([rows, HIDDEN], dtype=torch.float8_e4m3fn, device=x.device) + xsf = torch.empty([HIDDEN // QUANT_BLOCK, rows], dtype=torch.float32, device=x.device) + _launch( + x, + router_rows, + logits, + xq, + xsf, + _NUM_WARPS[triton.next_power_of_2(rows)], + _pdl(x.device.index), + ) + return logits, xq, xsf + + def reference( + self, x: torch.Tensor, router_rows: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """fp32 logits rounded once to bf16; the quantization with exact divisions.""" + with true_fp32_matmul(): + logits = (x.float() @ router_rows.float().t()).to(torch.bfloat16) + t, h = x.shape + blocks = x.float().reshape(t, h // QUANT_BLOCK, QUANT_BLOCK) + amax = blocks.abs().amax(dim=2).clamp(min=AMAX_FLOOR) + quant = E4M3_MAX / amax + xq = (blocks * quant.unsqueeze(2)).clamp(-E4M3_MAX, E4M3_MAX).to(torch.float8_e4m3fn) + return logits, xq.reshape(t, h), (1.0 / quant).t().contiguous() + + def is_valid(self, x: torch.Tensor, router_rows: torch.Tensor) -> None: + assert x.dtype == torch.bfloat16 and router_rows.dtype == torch.bfloat16, ( + x.dtype, + router_rows.dtype, + ) + assert x.is_contiguous() and router_rows.is_contiguous(), ( + "x and router_rows are read as dense row-major; pass the checkpoint's [512, 2560] " + "router (the target's `router.t()`), not its transposed view" + ) + assert x.dim() == 2 and x.shape[1] == HIDDEN and 1 <= x.shape[0] <= MAX_ROWS, tuple(x.shape) + assert tuple(router_rows.shape) == (EXPERTS, HIDDEN), tuple(router_rows.shape) + + +moe_front_router_quant = MoeFrontRouterQuant() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py new file mode 100644 index 000000000000..953053858c71 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py @@ -0,0 +1,723 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A prefill chunk's routed experts over a rank's window, pre-routed: the rows permuted into +expert-major slots padded to 128-row M tiles, FC1 and FC2 as contiguous grouped FP8 block-scale +GEMMs (`blockwise_gemm_grouped`, the checkpoint's 1x128 / 128x128 scales as they are), then each +row's routing-weighted sum over its slots (fp32, a fixed tree). + +No host synchronization, so a call is CUDA-graph capturable: buffers are sized for the worst case +of the call's rows and the GEMMs skip tiles past the live count. Slot ranks depend on scheduling, +the results do not.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +HIDDEN = 2560 +INTER = 640 +FC1_ROWS = 2 * INTER +TOPK = 10 +BLOCK = 128 +#: Rows of a grouped GEMM's M tile; every expert's rows are padded to a multiple of it. +M_TILE = 128 +E4M3_MAX = 448.0 +#: The runner's floor of a 1x128 block's amax before its scale (`fp8_block_scale_moe_runner`). +ACT_AMAX_FLOOR = 1.0e-10 +#: Columns of a finalize chunk (`_finalize_kernel`, one row a program). +FIN_BH = 512 +#: Rows of an activation program (`_act_q_kernel`). +ACT_BM = 16 +#: Counter stride (int32 words) of `_count_kernel`'s atomics: one 128-byte line an expert. +CNT_STRIDE = 32 +#: Whether the finalize keeps two chunks in flight (`_finalize_pipe_kernel`, the same sums). +FIN_PIPE = True + +#: (mma_tiler_mn, cluster_shape_mn) of FC1 / FC2, on 1-CTA instructions. +TACTICS = (((128, 128), (1, 1)), ((128, 128), (1, 1))) +#: FC1's epilogue stores the 1x128 e4m3 round trip (`epi_quant`), not fp32: same bits, 1/4 bytes. +EPI_QUANT = True + + +@triton.jit(do_not_specialize=["rows", "offset", "local"]) +def _count_kernel( + ids_ptr, + cnt_ptr, + rank_ptr, + rows, + offset, + local, + K: tl.constexpr, + KP: tl.constexpr, + RB: tl.constexpr, + CS: tl.constexpr, +): + """rank[r, k] = the next free rank of expert ids[r, k] (atomic), or -1 outside the window.""" + r = tl.program_id(0) * RB + tl.arange(0, RB) + kk = tl.arange(0, KP) + m = (r < rows)[:, None] & (kk < K)[None, :] + e = tl.load(ids_ptr + r[:, None] * K + kk[None, :], mask=m, other=-1) - offset + inw = m & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + pos = tl.atomic_add(cnt_ptr + ec * CS, tl.where(inw, 1, 0), mask=inw, sem="relaxed") + tl.store(rank_ptr + r[:, None] * K + kk[None, :], tl.where(inw, pos, -1), mask=m) + + +@triton.jit(do_not_specialize=["local"]) +def _layout_kernel( + cnt_ptr, + cnt_out_ptr, + roff_ptr, + group_ptr, + ntiles_ptr, + local, + LP: tl.constexpr, + TB: tl.constexpr, + MT: tl.constexpr, + CS: tl.constexpr, +): + """Program e: expert e's rows start at roff[e] (a multiple of MT), its M tiles name it in + `group`; program 0 stores the live tile count; with CS > 1, e's count goes to `cnt_out`.""" + pe = tl.program_id(0) + e = tl.arange(0, LP) + c = tl.load(cnt_ptr + e * CS, mask=e < local, other=0) + if CS > 1: + tl.store(cnt_out_ptr + pe, tl.sum(tl.where(e == pe, c, 0), axis=0)) + t = (c + MT - 1) // MT + tend = tl.cumsum(t, axis=0) + mine = tl.sum(tl.where(e == pe, t, 0), axis=0) + start = tl.sum(tl.where(e == pe, tend, 0), axis=0) - mine + tl.store(roff_ptr + pe, start * MT) + if pe == 0: + tl.store(ntiles_ptr, tl.sum(t, axis=0)) + for j0 in range(0, mine, TB): + j = j0 + tl.arange(0, TB) + tl.store(group_ptr + start + j, tl.full([TB], 0, tl.int32) + pe, mask=j < mine) + + +@triton.jit(do_not_specialize=["xs_kb", "xs_t", "sf_m", "offset", "local"]) +def _permute_kernel( + xq_ptr, + xsf_ptr, + xs_kb, + xs_t, + ids_ptr, + rank_ptr, + roff_ptr, + a_ptr, + sfa_ptr, + sf_m, + offset, + local, + K: tl.constexpr, + HW: tl.constexpr, + HWP: tl.constexpr, + NKB: tl.constexpr, + NKBP: tl.constexpr, +): + """Row t's e4m3 codes and NKB scales to each of its window slots (row roff[e] + rank).""" + t = tl.program_id(0).to(tl.int64) + w = tl.arange(0, HWP) + wm = w < HW + v = tl.load(xq_ptr + t * HW + w, mask=wm, other=0) + kb = tl.arange(0, NKBP) + kbm = kb < NKB + sc = tl.load(xsf_ptr + kb * xs_kb + t * xs_t, mask=kbm, other=0.0) + for k in tl.static_range(K): + e = tl.load(ids_ptr + t * K + k) - offset + if (e >= 0) & (e < local): + dst = (tl.load(roff_ptr + e) + tl.load(rank_ptr + t * K + k)).to(tl.int64) + tl.store(a_ptr + dst * HW + w, v, mask=wm) + tl.store(sfa_ptr + kb.to(tl.int64) * sf_m + dst, sc, mask=kbm) + + +@triton.jit +def _requant(x, AMAX_MIN: tl.constexpr, QMAX: tl.constexpr): + """1x128 e4m3 round trip along axis 1.""" + amax = tl.maximum(tl.max(tl.abs(x), axis=1), AMAX_MIN) + sc = tl.math.div_rn(amax, tl.full(amax.shape, QMAX, tl.float32)) + inv = tl.math.div_rn(tl.full(amax.shape, QMAX, tl.float32), amax) + q = tl.minimum(tl.maximum(x * inv[:, None], -QMAX), QMAX).to(tl.float8e4nv) + return q, sc + + +@triton.jit(do_not_specialize=["sf_m"]) +def _act_kernel( + c1_ptr, + group_ptr, + roff_ptr, + cnt_ptr, + ntiles_ptr, + a2_ptr, + sfa2_ptr, + sf_m, + I_: tl.constexpr, + NB: tl.constexpr, + NBP: tl.constexpr, + QB: tl.constexpr, + BM: tl.constexpr, + MT: tl.constexpr, + AMAX_MIN: tl.constexpr, + QMAX: tl.constexpr, +): + """BM rows of one M tile: fp32 up / gate -> e4m3 round trip -> silu(gate) * up -> e4m3 codes + `a2[r, I_]` and M-major scales `sfa2[b, r]`; rows and tiles past the live ones skipped.""" + r0 = tl.program_id(0) * BM + tile = r0 // MT + if tile < tl.load(ntiles_ptr): + g = tl.load(group_ptr + tile) + lim = tl.load(roff_ptr + g) + tl.load(cnt_ptr + g) + if r0 < lim: + r2 = tl.arange(0, BM * NBP) + r = r0 + r2 // NBP + b = r2 % NBP + ok = (r < lim) & (b < NB) + row = r.to(tl.int64) + cols = tl.arange(0, QB) + off = row[:, None] * (2 * I_) + b[:, None] * QB + cols[None, :] + up = tl.load(c1_ptr + off, mask=ok[:, None], other=0.0) + gate = tl.load(c1_ptr + off + I_, mask=ok[:, None], other=0.0) + uq, us = _requant(up, AMAX_MIN, QMAX) + gq, gs = _requant(gate, AMAX_MIN, QMAX) + u = uq.to(tl.float32) * us[:, None] + gv = gq.to(tl.float32) * gs[:, None] + h = gv * tl.sigmoid(gv) * u + hq, hs = _requant(h, AMAX_MIN, QMAX) + tl.store( + a2_ptr + row[:, None] * I_ + b[:, None] * QB + cols[None, :], + hq.to(tl.uint8, bitcast=True), + mask=ok[:, None], + ) + tl.store(sfa2_ptr + b.to(tl.int64) * sf_m + row, hs, mask=ok) + + +@triton.jit(do_not_specialize=["sf_m"]) +def _act_q_kernel( + q1_ptr, + s1_ptr, + group_ptr, + roff_ptr, + cnt_ptr, + ntiles_ptr, + a2_ptr, + sfa2_ptr, + sf_m, + I_: tl.constexpr, + NB: tl.constexpr, + QB: tl.constexpr, + BM: tl.constexpr, + MT: tl.constexpr, + AMAX_MIN: tl.constexpr, + QMAX: tl.constexpr, +): + """`_act_kernel` over FC1's stored codes `q1[r, 2 I_]` and M-major scales `s1[b, r]`.""" + r0 = tl.program_id(0) * BM + tile = r0 // MT + if tile < tl.load(ntiles_ptr): + g = tl.load(group_ptr + tile) + lim = tl.load(roff_ptr + g) + tl.load(cnt_ptr + g) + if r0 < lim: + r = r0 + tl.arange(0, BM) + ok = r < lim + row = r.to(tl.int64) + cols = tl.arange(0, QB) + for b in tl.static_range(NB): + off = row[:, None] * (2 * I_) + b * QB + cols[None, :] + uq = tl.load(q1_ptr + off, mask=ok[:, None], other=0).to( + tl.float8e4nv, bitcast=True + ) + gq = tl.load(q1_ptr + off + I_, mask=ok[:, None], other=0).to( + tl.float8e4nv, bitcast=True + ) + us = tl.load(s1_ptr + b * sf_m + row, mask=ok, other=0.0) + gs = tl.load(s1_ptr + (b + NB) * sf_m + row, mask=ok, other=0.0) + u = uq.to(tl.float32) * us[:, None] + gv = gq.to(tl.float32) * gs[:, None] + h = gv * tl.sigmoid(gv) * u + hq, hs = _requant(h, AMAX_MIN, QMAX) + tl.store( + a2_ptr + row[:, None] * I_ + b * QB + cols[None, :], + hq.to(tl.uint8, bitcast=True), + mask=ok[:, None], + ) + tl.store(sfa2_ptr + b * sf_m + row, hs, mask=ok) + + +@triton.jit(do_not_specialize=["offset", "local", "row0"]) +def _finalize_kernel( + c2_ptr, + ids_ptr, + w_ptr, + rank_ptr, + roff_ptr, + out_ptr, + offset, + local, + row0, + H: tl.constexpr, + K: tl.constexpr, + KP: tl.constexpr, + BH: tl.constexpr, + NCH: tl.constexpr, +): + """Row t = row0 + program: its window slots' FC2 rows times their routing weights, summed in + fp32 (a fixed tree), stored bf16 (zero with no slot in the window). A row reads only its own + slots, so a launch over any row range writes the whole launch's bits there.""" + t = row0 + tl.program_id(0).to(tl.int64) + kk = tl.arange(0, KP) + km = kk < K + e = tl.load(ids_ptr + t * K + kk, mask=km, other=-1) - offset + inw = km & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + rows = ( + tl.load(roff_ptr + ec, mask=inw, other=0) + + tl.load(rank_ptr + t * K + kk, mask=inw, other=0) + ).to(tl.int64) + w = tl.where(inw, tl.load(w_ptr + t * K + kk, mask=km, other=0.0).to(tl.float32), 0.0) + for ch in tl.static_range(NCH): + cols = ch * BH + tl.arange(0, BH) + cm = cols < H + v = tl.load( + c2_ptr + rows[:, None] * H + cols[None, :], mask=inw[:, None] & cm[None, :], other=0.0 + ) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + cols, acc.to(tl.bfloat16), mask=cm) + + +@triton.jit(do_not_specialize=["offset", "local", "row0"]) +def _finalize_pipe_kernel( + c2_ptr, + ids_ptr, + w_ptr, + rank_ptr, + roff_ptr, + out_ptr, + offset, + local, + row0, + H: tl.constexpr, + K: tl.constexpr, + KP: tl.constexpr, + BH: tl.constexpr, + NCH: tl.constexpr, +): + """`_finalize_kernel` with chunk c + 1's loads issued before chunk c's store; the same sums.""" + t = row0 + tl.program_id(0).to(tl.int64) + kk = tl.arange(0, KP) + km = kk < K + e = tl.load(ids_ptr + t * K + kk, mask=km, other=-1) - offset + inw = km & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + rows = ( + tl.load(roff_ptr + ec, mask=inw, other=0) + + tl.load(rank_ptr + t * K + kk, mask=inw, other=0) + ).to(tl.int64) + w = tl.where(inw, tl.load(w_ptr + t * K + kk, mask=km, other=0.0).to(tl.float32), 0.0) + cols = tl.arange(0, BH) + v = tl.load( + c2_ptr + rows[:, None] * H + cols[None, :], + mask=inw[:, None] & (cols < H)[None, :], + other=0.0, + ) + for ch in tl.static_range(1, NCH): + nc = ch * BH + tl.arange(0, BH) + vn = tl.load( + c2_ptr + rows[:, None] * H + nc[None, :], + mask=inw[:, None] & (nc < H)[None, :], + other=0.0, + ) + pc = (ch - 1) * BH + tl.arange(0, BH) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + pc, acc.to(tl.bfloat16), mask=pc < H) + v = vn + lc = (NCH - 1) * BH + tl.arange(0, BH) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + lc, acc.to(tl.bfloat16), mask=lc < H) + + +_GEMMS: dict = {} + + +def _grouped_gemm(a, w, sfa, sfw, c, group, ntiles, tiles, sf_m, tactic): + """c[r, n] = sum_k a[r, k] w[group(r // M_TILE), n, k] with a's 1x128 (M-major `sfa[kb, r]`) + and w's 128x128 fp32 scales, over `tiles` M tiles of which the first `ntiles[0]` are live.""" + import cutlass + import cutlass.cute as cute + from cutlass.cute.runtime import from_dlpack, make_ptr + + from .blockwise_gemm_grouped import Sm100BlockwiseGemmKernel + + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + tiler, cluster = tactic + c3 = c.view(tiles, M_TILE, n).permute(1, 2, 0) + key = (c.dtype, tactic) + fn = _GEMMS.get(key) + if fn is None: + gemm = Sm100BlockwiseGemmKernel(mma_tiler_mn=tiler, cluster_shape_mn=cluster) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + + def ptr(t, dt, align=16): + return make_ptr(dt, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + fn = cute.compile( + gemm.wrapper_grouped, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + ptr(a, cutlass.Float8E4M3FN), + ptr(w, cutlass.Float8E4M3FN), + ptr(sfa, cutlass.Float32), + ptr(sfw, cutlass.Float32), + from_dlpack(c3).mark_layout_dynamic(leading_dim=1), + ptr(group, cutlass.Int32, 4), + ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + c3, + group.data_ptr(), + ntiles.data_ptr(), + ) + + +def _grouped_gemm_q(a, w, sfa, sfw, cq, cs, group, ntiles, tiles, sf_m, tactic): + """`_grouped_gemm` with the 1x128 e4m3 round trip in the epilogue (`epi_quant`): `cq[r, n]` + e4m3 codes, `cs[n // 128, r]` fp32 scales (row-major `[n / 128, sf_m]`).""" + import cutlass + import cutlass.cute as cute + from cutlass.cute.runtime import from_dlpack, make_ptr + + from .blockwise_gemm_grouped import Sm100BlockwiseGemmKernel + + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + key = ("q", tactic) + fn = _GEMMS.get(key) + if fn is None: + tiler, cluster = tactic + gemm = Sm100BlockwiseGemmKernel( + mma_tiler_mn=tiler, cluster_shape_mn=cluster, epi_quant=True + ) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + + def ptr(t, dt, align=16): + return make_ptr(dt, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + fn = cute.compile( + gemm.wrapper_grouped_quant, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + ptr(a, cutlass.Float8E4M3FN), + ptr(w, cutlass.Float8E4M3FN), + ptr(sfa, cutlass.Float32), + ptr(sfw, cutlass.Float32), + ptr(cq, cutlass.Float8E4M3FN), + from_dlpack(cs).mark_layout_dynamic(leading_dim=1), + ptr(group, cutlass.Int32, 4), + ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + cq.data_ptr(), + cs, + group.data_ptr(), + ntiles.data_ptr(), + ) + + +def tile_bound(rows: int, local: int, topk: int = TOPK) -> int: + """M tiles that hold any routing of `rows` rows over a `local`-expert window.""" + return -(-rows * min(topk, local) // M_TILE) + local + + +def _plan(xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics=None, fin_first=None): + """One call's buffers and its launches in order as (name, launch) pairs, the last writing `out`; + `fin_first = (r0, r1)` finalizes rows `[r0, r1)` before the rest (the same bits).""" + t1, t2 = tactics or TACTICS + rows = xq.shape[0] + local = fc1.shape[0] + dev = xq.device + assert xq.shape[1] == HIDDEN and xq.is_contiguous() and ids.shape == (rows, TOPK) + ids = ids.contiguous() + wts = wts.contiguous() + out = torch.empty(rows, HIDDEN, dtype=torch.bfloat16, device=dev) + if rows == 0: + return out, [] + tiles = tile_bound(rows, local) + m = tiles * M_TILE + cs = CNT_STRIDE + cntp = torch.empty(local * cs, dtype=torch.int32, device=dev) + cnt = cntp if cs == 1 else torch.empty(local, dtype=torch.int32, device=dev) + rank = torch.empty(rows, TOPK, dtype=torch.int32, device=dev) + roff = torch.empty(local, dtype=torch.int32, device=dev) + group = torch.empty(tiles, dtype=torch.int32, device=dev) + ntiles = torch.empty(1, dtype=torch.int32, device=dev) + # A (later the FC2 operand) and C (FC1's codes or fp32 accumulator, later FC2's bf16) are + # reused in place: each is dead by the time its second use writes it (one stream). + a = torch.empty(m * HIDDEN, dtype=torch.uint8, device=dev) + sfa = torch.empty((HIDDEN // BLOCK) * m, dtype=torch.float32, device=dev) + c = torch.empty(m * FC1_ROWS, dtype=torch.float32, device=dev) + a1 = a.view(torch.float8_e4m3fn).view(m, HIDDEN) + a2 = a[: m * INTER].view(m, INTER) + sfa2 = sfa[: (INTER // BLOCK) * m].view(INTER // BLOCK, m) + c2 = c.view(torch.bfloat16)[: m * HIDDEN].view(m, HIDDEN) + epiq = EPI_QUANT and t1[0][1] == BLOCK + q1 = c.view(torch.uint8)[: m * FC1_ROWS].view(m, FC1_ROWS) + s1q = torch.empty(FC1_ROWS // BLOCK, m, dtype=torch.float32, device=dev) if epiq else None + c1 = c.view(m, FC1_ROWS) + rb = 16 + bm = 4 + + def count(): + cntp.zero_() + _count_kernel[(triton.cdiv(rows, rb),)]( + ids, cntp, rank, rows, offset, local, K=TOPK, KP=16, RB=rb, CS=cs, num_warps=4 + ) + + def layout(): + _layout_kernel[(local,)]( + cntp, + cnt, + roff, + group, + ntiles, + local, + LP=triton.next_power_of_2(local), + TB=128, + MT=M_TILE, + CS=cs, + num_warps=4, + ) + + def permute(): + _permute_kernel[(rows,)]( + xq.view(torch.int32), + xsf, + xsf.stride(0), + xsf.stride(1), + ids, + rank, + roff, + a.view(torch.int32), + sfa, + m, + offset, + local, + K=TOPK, + HW=HIDDEN // 4, + HWP=triton.next_power_of_2(HIDDEN // 4), + NKB=HIDDEN // BLOCK, + NKBP=32, + num_warps=4, + ) + + def fc1g(): + if epiq: + _grouped_gemm_q( + a1, + fc1, + sfa.view(HIDDEN // BLOCK, m), + fc1_scale, + q1, + s1q, + group, + ntiles, + tiles, + m, + t1, + ) + else: + _grouped_gemm( + a1, fc1, sfa.view(HIDDEN // BLOCK, m), fc1_scale, c1, group, ntiles, tiles, m, t1 + ) + + def act(): + if epiq: + _act_q_kernel[(m // ACT_BM,)]( + q1, + s1q, + group, + roff, + cnt, + ntiles, + a2, + sfa2, + m, + I_=INTER, + NB=INTER // BLOCK, + QB=BLOCK, + BM=ACT_BM, + MT=M_TILE, + AMAX_MIN=ACT_AMAX_FLOOR, + QMAX=E4M3_MAX, + num_warps=4, + ) + else: + _act_kernel[(m // bm,)]( + c1, + group, + roff, + cnt, + ntiles, + a2, + sfa2, + m, + I_=INTER, + NB=INTER // BLOCK, + NBP=8, + QB=BLOCK, + BM=bm, + MT=M_TILE, + AMAX_MIN=ACT_AMAX_FLOOR, + QMAX=E4M3_MAX, + num_warps=4, + ) + + def fc2g(): + _grouped_gemm( + a2.view(torch.float8_e4m3fn), fc2, sfa2, fc2_scale, c2, group, ntiles, tiles, m, t2 + ) + + fin = _finalize_pipe_kernel if FIN_PIPE else _finalize_kernel + + def finalize_rows(r0, r1): + if r1 <= r0: + return + fin[(r1 - r0,)]( + c2, + ids, + wts, + rank, + roff, + out, + offset, + local, + r0, + H=HIDDEN, + K=TOPK, + KP=16, + BH=FIN_BH, + NCH=triton.cdiv(HIDDEN, FIN_BH), + num_warps=4, + ) + + stages = [ + ("count", count), + ("layout", layout), + ("permute", permute), + ("fc1", fc1g), + ("act", act), + ("fc2", fc2g), + ] + if fin_first is None: + return out, stages + [("finalize", lambda: finalize_rows(0, rows))] + r0, r1 = fin_first + assert 0 <= r0 <= r1 <= rows and (r0 == 0 or r1 == rows), (fin_first, rows) + rest = (r1, rows) if r0 == 0 else (0, r0) + return out, stages + [ + ("finalize_first", lambda: finalize_rows(r0, r1)), + ("finalize_rest", lambda: finalize_rows(*rest)), + ] + + +def moe_prefill_grouped( + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, +) -> torch.Tensor: + """This window's partial of the routed experts, `[rows, HIDDEN]` bf16: `xq` `[rows, HIDDEN]` + e4m3 (contiguous), `xsf` `[HIDDEN / 128, rows]` fp32, `ids` / `wts` `[rows, TOPK]` over the + whole routing space (ids outside `[offset, offset + local)` add nothing), the window's `fc1` + `[local, 2I, H]` / `fc2` `[local, H, I]` e4m3 with fp32 128x128 scales. `fin_first = (r0, r1)` + (rows at either end) finalizes those rows, calls `after_first(out)`, then finalizes the rest.""" + out, stages = _plan( + xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics, fin_first + ) + for name, launch in stages: + launch() + if name == "finalize_first" and after_first is not None: + after_first(out) + return out + + +_READY: set = set() + + +def prefill_grouped_ready(local: int) -> bool: + """Whether `warm_prefill_grouped` compiled this window's kernels (a call can be captured).""" + return (TACTICS, local, EPI_QUANT) in _READY + + +def warm_prefill_grouped(device, local: int) -> None: + """Compile every kernel of `moe_prefill_grouped` for a `local`-expert window with one eager + call whose rows all go to expert 0 (the GEMMs' shapes are dynamic). Idempotent.""" + if (TACTICS, local, EPI_QUANT) in _READY: + return + dev = torch.device("cuda", device) if isinstance(device, int) else torch.device(device) + rows = 2 * M_TILE + fc1 = torch.zeros(1, FC1_ROWS, HIDDEN, dtype=torch.float8_e4m3fn, device=dev).expand( + local, -1, -1 + ) + fc2 = torch.zeros(1, HIDDEN, INTER, dtype=torch.float8_e4m3fn, device=dev).expand(local, -1, -1) + s1 = torch.ones(1, FC1_ROWS // BLOCK, HIDDEN // BLOCK, dtype=torch.float32, device=dev).expand( + local, -1, -1 + ) + s2 = torch.ones(1, HIDDEN // BLOCK, INTER // BLOCK, dtype=torch.float32, device=dev).expand( + local, -1, -1 + ) + xq = torch.zeros(rows, HIDDEN, dtype=torch.float8_e4m3fn, device=dev) + xsf = torch.ones(HIDDEN // BLOCK, rows, dtype=torch.float32, device=dev) + ids = torch.zeros(rows, TOPK, dtype=torch.int32, device=dev) + ids[:, 1:] = -1 + wts = torch.ones(rows, TOPK, dtype=torch.bfloat16, device=dev) + moe_prefill_grouped(xq, xsf, ids, wts, fc1, s1, fc2, s2, 0) + torch.cuda.synchronize(dev) + _READY.add((TACTICS, local, EPI_QUANT)) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py new file mode 100644 index 000000000000..664cb25cefc8 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's routed experts over 128-row intermediate blocks, in one launch of 4- or 8-CTA +clusters (`native_ops.fp8_block_moe_tpb`, CUDA C++ in this target's `csrc/fp8BlockMoeTpb.cu`). + +An expert's intermediate (640) is five 128-row blocks and its activation needs nothing outside a +block, so the routed experts are dealt by blocks over the four tep4 ranks: expert e's block b lives +on rank (e + b) % 4, and each rank returns its partial for the TP sum after the layer. A cluster +runs a block's FC1 -> activation -> FC2, sharing the FC1 outputs through distributed shared +memory; partials are summed in cluster order (deterministic). + +Operands (`BlockExperts`), block-major, a rank's 640 blocks: +`w1 [640, 256, 2560]` e4m3 (rows: the block's 128 up rows, then its 128 gate rows), `s1 [640, 2, 20]` +fp32, `w2 [640, 2560, 128]` e4m3 (the block's 128 down_proj columns), `s2 [640, 20]` fp32. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( + renorm_moe_routing_op, +) +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl + +from ... import native_ops +from .fp8_block_scale_moe_runner import dequantize_block_weight, requantize_1x128 + +EXPERTS = 512 +TOP_K = 10 +HIDDEN = 2560 +INTER = 640 +#: 128-row intermediate blocks an expert has. +EXPERT_BLOCKS = INTER // 128 +#: Blocks a rank holds (tep4 round robin: 512 * 5 / 4). +BLOCKS = 640 +RANKS = 4 +MAX_ROWS = 8 + + +class BlockExperts(NamedTuple): + """One MoE layer's routed experts at this rank, block-major (see the module docstring).""" + + w1: torch.Tensor + s1: torch.Tensor + w2: torch.Tensor + s2: torch.Tensor + rank: int + + +#: The shared expert's intermediate, and the work items its TP slice is folded in as (`SharedBlocks`). +SHARED_INTER = 640 +SHARED_ITEMS = 5 +SHARED_ITEM_COLS = 32 + + +class SharedBlocks(NamedTuple): + """This rank's TP4 slice of the bf16 shared expert, folded into the launch (3..8 rows): `x` the + MoE site's `[T, 2560]` bf16 rows, `gu` `[5, 64, 2560]` the slice's gate / up rows (item sb = columns + `[160 rank + 32 sb, +32)` as 8 groups of 4 columns, each `[gate 4 ; up 4]`), `gl` the `[2560]` gate-logit row, + `dn` `[5, 2560, 32]` the slice's down_proj columns (`shared_blocks`).""" + + x: torch.Tensor + gu: torch.Tensor + gl: torch.Tensor + dn: torch.Tensor + + +def shared_blocks(gate, up, dn, rank: int): + """The checkpoint's shared-expert gate / up rows `[640, 2560]` and down `[2560, 640]` (bf16) -> this rank's + `(gu [5, 64, 2560], dn [5, 2560, 32])`.""" + w = SHARED_INTER // RANKS + lo = rank * w + g = gate[lo : lo + w].reshape(SHARED_ITEMS, 8, 4, -1) + u = up[lo : lo + w].reshape(SHARED_ITEMS, 8, 4, -1) + gu = torch.stack([g, u], dim=2).reshape(SHARED_ITEMS, 64, -1).contiguous() + d = ( + dn[:, lo : lo + w] + .reshape(dn.shape[0], SHARED_ITEMS, SHARED_ITEM_COLS) + .permute(1, 0, 2) + .contiguous() + ) + return gu, d + + +def block_ids(rank: int) -> tuple[list[int], list[int]]: + """(expert, block) of each of a rank's 640 blocks: j < 512 is expert j's block (rank - j) % 4, + j >= 512 is expert 4 (j - 512) + rank's block 4.""" + experts = [j if j < EXPERTS else 4 * (j - EXPERTS) + rank for j in range(BLOCKS)] + blocks = [(rank - j) % RANKS if j < EXPERTS else EXPERT_BLOCKS - 1 for j in range(BLOCKS)] + return experts, blocks + + +def block_major(fc1_w, fc1_s, fc2_w, fc2_s, rank: int) -> BlockExperts: + """Whole-expert operands of all 512 experts indexed by routing id (`[512, 1280, 2560]` rows + `[up ; gate]`, `[512, 10, 20]`, `[512, 2560, 640]`, `[512, 20, 5]`) -> the rank's block-major + operands.""" + experts, blocks = block_ids(rank) + e = torch.tensor(experts, device=fc1_w.device) + b = torch.tensor(blocks, device=fc1_w.device) + n = fc1_w.shape[0] + w1 = fc1_w.view(n, 2, EXPERT_BLOCKS, 128, HIDDEN)[e, :, b].reshape(BLOCKS, 256, HIDDEN) + s1 = fc1_s.view(n, 2, EXPERT_BLOCKS, HIDDEN // 128)[e, :, b].reshape(BLOCKS, 2, HIDDEN // 128) + w2 = fc2_w.view(n, HIDDEN, EXPERT_BLOCKS, 128)[e, :, b] + s2 = fc2_s[e, :, b] + return BlockExperts(w1.contiguous(), s1.contiguous(), w2.contiguous(), s2.contiguous(), rank) + + +_WORKSPACE: dict = {} + + +def _workspace(device: torch.device) -> torch.Tensor: + index = device.index if device.index is not None else torch.cuda.current_device() + ws = _WORKSPACE.get(index) + if ws is None: + nbytes = native_ops.fp8_block_moe_tpb_workspace_bytes() + ws = torch.zeros(nbytes, dtype=torch.uint8, device=torch.device("cuda", index)) + _WORKSPACE[index] = ws + return ws + + +@functools.cache +def _pdl(device_index: int) -> bool: + return get_env_enable_pdl() and torch.cuda.get_device_capability(device_index)[0] >= 9 + + +def _cells(): + eager = tuple( + Cell( + why=f"decode routed experts at {r} rows, round-robin blocks of tep4 rank {rank}", + spec=dict(rows=r, rank=rank, capture=False), + ) + for r, rank in ((1, 0), (8, 3)) + ) + captured = tuple( + Cell( + why=f"captured decode routed experts at {r} rows, round-robin blocks, replayed twice", + spec=dict(rows=r, rank=2, capture=True), + ) + for r in (3,) + ) + return eager + captured + + +class MoeTpbDecode(OpWrapper): + """`partial = moe_tpb_decode(logits, xq, xsf, blocks, shared=None)`. + + `logits` `[T, 512]` bf16 (the front end's), `xq` `[T, 2560]` e4m3 and `xsf` `[20, >= T]` fp32 its + 1x128 scales transposed (`moe_front_router_quant`'s), `blocks` this rank's `BlockExperts`, `shared` + this rank's slice of the shared expert to fold in (`SharedBlocks`, 3..8 rows). Returns the + `[T, 2560]` bf16 partial of this rank's blocks: the ranks' partials sum to the layer's routed + experts (plus `sigmoid(gate logit) * shared` with `shared`).""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: `torch_ext::fp8BlockMoeTpb` (`csrc/fp8BlockMoeTpbOp.cpp`, kernel in + `csrc/fp8BlockMoeTpb.cu`). The clusters' partials meet through counters in the op's + per-device workspace, so calls on one device must not overlap in time. PDL per + `TRTLLM_ENABLE_PDL` on sm_90+. + """ + + def __call__(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + extra = {} + if shared is not None: + extra.update(sh_x=shared.x, sh_gu=shared.gu, sh_gl=shared.gl, sh_dn=shared.dn) + return native_ops.fp8_block_moe_tpb( + logits, + xq, + xsf, + blocks.w1, + blocks.s1, + blocks.w2, + blocks.s2, + int(blocks.rank), + _workspace(logits.device), + _pdl(logits.device.index), + **extra, + ) + + def reference(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + """The runner reference's arithmetic over this rank's blocks of `renorm_moe_routing_op`'s + routing, fp32, rounded once; with `shared`, plus `sigmoid(gate logit) * slice`.""" + routed = self._routed(logits, xq, xsf, blocks) + if shared is None: + return routed.to(torch.bfloat16) + scale, part = shared_reference(shared) + return (routed + scale * part).to(torch.bfloat16) + + @staticmethod + def _routed(logits, xq, xsf, blocks: BlockExperts): + ids, w = renorm_moe_routing_op(logits, topk=TOP_K, output_dtype=torch.bfloat16) + T = xq.shape[0] + nb = HIDDEN // 128 + act_in = ( + xq.float().reshape(T, nb, 128) * xsf[:, :T].t().reshape(T, nb, 1).float() + ).reshape(T, HIDDEN) + out = torch.zeros(T, HIDDEN, dtype=torch.float32, device=xq.device) + experts, _ = block_ids(blocks.rank) + by_expert: dict[int, list[int]] = {} + for j, e in enumerate(experts): + by_expert.setdefault(e, []).append(j) + for e in torch.unique(ids).tolist(): + tok, slot = (ids.long() == e).nonzero(as_tuple=True) + for j in by_expert.get(e, []): + w1 = dequantize_block_weight(blocks.w1[j], blocks.s1[j]) + y1 = requantize_1x128(act_in[tok] @ w1.t()) + act = requantize_1x128(torch.nn.functional.silu(y1[:, 128:]) * y1[:, :128]) + w2 = dequantize_block_weight(blocks.w2[j], blocks.s2[j].view(-1, 1)) + out.index_add_(0, tok, (act @ w2.t()) * w[tok, slot].float().unsqueeze(1)) + return out + + def is_valid(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + rows = logits.shape[0] + assert logits.dtype == torch.bfloat16 and logits.is_contiguous(), logits.dtype + assert tuple(logits.shape[1:]) == (EXPERTS,) and 1 <= rows <= MAX_ROWS, tuple(logits.shape) + assert xq.dtype == torch.float8_e4m3fn and tuple(xq.shape) == (rows, HIDDEN) + assert xq.is_contiguous() + assert xsf.dtype == torch.float32 and xsf.shape[0] == HIDDEN // 128 and xsf.shape[1] >= rows + assert xsf.stride(1) == 1, "xsf is read as [20, stride] with unit column stride" + assert tuple(blocks.w1.shape) == (BLOCKS, 256, HIDDEN) and blocks.w1.is_contiguous() + assert tuple(blocks.s1.shape) == (BLOCKS, 2, HIDDEN // 128) and blocks.s1.is_contiguous() + assert tuple(blocks.w2.shape) == (BLOCKS, HIDDEN, 128) and blocks.w2.is_contiguous() + assert tuple(blocks.s2.shape) == (BLOCKS, HIDDEN // 128) and blocks.s2.is_contiguous() + assert 0 <= blocks.rank < RANKS, blocks.rank + if shared is not None: + assert 3 <= rows <= MAX_ROWS, rows + assert shared.x.dtype == torch.bfloat16 and tuple(shared.x.shape) == (rows, HIDDEN) + assert shared.x.is_contiguous() + assert shared.gu.dtype == torch.bfloat16 and tuple(shared.gu.shape) == ( + SHARED_ITEMS, + 64, + HIDDEN, + ) + assert shared.gl.dtype == torch.bfloat16 and shared.gl.numel() == HIDDEN + assert shared.dn.dtype == torch.bfloat16 and tuple(shared.dn.shape) == ( + SHARED_ITEMS, + HIDDEN, + SHARED_ITEM_COLS, + ) + assert ( + shared.gu.is_contiguous() + and shared.gl.is_contiguous() + and shared.dn.is_contiguous() + ) + + +def shared_reference(shared: SharedBlocks) -> tuple[torch.Tensor, torch.Tensor]: + """`(sigmoid(gate logit) [T, 1], slice [T, 2560])` in fp32 with the unfused path's bf16 roundings + (the gate|up GEMM's outputs, `silu_and_mul_rows`' activation); the slice summed over the ranks + is the shared expert's output before its bf16 rounding.""" + x = shared.x.float() + gu = shared.gu.float().reshape(SHARED_ITEMS, 8, 2, 4, HIDDEN) + gate = gu[:, :, 0].reshape(-1, HIDDEN) + up = gu[:, :, 1].reshape(-1, HIDDEN) + g = (x @ gate.t()).to(torch.bfloat16).float() + u = (x @ up.t()).to(torch.bfloat16).float() + logit = (x @ shared.gl.float().reshape(HIDDEN, 1)).to(torch.bfloat16).float() + act = (torch.nn.functional.silu(g) * u).to(torch.bfloat16).float() + dn = shared.dn.float().permute(1, 0, 2).reshape(HIDDEN, -1) + return torch.sigmoid(logit), act @ dn.t() + + +moe_tpb_decode = MoeTpbDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py new file mode 100644 index 000000000000..19893a7f2c02 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The draft layer's MoE combination and hyper-connection injection in one Triton launch. + +Bitwise the composed `fused_sigmoid_gate_mul_add` + `qwen4_exp_post_inject` path: one fp32 +`fma.rn` rounded once to bf16 for the combination, and an exact bf16 x bf16 product for the +injection, rounded once on the store as `addcmul` does. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Columns a program forms. +BLOCK = 512 + + +# The gate logit is one scalar a row: its row stride and address carry no vector width. +@triton.jit(do_not_specialize=["s_gate"], do_not_specialize_on_alignment=["gate"]) +def _combine_inject_kernel( + x, + routed, + shared, + gate, + inject, + out, + s_x, + s_routed, + s_shared, + s_gate, + s_inject, + s_out, + H: tl.constexpr, + HC: tl.constexpr, + BLOCK_: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.program_id(1) * BLOCK_ + tl.arange(0, BLOCK_) + mask = cols < H + s = tl.sigmoid(tl.load(gate + row * s_gate).to(tl.float32)) + f = tl.load(routed + row * s_routed + cols, mask=mask, other=0.0).to(tl.float32) + sh = tl.load(shared + row * s_shared + cols, mask=mask, other=0.0).to(tl.float32) + b = tl.fma(s, sh, f).to(routed.dtype.element_ty).to(tl.float32) + for c in tl.static_range(HC): + w = tl.load(inject + row * s_inject + c).to(tl.float32) + xv = tl.load(x + row * s_x + c * H + cols, mask=mask, other=0.0).to(tl.float32) + tl.store(out + row * s_out + c * H + cols, (xv + b * w).to(out.dtype.element_ty), mask=mask) + + +def _ckpt(rows: int, gate: str = "strided") -> dict: + return dict(rows=rows, hc_count=4, hidden=2560, gate=gate) + + +class MtpMoeCombineInject(OpWrapper): + """`out = mtp_moe_combine_inject(x, routed, gate_logits, shared, inject)`: + `out[t, s * H + i] = x[t, s * H + i] + bf16(routed[t, i] + sigmoid(gate_logits[t]) * + shared[t, i]) * inject[t, s]`, a fresh `[T, hc_count * H]` tensor. + + - `x` `[T, hc_count * H]` bf16, unit column stride (the pre-norm residual, stream-major); + - `routed`, `shared` `[T, H]` bf16, unit column stride; + - `gate_logits` `[T, 1]` / `[T]` (any row stride: the column the shared expert's fused GEMM + leaves); `inject` `[T, hc_count]` bf16, unit column stride. + + No input is written. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep4 draft step 1: 1 row", spec=_ckpt(1)), + Cell(why="batch-4 draft step 0: 12 rows", spec=_ckpt(12)), + Cell(why="contiguous gate vector, 4 rows", spec=_ckpt(4, "contiguous")), + ) + + note = """ + Source: one Triton kernel (`_combine_inject_kernel`), grid `(T, ceil(H / 512))`, JIT-compiled + per (H, hc_count, dtypes); the engine's warmup forward compiles it before capture. + """ + + def __call__(self, x, routed, gate_logits, shared, inject) -> torch.Tensor: + rows, width = x.shape + hidden = routed.shape[1] + hc = inject.shape[1] + out = torch.empty([rows, width], dtype=x.dtype, device=x.device) + if rows: + gate = gate_logits.reshape(rows, -1) if gate_logits.dim() == 1 else gate_logits + _combine_inject_kernel[(rows, triton.cdiv(hidden, BLOCK))]( + x, + routed, + shared, + gate, + inject, + out, + x.stride(0), + routed.stride(0), + shared.stride(0), + gate.stride(0), + inject.stride(0), + out.stride(0), + H=hidden, + HC=hc, + BLOCK_=BLOCK, + num_warps=4, + ) + return out + + def reference(self, x, routed, gate_logits, shared, inject) -> torch.Tensor: + """fp32 combination rounded once to bf16, then the fp32 injection rounded once.""" + rows = x.shape[0] + g = torch.sigmoid(gate_logits.reshape(rows, 1).float()) + b = (routed.float() + g * shared.float()).to(routed.dtype).float() + streams = x.float().unflatten(-1, (inject.shape[1], routed.shape[1])) + return (streams + b.unsqueeze(-2) * inject.float().unsqueeze(-1)).flatten(-2).to(x.dtype) + + def is_valid(self, x, routed, gate_logits, shared, inject) -> None: + rows = x.shape[0] + assert x.dim() == 2 and routed.dim() == 2 and shared.dim() == 2 and inject.dim() == 2 + assert x.dtype == routed.dtype == shared.dtype == inject.dtype == torch.bfloat16 + hidden, hc = routed.shape[1], inject.shape[1] + assert x.shape[1] == hc * hidden, (tuple(x.shape), hc, hidden) + assert routed.shape[0] == shared.shape[0] == inject.shape[0] == rows + assert tuple(shared.shape) == (rows, hidden) + for t in (x, routed, shared, inject): + assert t.stride(1) == 1 or t.shape[1] == 1, (tuple(t.shape), t.stride()) + assert gate_logits.numel() == rows and (gate_logits.dim() == 1 or gate_logits.shape[1] == 1) + assert x.device == routed.device == shared.device == inject.device == gate_logits.device + + +mtp_moe_combine_inject = MtpMoeCombineInject() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py new file mode 100644 index 000000000000..304eb33eade5 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A verify step's multi-token causal conv1d written token-major (`[n * p, C]`, the layout the GDN +cached replay reads): bitwise the family `causal_conv1d_triton_update` plus a token-major copy. +With `wait_producer` the kernel reads the slots and the conv pool before `griddepcontrol.wait` +and `x` after it. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_triton_update import ( # noqa: E501 + causal_conv1d_triton_update, +) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber +from tensorrt_llm._torch.modules.mamba.causal_conv1d_triton import _causal_conv1d_update_kernel +from tensorrt_llm._utils import get_sm_version + +_BLOCK_N = 128 +_NUM_WARPS = 4 +#: Sequences a `_verify_conv_seqs_kernel` program (taps loaded once): 0 takes `_AUTO_SEQS` from +#: `_AUTO_MIN_SEQUENCES` sequences on and the family kernel below; a positive value forces it. +_SEQS = 0 +_AUTO_SEQS = 2 +_AUTO_MIN_SEQUENCES = 104 + + +def _seqs_a_program(batch: int) -> int: + if _SEQS > 0: + return _SEQS + return _AUTO_SEQS if batch >= _AUTO_MIN_SEQUENCES else 1 + + +@triton.jit(do_not_specialize=["batch"]) +def _verify_conv_seqs_kernel( + x_ptr, + w_ptr, + conv_state_ptr, + slots_ptr, + window_ptr, + window_rows_ptr, + o_ptr, + batch, + dim: tl.constexpr, + stride_x_seq: tl.constexpr, + stride_x_dim: tl.constexpr, + stride_x_token: tl.constexpr, + stride_w_dim: tl.constexpr, + stride_w_width: tl.constexpr, + stride_cs_seq: tl.constexpr, + stride_cs_dim: tl.constexpr, + stride_cs_tok: tl.constexpr, + stride_slots: tl.constexpr, + stride_win_seq: tl.constexpr, + stride_win_step: tl.constexpr, + stride_win_dim: tl.constexpr, + stride_win_win: tl.constexpr, + stride_wrows: tl.constexpr, + stride_o_seq: tl.constexpr, + stride_o_dim: tl.constexpr, + stride_o_token: tl.constexpr, + pad_slot_id: tl.constexpr, + SEQLEN: tl.constexpr, + SEQS: tl.constexpr, + BLOCK_N: tl.constexpr, + LAUNCH_DEPENDENT_KERNELS: tl.constexpr, + WAIT_PRODUCER: tl.constexpr, +): + """`_causal_conv1d_update_kernel` (width 4, SiLU, pad slots, snapshots, `SEQLEN >= 3`) over + `SEQS` sequences a program, per element the same arithmetic and stores.""" + if LAUNCH_DEPENDENT_KERNELS: + tl.extra.cuda.gdc_launch_dependents() + seq0 = tl.program_id(0) * SEQS + idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask_w = idx_feats < dim + w_base = w_ptr + (idx_feats * stride_w_dim) + w_col0 = tl.load(w_base + (0 * stride_w_width), mask_w, other=0.0) + w_col1 = tl.load(w_base + (1 * stride_w_width), mask_w, other=0.0) + w_col2 = tl.load(w_base + (2 * stride_w_width), mask_w, other=0.0) + w_col3 = tl.load(w_base + (3 * stride_w_width), mask_w, other=0.0) + # Slots, snapshot rows and pre-step windows (earlier steps' data) before the dependency wait. + slots = () + wrows = () + lives = () + cols = () + for s in tl.static_range(SEQS): + seq = seq0 + s + in_batch = seq < batch + slot = tl.load(slots_ptr + seq * stride_slots, mask=in_batch, other=pad_slot_id).to( + tl.int64 + ) + wrow = tl.load(window_rows_ptr + seq * stride_wrows, mask=in_batch, other=0).to(tl.int64) + live = in_batch & (slot != pad_slot_id) + m = mask_w & live + prior = conv_state_ptr + (slot * stride_cs_seq) + (idx_feats * stride_cs_dim) + cols = cols + ( + tl.load(prior, m, 0.0), + tl.load(prior + 1 * stride_cs_tok, m, 0.0), + tl.load(prior + 2 * stride_cs_tok, m, 0.0), + ) + slots = slots + (slot,) + wrows = wrows + (wrow,) + lives = lives + (live,) + if WAIT_PRODUCER: + gdc_wait_with_memory_clobber() + xs = () + for s in tl.static_range(SEQS): + x_base = x_ptr + ((seq0 + s) * stride_x_seq) + (idx_feats * stride_x_dim) + for t in tl.static_range(SEQLEN): + xs = xs + (tl.load(x_base + t * stride_x_token, mask=mask_w & lives[s]),) + tl.debug_barrier() + for s in tl.static_range(SEQS): + seq = seq0 + s + m = mask_w & lives[s] + # The advanced pool: the last three inputs. + state_base = conv_state_ptr + (slots[s] * stride_cs_seq) + (idx_feats * stride_cs_dim) + for k in tl.static_range(3): + tl.store(state_base + k * stride_cs_tok, xs[s * SEQLEN + SEQLEN - 3 + k], m) + col0 = cols[3 * s] + col1 = cols[3 * s + 1] + col2 = cols[3 * s + 2] + for t in tl.static_range(SEQLEN): + acc = tl.zeros((BLOCK_N,), dtype=tl.float32) + acc += col0 * w_col0 + acc += col1 * w_col1 + acc += col2 * w_col2 + x_t = xs[s * SEQLEN + t] + acc += x_t * w_col3 + col0 = col1 + col1 = col2 + col2 = x_t + win = ( + window_ptr + + wrows[s] * stride_win_seq + + t * stride_win_step + + idx_feats * stride_win_dim + ) + tl.store(win + 0 * stride_win_win, col0, mask=m) + tl.store(win + 1 * stride_win_win, col1, mask=m) + tl.store(win + 2 * stride_win_win, col2, mask=m) + acc = acc / (1 + tl.exp(-acc)) + tl.store( + o_ptr + seq * stride_o_seq + t * stride_o_token + idx_feats * stride_o_dim, + acc, + mask=m, + ) + + +def _launch_seqs( + x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait, seqs +): + batch, dim, seqlen = x3.shape + _verify_conv_seqs_kernel[(triton.cdiv(batch, seqs), triton.cdiv(dim, _BLOCK_N))]( + x3, + weight, + conv_state, + slots, + window, + window_rows, + out3, + batch, + dim, + *x3.stride(), + *weight.stride(), + *conv_state.stride(), + slots.stride(0), + *window.stride(), + window_rows.stride(0), + *out3.stride(), + pad_slot_id, + SEQLEN=seqlen, + SEQS=seqs, + BLOCK_N=_BLOCK_N, + LAUNCH_DEPENDENT_KERNELS=ldk, + WAIT_PRODUCER=wait, + num_warps=_NUM_WARPS, + **({"launch_pdl": True} if wait else {}), + ) + + +def _launch(x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait): + batch, dim, seqlen = x3.shape + width = weight.shape[1] + seqs = _seqs_a_program(batch) + if seqs > 1 and width == 4 and seqlen >= 3 and conv_state.shape[-1] == 3: + return _launch_seqs( + x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait, seqs + ) + state_len = width - 1 + # `HOIST_X` (width 4) loads every token's x row before the stores, which ptxas cannot prove + # do not alias them: one L2 round trip for x a row instead of p. Bitwise. + hoist = width == 4 + _causal_conv1d_update_kernel[(batch, triton.cdiv(dim, _BLOCK_N))]( + x3, + weight, + None, + conv_state, + None, + slots, + None, + window, + window_rows, + None, + None, + None, + out3, + batch, + dim, + seqlen, + state_len, + conv_state.shape[0], + *x3.stride(), + *weight.stride(), + *conv_state.stride(), + slots.stride(0), + *window.stride(), + window_rows.stride(0), + 0, + 0, + 0, + 0, + 0, + 0, + *out3.stride(), + pad_slot_id, + HAS_BIAS=False, + KERNEL_WIDTH=width, + SILU_ACTIVATION=True, + IS_CONTINUOUS_BATCHING=True, + IS_SPEC_DECODING=False, + NP2_STATELEN=triton.next_power_of_2(state_len), + NP2_SEQLEN=triton.next_power_of_2(seqlen), + USE_PAD_SLOT=True, + BLOCK_N=_BLOCK_N, + SAVE_INTERMEDIATE=True, + HAS_EAGLE_TREE_CUSTOM_ATTN_MASK=False, + LAUNCH_DEPENDENT_KERNELS=ldk, + num_warps=_NUM_WARPS, + **({"WAIT_PRODUCER": True, "launch_pdl": True} if wait else {}), + **({"HOIST_X": True} if hoist else {}), + ) + + +class CausalConv1dTokenMajor(OpWrapper): + """`out = causal_conv1d_token_major(x, rows, conv_state, weight, slots, window, + window_rows, pad_slot_id, launch_dependent_kernels, wait_producer)`. + + `x` is the `[rows * p, C]` q|k|v column view of the projection; the step is the family's + `causal_conv1d_triton_update` over its `[rows, C, p]` view (SiLU, no bias, + `conv_state_indices=slots`, snapshots into `window` at `window_rows`). Returns the + contiguous token-major `[rows * p, C]` output. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep4 verify, 1 request x 3 tokens (MTP-2 c=1)", spec=dict(rows=1, p=3)), + Cell(why="tep4 verify, 4 requests x 4 tokens (MTP-3)", spec=dict(rows=4, p=4)), + Cell(why="tep4 verify, 1 request x 3 tokens, PDL wait", spec=dict(rows=1, p=3, wait=True)), + ) + + note = """ + Source: the Triton `_causal_conv1d_update_kernel` (`modules/mamba/causal_conv1d_triton.py`) + with token-major output strides, and `_verify_conv_seqs_kernel` above. Certified bitwise + against the family entry plus the copy, pool and snapshots included. + """ + + def __call__( + self, + x: torch.Tensor, + rows: int, + conv_state: torch.Tensor, + weight: torch.Tensor, + slots: torch.Tensor, + window: torch.Tensor, + window_rows: torch.Tensor, + pad_slot_id: int, + launch_dependent_kernels: bool = False, + wait_producer: bool = False, + ) -> torch.Tensor: + tokens, dim = x.shape + p = tokens // rows + out = torch.empty([tokens, dim], dtype=x.dtype, device=x.device) + pdl = get_sm_version() >= 90 + _launch( + torch.transpose(torch.reshape(x, [rows, p, dim]), 1, 2), + conv_state, + weight, + slots, + window, + window_rows, + pad_slot_id, + torch.transpose(torch.reshape(out, [rows, p, dim]), 1, 2), + launch_dependent_kernels and pdl, + wait_producer and pdl, + ) + return out + + def reference( + self, + x: torch.Tensor, + rows: int, + conv_state: torch.Tensor, + weight: torch.Tensor, + slots: torch.Tensor, + window: torch.Tensor, + window_rows: torch.Tensor, + pad_slot_id: int, + launch_dependent_kernels: bool = False, + wait_producer: bool = False, + ) -> torch.Tensor: + """The family entry over `x`'s `[rows, C, p]` view, copied token-major (it advances + `conv_state` and writes `window` as the op does: call it on copies).""" + tokens, dim = x.shape + p = tokens // rows + out = causal_conv1d_triton_update( + x=torch.transpose(torch.reshape(x, [rows, p, dim]), 1, 2), + conv_state=conv_state, + weight=weight, + bias=None, + activation="silu", + conv_state_indices=slots, + pad_slot_id=pad_slot_id, + num_accepted_tokens=None, + intermediate_conv_window=window, + intermediate_state_indices=window_rows, + ) + return torch.reshape(torch.transpose(out, 1, 2), [tokens, dim]) + + def is_valid(self, x, rows, conv_state, weight, slots, window, window_rows, *args, **kwargs): + assert x.dim() == 2 and x.stride(1) == 1 and x.shape[0] % rows == 0, ( + "x is the [rows * p, C] column view of the projection, unit column stride" + ) + # The pool is read through its strides; the taps must be unit-stride along the width. + assert weight.stride(1) == 1, "conv taps [C, W] with a unit width stride" + + +causal_conv1d_token_major = CausalConv1dTokenMajor() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py new file mode 100644 index 000000000000..91dfe24f539a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py @@ -0,0 +1,646 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Chunked gated delta rule on FlashInfer's Blackwell kernels: the GDN prefill core. + +The FP8 target's copy of the family entry +`models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule`, with in-place paths. Every call +is the target's form: `has_initial_state` (a fresh sequence starts from zeros and its pool row +is not read, so the caller need not reset it), the in-place indexed state update, and +`flashinfer.chunk_gated_delta_rule` (CuTe-DSL tcgen05 kernels on sm_100/103) on one of the three +paths `__call__` picks. +""" + +from __future__ import annotations + +import functools +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.fused_state_io import cast_scatter_fp32_vk_to_vk + +#: The Gated DeltaNet at tp1 and dep4: (H, HV, K, V). +QWEN38_GEOMETRY = (16, 48, 128, 128) +#: Slot count of the dense-pool cell. +ENGINE_SLOTS = 461 + + +#: A tp4/tep4 rank: tp1's heads split four ways. +RANK4_GEOMETRY = (4, 12, 128, 128) +#: Per-rank slot counts the V2 hybrid manager allocates on this checkpoint (bf16 KV and state, +#: KV fraction 0.6). It sizes the pool from the rank's memory budget, so a dep4 rank (every GDN +#: head) holds fewer slots than a tep4 rank. +TEP4_SLOTS = 1874 +DEP4_SLOTS = 850 +#: Not measured: a tp4 rank's state per slot is tep4's. +TP4_SLOTS = TEP4_SLOTS +#: GDN layers whose states share one slot page: a layer's pool view has a slot stride of 36 +#: `[HV, V, K]` states (a tep4 rank's last slot starts 1.3e10 elements past the base). +PAGE_LAYERS = 36 + +#: A tp2/tep2 rank: tp1's heads split two ways. +RANK2_GEOMETRY = (8, 24, 128, 128) +#: Not measured: a tp2/tep2 slot holds half a dep4 state, so twice dep4's count spans dep4's +#: page (2.4e10 elements, the largest certified). +TP2_SLOTS = 2 * DEP4_SLOTS + + +def _engine_slots(num_seqs: int) -> tuple[int, ...]: + """`num_seqs` distinct slots of the engine pool, slot 0 and the last live.""" + return ((0, ENGINE_SLOTS - 1) + tuple(i * 13 + 4 for i in range(num_seqs)))[:num_seqs] + + +def _rank_slots(num_seqs: int, slots: int) -> tuple[int, ...]: + """`num_seqs` distinct slots: the last, the first, then a stride-37 walk.""" + rest = tuple((i * 37 + 4) % (slots - 2) + 1 for i in range(max(num_seqs - 2, 0))) + picked = ((slots - 1, 0) + rest)[:num_seqs] + assert len(set(picked)) == num_seqs + return picked + + +def _rank_cell(why: str, geometry, seqlens, slots: int, initial, fi_path: str) -> Cell: + """The target's call at one rank: the layer's view of the bf16 state page at the member's + slot count, a prefix view of a wider output buffer. `fi_path` is the path it reaches on + sm_103 (`cp`, `chunked` or `inplace`).""" + return Cell( + why=why, + spec=dict( + geometry=geometry, + seqlens=seqlens, + slots=slots, + page_layers=PAGE_LAYERS, + indices=_rank_slots(len(seqlens), slots), + pool_dtype=torch.bfloat16, + output="slice", + initial=initial, + linear_gate=True, + fi_path=fi_path, + ), + ) + + +#: Elements of a `[V, K]` state plane a gather program copies. +_GATHER_BLOCK = 4096 + + +# Slot ids and carry flags are scalar-read views of the step's metadata. +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "init_ptr"]) +def _gather_carried_states_kernel( + pool_ptr, + dst_ptr, + slot_ptr, + init_ptr, + stride_ps, + stride_ph, + stride_pv, + stride_pk, + H: tl.constexpr, + K: tl.constexpr, + PLANE: tl.constexpr, + BLOCK: tl.constexpr, +): + """`BLOCK` elements of sequence `n`'s head `h` plane into the packed `[N, H, V, K]` buffer: + pool row `slot_ptr[n]` when `init_ptr[n]`, else +0.0 (the row is not read).""" + n = tl.program_id(0) + h = tl.program_id(1) + i = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + mask = i < PLANE + dst = dst_ptr + (n.to(tl.int64) * H + h) * PLANE + i + if tl.load(init_ptr + n) != 0: + slot = tl.load(slot_ptr + n).to(tl.int64) + src = ( + pool_ptr + slot * stride_ps + h * stride_ph + (i // K) * stride_pv + (i % K) * stride_pk + ) + tl.store(dst, tl.load(src, mask=mask), mask=mask) + else: + tl.store(dst, tl.zeros([BLOCK], dtype=dst_ptr.dtype.element_ty), mask=mask) + + +def _gather_carried_states( + pool: torch.Tensor, indices: torch.Tensor, has_initial_state: torch.Tensor +) -> torch.Tensor: + """`[N, H, V, K]` in the pool's dtype: `pool[indices[n]]` for a carried sequence, zeros + for a fresh one.""" + num_seqs = indices.shape[0] + _, h, v, k = pool.shape + out = torch.empty(num_seqs, h, v, k, dtype=pool.dtype, device=pool.device) + _gather_carried_states_kernel[(num_seqs, h, triton.cdiv(v * k, _GATHER_BLOCK))]( + pool, out, indices, has_initial_state, *pool.stride(), + H=h, K=k, PLANE=v * k, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + return out + + +#: Context tokens up to which a call runs in place on the chunked kernel: at most one 64-token +#: chunk a sequence, one launch, where the CP pipeline buys nothing. +INPLACE_MAX_TOKENS = 64 + + +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "init_ptr"]) +def _zero_fresh_states_kernel( + pool_ptr, + slot_ptr, + init_ptr, + stride_ps, + stride_ph, + stride_pv, + stride_pk, + K: tl.constexpr, + PLANE: tl.constexpr, + BLOCK: tl.constexpr, +): + """Zero `BLOCK` elements of sequence `n`'s head `h` plane of pool row `slot_ptr[n]` when + `init_ptr[n]` is false; a carried sequence's row is not touched.""" + n = tl.program_id(0) + if tl.load(init_ptr + n) == 0: + h = tl.program_id(1) + i = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + slot = tl.load(slot_ptr + n).to(tl.int64) + dst = ( + pool_ptr + slot * stride_ps + h * stride_ph + (i // K) * stride_pv + (i % K) * stride_pk + ) + tl.store(dst, tl.zeros([BLOCK], dtype=pool_ptr.dtype.element_ty), mask=i < PLANE) + + +def _chunk_inplace( + q, k, v, g, beta, scale, pool, indices, has_initial_state, cu_seqlens, output, linear_gate +) -> torch.Tensor: + """The fresh rows zeroed, then the chunked kernel (`use_cp=False`) reading and writing the + named pool rows itself (`state_indices`, the pool as `output_state`): no packed copies.""" + import flashinfer + + num_seqs = indices.shape[0] + _, h, v_dim, k_dim = pool.shape + _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( + pool, indices, has_initial_state, *pool.stride(), + K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=pool, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=pool, + use_cp=False, + state_indices=indices, + ) + return out.unsqueeze(0) + + +#: FlashInfer's sm_10x rule for its context-parallel pipeline (`should_use_cp_host`): CP when +#: `num_seqs x num_v_heads x 4` is below the SM count. +_CP_PARALLELISM_DENOMINATOR = 4 + + +@functools.lru_cache(maxsize=None) +def _sm_count(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +@functools.lru_cache(maxsize=1024) +def _cp_chunk_len_cached(seqlen: int, num_heads: int, device_index: int) -> int: + from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import choose_cp_chunk_len_host + from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + + device = torch.device("cuda", device_index) + return int( + choose_cp_chunk_len_host( + seqlen, + num_heads, + get_device_sm_count(device), + device_capability=get_compute_capability(device), + total_seqlen=seqlen, + num_seqs=1, + device_name=get_device_name(device), + ) + ) + + +def cp_chunk_len_for(seqlen: int, num_heads: int, device: torch.device) -> int: + """The CP chunk length FlashInfer picks by itself for one `seqlen`-token sequence over + `num_heads` (max of q and v) heads. A call over a padded row count passes it + (`_cp_chunk_len`) to cut the sequence as the unpadded call does.""" + index = device.index if device.index is not None else torch.cuda.current_device() + return _cp_chunk_len_cached(seqlen, num_heads, index) + + +def _reaches_cp(num_seqs: int, num_v_heads: int, device: torch.device) -> bool: + """Whether FlashInfer's `use_cp="auto"` routes `num_seqs` sequences to the CP pipeline.""" + return num_seqs * num_v_heads * _CP_PARALLELISM_DENOMINATOR < _sm_count(device.index) + + +def _chunk_cp_inplace( + q, + k, + v, + g, + beta, + scale, + pool, + indices, + has_initial_state, + cu_seqlens, + output, + linear_gate, + fi_kwargs=None, +) -> torch.Tensor: + """The CP pipeline in place over the pool: the fresh rows zeroed, then the pipeline reading + and writing the named rows itself. It keeps the state in its own workspace between its + kernels and writes each final state once, so the result is bitwise `_chunk_carried`'s + without its gather and scatter.""" + import flashinfer + + num_seqs = indices.shape[0] + _, h, v_dim, k_dim = pool.shape + _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( + pool, indices, has_initial_state, *pool.stride(), + K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=pool, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=pool, + use_cp=True, + state_indices=indices, + **(fi_kwargs or {}), + ) + return out.unsqueeze(0) + + +def _chunk_carried( + q, + k, + v, + g, + beta, + scale, + pool, + indices, + has_initial_state, + cu_seqlens, + output, + linear_gate, + keep_rows=None, + keep_slots=None, + use_cp=None, + fi_kwargs=None, +) -> torch.Tensor: + """Packed path: the named states gathered in the pool's dtype (zeros for a fresh sequence), + FlashInfer's kernel (`g` exponentiated unless `linear_gate`), and the scatter of the final + states into the same slots, or with `keep_rows` only those sequences' into `keep_slots`. + `use_cp` None is FlashInfer's heuristic; `fi_kwargs` are passed through (a padded call's + `max_seqlen` and `_cp_chunk_len`).""" + import flashinfer + + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + packed = _gather_carried_states(pool, indices, has_initial_state) + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + final = torch.empty_like(packed) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=packed, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=final, + **({} if use_cp is None else {"use_cp": use_cp}), + **(fi_kwargs or {}), + ) + if keep_rows is None: + cast_scatter_fp32_vk_to_vk(final, pool, indices) + else: + cast_scatter_fp32_vk_to_vk(final.index_select(0, keep_rows), pool, keep_slots) + return out.unsqueeze(0) + + +def chunk_split_first_pass( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + pool: torch.Tensor, + gather_slots: torch.Tensor, + has_initial_state: torch.Tensor, + cu_seqlens: torch.Tensor, + keep_rows: torch.Tensor, + keep_slots: torch.Tensor, + output: torch.Tensor, + num_whole_seqs: int, + whole: Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None, +) -> torch.Tensor: + """First pass of an in-forward snapshot split (`PrefillTarget._gdn_split`): `_chunk_carried` + over `cu_seqlens` with extra sequence boundaries (`linear_gate`), only sequences `keep_rows` + writing their final states, to pool rows `keep_slots`. Bitwise the unsplit two-iteration + schedule: on the chunked kernel every sequence runs on its own; the CP pipeline segments a + sequence by how many sequences the call has, so when FlashInfer picks CP for the + `num_whole_seqs` unsplit sequences the pass runs over the kept rows alone, `whole` = (their + rows, boundaries, carry flags), copied in and out, or on views when `rows` is an int (the + kept rows lead the step).""" + assert q.shape[1] > INPLACE_MAX_TOKENS, ( + f"a split first pass of {q.shape[1]} tokens would leave the carried path" + ) + if whole is not None and _use_cp(num_whole_seqs * pool.shape[1], q.device): + rows, cu, carry = whole + if isinstance(rows, int): + # The kept rows lead: the same operands as their gather, without the copies. + _chunk_carried( + *(x[:, :rows] for x in (q, k, v, g, beta)), + None, pool, keep_slots, carry, cu, output[:, :rows], True, + ) # fmt: skip + return output + o = _chunk_carried( + *(x.index_select(1, rows) for x in (q, k, v, g, beta)), + None, pool, keep_slots, carry, cu, None, True, + ) # fmt: skip + output[0].index_copy_(0, rows, o[0]) + return output + return _chunk_carried( + q, k, v, g, beta, None, pool, gather_slots, has_initial_state, cu_seqlens, output, True, + keep_rows=keep_rows, keep_slots=keep_slots, + use_cp=_use_cp(num_whole_seqs * pool.shape[1], q.device), + ) # fmt: skip + + +@functools.lru_cache(maxsize=64) +def _use_cp_cached(units: int, device_index: int) -> bool: + from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import should_use_cp_host + from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + + device = torch.device("cuda", device_index) + return bool( + should_use_cp_host( + units, + get_device_sm_count(device), + get_device_name(device), + device_capability=get_compute_capability(device), + ) + ) + + +def split_takes_cp(num_seqs: int, v_heads: int, device: torch.device) -> bool: + """Whether `chunk_split_first_pass` runs over the kept rows alone (FlashInfer picks CP).""" + return _use_cp(num_seqs * v_heads, device) + + +def _use_cp(units: int, device: torch.device) -> bool: + """FlashInfer's CP choice (`should_use_cp_host`) for `units` = sequences x value heads.""" + index = device.index if device.index is not None else torch.cuda.current_device() + return _use_cp_cached(units, index) + + +class FlashinferChunkGatedDeltaRule(OpWrapper): + """The gated delta rule over whole sequences, in place over the state pool. + + Per token `t` and value head, `h` the `[V, K]` state, `q_h`/`k_h` the key heads broadcast + (value head `i` reads key head `i // (HV // H)`), fp32 accumulation: + + h *= exp(g[t]); u = (v[t] - h k_h[t]) * beta[t]; h += u k_h[t]^T + o[t] = h (scale * q_h[t]) + + computed 64 tokens at a time on tensor cores (bf16 operands); `g` is in log space, or with + `linear_gate` the factor `exp(g)`; `scale=None` is `K ** -0.5`. One packed row split at + `cu_seqlens`; sequence `n` starts from `initial_state[initial_state_indices[n]]` (the pool + `[S, HV, V, K]`), or from zeros when `has_initial_state[n]` is false, and its final state is + written back into that slot. The conv, q/k/v/g/beta preparation and the gated norm are the + caller's. Returns `(o, None)`, `o` `[1, T, HV, V]` in `q.dtype` (`output` when given). + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: geometry (H, HV, K, V); seqlens; `slots`-row pool with `indices`; pool_dtype; + # output (None | "buffer" | "slice", a prefix view of a wider buffer); page_layers (the + # pool is the layer's view of the state page); initial; linear_gate; fi_path. Each head + # layout on each path `__call__` routes it to: in place (with the 64 / 65-token edge), + # CP and chunked. + CELLS: tuple[Cell, ...] = ( + Cell( + why="target call: bf16 pool, has_initial_state [1, 0, 1], seqlens [300, 1024, 500]", + spec=dict( + geometry=QWEN38_GEOMETRY, + seqlens=(300, 1024, 500), + slots=ENGINE_SLOTS, + indices=_engine_slots(3), + pool_dtype=torch.bfloat16, + output="buffer", + initial=(True, False, True), + linear_gate=True, + ), + ), + _rank_cell( + "tep4 4/12, [200, 37, 91] carried/fresh/carried, CP path, 1874 slots", + RANK4_GEOMETRY, (200, 37, 91), TEP4_SLOTS, (True, False, True), "cp", + ), + _rank_cell( + "tp4 4/12, carried 8192-token chunk, CP path, 1874-slot bf16 page", + RANK4_GEOMETRY, (8192,), TP4_SLOTS, (True,), "cp", + ), + _rank_cell( + "dep4 16/48, carried 512-token prompt, 850-slot bf16 page", + QWEN38_GEOMETRY, (512,), DEP4_SLOTS, (True,), "chunked", + ), + _rank_cell( + "tep4 4/12 envelope, 8192 rows over 128 seqs of 63/65, chunked path", + RANK4_GEOMETRY, (63, 65) * 64, TEP4_SLOTS, (True, False) * 64, "chunked", + ), + _rank_cell( + "tep4 4/12, one carried 64-token chunk (the in-place bound), 1874 slots", + RANK4_GEOMETRY, (64,), TEP4_SLOTS, (True,), "inplace", + ), + _rank_cell( + "tep4 4/12, one carried 65-token chunk, past the in-place bound: CP path", + RANK4_GEOMETRY, (65,), TEP4_SLOTS, (True,), "cp", + ), + _rank_cell( + "dep4 16/48, carried 2 + fresh 30 + carried 1 tokens, in place, 850 slots", + QWEN38_GEOMETRY, (2, 30, 1), DEP4_SLOTS, (True, False, True), "inplace", + ), + # tp2/tep2: 24 v heads reach the CP pipeline for one context sequence only + # (24 x 4 < 152 SMs, 48 x 4 is not). + _rank_cell( + "tp2/tep2 8/24, fresh 8192-token prompt, CP path, 1700-slot bf16 page", + RANK2_GEOMETRY, (8192,), TP2_SLOTS, (False,), "cp", + ), + _rank_cell( + "tp2/tep2 8/24, [200, 37, 91] carried/fresh/carried, chunked path, 1700 slots", + RANK2_GEOMETRY, (200, 37, 91), TP2_SLOTS, (True, False, True), "chunked", + ), + _rank_cell( + "tp2/tep2 8/24, fresh 5 + carried 2 tokens, in place, 1700 slots", + RANK2_GEOMETRY, (5, 2), TP2_SLOTS, (False, True), "inplace", + ), + ) # fmt: skip + + note = """ + Source: `flashinfer.chunk_gated_delta_rule` (`flashinfer.gdn_prefill`), CuTe-DSL kernels + JIT-compiled in-process at the first call per path and device; Triton kernels for the + fresh-row zeroing, the packed gather and the scatter (`cast_scatter_fp32_vk_to_vk`). + + Paths (`__call__`): at most `INPLACE_MAX_TOKENS` context tokens run in place on the chunked + kernel; calls FlashInfer routes to its CP pipeline (`_reaches_cp`: sequences x HV x 4 below + the SM count, so on GB300 one sequence at 8/24, 1-3 at 4/12, never at 16/48) run in place + on it; the rest take the packed path (`_chunk_carried`: a copy of the named states in and + out). The state is fp32 inside and rounded once to the pool's dtype. + + Preconditions: q/k L2-normalized by the caller, `g` fp32, `cu_seqlens` given, slots + distinct and in range (not checked), context rows only (single-token rows belong to the + decode kernel). FlashInfer keeps one TMA workspace per device: two streams must not run + this op concurrently. Slot offsets are int64 (every certified page's last slot is past + int32). + + Certified on GB300 against the fp64 reference at the cells: bf16 pools at 16/48, 8/24 and + 4/12 heads on each path they reach. Not certified: fp32 / fp16 pools, `scale` other than the + default, other head layouts, K or V other than 128, 2 context sequences on CP at 4/12, a + carried sequence on CP at 8/24, the measured slot counts of tp4, tp2 and tep2 (stand-ins + above), a call inside a CUDA graph (the target calls it eagerly). + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: Optional[float] = None, + initial_state: Optional[torch.Tensor] = None, + initial_state_indices: Optional[torch.Tensor] = None, + inplace_indexed_state_update: bool = False, + output_final_state: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, + output: Optional[torch.Tensor] = None, + state_workspace: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + has_initial_state: Optional[torch.Tensor] = None, + linear_gate: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + assert has_initial_state is not None, ( + "the target's call only: has_initial_state, the in-place indexed update" + ) + assert ( + inplace_indexed_state_update + and initial_state_indices is not None + and not output_final_state + and not head_first + and not use_qk_l2norm_in_kernel + and state_workspace is None + and cu_seqlens is not None + and g.dtype == torch.float32 + ), "has_initial_state: the in-place indexed call only, q/k normalized by the caller" + if q.shape[1] <= INPLACE_MAX_TOKENS: + path = _chunk_inplace + elif _reaches_cp(initial_state_indices.shape[0], v.shape[2], q.device): + path = _chunk_cp_inplace + else: + path = _chunk_carried + o = path( + q, k, v, g, beta, scale, initial_state, initial_state_indices, + has_initial_state, cu_seqlens, output, linear_gate, + ) # fmt: skip + return o, None + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: Optional[float] = None, + initial_state: Optional[torch.Tensor] = None, + initial_state_indices: Optional[torch.Tensor] = None, + inplace_indexed_state_update: bool = False, + output_final_state: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, + output: Optional[torch.Tensor] = None, + state_workspace: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + has_initial_state: Optional[torch.Tensor] = None, + linear_gate: bool = False, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], torch.Tensor]: + """`(o, final_state, initial_state as the call leaves it)`: an fp64 token-at-a-time + recurrence, `o` rounded once to `q.dtype`, the states to the pool's dtype. Mutates + nothing; `output`, `head_first` and `state_workspace` are ignored.""" + dt = torch.float64 + _, _, num_k_heads, head_k = q.shape + num_v_heads = v.shape[2] + repeats = num_v_heads // num_k_heads + scale = head_k**-0.5 if scale is None else scale + bounds = cu_seqlens.tolist() + slots = initial_state_indices.tolist() + carried = [True] * len(slots) if has_initial_state is None else has_initial_state.tolist() + out = torch.zeros(1, q.shape[1], num_v_heads, v.shape[3], dtype=dt, device=q.device) + finals = [] + for n in range(len(bounds) - 1): + state = initial_state[slots[n]].to(dt) + if not carried[n]: + state = torch.zeros_like(state) + for t in range(bounds[n], bounds[n + 1]): + q_t, k_t = q[0, t].to(dt), k[0, t].to(dt) + if use_qk_l2norm_in_kernel: + q_t = q_t / torch.sqrt(q_t.pow(2).sum(-1, keepdim=True) + 1e-6) + k_t = k_t / torch.sqrt(k_t.pow(2).sum(-1, keepdim=True) + 1e-6) + q_t = q_t.repeat_interleave(repeats, dim=0) * scale + k_t = k_t.repeat_interleave(repeats, dim=0) + decay = g[0, t].to(dt) if linear_gate else g[0, t].to(dt).exp() + state = state * decay[:, None, None] + u = (v[0, t].to(dt) - (state * k_t[:, None, :]).sum(-1)) * beta[0, t].to(dt)[ + :, None + ] + state = state + u[:, :, None] * k_t[:, None, :] + out[0, t] = (state * q_t[:, None, :]).sum(-1) + finals.append(state) + pool = initial_state.clone() + final_state = None + if inplace_indexed_state_update: + for slot, state in zip(slots, finals): + pool[slot] = state.to(pool.dtype) + elif output_final_state: + final_state = torch.stack(finals).to(initial_state.dtype) + return out.to(q.dtype), final_state, pool + + +flashinfer_chunk_gated_delta_rule = FlashinferChunkGatedDeltaRule() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py new file mode 100644 index 000000000000..fbd39a5303f7 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -0,0 +1,2780 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GDN cached-replay verify step on warp tensor cores (CuTe DSL): the Triton +`_cached_replay_kernel` (`fused_recurrent_gated_delta_rule_cached_replay_update`) at this +checkpoint's geometry (K=V=128, history 16, bf16 state, T=1..8 new tokens) over H q/k heads and +HV = 3 H value heads (`GEOMETRIES`, compile-time parameters of `_Replay`). + +A CTA serves one (request, q/k head, split): the q/k head's 384 state rows (3 v heads x 128, K +innermost) are 24 m-tiles of 16 rows, `tpc` of them a CTA. Per m-tile a warp computes the state +dots D[v, k_t] = h0[v, :] . k_t and D[v, q_t] = h0[v, :] . q_t on mma.sync m16n8k16 (bf16 in, +fp32 out), and the history term as one more MMA with the [16 x 2T] coefficients rounded to bf16 +as the Triton kernel rounds them; the T x T forward substitution runs per state row on CUDA +cores. With `commit`, a row whose history wraps (`p + T > 16`) writes the advanced checkpoint over +the tile it just used (u split into bf16 hi and lo, as the Triton commit does). + +With PDL the state and history (not written by the previous kernel) are read before +`griddepcontrol.wait`, the step's own inputs after it. +""" + +from __future__ import annotations + +from typing import Optional + +import cutlass +import cutlass.cute as cute +import torch +import torch.nn.functional as F +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T as _ty +from cutlass.cutlass_dsl import dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +K, V, HIST = 128, 128, 16 +#: Value heads a q/k head serves. +GRP = 3 +#: `(q/k heads, value heads)` compiled for: a whole layer (tp1, attention DP), a tp2/tep2 and a +#: tp4/tep4 rank. +GEOMETRIES = ((16, 48), (8, 24), (4, 12)) +TILES = GRP * (V // 16) # m-tiles of one (request, q/k head) +PU = 272 # shared row pitch (bytes) of 128-element bf16 rows read by ldmatrix (conflict-free) +PK = 320 # shared row pitch of the history keys read as permuted 16-byte fragments +LOG2E = 1.4426950408889634 +LN2 = 0.6931471805599453 + +# -- inline PTX helpers ------------------------------------------------------------------------ + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _unpack(res, n, ty, loc, ip): + return [ + ty( + llvm.extractvalue( + _ty.i32() if ty is cutlass.Int32 else _ty.f32(), res, [i], loc=loc, ip=ip + ) + ) + for i in range(n) + ] + + +def _make_ldg4(qual): + @dsl_user_op + def _ldg4(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + f"ld.global{qual}.v4.b32 {{$0, $1, $2, $3}}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + return _ldg4 + + +@dsl_user_op +def _policy_evict_first(*, loc=None, ip=None): + return cutlass.Int64( + _asm(_ty.i64(), [], "createpolicy.fractional.L2::evict_first.b64 $0, 1.0;", "=l", loc, ip) + ) + + +@dsl_user_op +def _ldg4_hint(gaddr, pol, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pol, cutlass.Int64, loc, ip)], + "ld.global.nc.L1::no_allocate.L2::cache_hint.v4.b32 {$0, $1, $2, $3}, [$4], $5;", + "=r,=r,=r,=r,l,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +_LDG4 = { + "plain": _make_ldg4(""), + "na": _make_ldg4(".L1::no_allocate"), + "nc": _make_ldg4(".nc.L1::no_allocate"), + "nc256": _make_ldg4(".nc.L1::no_allocate.L2::256B"), + "cs": _make_ldg4(".cs"), + "lu": _make_ldg4(".lu"), +} +_ldg4 = _LDG4["plain"] + + +@dsl_user_op +def _ldg4p(gaddr, pred, *, loc=None, ip=None): + """16 bytes, or zeros where `pred` is 0 (nothing read).""" + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred p; setp.ne.b32 p, $5, 0; mov.b32 $0, 0; mov.b32 $1, 0; mov.b32 $2, 0; " + "mov.b32 $3, 0; @p ld.global.v4.b32 {$0, $1, $2, $3}, [$4]; }", + "=r,=r,=r,=r,l,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _stg4(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(x, cutlass.Int32, loc, ip) for x in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg_bf16(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg_f32(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ st.global.f32 [$1], $2; mov.u32 $0, 0; }", + "=r,l,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg_i32(gaddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_f32(gaddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.f32 $0, [$1];", + "=f,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_f32p(gaddr, pred, *, loc=None, ip=None): + """An f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.f32 $0, 0f00000000; @q ld.global.f32 $0, [$1]; }", + "=f,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_b16p(gaddr, pred, *, loc=None, ip=None): + """16 bits zero-extended (0 where `pred` is 0), converted where used so the load does not stall.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.b32 $0, 0; @q ld.global.u16 $0, [$1]; }", + "=r,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _b16_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b32 t; shl.b32 t, $1, 16; mov.b32 $0, t; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _f32_bits(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(_ty.i32(), [_ir(x, cutlass.Float32, loc, ip)], "mov.b32 $0, $1;", "=r,f", loc, ip) + ) + + +@dsl_user_op +def _bits_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Int32, loc, ip)], "mov.b32 $0, $1;", "=f,r", loc, ip) + ) + + +@dsl_user_op +def _ldg_bf16p(gaddr, pred, *, loc=None, ip=None): + """A bf16 widened to f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; .reg .b16 h; setp.ne.b32 q, $2, 0; mov.b16 h, 0; @q ld.global.b16 h, [$1]; " + "cvt.f32.bf16 $0, h; }", + "=f,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _cp16z(saddr, gaddr, nbytes, *, loc=None, ip=None): + """cp.async of 16 bytes, zero-filled past `nbytes` (0 or 16).""" + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + ], + "{ cp.async.cg.shared.global [$1], [$2], 16, $3; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _cp_commit(*, loc=None, ip=None): + _asm(_ty.i32(), [], "{ cp.async.commit_group; mov.u32 $0, 0; }", "=r", loc, ip, side=True) + + +@dsl_user_op +def _cp_wait_all(*, loc=None, ip=None): + _asm(_ty.i32(), [], "{ cp.async.wait_all; mov.u32 $0, 0; }", "=r", loc, ip, side=True) + + +@dsl_user_op +def _sts_i32(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts_f32(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Float32, loc, ip)], + "{ st.shared.f32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts2_f32(saddr, a, b, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(a, cutlass.Float32, loc, ip), + _ir(b, cutlass.Float32, loc, ip), + ], + "{ st.shared.v2.f32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,r,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts_bf16(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.shared.b16 [$1], h; mov.u32 $0, 0; }", + "=r,r,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds_i32(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds_f32(saddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.f32 $0, [$1];", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds_bf16(saddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; ld.shared.b16 h, [$1]; cvt.f32.bf16 $0, h; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds4_i32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds4_f32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.f32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _ldsm4t(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (bf16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([_ty.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _bf2_pack(lo, hi, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.bf16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_lo_v(w, *, loc=None, ip=None): + """`_bf2_lo` pinned in program order, so the conversion of a prefetched word is not hoisted + up to its load.""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_hi_v(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_lo(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_hi(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _ex2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "ex2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _lg2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "lg2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _div_approx(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "div.approx.ftz.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _div_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "div.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sqrt_rn(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], "sqrt.rn.f32 $0, $1;", "=f,f", loc, ip) + ) + + +@dsl_user_op +def _mul_bf16(a, b, *, loc=None, ip=None): + """bf16 x bf16 rounded to bf16 (`mul.rn.bf16`), bf16 bits in the low half of an i32.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(a, cutlass.Int32, loc, ip), _ir(b, cutlass.Int32, loc, ip)], + "{ .reg .b16 x, y, z; cvt.u16.u32 x, $1; cvt.u16.u32 y, $2; mul.rn.bf16 z, x, y; " + "cvt.u32.u16 $0, z; }", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _conv4_silu(p0, p1, p2, p3, *, loc=None, ip=None): + """The verify conv's per-element tail in the Triton `_causal_conv1d_update_kernel`'s PTX: + the bf16 tap products summed in fp32 in tap order, SiLU as `acc / (1 + ex2.approx(acc * + -log2e))` (`div.full.f32`), rounded to bf16 (bits in the low half of an i32).""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(x, cutlass.Int32, loc, ip) for x in (p0, p1, p2, p3)], + "{ .reg .b16 h0, h1, h2, h3, o; .reg .f32 a, t, e, d, y; " + "cvt.u16.u32 h0, $1; cvt.u16.u32 h1, $2; cvt.u16.u32 h2, $3; cvt.u16.u32 h3, $4; " + "cvt.f32.bf16 a, h0; add.f32 a, a, 0f00000000; add.rn.f32.bf16 a, h1, a; " + "add.rn.f32.bf16 a, h2, a; add.rn.f32.bf16 a, h3, a; " + "mul.f32 t, a, 0fBFB8AA3B; ex2.approx.f32 e, t; add.f32 d, e, 0f3F800000; " + "div.full.f32 y, a, d; cvt.rn.bf16.f32 o, y; cvt.u32.u16 $0, o; }", + "=r,r,r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sts4(saddr, a, b, c, d, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)] + + [_ir(x, cutlass.Int32, loc, ip) for x in (a, b, c, d)], + "{ st.shared.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _pack16(lo, hi, *, loc=None, ip=None): + """Two bf16 bit patterns (low halves of i32) packed into one word, `lo` in the low half.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(lo, cutlass.Int32, loc, ip), _ir(hi, cutlass.Int32, loc, ip)], + "prmt.b32 $0, $1, $2, 0x5410;", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _half16(w, hi, *, loc=None, ip=None): + """The low (`hi` False) or high half of a word, zero-extended.""" + sel = "0x4432" if hi else "0x4410" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(w, cutlass.Int32, loc, ip)], + f"prmt.b32 $0, $1, 0, {sel};", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg_b16(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; cvt.u16.u32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_init(saddr, count, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(count, cutlass.Int32, loc, ip)], + "{ mbarrier.init.shared::cta.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _fence_mbar_init(*, loc=None, ip=None): + _asm( + _ty.i32(), + [], + "{ fence.mbarrier_init.release.cluster; mov.u32 $0, 0; }", + "=r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s(saddr, gaddr, nbytes, mbar, *, loc=None, ip=None): + """mbarrier.arrive.expect_tx + one 1-D bulk copy global -> shared of `nbytes`.""" + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + ], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$4], $3; " + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], $3, [$4]; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s_hint(saddr, gaddr, nbytes, mbar, pol, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + _ir(pol, cutlass.Int64, loc, ip), + ], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$4], $3; " + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$1], [$2], $3, [$4], $5; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r,l", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_wait(mbar, phase, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(mbar, cutlass.Int32, loc, ip), _ir(phase, cutlass.Int32, loc, ip)], + "{ .reg .pred P1; LAB_WAIT: mbarrier.try_wait.parity.shared::cta.b64 P1, [$1], $2; " + "@P1 bra DONE; bra LAB_WAIT; DONE: mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_arrive(mbar, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(mbar, cutlass.Int32, loc, ip)], + "{ mbarrier.arrive.shared::cta.b64 _, [$1]; mov.u32 $0, 0; }", + "=r,r", + loc, + ip, + side=True, + ) + + +# -- the folded gated RMS norm (`_Replay(norm=True)`): the instructions of the Triton +# `_rms_norm_gated_fwd_multirow_kernel` (one row a warp, four columns a lane), so its bits. + + +@dsl_user_op +def _ldg2_i32(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 2), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v2.b32 {$0, $1}, [$2];", + "=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds2_i32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 2), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v2.b32 {$0, $1}, [$2];", + "=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _stg2_i32(gaddr, a, b, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Int32, loc, ip), + _ir(b, cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _gn_sumsq(x0, x1, x2, x3, *, loc=None, ip=None): + """A lane's sum of squares in the Triton kernel's order: (x0 x0 + x2 x2) + (x1 x1 + x3 x3).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip) for x in (x0, x1, x2, x3)], + "{ .reg .f32 p2, p3, a, b; mul.rn.f32 p2, $3, $3; mul.rn.f32 p3, $4, $4; " + "fma.rn.f32 a, $1, $1, p2; fma.rn.f32 b, $2, $2, p3; add.rn.f32 $0, a, b; }", + "=f,f,f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _add_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "add.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _mul_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "mul.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _gn_rstd(sumsq, eps, *, loc=None, ip=None): + """1 / sqrt(sum / 128 + eps): `div.full.f32`, `add.f32`, `sqrt.approx.ftz.f32`, `div.full.f32`.""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(sumsq, cutlass.Float32, loc, ip), _ir(eps, cutlass.Float32, loc, ip)], + "{ .reg .f32 n, v, a, q, one; mov.b32 n, 0f43000000; mov.b32 one, 0f3F800000; " + "div.full.f32 v, $1, n; add.rn.f32 a, $2, v; sqrt.approx.ftz.f32 q, a; " + "div.full.f32 $0, one, q; }", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _gn_sigmoid(z, *, loc=None, ip=None): + """sigmoid(z) as Triton's `tl.sigmoid`: 1 / (1 + ex2.approx(z * -log2(e))).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(z, cutlass.Float32, loc, ip)], + "{ .reg .f32 t, e, d, one; mov.b32 one, 0f3F800000; mul.rn.f32 t, $1, 0fBFB8AA3B; " + "ex2.approx.f32 e, t; add.rn.f32 d, e, one; div.full.f32 $0, one, d; }", + "=f,f", + loc, + ip, + ) + ) + + +def _exp(x): + return _ex2(x * LOG2E) + + +def _shfl_xor(x, m): + return cute.arch.shuffle_sync_bfly(x, offset=m) + + +# -- the kernel -------------------------------------------------------------------------------- + + +class _Replay: + def __init__( + self, + *, + T, + tpc, + warps, + commit, + pdl, + ld="nc", + pf=1, + minb=1, + l2norm=True, + gating=True, + g_f32=False, + b_f32=False, + early_trigger=True, + lo_split=True, + state_before_wait=True, + smem_tiles=True, + spec_hist=False, + qk_first=True, + use_wi=False, + evict_first=True, + early_acc=False, + h=16, + hv=48, + fold=False, + norm=False, + norm_eps=1e-6, + ): + assert 1 <= T <= 8 + # the gated RMS norm folded in: a CTA holds one value head's 128 outputs of its T rows + self.norm = bool(norm) + self.norm_eps = float(norm_eps) + assert not self.norm or (tpc == 8 and T <= warps), "the folded norm: one value head a CTA" + assert (h, hv) in GEOMETRIES, (h, hv) + # the verify conv folded in: a CTA computes its head's q / k and its value heads' v rows + # from the projection, the conv pool and the taps; a value head belongs to one CTA + assert not fold or tpc in (8, 24), "the folded conv needs whole value heads a CTA" + self.fold = bool(fold) + if self.fold: + qk_first = False + self.H, self.HV = h, hv + assert tpc in (24, 12, 8, 4, 2, 1) and tpc % warps == 0 + self.T = T + self.TP = 4 if T <= 4 else 8 + self.NT = self.TP // 4 + self.tpc, self.W = tpc, warps + self.tpw = tpc // warps + self.S = TILES // tpc + self.nhv = 3 if tpc == 24 else (2 if tpc == 12 else 1) + self.commit, self.pdl = commit, pdl + # pf = 0: every state tile by bulk copy as soon as the slot is known, none in registers + self.ld, self.pf, self.minb = ld, max(0 if smem_tiles else 1, min(pf, self.tpw)), minb + # the tiles' state dots before the first barrier, while the history is in flight + self.use_early = bool(early_acc) + self.l2norm, self.gating, self.g_f32, self.b_f32 = l2norm, gating, g_f32, b_f32 + self.early_trigger, self.lo_split = early_trigger, lo_split + self.state_before_wait = state_before_wait + # tiles past the first `pf` of a warp: bulk-copied into shared memory at the start + self.nsm = self.tpw - self.pf if smem_tiles else 0 + self.spec_hist = spec_hist + self.qk_first = qk_first + # (row, slot, pending count, half) per item from the engine's replay work items + self.use_wi = use_wi + self.evict_first = evict_first + nset = 2 if self.spec_hist else 1 + self.nset = nset + self.RP = (2 * self.TP + 4) * 4 # per-warp scratch row pitch (bytes) + o = 0 + self.o_st = o + o += warps * self.nsm * 4096 + self.o_mb = o + o += -(-(warps * self.nsm * 8) // 16) * 16 + self.o_u = o + o += nset * self.nhv * 16 * PU + self.o_kh = o + o += nset * 16 * PK + self.o_kp = o + o += 16 * PU if commit else 0 + self.o_v = o + o += self.nhv * T * 256 + self.o_x = o + o += warps * 16 * self.RP + self.o_kk = o + o += 16 * 8 * 4 + self.o_cf = o + o += self.nhv * 16 * 16 * 2 + self.o_gh = o + o += nset * self.nhv * 16 * 4 + self.o_dec = o + o += self.nhv * 16 * 4 + self.o_g = o + o += self.nhv * 8 * 4 + self.o_eg = o + o += self.nhv * 8 * 4 + self.o_beta = o + o += self.nhv * 8 * 4 + self.o_a = o + o += self.nhv * 64 * 4 + self.o_cq = o + o += self.nhv * 64 * 4 + self.o_egs = o + o += 16 + # the folded conv's q | k rows: T x 256 bf16 + self.o_cv = o + o += T * 512 if self.fold else 0 + # the folded norm's rows: T x 128 bf16 (the outputs, rounded as they would be stored) + self.o_on = o + o += T * 256 if self.norm else 0 + self.smem = -(-o // 128) * 128 + 128 + + @cute.jit + def __call__( + self, + sidx_t: cute.Tensor, + q_ptr: cutlass.Int64, + k_ptr: cutlass.Int64, + v_ptr: cutlass.Int64, + q_sn: cutlass.Int64, + q_st: cutlass.Int64, + k_sn: cutlass.Int64, + k_st: cutlass.Int64, + v_sn: cutlass.Int64, + v_st: cutlass.Int64, + g_ptr: cutlass.Int64, + g_st: cutlass.Int64, + b_ptr: cutlass.Int64, + b_st: cutlass.Int64, + alog_ptr: cutlass.Int64, + dtb_ptr: cutlass.Int64, + o_ptr: cutlass.Int64, + h_ptr: cutlass.Int64, + h_ss: cutlass.Int64, + pnat_ptr: cutlass.Int64, + buf_ptr: cutlass.Int64, + ou_ptr: cutlass.Int64, + ok_ptr: cutlass.Int64, + og_ptr: cutlass.Int64, + wi_ptr: cutlass.Int64, + cx_ptr: cutlass.Int64, + cx_st: cutlass.Int64, + cw_ptr: cutlass.Int64, + cs_ptr: cutlass.Int64, + cs_ss: cutlass.Int64, + win_ptr: cutlass.Int64, + win_sn: cutlass.Int64, + win_st: cutlass.Int64, + wrow_ptr: cutlass.Int64, + zn_ptr: cutlass.Int64, + zn_st: cutlass.Int64, + gw_ptr: cutlass.Int64, + on_ptr: cutlass.Int64, + n_rows: cutlass.Int32, + scale: cutlass.Float32, + stream, + ): + self.kernel( + sidx_t.iterator.toint(), + q_ptr, + k_ptr, + v_ptr, + q_sn, + q_st, + k_sn, + k_st, + v_sn, + v_st, + g_ptr, + g_st, + b_ptr, + b_st, + alog_ptr, + dtb_ptr, + o_ptr, + h_ptr, + h_ss, + pnat_ptr, + buf_ptr, + ou_ptr, + ok_ptr, + og_ptr, + wi_ptr, + cx_ptr, + cx_st, + cw_ptr, + cs_ptr, + cs_ss, + win_ptr, + win_sn, + win_st, + wrow_ptr, + zn_ptr, + zn_st, + gw_ptr, + on_ptr, + scale, + ).launch( + grid=[n_rows * (self.H * self.S), 1, 1], + block=[32 * self.W, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=self.minb, + ) + + # -- straight-line helpers (traced inline) ------------------------------------------------ + + def _tile_addr(self, h_ptr, slot64, h_ss, h, sp, warp, i, g, c): + mt = sp * self.tpc + warp + self.W * i + hv = h * GRP + mt // 8 + v0 = (mt % 8) * 16 + base = h_ptr + (slot64 * h_ss + cutlass.Int64(hv * V + v0) * K) * 2 + return base + cutlass.Int64(g * 256 + c * 16) + + def _qk_raw(self, nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, live): + """Raw bf16 words of key / query column nt * 8 + g at this lane's head dims, the + column's kind and its scale.""" + T, TP = self.T, self.TP + zi = cutlass.Int32(0) + t = g % 4 if TP == 4 else g + valid = (t < T) & live + tc = cutlass.min(t, T - 1) + qa = q_ptr + (n64 * q_sn + cutlass.Int64(tc) * q_st + cutlass.Int64(h * K + c * 8)) * 2 + ka = k_ptr + (n64 * k_sn + cutlass.Int64(tc) * k_st + cutlass.Int64(h * K + c * 8)) * 2 + if TP == 4: + is_q = g >= 4 + src = cutlass.Int64(cutlass.select_(is_q, qa, ka)) + mult = cutlass.Float32(cutlass.select_(is_q, scale, cutlass.Float32(1.0))) + elif nt == 1: + is_q = g >= 0 + src = qa + mult = scale + else: + is_q = g < 0 + src = ka + mult = cutlass.Float32(1.0) + vpred = cutlass.Int32(cutlass.select_(valid, cutlass.Int32(1), zi)) + raw = [] + for jj in range(4): + raw.extend(_ldg4p(src + jj * 64, vpred)) + return raw, is_q, mult + + def _qk_raw_smem(self, nt, s_cv, g, c, scale): + """`_qk_raw` over the folded conv's shared q | k rows (the kernel's fold prologue).""" + T, TP = self.T, self.TP + zi = cutlass.Int32(0) + t = g % 4 if TP == 4 else g + valid = t < T + tc = cutlass.min(t, T - 1) + if TP == 4: + is_q = g >= 4 + elif nt == 1: + is_q = g >= 0 + else: + is_q = g < 0 + mult = cutlass.Float32(cutlass.select_(is_q, scale, cutlass.Float32(1.0))) + src = ( + s_cv + tc * 512 + cutlass.Int32(cutlass.select_(is_q, zi, cutlass.Int32(256))) + c * 16 + ) + raw = [] + for jj in range(4): + w4 = _lds4_i32(src + jj * 64) + for e in range(4): + raw.append(cutlass.Int32(cutlass.select_(valid, w4[e], zi))) + return raw, is_q, mult + + def _qk_frag(self, raw, is_q, mult): + """L2-normalized (queries also scaled) bf16 B fragment from `_qk_raw`.""" + ss = cutlass.Float32(0.0) + fl = [] + for e in range(16): + lo = _bf2_lo_v(raw[e]) + hi = _bf2_hi_v(raw[e]) + fl.append(lo) + fl.append(hi) + ss = ss + lo * lo + hi * hi + if self.l2norm: + ss = ss + _shfl_xor(ss, 1) + ss = ss + _shfl_xor(ss, 2) + inv = _div_rn(mult, _sqrt_rn(ss + 1e-6)) + return [_bf2_pack(fl[2 * e] * inv, fl[2 * e + 1] * inv) for e in range(16)] + return [ + cutlass.Int32( + cutlass.select_(is_q, _bf2_pack(fl[2 * e] * mult, fl[2 * e + 1] * mult), raw[e]) + ) + for e in range(16) + ] + + def _gate_raw(self, pred, hv, n64, g_ptr, g_st, b_ptr, b_st, alog_ptr, dtb_ptr): + """Raw g / beta of the T tokens for v head `hv` and its A_log / dt_bias (predicated).""" + T = self.T + zf = cutlass.Float32(0.0) + pi = cutlass.Int32(cutlass.select_(pred, cutlass.Int32(1), cutlass.Int32(0))) + alv, dtb = zf, zf + if self.gating: + alv = _ldg_f32p(alog_ptr + cutlass.Int64(hv) * 4, pi) + dtb = _ldg_f32p(dtb_ptr + cutlass.Int64(hv) * 4, pi) + graws, braws = [], [] + for t in range(T): + if self.g_f32: + graws.append( + _ldg_f32p(g_ptr + (n64 * T + t) * g_st * 4 + cutlass.Int64(hv) * 4, pi) + ) + else: + graws.append( + _ldg_b16p(g_ptr + (n64 * T + t) * g_st * 2 + cutlass.Int64(hv) * 2, pi) + ) + if self.b_f32: + braws.append( + _ldg_f32p(b_ptr + (n64 * T + t) * b_st * 4 + cutlass.Int64(hv) * 4, pi) + ) + else: + braws.append( + _ldg_b16p(b_ptr + (n64 * T + t) * b_st * 2 + cutlass.Int64(hv) * 2, pi) + ) + return graws, braws, alv, dtb + + def _smem_tile(self, s_st, s_mb, i, g, c, zi): + """This lane's A fragment words of staged tile i: rows g and g + 8, chunk order by row + parity so each quarter-warp's two rows are on disjoint banks.""" + kslot = i - self.pf + _mbar_wait(s_mb + kslot * 8, zi) + tb = s_st + kslot * 4096 + g * 256 + c * 16 + godd = (g % 2) == 1 + raw = [] + for ii in range(4): + jsw = (g % 2) ^ ii + raw.append(_lds4_i32(tb + jsw * 64)) + raw.append(_lds4_i32(tb + 2048 + jsw * 64)) + cur = [] + for jj in range(4): + for rr in range(2): + for e in range(4): + cur.append( + cutlass.Int32( + cutlass.select_(godd, raw[2 * (jj ^ 1) + rr][e], raw[2 * jj + rr][e]) + ) + ) + return cur + + def _tile_dots(self, cur, bq, zf): + """D = h0 [16 x 128] times the T keys / scaled queries, two MMA chains (even / odd k-steps).""" + NT = self.NT + acc0 = [[zf, zf, zf, zf] for _ in range(NT)] + acc1 = [[zf, zf, zf, zf] for _ in range(NT)] + for jj in range(4): + for s in range(2): + a = ( + cur[8 * jj + 2 * s], + cur[8 * jj + 4 + 2 * s], + cur[8 * jj + 2 * s + 1], + cur[8 * jj + 4 + 2 * s + 1], + ) + for nt in range(NT): + bfr = (bq[nt][4 * jj + 2 * s], bq[nt][4 * jj + 2 * s + 1]) + if s == 0: + acc0[nt] = _mma(a, bfr, acc0[nt]) + else: + acc1[nt] = _mma(a, bfr, acc1[nt]) + return [[acc0[nt][e] + acc1[nt][e] for e in range(4)] for nt in range(NT)] + + def _g_f32(self, x): + return x if self.g_f32 else _b16_f32(x) + + def _b_f32(self, x): + return x if self.b_f32 else _b16_f32(x) + + def _load_tile(self, addr, pol=None): + regs = [] + for jj in range(4): + for off in (jj * 64, 2048 + jj * 64): + if pol is not None: + regs.extend(_ldg4_hint(addr + off, pol)) + else: + regs.extend(_LDG4[self.ld](addr + off)) + return regs + + @cute.kernel + def kernel( + self, + sidx_ptr: cutlass.Int64, + q_ptr: cutlass.Int64, + k_ptr: cutlass.Int64, + v_ptr: cutlass.Int64, + q_sn: cutlass.Int64, + q_st: cutlass.Int64, + k_sn: cutlass.Int64, + k_st: cutlass.Int64, + v_sn: cutlass.Int64, + v_st: cutlass.Int64, + g_ptr: cutlass.Int64, + g_st: cutlass.Int64, + b_ptr: cutlass.Int64, + b_st: cutlass.Int64, + alog_ptr: cutlass.Int64, + dtb_ptr: cutlass.Int64, + o_ptr: cutlass.Int64, + h_ptr: cutlass.Int64, + h_ss: cutlass.Int64, + pnat_ptr: cutlass.Int64, + buf_ptr: cutlass.Int64, + ou_ptr: cutlass.Int64, + ok_ptr: cutlass.Int64, + og_ptr: cutlass.Int64, + wi_ptr: cutlass.Int64, + cx_ptr: cutlass.Int64, + cx_st: cutlass.Int64, + cw_ptr: cutlass.Int64, + cs_ptr: cutlass.Int64, + cs_ss: cutlass.Int64, + win_ptr: cutlass.Int64, + win_sn: cutlass.Int64, + win_st: cutlass.Int64, + wrow_ptr: cutlass.Int64, + zn_ptr: cutlass.Int64, + zn_st: cutlass.Int64, + gw_ptr: cutlass.Int64, + on_ptr: cutlass.Int64, + scale: cutlass.Float32, + ): + T: cutlass.Constexpr = self.T + TP: cutlass.Constexpr = self.TP + NT: cutlass.Constexpr = self.NT + W: cutlass.Constexpr = self.W + TPW: cutlass.Constexpr = self.tpw + TPC: cutlass.Constexpr = self.tpc + NHV: cutlass.Constexpr = self.nhv + S: cutlass.Constexpr = self.S + NTHR: cutlass.Constexpr = 32 * self.W + RP: cutlass.Constexpr = self.RP + H: cutlass.Constexpr = self.H + HV: cutlass.Constexpr = self.HV + + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = tidx // 32 + lane = tidx % 32 + g = lane // 4 + c = lane % 4 + sp = bidx % S + nh = bidx // S + h = nh % H + if const_expr(self.use_wi): + wi = _LDG4["plain"](wi_ptr + cutlass.Int64(nh // H) * 16) + n = wi[0] + else: + n = nh // H + n64 = cutlass.Int64(n) + zf = cutlass.Float32(0.0) + zi = cutlass.Int32(0) + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_u = sbase + self.o_u + s_kh = sbase + self.o_kh + s_kp = sbase + self.o_kp + s_v = sbase + self.o_v + s_x = sbase + self.o_x + warp * (16 * RP) + s_kk = sbase + self.o_kk + s_cf = sbase + self.o_cf + s_gh = sbase + self.o_gh + s_dec = sbase + self.o_dec + s_g = sbase + self.o_g + s_eg = sbase + self.o_eg + s_beta = sbase + self.o_beta + s_a = sbase + self.o_a + s_cq = sbase + self.o_cq + s_egs = sbase + self.o_egs + s_cv = sbase + self.o_cv + s_on = sbase + self.o_on + + if const_expr(self.pdl and not self.state_before_wait and not self.qk_first): + cute.arch.griddepcontrol_wait() + + NSM: cutlass.Constexpr = self.nsm + s_st = sbase + self.o_st + warp * (NSM * 4096) + s_mb = sbase + self.o_mb + warp * (NSM * 8) + if const_expr(NSM > 0): + if lane == 0: + for k_ in cutlass.range_constexpr(NSM): + _mbar_init(s_mb + k_ * 8, cutlass.Int32(1)) + _fence_mbar_init() + + hv_first = (sp * TPC) // 8 + hv0 = h * GRP + hv_first + gi = tidx - (32 * (W - 1) if W > 1 else 0) + gpred = (gi >= 0) & (gi < NHV) + if const_expr(self.qk_first): + # the step's own (L2-resident) inputs first, ahead of the state stream + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + if const_expr(self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + pre = [ + self._qk_raw( + nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, tidx >= 0 + ) + for nt in range(NT) + ] + gpre = self._gate_raw( + gpred, + hv0 + cutlass.min(cutlass.max(gi, zi), NHV - 1), + n64, + g_ptr, + g_st, + b_ptr, + b_st, + alog_ptr, + dtb_ptr, + ) + n_v0: cutlass.Constexpr = NHV * T * 16 + for it0 in cutlass.range_constexpr(-(-n_v0 // NTHR)): + ev0 = tidx + it0 * NTHR + if ev0 < n_v0: + hl0 = ev0 // (T * 16) + tv0 = (ev0 // 16) % T + qv0 = ev0 % 16 + sv0 = ( + v_ptr + + ( + n64 * v_sn + + cutlass.Int64(tv0) * v_st + + cutlass.Int64((hv0 + hl0) * V + qv0 * 8) + ) + * 2 + ) + _cp16z(s_v + (hl0 * T + tv0) * 256 + qv0 * 16, sv0, cutlass.Int32(16)) + _cp_commit() + + if const_expr(self.use_wi): + slot = wi[1] + else: + slot = _ldg_i32(sidx_ptr + n64 * 4) + if slot >= 0: + slot64 = cutlass.Int64(slot) + # ---- 1. the state tiles: bulk copies first, then the register tile ----------------- + pol = _policy_evict_first() if const_expr(self.evict_first) else None + if const_expr(NSM > 0): + if lane == 0: + for k_ in cutlass.range_constexpr(NSM): + if const_expr(self.evict_first): + _bulk_g2s_hint( + s_st + k_ * 4096, + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, self.pf + k_, 0, 0 + ), + cutlass.Int32(4096), + s_mb + k_ * 8, + pol, + ) + else: + _bulk_g2s( + s_st + k_ * 4096, + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, self.pf + k_, 0, 0 + ), + cutlass.Int32(4096), + s_mb + k_ * 8, + ) + tiles = [] + for i in cutlass.range_constexpr(self.pf): + tiles.append( + self._load_tile(self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i, g, c), pol) + ) + + # ---- 2. history bookkeeping and the history rows into shared memory ------------ + if const_expr(self.use_wi): + p = wi[2] + rb = wi[3] + else: + p = _ldg_i32(pnat_ptr + slot64 * 4) + rb = _ldg_i32(buf_ptr + slot64 * 4) + wrap = (p + T) > HIST + wb = cutlass.Int32(cutlass.select_(wrap, 1 - rb, rb)) + woff = cutlass.Int32(cutlass.select_(wrap, zi, p)) + rslot = slot64 * 2 + cutlass.Int64(rb) + g_rd = og_ptr + rslot * (HV * HIST * 4) + wslot = slot64 * 2 + cutlass.Int64(wb) + u_rd = ou_ptr + rslot * (HIST * HV * V * 2) + k_rd = ok_ptr + rslot * (HIST * H * K * 2) + u_wr = ou_ptr + wslot * (HIST * HV * V * 2) + k_wr = ok_ptr + wslot * (HIST * H * K * 2) + g_wr = og_ptr + wslot * (HV * HIST * 4) + + n_u: cutlass.Constexpr = NHV * 256 + if const_expr(self.spec_hist): + # both halves, every row, before p / half are known; rows j >= p masked at use + for hb in cutlass.range_constexpr(2): + hslot = slot64 * 2 + hb + u_h = ou_ptr + hslot * (HIST * HV * V * 2) + k_h = ok_ptr + hslot * (HIST * H * K * 2) + g_h = og_ptr + hslot * (HV * HIST * 4) + for it in cutlass.range_constexpr(-(-n_u // NTHR)): + e = tidx + it * NTHR + if e < n_u: + hl = e // 256 + j = (e // 16) % 16 + qq = e % 16 + src = u_h + cutlass.Int64(((j * HV + hv0 + hl) * V + qq * 8) * 2) + _cp16z( + s_u + hb * (NHV * 16 * PU) + hl * (16 * PU) + j * PU + qq * 16, + src, + cutlass.Int32(16), + ) + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + qq = e % 16 + src = k_h + cutlass.Int64(((j * H + h) * K + qq * 8) * 2) + _cp16z(s_kh + hb * (16 * PK) + j * PK + qq * 16, src, cutlass.Int32(16)) + if tidx < NHV * 4: + hl = tidx // 4 + qq = tidx % 4 + src = g_h + cutlass.Int64(((hv0 + hl) * HIST + qq * 4) * 4) + _cp16z(s_gh + hb * (NHV * 64) + hl * 64 + qq * 16, src, cutlass.Int32(16)) + else: + for it in cutlass.range_constexpr(-(-n_u // NTHR)): + e = tidx + it * NTHR + if e < n_u: + hl = e // 256 + j = (e // 16) % 16 + qq = e % 16 + jc = cutlass.min( + j, cutlass.max(p - 1, zi) + ) # an in-bounds row for a zero fill + src = u_rd + cutlass.Int64(((jc * HV + hv0 + hl) * V + qq * 8) * 2) + _cp16z( + s_u + hl * (16 * PU) + j * PU + qq * 16, + src, + cutlass.Int32(cutlass.select_(j < p, cutlass.Int32(16), zi)), + ) + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + qq = e % 16 + jc = cutlass.min(j, cutlass.max(p - 1, zi)) + src = k_rd + cutlass.Int64(((jc * H + h) * K + qq * 8) * 2) + _cp16z( + s_kh + j * PK + qq * 16, + src, + cutlass.Int32(cutlass.select_(j < p, cutlass.Int32(16), zi)), + ) + if tidx < NHV * 4: + hl = tidx // 4 + qq = tidx % 4 + src = g_rd + cutlass.Int64(((hv0 + hl) * HIST + qq * 4) * 4) + _cp16z(s_gh + hl * 64 + qq * 16, src, cutlass.Int32(16)) + _cp_commit() + + if const_expr(self.spec_hist): + s_u = s_u + rb * (NHV * 16 * PU) + s_kh = s_kh + rb * (16 * PK) + s_gh = s_gh + rb * (NHV * 64) + + # ---- 3. the step's own inputs (written by the previous kernel) ------------------ + if const_expr(not self.qk_first): + if const_expr(self.pdl and self.state_before_wait): + cute.arch.griddepcontrol_wait() + if const_expr(self.pdl and self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + if const_expr(self.fold): + # The verify conv of this CTA's channels for the T rows, a thread an 8-channel + # group. The owning CTA writes the per-row windows (q/k: split 0); the pool is + # not advanced (the engine's acceptance promote writes it). + ngr: cutlass.Constexpr = 32 + NHV * 16 + wrow = cutlass.Int64(_ldg_i32(wrow_ptr + n64 * 4)) + for itc in cutlass.range_constexpr(-(-ngr // NTHR)): + gc = tidx + itc * NTHR + if gc < ngr: + ec = gc * 8 + is_v = ec >= 256 + vv = ec - 256 + hlc = vv // 128 + vc = vv - hlc * 128 + ch = cutlass.Int32( + cutlass.select_( + is_v, + 2 * H * K + (hv0 + hlc) * V + vc, + cutlass.select_( + ec < 128, h * K + ec, H * K + h * K + (ec - 128) + ), + ) + ) + dst = cutlass.Int32( + cutlass.select_(is_v, s_v + (hlc * T) * 256 + vc * 2, s_cv + ec * 2) + ) + pitch = cutlass.Int32( + cutlass.select_(is_v, cutlass.Int32(256), cutlass.Int32(512)) + ) + owns = is_v | (sp == 0) + ch64 = cutlass.Int64(ch) + wa = cw_ptr + ch64 * 8 + ww = [] + for q4 in cutlass.range_constexpr(4): + ww.extend(_LDG4["plain"](wa + q4 * 16)) + pa = cs_ptr + (slot64 * cs_ss + ch64 * 3) * 2 + pw = [] + for q4 in cutlass.range_constexpr(3): + pw.extend(_LDG4["plain"](pa + q4 * 16)) + xw = [] + for tt in cutlass.range_constexpr(T): + xw.append( + _LDG4["plain"](cx_ptr + ((n64 * T + tt) * cx_st + ch64) * 2) + ) + # per channel: taps = pool[0..2], x rows; weights w[0..3] + outs = [[] for _ in range(T)] + wins = [[] for _ in range(T)] + for cl in cutlass.range_constexpr(8): + wt = [ + _half16(ww[(cl * 4 + j) // 2], (cl * 4 + j) % 2 == 1) + for j in range(4) + ] + taps = [ + _half16(pw[(cl * 3 + j) // 2], (cl * 3 + j) % 2 == 1) + for j in range(3) + ] + for tt in cutlass.range_constexpr(T): + taps.append(_half16(xw[tt][cl // 2], cl % 2 == 1)) + for tt in cutlass.range_constexpr(T): + outs[tt].append( + _conv4_silu( + _mul_bf16(taps[tt], wt[0]), + _mul_bf16(taps[tt + 1], wt[1]), + _mul_bf16(taps[tt + 2], wt[2]), + _mul_bf16(taps[tt + 3], wt[3]), + ) + ) + for j in cutlass.range_constexpr(3): + wins[tt].append(taps[tt + 1 + j]) + for tt in cutlass.range_constexpr(T): + o = outs[tt] + _sts4( + dst + tt * pitch, + _pack16(o[0], o[1]), + _pack16(o[2], o[3]), + _pack16(o[4], o[5]), + _pack16(o[6], o[7]), + ) + if owns: + wv = wins[tt] + wbase = ( + win_ptr + + (wrow * win_sn + cutlass.Int64(tt) * win_st + ch64 * 3) + * 2 + ) + for q4 in cutlass.range_constexpr(3): + _stg4( + wbase + q4 * 16, + _pack16(wv[q4 * 8], wv[q4 * 8 + 1]), + _pack16(wv[q4 * 8 + 2], wv[q4 * 8 + 3]), + _pack16(wv[q4 * 8 + 4], wv[q4 * 8 + 5]), + _pack16(wv[q4 * 8 + 6], wv[q4 * 8 + 7]), + ) + cute.arch.sync_threads() + n_v: cutlass.Constexpr = 0 if self.fold else NHV * T * 16 + for it in cutlass.range_constexpr(-(-n_v // NTHR)): + e = tidx + it * NTHR + if e < n_v: + hl = e // (T * 16) + t = (e // 16) % T + qq = e % 16 + src = ( + v_ptr + + ( + n64 * v_sn + + cutlass.Int64(t) * v_st + + cutlass.Int64((hv0 + hl) * V + qq * 8) + ) + * 2 + ) + _cp16z(s_v + (hl * T + t) * 256 + qq * 16, src, cutlass.Int32(16)) + _cp_commit() + + # key / query B fragments: column nt * 8 + g; lane quad c holds dims 32 jj + 8 c + 0..7 + bq = [] + for nt in cutlass.range_constexpr(NT): + if const_expr(self.qk_first): + rq = pre[nt] + elif const_expr(self.fold): + rq = self._qk_raw_smem(nt, s_cv, g, c, scale) + else: + rq = self._qk_raw( + nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, tidx >= 0 + ) + bq.append(self._qk_frag(rq[0], rq[1], rq[2])) + + # the new keys into the history (one CTA of the head, warp 0) + if sp == 0: + if warp == 0: + # column g < T of tile 0 is key g (both column layouts) + if g < T: + kfr = bq[0] + dst = k_wr + cutlass.Int64((((woff + g) * H + h) * K + c * 8) * 2) + for jj in cutlass.range_constexpr(4): + _stg4( + dst + jj * 64, + kfr[4 * jj], + kfr[4 * jj + 1], + kfr[4 * jj + 2], + kfr[4 * jj + 3], + ) + + # gating, cumulative decays, the G history (a thread per v head, off warp 0) + if gpred: + hl = gi + hv = hv0 + hl + gs = _ldg_f32p( + g_rd + cutlass.Int64(hv * HIST + p - 1) * 4, + cutlass.Int32(cutlass.select_(p > 0, cutlass.Int32(1), zi)), + ) + if const_expr(self.qk_first): + graws, braws, alv, dtb = gpre + else: + graws, braws, alv, dtb = self._gate_raw( + gpred, hv, n64, g_ptr, g_st, b_ptr, b_st, alog_ptr, dtb_ptr + ) + aexp = _exp(alv) + owns = (hv_first + hl) * 8 >= sp * TPC + gl = zf + for t in cutlass.range_constexpr(T): + graw = self._g_f32(graws[t]) + braw = self._b_f32(braws[t]) + if const_expr(self.gating): + x = graw + dtb + spl = cutlass.Float32( + cutlass.select_(x <= 20.0, LN2 * _lg2(1.0 + _ex2(x * LOG2E)), x) + ) + gt = -aexp * spl + beta = _div_approx(cutlass.Float32(1.0), 1.0 + _ex2(-braw * LOG2E)) + else: + gt = graw + beta = braw + gl = gl + gt + gabs = gs + gl + _sts_f32(s_g + (hl * 8 + t) * 4, gabs) + _sts_f32(s_eg + (hl * 8 + t) * 4, _exp(gabs)) + _sts_f32(s_beta + (hl * 8 + t) * 4, beta) + if owns: + _stg_f32( + g_wr + cutlass.Int64((hv * HIST + woff + t) * 4), + cutlass.Float32(cutlass.select_(wrap, gl, gabs)), + ) + _sts_f32(s_egs + hl * 4, _exp(gs)) + + # new-token dot products: rows (keys | queries) x key columns + if warp == 0: + d = [zf, zf, zf, zf] + for jj in cutlass.range_constexpr(4): + for s in cutlass.range_constexpr(2): + b0 = bq[0][4 * jj + 2 * s] + b1 = bq[0][4 * jj + 2 * s + 1] + if const_expr(NT == 1): + a = (b0, zi, b1, zi) + else: + a = (b0, bq[1][4 * jj + 2 * s], b1, bq[1][4 * jj + 2 * s + 1]) + d = _mma(a, (b0, b1), d) + _sts2_f32(s_kk + (g * 8 + 2 * c) * 4, d[0], d[1]) + _sts2_f32(s_kk + ((g + 8) * 8 + 2 * c) * 4, d[2], d[3]) + + # the state dots need no history: run them while it is in flight + early = [] + if const_expr(self.use_early): + for i in cutlass.range_constexpr(TPW): + if const_expr(NSM == 0): + if const_expr(i + self.pf < TPW): + tiles.append( + self._load_tile( + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, i + self.pf, g, c + ) + ) + ) + if const_expr(i < self.pf or NSM == 0): + early.append(self._tile_dots(tiles[i], bq, zf)) + else: + early.append( + self._tile_dots(self._smem_tile(s_st, s_mb, i, g, c, zi), bq, zf) + ) + + _cp_wait_all() + cute.arch.sync_threads() + + # ---- 4. history-key dots, commit operand, per-head scalars ---------------------- + hk_warp: cutlass.Constexpr = 1 if W > 1 else 0 + if warp == hk_warp: + dh = [[zf, zf, zf, zf] for _ in range(NT)] + for jj in cutlass.range_constexpr(4): + rg = _lds4_i32(s_kh + g * PK + jj * 64 + c * 16) + rh = _lds4_i32(s_kh + (g + 8) * PK + jj * 64 + c * 16) + for s in cutlass.range_constexpr(2): + a = (rg[2 * s], rh[2 * s], rg[2 * s + 1], rh[2 * s + 1]) + for nt in cutlass.range_constexpr(NT): + dh[nt] = _mma( + a, (bq[nt][4 * jj + 2 * s], bq[nt][4 * jj + 2 * s + 1]), dh[nt] + ) + # history coefficients (k_t . k_j or q_t . k_j) exp(G_t - G_j), j < p, bf16 B fragments + for hl in cutlass.range_constexpr(NHV): + gh_g = _lds_f32(s_gh + (hl * 16 + g) * 4) + gh_h = _lds_f32(s_gh + (hl * 16 + g + 8) * 4) + for nt in cutlass.range_constexpr(NT): + for e in cutlass.range_constexpr(2): + col = nt * 8 + 2 * c + e + if const_expr(TP == 4): + tt = (2 * c + e) % 4 + else: + tt = 2 * c + e + tv = tt < T + gtt = _lds_f32(s_g + (hl * 8 + tt) * 4) + vg = cutlass.Float32( + cutlass.select_((g < p) & tv, dh[nt][e] * _exp(gtt - gh_g), zf) + ) + vh = cutlass.Float32( + cutlass.select_( + ((g + 8) < p) & tv, dh[nt][2 + e] * _exp(gtt - gh_h), zf + ) + ) + _sts_bf16(s_cf + ((hl * 16 + col) * 16 + g) * 2, vg) + _sts_bf16(s_cf + ((hl * 16 + col) * 16 + g + 8) * 2, vh) + + if const_expr(self.commit): + if wrap: + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + jj = (e // 4) % 4 + aa = e % 4 + w4 = _lds4_i32(s_kh + j * PK + jj * 64 + aa * 16) + for s4 in cutlass.range_constexpr(4): + wv = w4[s4] + if const_expr(self.spec_hist): + wv = cutlass.Int32(cutlass.select_(j < p, wv, zi)) + _sts_i32(s_kp + j * PU + (16 * jj + 4 * s4 + aa) * 4, wv) + + n_ts: cutlass.Constexpr = NHV * T * T + for it in cutlass.range_constexpr(-(-n_ts // NTHR)): + e = tidx + it * NTHR + if e < n_ts: + hl = e // (T * T) + t = (e // T) % T + s = e % T + gt = _lds_f32(s_g + (hl * 8 + t) * 4) + gsv = _lds_f32(s_g + (hl * 8 + s) * 4) + bt = _lds_f32(s_beta + (hl * 8 + t) * 4) + dec = _exp(gt - gsv) + kk = _lds_f32(s_kk + (t * 8 + s) * 4) + qk = _lds_f32(s_kk + ((TP + t) * 8 + s) * 4) + av = cutlass.Float32(cutlass.select_(s < t, bt * kk * dec, zf)) + cv = cutlass.Float32(cutlass.select_(s <= t, qk * dec, zf)) + _sts_f32(s_a + (hl * 64 + t * 8 + s) * 4, av) + _sts_f32(s_cq + (hl * 64 + t * 8 + s) * 4, cv) + if const_expr(self.commit): + if tidx < NHV * 16: + hl = tidx // 16 + j = tidx % 16 + gsv = cutlass.Float32(0.0) + if p > 0: + gsv = _lds_f32(s_gh + (hl * 16 + p - 1) * 4) + ghj = _lds_f32(s_gh + (hl * 16 + j) * 4) + _sts_f32( + s_dec + (hl * 16 + j) * 4, + cutlass.Float32(cutlass.select_(j < p, _exp(gsv - ghj), zf)), + ) + cute.arch.sync_threads() + + # the folded norm's gate and weight words (after any dependency wait), landing during the tiles + if const_expr(self.norm): + nz0 = zi + nz1 = zi + nw0 = zi + nw1 = zi + if warp < T: + hvp = h * GRP + sp + rowp = n64 * T + cutlass.Int64(warp) + colp = cutlass.Int64(4 * lane) + zpre = _ldg2_i32(zn_ptr + (rowp * zn_st + cutlass.Int64(hvp * V) + colp) * 2) + wpre = _ldg2_i32(gw_ptr + colp * 2) + nz0 = zpre[0] + nz1 = zpre[1] + nw0 = wpre[0] + nw1 = wpre[1] + + # ---- 5. the warp's m-tiles ------------------------------------------------------ + jrow = (lane % 8) + 8 * (lane // 16) + v8 = 8 * ((lane // 8) % 2) + r = lane % 16 + hf = lane // 16 + godd = (g % 2) == 1 + mask01 = cutlass.Int32( + cutlass.select_(2 * c < p, cutlass.Int32(0xFFFF), zi) + ) | cutlass.Int32(cutlass.select_(2 * c + 1 < p, cutlass.Int32(-65536), zi)) + mask89 = cutlass.Int32( + cutlass.select_(2 * c + 8 < p, cutlass.Int32(0xFFFF), zi) + ) | cutlass.Int32(cutlass.select_(2 * c + 9 < p, cutlass.Int32(-65536), zi)) + for i in cutlass.range_constexpr(TPW): + if const_expr(NSM == 0 and not self.use_early): + if const_expr(i + self.pf < TPW): + tiles.append( + self._load_tile( + self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i + self.pf, g, c) + ) + ) + if const_expr(i < self.pf or NSM == 0): + cur = tiles[i] + elif const_expr(self.use_early): + # a staged tile is read again from shared memory only for its commit + cur = None + else: + # rows g and g + 8 of the staged tile, chunk order by row parity (bank-disjoint) + kslot = i - self.pf + _mbar_wait(s_mb + kslot * 8, zi) + tb = s_st + kslot * 4096 + g * 256 + c * 16 + raw = [] + for ii in cutlass.range_constexpr(4): + jsw = (g % 2) ^ ii + raw.append(_lds4_i32(tb + jsw * 64)) + raw.append(_lds4_i32(tb + 2048 + jsw * 64)) + cur = [] + for jj in cutlass.range_constexpr(4): + for rr in cutlass.range_constexpr(2): + for e in cutlass.range_constexpr(4): + cur.append( + cutlass.Int32( + cutlass.select_( + godd, raw[2 * (jj ^ 1) + rr][e], raw[2 * jj + rr][e] + ) + ) + ) + mt = sp * TPC + warp + W * i + hvr = mt // 8 + hv = h * GRP + hvr + hl = hvr - hv_first + v0 = (mt % 8) * 16 + + if const_expr(self.use_early): + acc = early[i] + else: + acc = self._tile_dots(cur, bq, zf) + + au = _ldsm4t(s_u + hl * (16 * PU) + jrow * PU + (v0 + v8) * 2) + if const_expr(self.spec_hist): + au = [cutlass.Int32(au[q4] & (mask01 if q4 < 2 else mask89)) for q4 in range(4)] + hacc = [] + for nt in cutlass.range_constexpr(NT): + cfa = s_cf + ((hl * 16 + nt * 8 + g) * 16 + 2 * c) * 2 + bc = (_lds_i32(cfa), _lds_i32(cfa + 16)) + hacc.append(_mma(au, bc, [zf, zf, zf, zf])) + + for nt in cutlass.range_constexpr(NT): + col0 = nt * 8 + 2 * c + if const_expr(TP == 4): + t0 = (2 * c) % 4 + else: + t0 = 2 * c + eg0 = _lds_f32(s_eg + (hl * 8 + t0) * 4) + eg1 = _lds_f32(s_eg + (hl * 8 + t0 + 1) * 4) + _sts2_f32( + s_x + g * RP + col0 * 4, + eg0 * acc[nt][0] + hacc[nt][0], + eg1 * acc[nt][1] + hacc[nt][1], + ) + _sts2_f32( + s_x + (g + 8) * RP + col0 * 4, + eg0 * acc[nt][2] + hacc[nt][2], + eg1 * acc[nt][3] + hacc[nt][3], + ) + cute.arch.sync_warp() + + # per state row: the forward substitution and the outputs + xs = [] + ys = [] + for q4 in cutlass.range_constexpr(TP // 4): + xs.extend(_lds4_f32(s_x + r * RP + q4 * 16)) + ys.extend(_lds4_f32(s_x + r * RP + (TP + 4 * q4) * 4)) + us = [] + for t in cutlass.range_constexpr(T): + vt = _lds_bf16(s_v + ((hl * T + t) * 128 + v0 + r) * 2) + bt = _lds_f32(s_beta + (hl * 8 + t) * 4) + u = bt * (vt - xs[t]) + for s in cutlass.range_constexpr(t): + u = u - _lds_f32(s_a + (hl * 64 + t * 8 + s) * 4) * us[s] + us.append(u) + if hf == 0: + for t in cutlass.range_constexpr(T): + o = ys[t] + for s in cutlass.range_constexpr(t + 1): + o = o + _lds_f32(s_cq + (hl * 64 + t * 8 + s) * 4) * us[s] + if const_expr(self.norm): + # the CTA's one value head: staged for the norm below + _sts_bf16(s_on + (t * V + v0 + r) * 2, o) + else: + _stg_bf16(o_ptr + (((n64 * T + t) * HV + hv) * V + v0 + r) * 2, o) + else: + for t in cutlass.range_constexpr(T): + _stg_bf16( + u_wr + cutlass.Int64((((woff + t) * HV + hv) * V + v0 + r) * 2), us[t] + ) + cute.arch.sync_warp() + + # the checkpoint commit of a wrapping row, over the tile just used + if const_expr(self.commit): + if wrap: + dc = [_lds_f32(s_dec + (hl * 16 + 2 * c + jx) * 4) for jx in [0, 1, 8, 9]] + egs = _lds_f32(s_egs + hl * 4) + a_hi = [] + a_lo = [] + for q4 in cutlass.range_constexpr(4): + dl = dc[0] if q4 < 2 else dc[2] + dhh = dc[1] if q4 < 2 else dc[3] + x0 = _bf2_lo(au[q4]) * dl + x1 = _bf2_hi(au[q4]) * dhh + hw = _bf2_pack(x0, x1) + a_hi.append(hw) + a_lo.append(_bf2_pack(x0 - _bf2_lo(hw), x1 - _bf2_hi(hw))) + if const_expr(cur is None): + curc = self._smem_tile(s_st, s_mb, i, g, c, zi) + else: + curc = cur + taddr = self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i, g, c) + kp_row = ( + s_kp + ((lane % 8) + 8 * ((lane // 8) % 2)) * PU + 16 * (lane // 16) + ) + for jj in cutlass.range_constexpr(4): + bb0 = _ldsm4t(kp_row + (4 * jj) * 16) + bb1 = _ldsm4t(kp_row + (4 * jj + 2) * 16) + ng = [] + nh8 = [] + for s4 in cutlass.range_constexpr(4): + bsrc = bb0 if s4 < 2 else bb1 + bf = (bsrc[2 * (s4 % 2)], bsrc[2 * (s4 % 2) + 1]) + dcm = _mma(a_hi, bf, [zf, zf, zf, zf]) + if const_expr(self.lo_split): + dcm = _mma(a_lo, bf, dcm) + wg = curc[8 * jj + s4] + wh = curc[8 * jj + 4 + s4] + ng.append( + _bf2_pack( + egs * _bf2_lo(wg) + dcm[0], egs * _bf2_hi(wg) + dcm[1] + ) + ) + nh8.append( + _bf2_pack( + egs * _bf2_lo(wh) + dcm[2], egs * _bf2_hi(wh) + dcm[3] + ) + ) + _stg4(taddr + jj * 64, ng[0], ng[1], ng[2], ng[3]) + _stg4(taddr + (2048 + jj * 64), nh8[0], nh8[1], nh8[2], nh8[3]) + + # ---- 6. the folded gated RMS norm: warp t takes row t of the CTA's value head ------ + if const_expr(self.norm): + cute.arch.sync_threads() + if warp < T: + hvn = h * GRP + sp # tpc 8: the CTA's value head + row64 = n64 * T + cutlass.Int64(warp) + col = cutlass.Int64(4 * lane) + xw = _lds2_i32(s_on + (warp * V + 4 * lane) * 2) + zw = [nz0, nz1] + ww = [nw0, nw1] + xs4 = [_bf2_lo(xw[0]), _bf2_hi(xw[0]), _bf2_lo(xw[1]), _bf2_hi(xw[1])] + ssq = _gn_sumsq(xs4[0], xs4[1], xs4[2], xs4[3]) + for m_ in cutlass.range_constexpr(5): + ssq = _add_rn(ssq, _shfl_xor(ssq, 16 >> m_)) + rstd = _gn_rstd(ssq, cutlass.Float32(self.norm_eps)) + ws4 = [_bf2_lo(ww[0]), _bf2_hi(ww[0]), _bf2_lo(ww[1]), _bf2_hi(ww[1])] + zs4 = [_bf2_lo(zw[0]), _bf2_hi(zw[0]), _bf2_lo(zw[1]), _bf2_hi(zw[1])] + ys4 = [ + _mul_rn(_mul_rn(_mul_rn(rstd, xs4[e]), ws4[e]), _gn_sigmoid(zs4[e])) + for e in range(4) + ] + _stg2_i32( + on_ptr + ((row64 * HV + cutlass.Int64(hvn)) * V + col) * 2, + _bf2_pack(ys4[0], ys4[1]), + _bf2_pack(ys4[2], ys4[3]), + ) + + if const_expr(self.pdl and not self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + + +# -- host side --------------------------------------------------------------------------------- + +_COMPILED: dict = {} + + +def default_config(n: int, t: int, commit: bool = False) -> dict: + """Launch mapping by request count: smaller CTAs for small (latency-bound) batches; from 32 + requests (or 8 with the commit) CTAs of 8 m-tiles x 4 warps, at most 5 (4) an SM.""" + if n <= 2: + return dict(tpc=2, warps=2, pf=0, minb=4, early_acc=True) + if n <= 8: + return dict(tpc=4, warps=4, pf=0, minb=4, early_acc=True) + if commit: + return dict(tpc=8, warps=4, pf=1, minb=4) + if n <= 16: + return dict(tpc=24, warps=8, pf=1, minb=2) + return dict(tpc=8, warps=4, pf=1, minb=5) + + +def _compile(key, kern): + fn = _COMPILED.get(key) + if fn is None: + dummy = torch.zeros(4, dtype=torch.int32, device="cuda") + i64 = cutlass.Int64(0) + fn = cute.compile( + kern, + from_dlpack(dummy, assumed_align=4).mark_layout_dynamic(leading_dim=0), + *([i64] * 37), + cutlass.Int32(1), + cutlass.Float32(1.0), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + +def conv_fold_supported(x, weight, pool, window) -> bool: + """Whether `cached_replay_update(conv_fold=...)` takes these operands: bf16, the layouts it + reads, and 16-byte alignment of every 8-channel group (vector loads / stores).""" + tensors = (x, weight, pool, window) + if any(t.dtype != torch.bfloat16 for t in tensors): + return False + if x.stride(1) != 1 or not weight.is_contiguous() or weight.shape[1] != 4: + return False + if pool.stride()[1:] != (3, 1) or window.stride()[2:] != (3, 1): + return False + strides = (x.stride(0), pool.stride(0), window.stride(0), window.stride(1)) + return all(t.data_ptr() % 16 == 0 for t in tensors) and all( + (2 * st) % 16 == 0 for st in strides + ) + + +def cached_replay_update( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: Optional[torch.Tensor], + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + **ignored, +) -> torch.Tensor: + """Same contract as `fused_recurrent_gated_delta_rule_cached_replay_update` (bf16 state). + + A wrapping row commits its checkpoint in-kernel unless work items + `use_all_layer_commit` + are given. Work items must list all N rows (`config={"use_wi": False}` reads + `state_indices` instead). The Triton launch knobs are ignored. + + `conv_fold` folds the verify conv in, bitwise the conv kernel: `x` (the q|k|v projection + columns), `weight` `[C, 4]`, `pool` `[slots, C, 3]` (read, not advanced: the engine's + acceptance promote rewrites it from `window`), `window` `[rows, T, C, 3]` and `window_rows`; + the q/k/v arguments then give shapes only. Needs `tpc` 8 or 24. + + `gated_norm` folds `rms_norm_gated_token_major` (sigmoid gate) in, bitwise: `z`, `weight`, + `eps`; the un-normed outputs are not stored. Needs `tpc` 8, 4 warps and T <= 4. + """ + del old_beta + N, T, H, Kd = k.shape + HV = v.shape[2] + assert (H, HV) in GEOMETRIES and Kd == K and v.shape[3] == V and history_size == HIST, ( + f"geometry: {H} q/k x {Kd}, {HV} v x {v.shape[3]}, history {history_size}" + ) + qkv_width = 2 * H * K + HV * V + assert 1 <= T <= 8 + assert ssm_states.dtype == torch.bfloat16, "bf16 checkpoint state only" + assert ssm_states.stride()[1:] == (V * K, K, 1) + assert old_u.dtype == torch.bfloat16 and old_k.dtype == torch.bfloat16 + assert old_G.dtype == torch.float32 + for x in (old_u, old_k, old_G): + assert x.is_contiguous() + assert g.shape == (N, T, HV) and beta.shape == (N, T, HV) + assert g.stride(2) == 1 and beta.stride(2) == 1 + assert g.stride(0) == T * g.stride(1) and beta.stride(0) == T * beta.stride(1) + assert g.dtype in (torch.bfloat16, torch.float32) and beta.dtype in ( + torch.bfloat16, + torch.float32, + ) + if (replay_work_items is None) != (n_writes is None): + raise ValueError("replay_work_items and n_writes must either both be set or both be None") + if replay_work_items is not None and not use_all_layer_commit: + raise ValueError("Partitioned replay requires the all-layer commit") + if replay_work_items is None and use_all_layer_commit: + raise ValueError("use_all_layer_commit requires replay work items") + commit = replay_work_items is None + if scale is None: + scale = K**-0.5 + if output is None: + output = torch.empty((N, T, HV, V), dtype=q.dtype, device=q.device) + assert output.is_contiguous() and output.dtype == torch.bfloat16 + if N == 0: + return output + if packed_qkv is not None: + assert packed_qkv.shape == (N * T, qkv_width) and packed_qkv.is_contiguous() + base = packed_qkv.data_ptr() + es = packed_qkv.element_size() + qp, kp, vp = base, base + H * K * es, base + 2 * H * K * es + qs = ks = vs = (T * qkv_width, qkv_width) + else: + for x, nh, d in ((q, H, K), (k, H, K), (v, HV, V)): + assert x.stride(3) == 1 and x.stride(2) == d, "head rows must be dense" + qp, kp, vp = q.data_ptr(), k.data_ptr(), v.data_ptr() + qs, ks, vs = q.stride()[:2], k.stride()[:2], v.stride()[:2] + gating = A_log is not None + if gating: + assert dt_bias is not None + A_log = A_log.float().contiguous() + dt_bias = dt_bias.float().contiguous() + ap, dp = A_log.data_ptr(), dt_bias.data_ptr() + else: + ap = dp = 0 + cfg = dict(default_config(N, T, commit)) + if config: + cfg.update(config) + fold_args = (0,) * 9 + if conv_fold is not None: + cx, cw, cs = conv_fold["x"], conv_fold["weight"], conv_fold["pool"] + win, wrows = conv_fold["window"], conv_fold["window_rows"] + assert cx.shape == (N * T, qkv_width) and cx.stride(1) == 1 and cx.dtype == torch.bfloat16 + assert cw.shape == (qkv_width, 4) and cw.is_contiguous() and cw.dtype == torch.bfloat16 + assert cs.shape[1:] == (qkv_width, 3) and cs.stride()[1:] == (3, 1) + assert cs.dtype == torch.bfloat16 + assert win.shape[1:] == (T, qkv_width, 3) and win.stride()[2:] == (3, 1) + assert win.dtype == torch.bfloat16 + assert wrows.dtype == torch.int32 and wrows.shape[0] >= N and wrows.is_contiguous() + assert conv_fold_supported(cx, cw, cs, win), ( + "conv_fold operands: layout / 16-byte alignment" + ) + if cfg.get("tpc") not in (8, 24): + # small batches' mappings split a value head over CTAs: take the whole-head ones + whole = ( + dict(tpc=8, warps=4, pf=1, minb=5) + if N > 16 + else dict(tpc=24, warps=8, pf=1, minb=2) + ) + cfg.update(whole) + cfg.pop("early_acc", None) + cfg["fold"] = True + fold_args = ( + cx.data_ptr(), cx.stride(0), cw.data_ptr(), cs.data_ptr(), cs.stride(0), + win.data_ptr(), win.stride(0), win.stride(1), wrows.data_ptr(), + ) # fmt: skip + norm_args = (0,) * 4 + if gated_norm is not None: + zn, gw = gated_norm["z"], gated_norm["weight"] + assert T <= 4, "the folded norm takes a warp a row" + assert ( + zn.shape == (N * T, HV, V) and zn.stride()[1:] == (V, 1) and zn.dtype == torch.bfloat16 + ) + assert gw.shape == (V,) and gw.is_contiguous() and gw.dtype == torch.bfloat16 + assert zn.data_ptr() % 8 == 0 and zn.stride(0) % 4 == 0 and gw.data_ptr() % 8 == 0 + if cfg.get("tpc") != 8 or cfg.get("warps") != 4: + cfg.update(tpc=8, warps=4) + cfg["pf"] = min(int(cfg.get("pf", 1)), 2) + cfg.setdefault("minb", 5) + cfg["norm"] = True + cfg["norm_eps"] = float(gated_norm["eps"]) + norm_args = (zn.data_ptr(), zn.stride(0), gw.data_ptr(), output.data_ptr()) + # PDL: state / history before the wait; without it, the step's own inputs first + cfg.setdefault("qk_first", not bool(launch_with_pdl)) + use_wi = bool(cfg.pop("use_wi", True)) and replay_work_items is not None + cfg["use_wi"] = use_wi + pdl = bool(launch_with_pdl) + kw = dict( + T=T, + h=H, + hv=HV, + commit=commit, + pdl=pdl, + l2norm=bool(use_qk_l2norm_in_kernel), + gating=gating, + g_f32=g.dtype == torch.float32, + b_f32=beta.dtype == torch.float32, + **cfg, + ) + key = tuple(sorted(kw.items())) + fn = _COMPILED.get(key) + if fn is None: + fn = _compile(key, _Replay(**kw)) + sidx = state_indices if state_indices.dtype == torch.int32 else state_indices.to(torch.int32) + fn( + sidx, + qp, + kp, + vp, + qs[0], + qs[1], + ks[0], + ks[1], + vs[0], + vs[1], + g.data_ptr(), + g.stride(1), + beta.data_ptr(), + beta.stride(1), + ap, + dp, + output.data_ptr(), + ssm_states.data_ptr(), + ssm_states.stride(0), + prev_num_accepted_tokens.data_ptr(), + cache_buf_idx.data_ptr(), + old_u.data_ptr(), + old_k.data_ptr(), + old_G.data_ptr(), + replay_work_items.data_ptr() if use_wi else 0, + *fold_args, + *norm_args, + N, + float(scale), + ) + return output + + +#: A tp4/tep4 rank's GDN heads. +RANK_K_HEADS, RANK_V_HEADS = 4, 12 +#: A tep4 engine rank's state slots, and the GDN layers sharing one slot page (slot stride 36 +#: layers' states). +TEP4_SLOTS, PAGE_LAYERS = 1874, 36 +#: The same 26.5 GB page at a dep4 rank's (48-head) and a tp2/tep2 rank's (24-head) slot +#: stride: the top slot sits 1.3e10 elements past the base, as tep4's does. +DEP4_SLOTS = 468 +TEP2_SLOTS = 937 + + +def _tep2_spec(path, n, t, **spec) -> dict: + """A tp2/tep2 rank's heads, 8 / 24, over a bf16 stock-manager page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=8, + HV=24, + history_size=HIST, + slots=TEP2_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +def _tep4_spec(path, n, t, **spec) -> dict: + """A tp4/tep4 rank's heads over the stock tep4 engine's bf16 state page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=RANK_K_HEADS, + HV=RANK_V_HEADS, + history_size=HIST, + slots=TEP4_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +def _dep4_spec(path, n, t, **spec) -> dict: + """A dep4 rank's heads, all 16 / 48 under attention DP, over a bf16 stock-manager page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=16, + HV=48, + history_size=HIST, + slots=DEP4_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +class GdnCachedReplayCute(OpWrapper): + """`cached_replay_update`: one verify step of `N` sequences x `T` tokens through one GDN + layer, the contract of `fused_recurrent_gated_delta_rule_cached_replay_update` (bf16 state). + Head counts from the call's shapes (`GEOMETRIES`); `g` / `beta` may be strided column views + of the input projection (unit head stride). + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: the test harness's `_Bench` spec (path, n x t, steps with random acceptance, start, + # accept, dup, H / HV, slots, page_layers) plus `strided_gates` (g / beta as column views of + # the projection). The small (in-kernel commit) and all_layer paths at each geometry. + CELLS: tuple[Cell, ...] = ( + # tep4: 4 / 12 heads. + Cell( + why="tep4 small n=1 T=4: MTP3 verify at con1, in-kernel commit on the page's last slot", + spec=_tep4_spec("small", 1, 4, steps=8, strided_gates=True), + ), + Cell( + why="tep4 small n=1 T=8 full accept: MTP7 appends to the history's end, then wraps", + spec=_tep4_spec( + "small", 1, 8, steps=8, start="fresh", accept="full", strided_gates=True + ), + ), + Cell( + why="tep4 all_layer n=16 T=4: the smallest partitioned batch, all-layer commit", + spec=_tep4_spec("all_layer", 16, 4, steps=6, strided_gates=True), + ), + Cell( + why="tep4 all_layer n=32 T=8: the widest verify below N=128", + spec=_tep4_spec("all_layer", 32, 8, steps=6, strided_gates=True), + ), + Cell( + why="tep4 all_layer n=128 T=4: a large partitioned batch", + spec=_tep4_spec("all_layer", 128, 4, steps=4), + ), + # dep4: 16 / 48 heads. + Cell( + why="dep4 small n=1 T=4: the in-kernel commit of the page's last slot (467)", + spec=_dep4_spec("small", 1, 4, steps=8, strided_gates=True), + ), + Cell( + why="dep4 small n=8 T=4 dup: rows 5..7 share one CUDA-graph padding slot", + spec=_dep4_spec("small", 8, 4, steps=6, dup=(5, 6, 7), strided_gates=True), + ), + # tep2: 8 / 24 heads, in-kernel commit below 16 requests, all-layer from 16. + Cell( + why="tep2 small n=4 T=4: a 16-row verify, in-kernel commit", + spec=_tep2_spec("small", 4, 4, steps=6, strided_gates=True), + ), + Cell( + why="tep2 all_layer n=32 T=4: a 128-row verify, all-layer commit", + spec=_tep2_spec("all_layer", 32, 4, steps=4, strided_gates=True), + ), + Cell( + why="dep4 all_layer n=128 T=4: a rank's largest verify", + spec=_dep4_spec("all_layer", 128, 4, steps=4, strided_gates=True), + ), + ) + + note = """ + Source: this file's CuTe DSL kernel (`_Replay`), one compile per (geometry, T, launch + mapping, commit, PDL, folds) on first use (5-10 s): the warm-up before CUDA-graph capture + must cover it. Called by `_gdn_verify` over the bf16 recurrent pool (slot stride 36 layers' + states) when the rank's GDN heads are a `GEOMETRIES` pair, with work items + + `use_all_layer_commit` exactly when the Triton entry gets them (N >= 16). PDL only at up to + 24 (request x value head) pairs: the conv before it triggers its dependents at its start. + + Numerics: the Triton kernel's rounding points (bf16 L2-normalized q / k and history + coefficients, fp32 recurrence, fp32 commit rounded once to the bf16 pool, bf16 history u / k, + fp32 G); the dot order differs, so outputs match it to a few bf16 ulps, not bitwise. Launch + mappings are bitwise each other, and CUDA-graph replay is bitwise eager. + + Certified on GB300 at the cells (rows at both ends of the page, consecutive steps from warm + histories with random acceptance) within the Triton entry's test bands. Not supported: fp32 + state, T > 8, other geometries, K or V other than 128, history other than 16, pre-gated + g / beta (`A_log=None` compiles but is not certified). + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + ) -> torch.Tensor: + return cached_replay_update( + q=q, + k=k, + v=v, + g=g, + beta=beta, + ssm_states=ssm_states, + state_indices=state_indices, + old_u=old_u, + old_k=old_k, + old_G=old_G, + old_beta=old_beta, + cache_buf_idx=cache_buf_idx, + prev_num_accepted_tokens=prev_num_accepted_tokens, + history_size=history_size, + scale=scale, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + A_log=A_log, + dt_bias=dt_bias, + launch_with_pdl=launch_with_pdl, + replay_work_items=replay_work_items, + n_writes=n_writes, + packed_qkv=packed_qkv, + use_all_layer_commit=use_all_layer_commit, + output=output, + config=config, + conv_fold=conv_fold, + gated_norm=gated_norm, + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """`(out, ssm_states, old_u, old_k, old_G)` as the call leaves them; mutates nothing. + The state is rebuilt from the checkpoint and history in fp32 and the new tokens stepped + one at a time in fp32 (no matmul, so no TF32), every store rounded once to its buffer's + dtype. Rows with a negative slot keep `output`'s prior content (NaN without `output`). + `replay_work_items` only decides whether a wrap commits here; the launch knobs, `config`, + `old_beta` and `n_writes` are ignored.""" + N, T, H, K = k.shape + HV, V = v.shape[2], v.shape[3] + rep = HV // H + dev = v.device + scale = K**-0.5 if scale is None else scale + if packed_qkv is not None: + rows = packed_qkv.view(N, T, -1) + q = rows[..., : H * K].view(N, T, H, K) + k = rows[..., H * K : 2 * H * K].view(N, T, H, K) + v = rows[..., 2 * H * K :].view(N, T, HV, V) + qf, kf, vf = q.float(), k.float(), v.float() + if use_qk_l2norm_in_kernel: + qf = qf / torch.sqrt(qf.square().sum(-1, keepdim=True) + 1e-6) + kf = kf / torch.sqrt(kf.square().sum(-1, keepdim=True) + 1e-6) + qf = (qf * scale).repeat_interleave(rep, dim=2) # [N, T, HV, K] + kv = kf.repeat_interleave(rep, dim=2) # [N, T, HV, K] + if A_log is not None: + gf = -torch.exp(A_log.float()) * F.softplus(g.float() + dt_bias.float()) + bf = torch.sigmoid(beta.float()) + else: + gf, bf = g.float(), beta.float() + + out_dtype = q.dtype if output is None else output.dtype + out = ( + torch.full((N, T, HV, V), float("nan"), device=dev) + if output is None + else output.float().clone() + ) + pool, u_buf, k_buf, G_buf = (x.clone() for x in (ssm_states, old_u, old_k, old_G)) + live = (state_indices[:N] >= 0).nonzero().flatten() + if live.numel() == 0: + return out.to(out_dtype), pool, u_buf, k_buf, G_buf + slot = state_indices[:N][live].long() + p = prev_num_accepted_tokens[slot].long() + b = cache_buf_idx[slot].long() + M, depth = live.numel(), old_u.shape[2] + + # 1. the pending state: checkpoint plus the p history entries of half b + pending = torch.arange(depth, device=dev)[None, :] < p[:, None] # [M, depth] + Gh = torch.where(pending[:, None, :], old_G[slot, b].float(), 0.0) # [M, HV, depth] + Uh = torch.where(pending[:, :, None, None], old_u[slot, b].float(), 0.0) + Kh = torch.where(pending[:, :, None, None], old_k[slot, b].float(), 0.0) + Kh = Kh.repeat_interleave(rep, dim=2) # [M, depth, HV, K] + last = (p - 1).clamp(min=0)[:, None, None].expand(M, HV, 1) + g_start = torch.where(p[:, None] > 0, Gh.gather(2, last).squeeze(2), 0.0) # [M, HV] + weight = torch.where(pending[:, None, :], torch.exp(g_start[:, :, None] - Gh), 0.0) + h = torch.exp(g_start)[:, :, None, None] * ssm_states[slot].float() # [M, HV, V, K] + for j in range(depth): + h = h + weight[:, :, j, None, None] * Uh[:, j, :, :, None] * Kh[:, j, :, None, :] + checkpoint = h + + # 2. the new tokens, in order + gl, bl = gf[live], bf[live] # [M, T, HV] + ql, kl, vl = qf[live], kv[live], vf[live] + G_local = torch.zeros(M, HV, device=dev) + u_new, G_new = [], [] + for t in range(T): + h = h * torch.exp(gl[:, t])[:, :, None, None] + u = bl[:, t, :, None] * (vl[:, t] - (h * kl[:, t, :, None, :]).sum(-1)) # [M, HV, V] + h = h + u[:, :, :, None] * kl[:, t, :, None, :] + out[live, t] = (h * ql[:, t, :, None, :]).sum(-1) + G_local = G_local + gl[:, t] + u_new.append(u) + G_new.append(G_local.clone()) + + # 3. the history append, and the checkpoint commit on a wrap + wrap = p + T > history_size + w_buf = torch.where(wrap, 1 - b, b) + w_off = torch.where(wrap, 0, p) + for t in range(T): + pos = w_off + t + u_buf[slot, w_buf, pos] = u_new[t].to(u_buf.dtype) + k_buf[slot, w_buf, pos] = kf[live, t].to(k_buf.dtype) + G_store = torch.where(wrap[:, None], G_new[t], g_start + G_new[t]) + G_buf[slot, w_buf, :, pos] = G_store.to(G_buf.dtype) + commit = replay_work_items is None + if commit and bool(wrap.any()): + pool[slot[wrap]] = checkpoint[wrap].to(pool.dtype) + return out.to(out_dtype), pool, u_buf, k_buf, G_buf + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + ) -> None: + N, _, H, K = k.shape + HV, V = v.shape[2], v.shape[3] + assert state_indices.dim() == 1 and state_indices.shape[0] >= N, ( + f"state_indices has shape {tuple(state_indices.shape)} for {N} sequences -- the " + "kernel reads one slot per sequence with no bound, so a shorter tensor hands the " + "trailing rows whatever follows it in memory, and they write that slot's history" + ) + pool_slots, slots = ssm_states.shape[0], old_u.shape[0] + assert slots >= pool_slots, ( + f"the history buffers have {slots} slots for a {pool_slots}-slot pool -- a slot " + "past the history's end is read and written out of bounds" + ) + for name, tensor in ( + ("prev_num_accepted_tokens", prev_num_accepted_tokens), + ("cache_buf_idx", cache_buf_idx), + ): + assert tensor.dim() == 1 and tensor.is_contiguous() and tensor.shape[0] >= pool_slots, ( + f"{name} has shape {tuple(tensor.shape)} and stride {tensor.stride()}; the kernel " + f"reads one element per slot of the {pool_slots}-slot pool, at the slot's index" + ) + expected = { + "old_u": (old_u, (slots, 2, history_size, HV, V)), + "old_k": (old_k, (slots, 2, history_size, H, K)), + "old_G": (old_G, (slots, 2, HV, history_size)), + } + for name, (tensor, shape) in expected.items(): + assert tuple(tensor.shape) == shape, ( + f"{name} has shape {tuple(tensor.shape)}; history_size={history_size} and this " + f"geometry need {shape} -- the kernel addresses the history by history_size and " + "the head geometry, never by the buffer's own shape" + ) + + +gdn_cached_replay_cute = GdnCachedReplayCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py new file mode 100644 index 000000000000..b0486477c914 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py @@ -0,0 +1,743 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A verify step's Gated-DeltaNet input projection with the causal conv of its q|k|v columns in +the GEMM's epilogue: `low_m_bf16_mm` then `causal_conv1d_token_major`, in one launch. + +The GEMM is FlashInfer's warp split-K kernel (`dense_bf16_gemm_warp_splitk`), subclassed with +its body unchanged. A CTA owns 16 output columns for every row of the step (one token tile), so +the reduce warp keeps the bf16-rounded tile in shared memory and its first 16 lanes run the conv +of their channel as `_causal_conv1d_update_kernel` does (window snapshots, SiLU, the token-major +`[n * p, C]` output, the pool advanced). No two CTAs touch one channel. +""" + +from __future__ import annotations + +import functools + +import cuda.bindings.driver as _cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute import experimental as cute_ext +from cutlass.cute.nvgpu import warp +from cutlass.cutlass_dsl import T as _ty +from cutlass.cutlass_dsl import dsl_user_op +from cutlass.experimental import primitives as prims +from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fw + +from tensorrt_llm._torch.modeling_v2.catalog._op import ( + Arch, + Cell, + OpWrapper, + true_fp32_matmul, +) + +from ..gemm import low_m_bf16_mm__decode as _lowm + +#: The GDN conv (width 4) over a tp4/tep4 rank's q|k|v channels. +CONV_WIDTH = 4 +CONV_DIM = 2560 +#: The projection a tp4/tep4 rank runs: `[q k v | z | a | b | pad]` = 4128 over hidden 2560. +N_OUT, K_IN = 4128, 2560 + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ldg_i32p(gaddr, pred, other: int, *, loc=None, ip=None): + """An int32, or `other` where `pred` is 0 (nothing read).""" + return cutlass.Int32( + _asm(_ty.i32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.b32 $0, " + str(int(other)) + "; " + "@q ld.global.b32 $0, [$1]; }", "=r,l,r", loc, ip, side=True) + ) # fmt: skip + + +@dsl_user_op +def _ldg_bf16p(gaddr, pred, *, loc=None, ip=None): + """One bf16 widened to f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; .reg .b16 h; setp.ne.b32 q, $2, 0; mov.b16 h, 0; " + "@q ld.global.b16 h, [$1]; cvt.f32.bf16 $0, h; }", "=f,l,r", loc, ip, side=True) + ) # fmt: skip + + +@dsl_user_op +def _stg_bf16(gaddr, x, *, loc=None, ip=None): + """One f32 rounded to bf16 (round to nearest even) and stored.""" + _asm(_ty.i32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,f", loc, ip, side=True) # fmt: skip + + +@dsl_user_op +def _round_bf16(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $1; cvt.f32.bf16 $0, h; }", "=f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _mul_bf16(a, b, *, loc=None, ip=None): + """A bf16 x bf16 product as the Triton kernel forms it: exact in f32, rounded to bf16.""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; .reg .f32 p; mul.rn.f32 p, $1, $2; cvt.rn.bf16.f32 h, p; " + "cvt.f32.bf16 $0, h; }", "=f,f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _add_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "add.rn.f32 $0, $1, $2;", "=f,f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _silu(x, *, loc=None, ip=None): + """acc / (1 + ex2(-acc * log2(e))): bitwise the Triton kernel's SiLU.""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .f32 t, e, d; neg.f32 t, $1; mul.rn.f32 t, t, 0f3FB8AA3B; " + "ex2.approx.f32 e, t; add.rn.f32 d, e, 0f3F800000; div.full.f32 $0, $1, d; }", + "=f,f", loc, ip) + ) # fmt: skip + + +class _InProjConvKernel(_fw.CpAsyncWarpSplitKKernel): + """FlashInfer's warp split-K kernel with the conv epilogue (see the module docstring).""" + + def __init__(self, tactic, use_pdl: bool, rows_per_request: int) -> None: + super().__init__(tactic, use_pdl, False) + self.P = rows_per_request + + @cute.experimental.jit + def __call__( + self, + weight, + activation, + bias, + output, + conv_w: cutlass.Int64, + pool: cutlass.Int64, + pool_s_seq: cutlass.Int64, + pool_s_dim: cutlass.Int64, + pool_s_tok: cutlass.Int64, + slots: cutlass.Int64, + win: cutlass.Int64, + win_s_seq: cutlass.Int64, + win_s_step: cutlass.Int64, + win_s_dim: cutlass.Int64, + win_s_win: cutlass.Int64, + win_rows: cutlass.Int64, + conv_out: cutlass.Int64, + pad_slot: cutlass.Int32, + stream: _cuda.CUstream, + ): + sA_layout = _fw._make_smem_layout( + weight.element_type, self.output_tile, self.k_tile, self.stages + ) + sB_layout = _fw._make_smem_layout( + activation.element_type, self.token_tile, self.k_tile, self.stages + ) + tiled_mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(weight.element_type, cutlass.Float32, _fw._MMA_SHAPE), + cute.make_layout((1, 1, _fw._COMPUTE_WARPS)), + permutation_mnk=(self.output_tile, self.token_tile, self.k_tile), + ) + self.kernel( + weight, + activation, + bias, + output, + sA_layout, + sB_layout, + tiled_mma, + conv_w, + pool, + pool_s_seq, + pool_s_dim, + pool_s_tok, + slots, + win, + win_s_seq, + win_s_step, + win_s_dim, + win_s_win, + win_rows, + conv_out, + pad_slot, + ).launch( + grid=( + cute.ceil_div(weight.shape[0], self.output_tile), + cute.ceil_div(activation.shape[0], self.token_tile), + 1, + ), + block=(self.threads, 1, 1), + smem=cute.Int64(self.smem_bytes + self.output_tile * self.token_tile * 4 + 64), + stream=stream, + use_pdl=self.use_pdl, + ) + + @cute.experimental.kernel + def kernel( + self, + weight: cute.Tensor, + activation: cute.Tensor, + bias: cute.Tensor, + output: cute.Tensor, + sA_layout: cute.ComposedLayout, + sB_layout: cute.ComposedLayout, + tiled_mma: cute.TiledMma, + conv_w: cutlass.Int64, + pool: cutlass.Int64, + pool_s_seq: cutlass.Int64, + pool_s_dim: cutlass.Int64, + pool_s_tok: cutlass.Int64, + slots: cutlass.Int64, + win: cutlass.Int64, + win_s_seq: cutlass.Int64, + win_s_step: cutlass.Int64, + win_s_dim: cutlass.Int64, + win_s_win: cutlass.Int64, + win_rows: cutlass.Int64, + conv_out: cutlass.Int64, + pad_slot: cutlass.Int32, + ): + # ---- FlashInfer's kernel body, unchanged (dense_bf16_gemm_warp_splitk.py) ---------- + _COMPUTE_WARPS = _fw._COMPUTE_WARPS + _A_LOADER_WARPS = _fw._A_LOADER_WARPS + _MMA_SHAPE = _fw._MMA_SHAPE + _MAILBOX_GROUP_VALUES = _fw._MAILBOX_GROUP_VALUES + _MAILBOX_PADDED_GROUP_VALUES = _fw._MAILBOX_PADDED_GROUP_VALUES + tidx, _, _ = cute.arch.thread_idx() + output_tile_idx, token_tile_idx, _ = cute.arch.block_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane_idx = tidx % cute.arch.WARP_SIZE + reuse_stages = cute.size(weight, mode=[1]) > self.stages * self.k_tile + + sA = cute_ext.allocate( + weight.element_type, cute.AddressSpace.smem, sA_layout, alignment=1024 + ) + sB = cute_ext.allocate( + activation.element_type, cute.AddressSpace.smem, sB_layout, alignment=1024 + ) + values_per_lane = self.output_tile * self.token_tile // cute.arch.WARP_SIZE + partial_stride = ( + self.output_tile * self.token_tile // _MAILBOX_GROUP_VALUES + ) * _MAILBOX_PADDED_GROUP_VALUES + partials = cute_ext.allocate( + cutlass.Float32, + cute.AddressSpace.smem, + cute.make_layout(_COMPUTE_WARPS * partial_stride), + alignment=16, + ) + bar_full_arr = cute_ext.allocate( + cutlass.Int64, cute.AddressSpace.smem, cute.make_layout(self.stages), alignment=8 + ) + bar_empty_arr = cute_ext.allocate( + cutlass.Int64, cute.AddressSpace.smem, cute.make_layout(self.stages), alignment=8 + ) + # The reduced tile, bf16-rounded, for the conv epilogue. + xtile = cute_ext.allocate( + cutlass.Float32, + cute.AddressSpace.smem, + cute.make_layout(self.output_tile * self.token_tile), + alignment=16, + ) + bar_full = bar_full_arr.iterator + bar_empty = bar_empty_arr.iterator + + k_tile_count = cute.size(weight, mode=[1]) // self.k_tile + + # ---- the conv's operands, read before the GEMM (nothing earlier in the step writes them) + P = self.P + MAX_REQ = self.token_tile // P + n_req = cute.size(output, mode=[0]) // P + conv_lane = cutlass.Int32(0) + if warp_idx == self.compute_warp_base: + if lane_idx < self.output_tile: + if output_tile_idx * self.output_tile < CONV_DIM: + conv_lane = cutlass.Int32(1) + c64 = cutlass.Int64(output_tile_idx * self.output_tile + lane_idx % self.output_tile) + wbase = conv_w + c64 * (CONV_WIDTH * 2) + w0 = _ldg_bf16p(wbase, conv_lane) + w1 = _ldg_bf16p(wbase + 2, conv_lane) + w2 = _ldg_bf16p(wbase + 4, conv_lane) + w3 = _ldg_bf16p(wbase + 6, conv_lane) + req_live = [] + req_cols = [] + req_win = [] + for s in cutlass.range_constexpr(MAX_REQ): + ok = conv_lane + if s >= n_req: + ok = cutlass.Int32(0) + slot = _ldg_i32p(slots + s * 4, ok, -1) + live = ok + if slot == pad_slot: + live = cutlass.Int32(0) + prow = pool + (cutlass.Int64(slot) * pool_s_seq + c64 * pool_s_dim) * 2 + req_cols.append( + ( + _ldg_bf16p(prow, live), + _ldg_bf16p(prow + pool_s_tok * 2, live), + _ldg_bf16p(prow + pool_s_tok * 4, live), + prow, + ) + ) + req_win.append(cutlass.Int64(_ldg_i32p(win_rows + s * 4, live, 0))) + req_live.append(live) + + if warp_idx < _A_LOADER_WARPS: + _fw._issue_weight_tile( + sA, + weight, + tidx, + output_tile_idx, + 0, + 0, + cutlass.Int64(0), + self.output_tile, + self.k_tile, + False, + ) + if warp_idx == self.compute_warp_base: + if lane_idx < self.stages: + cute.arch.mbarrier_init(bar_full + lane_idx, self.load_threads) + if const_expr(reuse_stages): + cute.arch.mbarrier_init(bar_empty + lane_idx, _COMPUTE_WARPS) + cute.arch.mbarrier_init_fence() + cute.arch.barrier() + + if warp_idx < _A_LOADER_WARPS: + weight_policy = _fw._l2_policy_evict_first() + empty_phase = cutlass.Int32(1) + prims.cp_async_mbarrier_arrive(bar_full.llvm_ptr, noinc=True) + for k_tile in cutlass.range(1, k_tile_count, unroll=1): + stage = k_tile % self.stages + if const_expr(reuse_stages): + cute.arch.mbarrier_wait(bar_empty + stage, empty_phase) + _fw._issue_weight_tile( + sA, + weight, + tidx, + output_tile_idx, + k_tile, + stage, + weight_policy, + self.output_tile, + self.k_tile, + True, + ) + prims.cp_async_mbarrier_arrive((bar_full + stage).llvm_ptr, noinc=True) + if stage == self.stages - 1: + empty_phase = empty_phase ^ 1 + elif warp_idx < self.compute_warp_base: + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_wait() + b_loader_threads = self.b_loader_warps * 32 + local_tid = tidx - _A_LOADER_WARPS * 32 + copy_count = self.token_tile * self.k_tile // (8 * b_loader_threads) + empty_phase = cutlass.Int32(1) + for k_tile in cutlass.range(k_tile_count, unroll=1): + stage = k_tile % self.stages + if const_expr(reuse_stages): + cute.arch.mbarrier_wait(bar_empty + stage, empty_phase) + for copy_idx in cutlass.range_constexpr(copy_count): + linear = local_tid * 8 + copy_idx * b_loader_threads * 8 + row = linear // self.k_tile + col = linear % self.k_tile + global_row = token_tile_idx * self.token_tile + row + if global_row < cute.size(activation, mode=[0]): + dst = ( + sB.iterator.raw_ptr() + + stage * self.token_tile * self.k_tile + + row * self.k_tile + + (col ^ ((row % 8) * 8)) + ) + src = activation.iterator.raw_ptr() + activation.layout( + (global_row, k_tile * self.k_tile + col) + ) + prims.cp_async_shared_global(dst, src, 16, "cg") + prims.cp_async_mbarrier_arrive((bar_full + stage).llvm_ptr, noinc=True) + if stage == self.stages - 1: + empty_phase = empty_phase ^ 1 + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_launch_dependents() + else: + mma_tid = tidx - self.compute_warp_base * cute.arch.WARP_SIZE + compute_warp = warp_idx - self.compute_warp_base + thr_mma = tiled_mma.get_slice(mma_tid) + acc = cute.make_rmem_tensor( + thr_mma.partition_shape_C((self.output_tile, self.token_tile)), cutlass.Float32 + ) + acc.fill(0.0) + k_phases = self.k_tile // (_COMPUTE_WARPS * _MMA_SHAPE[2]) + m_iters = self.output_tile // _MMA_SHAPE[0] + n_iters = self.token_tile // _MMA_SHAPE[1] + a_offsets = tuple( + tuple( + _fw._a_fragment_offset(lane_idx, compute_warp, m_iter, phase, self.k_tile) + for phase in range(k_phases) + ) + for m_iter in range(m_iters) + ) + b_offsets = tuple( + tuple( + _fw._b_pair_offset( + lane_idx, compute_warp, n_iter, pair, self.token_tile, self.k_tile + ) + for pair in range(k_phases // 2) + ) + for n_iter in range(n_iters) + ) + a_stage_elems = self.output_tile * self.k_tile + b_stage_elems = self.token_tile * self.k_tile + full_phase = cutlass.Int32(0) + for k_tile in cutlass.range(k_tile_count, unroll_full=True): + stage = k_tile % self.stages + cute.arch.mbarrier_wait(bar_full + stage, full_phase) + a_base = stage * a_stage_elems + b_base = stage * b_stage_elems + a_registers = tuple( + tuple( + _fw._ldmatrix_x4(sA, a_base + a_offsets[m_iter][phase]) + for phase in range(k_phases) + ) + for m_iter in range(m_iters) + ) + b_registers = tuple( + tuple( + _fw._ldmatrix_x4(sB, b_base + b_offsets[n_iter][pair]) + for pair in range(k_phases // 2) + ) + for n_iter in range(n_iters) + ) + for phase in cutlass.range_constexpr(k_phases): + for n_iter in cutlass.range_constexpr(n_iters): + packed_b = b_registers[n_iter][phase // 2] + b_offset = (phase % 2) * 2 + for m_iter in cutlass.range_constexpr(m_iters): + acc_base = (m_iter * n_iters + n_iter) * 4 + result = prims.mma_sync( + llvm.StructType.get_literal([cutlass.Float32.mlir_type] * 4), + shape=_MMA_SHAPE, + layout_a=prims.MMALayout.ROW, + layout_b=prims.MMALayout.COL, + operand_a=[ + a_registers[m_iter][phase][i].ir_value() for i in range(4) + ], + operand_b=[packed_b[b_offset + i].ir_value() for i in range(2)], + operand_c=[acc[acc_base + i].ir_value() for i in range(4)], + multiplicand_a_ptx_type=prims.MMAType.BF16, + multiplicand_b_ptx_type=prims.MMAType.BF16, + ) + for i in cutlass.range_constexpr(4): + acc[acc_base + i] = cutlass.Float32( + llvm.extractvalue(cutlass.Float32.mlir_type, result, [i]) + ) + if const_expr(reuse_stages): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bar_empty + stage) + if stage == self.stages - 1: + full_phase = full_phase ^ 1 + for value in cutlass.range_constexpr(values_per_lane): + atom = value % 4 + fragment = value // 4 + m_iter = fragment // n_iters + n_iter = fragment % n_iters + feature = lane_idx // 4 + 8 * (atom // 2) + m_iter * _MMA_SHAPE[0] + token = 8 * n_iter + 2 * (lane_idx % 4) + atom % 2 + partials[ + compute_warp * partial_stride + + _fw._partial_offset(feature, token, self.token_tile) + ] = acc[value] + prims.barrier_cta_sync(1, thread_count=_COMPUTE_WARPS * 32) + + if warp_idx == self.compute_warp_base: + final_acc = cute.make_rmem_tensor(cute.make_layout(values_per_lane), cutlass.Float32) + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + total = cutlass.Float32(0) + for peer in cutlass.range_constexpr(_COMPUTE_WARPS): + total = ( + total + + partials[ + peer * partial_stride + + _fw._partial_offset(feature, token, self.token_tile) + ] + ) + final_acc[value] = total + + output_base = output_tile_idx * self.output_tile + token_base = token_tile_idx * self.token_tile + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + if token_base + token < cute.size(output, mode=[0]): + output[token_base + token, output_base + feature] = final_acc[value].to( + output.element_type + ) + # ---- the conv epilogue ----------------------------------------------------------- + if output_base < CONV_DIM: + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + xtile[token * self.output_tile + feature] = _round_bf16(final_acc[value]) + cute.arch.sync_warp() + for s in cutlass.range_constexpr(MAX_REQ): + if req_live[s] != 0: + # One channel's conv over request s. + col0, col1, col2, prow = req_cols[s] + wb = win + (req_win[s] * win_s_seq + c64 * win_s_dim) * 2 + for t in cutlass.range_constexpr(P): + x = xtile[(s * P + t) * self.output_tile + lane_idx] + a = _add_rn(cutlass.Float32(0.0), _mul_bf16(col0, w0)) + a = _add_rn(a, _mul_bf16(col1, w1)) + a = _add_rn(a, _mul_bf16(col2, w2)) + a = _add_rn(a, _mul_bf16(x, w3)) + col0 = col1 + col1 = col2 + col2 = x + wt = wb + cutlass.Int64(t) * win_s_step * 2 + _stg_bf16(wt, col0) + _stg_bf16(wt + win_s_win * 2, col1) + _stg_bf16(wt + win_s_win * 4, col2) + _stg_bf16( + conv_out + (cutlass.Int64(s * P + t) * CONV_DIM + c64) * 2, + _silu(a), + ) + # The pool advanced to the last 3 inputs of [window | rows]. + _stg_bf16(prow, col0) + _stg_bf16(prow + pool_s_tok * 2, col1) + _stg_bf16(prow + pool_s_tok * 4, col2) + + +def fused_tactic(rows: int): + """The FlashInfer warp split-K tactic `low_m_bf16_mm` runs this projection with at `rows` + rows, when it is that kernel in one token tile (the fused launch's domain), else None.""" + p = _lowm.plan(rows, N_OUT, K_IN) + if p.kind != "warp": + return None + tactic = _fw.WarpSplitKTactic(*p.tactic) + return tactic if rows <= tactic.token_tile else None + + +def _from_dlpack(tensor: torch.Tensor, *, dynamic_m: bool = False): + return _fw._from_dlpack(tensor, dynamic_m=dynamic_m) + + +@functools.cache +def _compile(device_index: int, tactic, use_pdl: bool, rows_per_request: int): + with torch.cuda.device(device_index): + i64 = cutlass.Int64(0) + return cute_ext.compile( + _InProjConvKernel(tactic, use_pdl, rows_per_request), + *( + _from_dlpack( + torch.empty(shape, device="cuda", dtype=torch.bfloat16), dynamic_m=dynamic_m + ) + for shape, dynamic_m in ( + ((N_OUT, K_IN), False), + ((_fw._MAX_M, K_IN), True), + ((N_OUT,), False), + ((_fw._MAX_M, N_OUT), True), + ) + ), # fmt: skip + *([i64] * 13), + cutlass.Int32(0), + _cuda.CUstream(torch.cuda.current_stream().cuda_stream), + ) + + +def _launch( + mat_a, + mat_b, + out, + conv_out, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, +): + compiled = _compile(mat_a.device.index, tactic, pdl, rows_per_request) + compiled( + _from_dlpack(mat_b.T), + _from_dlpack(mat_a, dynamic_m=True), + _from_dlpack(out[0]), + _from_dlpack(out, dynamic_m=True), + conv_weight.data_ptr(), + conv_state.data_ptr(), + conv_state.stride(0), + conv_state.stride(1), + conv_state.stride(2), + slots.data_ptr(), + window.data_ptr(), + window.stride(0), + window.stride(1), + window.stride(2), + window.stride(3), + window_rows.data_ptr(), + conv_out.data_ptr(), + pad_slot_id, + _cuda.CUstream(torch.cuda.current_stream(mat_a.device).cuda_stream), + ) + + +def _cells(): + return tuple( + Cell( + why=f"tep4 MTP verify, {n} request(s) x {p} rows: in_proj + conv in one launch", + spec=dict(requests=n, p=p, capture=capture), + ) + # One per warp split-K tactic `fused_tactic` picks (3, 6 and 16 rows), the first captured. + for n, p, capture in ((1, 3, True), (2, 3, False), (4, 4, False)) + ) + + +class GdnInProjConv(OpWrapper): + """`(proj, conv) = gdn_in_proj_conv(x, in_proj, conv_weight, conv_state, slots, window, + window_rows, pad_slot_id, rows_per_request, tactic, pdl)` for a verify step's rows. + + `x` `[T, 2560]` bf16 contiguous, `T = n * rows_per_request`; `in_proj` the `[2560, 4128]` + column-major operand; `conv_weight` `[2560, 4]`; `conv_state` the `[slots, 2560, 3]` pool; + `slots` `[n]` int32 (`pad_slot_id` skipped); `window` the `[rows, p, 2560, 3]` snapshot + buffer and `window_rows` `[n]` int32; `tactic` the warp split-K tactic `low_m_bf16_mm` picks + at `T` rows (`fused_tactic`). Returns `proj` `[T, 4128]`, bitwise `low_m_bf16_mm`'s, and + `conv` `[T, 2560]` token-major with `conv_state` and `window` written, bitwise + `causal_conv1d_token_major` over `proj`'s q|k|v columns. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: FlashInfer's `CpAsyncWarpSplitKKernel` (CuTe DSL) plus the conv epilogue, compiled + per (tactic, PDL, rows per request) with M dynamic: the warmup forward must compile it + before capture. Preconditions, refused by `is_valid`: the shapes above, `T` a multiple of + `rows_per_request`, `T <= tactic.token_tile`. + """ + + def __call__( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ): + rows = x.shape[0] + out = torch.empty([rows, N_OUT], dtype=torch.bfloat16, device=x.device) + conv = torch.empty([rows, CONV_DIM], dtype=torch.bfloat16, device=x.device) + _launch( + x, in_proj, out, conv, conv_weight, conv_state, slots, window, window_rows, + pad_slot_id, rows_per_request, tactic, pdl, + ) # fmt: skip + return out, conv + + def reference( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ): + """fp32 projection rounded to bf16, then the conv's per-element arithmetic in torch + (writes `conv_state` and `window`: call it on copies).""" + with true_fp32_matmul(): + out = (x.float() @ in_proj.float()).to(torch.bfloat16) + rows = x.shape[0] + p = rows_per_request + conv = torch.zeros([rows, CONV_DIM], dtype=torch.bfloat16, device=x.device) + w = conv_weight.float() + for s in range(rows // p): + slot = int(slots[s]) + if slot == pad_slot_id: + continue + cols = [conv_state[slot, :, j].float() for j in range(CONV_WIDTH - 1)] + for t in range(p): + xv = out[s * p + t, :CONV_DIM].float() + acc = torch.zeros(CONV_DIM, dtype=torch.float32, device=x.device) + for j, v in enumerate(cols + [xv]): + acc = acc + (v * w[:, j]).to(torch.bfloat16).float() + cols = cols[1:] + [xv] + for j in range(CONV_WIDTH - 1): + window[int(window_rows[s]), t, :, j] = cols[j].to(window.dtype) + conv[s * p + t] = (acc / (1 + torch.exp(-acc))).to(torch.bfloat16) + for j in range(CONV_WIDTH - 1): + conv_state[slot, :, j] = cols[j].to(conv_state.dtype) + return out, conv + + def is_valid( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ) -> None: + rows = x.shape[0] + assert x.dtype == in_proj.dtype == torch.bfloat16 and x.is_contiguous() + assert tuple(x.shape) == (rows, K_IN) and tuple(in_proj.shape) == (K_IN, N_OUT) + assert in_proj.stride() == (1, K_IN), "in_proj is the column-major [2560, 4128] operand" + assert rows % rows_per_request == 0 and 1 <= rows <= tactic.token_tile, (rows, tactic) + assert tuple(conv_weight.shape) == (CONV_DIM, CONV_WIDTH) and conv_weight.is_contiguous() + assert conv_state.dim() == 3 and tuple(conv_state.shape[1:]) == (CONV_DIM, CONV_WIDTH - 1) + assert window.dim() == 4 and tuple(window.shape[2:]) == (CONV_DIM, CONV_WIDTH - 1) + assert window.shape[1] >= rows_per_request + assert slots.dtype == window_rows.dtype == torch.int32 + n = rows // rows_per_request + assert slots.shape[0] >= n and window_rows.shape[0] >= n + + +gdn_in_proj_conv = GdnInProjConv() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py new file mode 100644 index 000000000000..733c9e9471e9 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The tails of an in-forward Mamba snapshot split in one launch: each split request's few tokens of +gated delta rule from its state at P (its live pool slot) into its staging row, read in place. + +Per value head `i` (key head `i // (HV // H)`), fp32 state `h` `[V, K]` stored in bf16, `g` the decay: + + h *= g[t]; u = (v[t] - h k[t]) * beta[t]; h += u k[t]^T; o[t] = h (K^-0.5 q[t]) + +token by token, so it matches FlashInfer's chunked form to rounding, not bitwise. q and k arrive +L2-normalized. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +# The tail's token, sequence, slot and stage-row lists are scalar-read views. +@triton.jit(do_not_specialize_on_alignment=["tok_ptr", "cu_ptr", "slot_ptr", "row_ptr"]) +def _gdn_tail_kernel( + q_ptr, + k_ptr, + v_ptr, + g_ptr, + beta_ptr, + tok_ptr, + cu_ptr, + pool_ptr, + slot_ptr, + stage_ptr, + row_ptr, + out_ptr, + stride_qt, + stride_qh, + stride_kt, + stride_kh, + stride_vt, + stride_vh, + stride_gt, + stride_bt, + stride_ps, + stride_ph, + stride_pv, + stride_ss, + stride_sh, + stride_sv, + stride_ot, + stride_oh, + scale, + GROUP: tl.constexpr, + K: tl.constexpr, + BV: tl.constexpr, +): + """One program: sequence `j`, value head `h`, value rows `[vb * BV, (vb + 1) * BV)` of the state.""" + j = tl.program_id(0) + h = tl.program_id(1) + vb = tl.program_id(2) + kh = h // GROUP + rv = vb * BV + tl.arange(0, BV) + rk = tl.arange(0, K) + slot = tl.load(slot_ptr + j).to(tl.int64) + src = pool_ptr + slot * stride_ps + h * stride_ph + rv[:, None] * stride_pv + rk[None, :] + state = tl.load(src).to(tl.float32) + t0 = tl.load(cu_ptr + j) + t1 = tl.load(cu_ptr + j + 1) + for t in range(t0, t1): + row = tl.load(tok_ptr + t).to(tl.int64) + kv = tl.load(k_ptr + row * stride_kt + kh * stride_kh + rk).to(tl.float32) + qv = tl.load(q_ptr + row * stride_qt + kh * stride_qh + rk).to(tl.float32) * scale + vv = tl.load(v_ptr + row * stride_vt + h * stride_vh + rv).to(tl.float32) + gt = tl.load(g_ptr + row * stride_gt + h).to(tl.float32) + bt = tl.load(beta_ptr + row * stride_bt + h).to(tl.float32) + state = state * gt + u = (vv - tl.sum(state * kv[None, :], axis=1)) * bt + state = state + u[:, None] * kv[None, :] + o = tl.sum(state * qv[None, :], axis=1) + tl.store(out_ptr + row * stride_ot + h * stride_oh + rv, o.to(out_ptr.dtype.element_ty)) + srow = tl.load(row_ptr + j).to(tl.int64) + dst = stage_ptr + srow * stride_ss + h * stride_sh + rv[:, None] * stride_sv + rk[None, :] + tl.store(dst, state.to(stage_ptr.dtype.element_ty)) + + +def gdn_inforward_tail( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, +) -> None: + """Tail `j` covers rows `tail_tok[cu_tail[j] : cu_tail[j + 1]]` of q / k `[1, T, H, K]`, v, g / beta + (fp32) and starts from `pool[tail_slots[j]]` (any slot stride); its final state goes to + `stage[stage_rows[j]]` and its outputs to rows `tail_tok` of `out` `[1, T, HV, V]`.""" + num = tail_slots.numel() + if num == 0: + return + _, _, heads, kdim = q.shape + _, _, vheads, vdim = v.shape + assert kdim == 128 and vdim % 32 == 0, (kdim, vdim) + assert pool.stride(3) == 1 and stage.stride(3) == 1 and q.stride(3) == 1 and k.stride(3) == 1 + assert v.stride(3) == 1 and out.stride(3) == 1 and g.stride(2) == 1 and beta.stride(2) == 1 + bv = 32 + _gdn_tail_kernel[(num, vheads, vdim // bv)]( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out, + q.stride(1), q.stride(2), k.stride(1), k.stride(2), v.stride(1), v.stride(2), + g.stride(1), beta.stride(1), + pool.stride(0), pool.stride(1), pool.stride(2), + stage.stride(0), stage.stride(1), stage.stride(2), + out.stride(1), out.stride(2), + kdim**-0.5, + GROUP=vheads // heads, K=kdim, BV=bv, num_warps=4, + ) # fmt: skip + + +def gdn_inforward_tail_reference( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out +) -> None: + """The same recurrence in float64 on the host side of torch (tests).""" + heads, vheads = q.shape[2], v.shape[2] + group = vheads // heads + scale = q.shape[3] ** -0.5 + cu = cu_tail.tolist() + toks = tail_tok.tolist() + for j, slot in enumerate(tail_slots.tolist()): + state = pool[slot].double() + for t in range(cu[j], cu[j + 1]): + r = toks[t] + kk = k[0, r].double().repeat_interleave(group, 0) # [HV, K] + qq = q[0, r].double().repeat_interleave(group, 0) * scale + state = state * g[0, r].double()[:, None, None] + u = (v[0, r].double() - torch.einsum("hvk,hk->hv", state, kk)) * beta[0, r].double()[ + :, None + ] + state = state + u[:, :, None] * kk[:, None, :] + out[0, r] = torch.einsum("hvk,hk->hv", state, qq).to(out.dtype) + stage[int(stage_rows[j])] = state.to(stage.dtype) + + +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "row_ptr", "win_ptr"]) +def _conv_swap_kernel( + conv_ptr, + stage_ptr, + x_ptr, + slot_ptr, + row_ptr, + win_ptr, + C, + stride_cs, + stride_cc, + stride_ss, + stride_sc, + stride_xt, + W: tl.constexpr, + BW: tl.constexpr, + BLOCK: tl.constexpr, +): + """One program: split request `j`, channels `[cb * BLOCK, (cb + 1) * BLOCK)`.""" + j = tl.program_id(0) + cb = tl.program_id(1) + c = cb * BLOCK + tl.arange(0, BLOCK) + w = tl.arange(0, BW) + m = (c[:, None] < C) & (w[None, :] < W) + slot = tl.load(slot_ptr + j).to(tl.int64) + srow = tl.load(row_ptr + j).to(tl.int64) + live = conv_ptr + slot * stride_cs + c[:, None] * stride_cc + w[None, :] + tl.store( + stage_ptr + srow * stride_ss + c[:, None] * stride_sc + w[None, :], + tl.load(live, mask=m), + mask=m, + ) + rows = tl.load(win_ptr + j * W + w, mask=w < W, other=0).to(tl.int64) + tl.store(live, tl.load(x_ptr + rows[None, :] * stride_xt + c[:, None], mask=m), mask=m) + + +def gdn_inforward_conv_swap( + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, +) -> None: + """For split request `j`: `stage[stage_rows[j]] = conv[tail_slots[j]]` (the window ending at the prompt end the + conv pass left), then `conv[tail_slots[j]][:, w] = x[conv_win[j * W + w]]` (the raw rows before P), with `conv` + `[S, C, W]` (any slot stride, rows dense), `stage` `[rows, C, W]` and `x` the `[T, C]` projection columns.""" + num = tail_slots.numel() + if num == 0: + return + _, chans, width = conv.shape + assert ( + conv.stride(2) == 1 + and conv.stride(1) == width + and stage.stride(2) == 1 + and x.stride(1) == 1 + ) + assert x.dtype == conv.dtype == stage.dtype + block = 256 + _conv_swap_kernel[(num, triton.cdiv(chans, block))]( + conv, stage, x, tail_slots, stage_rows, conv_win, chans, + conv.stride(0), conv.stride(1), stage.stride(0), stage.stride(1), x.stride(0), + W=width, BW=triton.next_power_of_2(width), BLOCK=block, num_warps=4, + ) # fmt: skip diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py new file mode 100644 index 000000000000..03f3efab6515 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Vision position-embedding bilinear interpolation onto one image's patch grid, in +spatial-merge order, by the stock Qwen3-VL tower's Triton kernel +(`_triton_pos_embed_interpolate` in `_torch/models/modeling_qwen3vl.py`), one call per image. +The NVFP4 default target's entry; the two checkpoints share the bf16 vision tower. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.models.modeling_qwen3vl import _triton_pos_embed_interpolate + +NUM_GRID = 48 +HIDDEN = 1152 +MERGE = 2 + + +def _ckpt(t: int, h: int, w: int) -> dict: + """The checkpoint's bf16 table at one `(t, h, w)` grid.""" + return dict( + t=t, + h=h, + w=w, + num_grid_per_side=NUM_GRID, + m_size=MERGE, + dtype=torch.bfloat16, + hidden=HIDDEN, + ) + + +class TritonPosEmbedInterpolate(OpWrapper): + """Resample a square row-major `[N*N, hidden]` table onto one `(t, h, w)` + patch grid; return a fresh `[t*h*w, hidden]` tensor in `dtype`. + + Corner-aligned bilinear (`align_corners=True`): row `i` of `h` samples the + table at `i * (N - 1) / (h - 1)` (0 when `h == 1`), `lo = floor`, + `hi = min(lo + 1, N - 1)`; `h` indexes table rows (stride `N`), `w` + columns. Rows come out in spatial-merge order -- whole `m_size x m_size` + blocks row-major, row-major inside a block -- which is what the patch + merger's 4-to-1 reshape consumes. `t` repeats the frame's rows. + + Arithmetic is the HF reference's (`fast_pos_embed_interpolate`): fp32 corner + weights rounded to `dtype`, multiply-adds in `dtype` (not an fp32 accumulation). + """ + + ARCHS = frozenset({Arch.SM_103}) + + # The tower is replicated on every rank, so these are every member's shapes. + CELLS: tuple[Cell, ...] = ( + Cell(why="one 4x8 image: 32 rows, both sides below the table's 48", spec=_ckpt(1, 4, 8)), + Cell(why="the 62x90 serving grid: 5580 rows, one image per call", spec=_ckpt(1, 62, 90)), + Cell(why="a 3-frame 6x10 video: t repeats one frame's 60 rows", spec=_ckpt(3, 6, 10)), + Cell( + why="128x256: 32768 rows, the tower's per-call cap and the envelope's largest", + spec=_ckpt(1, 128, 256), + ), + ) + + note = """ + Source: the Triton kernel `_bilinear_pos_embed_kernel` behind the stock + `_triton_pos_embed_interpolate` (not a registered op). Measured on GB300 within 4 / 1 + bf16 ulp (element / relative RMS) of the fp64 reference. `is_valid` refuses what the + kernel computes silently wrong: a strided table, a `num_grid_per_side` that does not + match the table, an empty side, `h` or `w` not divisible by `m_size`. Hard limit: + `t*h*w*hidden` below 2^31 (int32 output offset). + Not certified: tables with uncorrelated neighbouring rows (can exceed the + 4-ulp band), fp16 or fp32 tables, a `dtype` other than the table's, `m_size` + other than 2, a hidden size other than 1152, a CPU table, and CUDA-graph + capture (the vision tower is eager only). + """ + + def __call__( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> torch.Tensor: + return _triton_pos_embed_interpolate( + embed_weight, t, h, w, num_grid_per_side, m_size, dtype + ) + + def reference( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """The four-corner weighted sum in fp64 at the exact sample positions, rounded once to + fp32 (not to `dtype`, so a band in `dtype` ulp measures the op's error alone).""" + n = num_grid_per_side + hidden = embed_weight.shape[1] + device = embed_weight.device + grid = embed_weight.double().reshape(n, n, hidden) + + def positions(size: int) -> torch.Tensor: + index = torch.arange(size, dtype=torch.float64, device=device) + return index * ((n - 1) / (size - 1)) if size > 1 else torch.zeros_like(index) + + h_pos, w_pos = positions(h), positions(w) + h_lo, w_lo = h_pos.floor().long(), w_pos.floor().long() + h_hi = (h_lo + 1).clamp(max=n - 1) + w_hi = (w_lo + 1).clamp(max=n - 1) + d_h = (h_pos - h_lo)[:, None, None] + d_w = (w_pos - w_lo)[None, :, None] + raster = ( + grid[h_lo][:, w_lo] * ((1 - d_h) * (1 - d_w)) + + grid[h_lo][:, w_hi] * ((1 - d_h) * d_w) + + grid[h_hi][:, w_lo] * (d_h * (1 - d_w)) + + grid[h_hi][:, w_hi] * (d_h * d_w) + ) + rows = ( + raster.reshape(h // m_size, m_size, w // m_size, m_size, hidden) + .permute(0, 2, 1, 3, 4) + .reshape(h * w, hidden) + ) + return (rows.repeat(t, 1) if t > 1 else rows).float() + + def is_valid( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> None: + assert embed_weight.is_contiguous(), ( + "embed_weight must be contiguous; the kernel indexes it as dense rows " + "and reads a strided table wrongly without raising" + ) + assert num_grid_per_side * num_grid_per_side == embed_weight.shape[0], ( + f"num_grid_per_side {num_grid_per_side} does not describe a " + f"{embed_weight.shape[0]}-row table; the kernel resamples the grid it is " + f"told, in or out of the table" + ) + assert t > 0 and h > 0 and w > 0, ( + f"empty grid side (t, h, w) = ({t}, {h}, {w}); the kernel returns no rows " + f"rather than raising" + ) + assert h % m_size == 0 and w % m_size == 0, ( + f"grid sides h={h}, w={w} are not whole m_size={m_size} merge blocks; " + f"the kernel's bare assert is all that stops an out-of-grid read" + ) + + +triton_pos_embed_interpolate = TritonPosEmbedInterpolate() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py new file mode 100644 index 000000000000..6fd62cd8dfc6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py @@ -0,0 +1,574 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The KV cache manager's pools as one rank of this target sees them, and their contract. + +Every rank builds the stock ``Qwen4ExpHybridCacheManager``, a KV cache manager V2 (which QSA +requires). `pool_geometry` derives the per-rank shapes it allocates: + +=================== ============================================ ============================= +pool shape after the slot dim split over ranks by +=================== ============================================ ============================= +paged K/V (HND) ``[2, H_kv, tokens_per_block, kv_width]`` attention TP (a) +NVFP4 block scales ``[2, H_kv, tokens_per_block, head_dim/16]`` attention TP (a) +QSA index plane ``[tokens_per_block, 1, index_head_dim]`` replicated (b) +QSA position plane ``[tokens_per_block, 3]`` int32 replicated (b) +GDN conv state ``[2 * H_k * D_k + H_v * D_v, kernel - 1]`` recurrent-state TP (c) +GDN recurrent state ``[H_v, D_v, D_k]`` recurrent-state TP (c) +PLE conv state ``[hc_count * hidden, (ple_kernel-1) * n]`` replicated (d) +PLE n-gram history ``[n - 1]`` int64 replicated (d) +=================== ============================================ ============================= + +``H_kv`` is ``num_key_value_heads``, ``H_k``/``D_k`` and ``H_v``/``D_v`` the GDN key and value +heads, ``kernel`` ``linear_conv_kernel_dim``, ``ple_kernel`` ``ple_conv_kernel_size`` and ``n`` +``ngram_size``; head counts are the rank's share. + +(a) Over ``tp_size`` (1 under attention DP), rounding up: with more ranks than KV heads, rank + ``r`` holds global KV head ``r * H / tp``, the order ``duplicate_kv_weight`` repeats them in. +(b) No mapping divides the indexer. When an index or position page could equal a K/V or + block-scale page, which V2 would coalesce into that pool, both are packed into one per-slot + ``QSA_SIDE_PLANES`` page (`PoolGeometry.qsa_side_planes`), as strided views of these shapes. +(c) Over ``mamba_effective_tp_size`` (1 under attention DP). Conv channels are ``[q | k | v]`` of + the rank's own heads; rank ``r`` holds key heads from ``r * H_k / tp`` and value heads from + ``r * H_v / tp``, the contiguous ranges a disaggregated transfer between layouts moves. The + recurrent state is served in bf16 only (`SSM_STATE_DTYPES`), not the checkpoint's fp32. +(d) The PLE layer reads the whole hyper-connection stream on every rank. + +With one-model MTP the draft layer (global layer ``num_hidden_layers``) is one more QSA attention +layer at the trunk's per-rank shapes (`draft_kv_cache_is_separate`). + +The ``check_*`` functions read only ``.shape``, ``.dtype`` and ``.stride()``, so they also run on +meta tensors. Each rejects a pool its consuming kernel would accept and mis-address. +""" + +from __future__ import annotations + +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.pyexecutor.config_utils import mamba_effective_tp_size +from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache +from tensorrt_llm.quantization.mode import QuantAlgo + +#: Coordinates per token in the QSA position plane (the mRoPE layout's three rows). +QSA_POSITION_AXES = 3 + +#: Values per e4m3 block scale of an NVFP4 K/V pool. +NVFP4_BLOCK = 16 + +#: Recurrent-state pool dtypes served (`kv_cache_config.mamba_ssm_cache_dtype`). +SSM_STATE_DTYPES = (torch.bfloat16,) + +#: The GDN cached replay's history length the replay entry is certified at. +GDN_REPLAY_HISTORY = 16 + +#: Most query rows per generation request in a verify step (`max_draft_len + 1`): the replay +#: kernel asserts `T <= 8` and `thop_attention`'s `predicted_tokens_per_seq` is certified to it. +MAX_VERIFY_WIDTH = 8 + +#: CUDA-graph padding dummy slots: one per draft length, and routing admits one fixed length. +CUDA_GRAPH_PADDING_DUMMY_SLOTS = 1 + +#: Bytes per element of the QSA index (bf16) and position (int32) planes. +_INDEX_ELEMENT_BYTES = 2 +_POSITION_ELEMENT_BYTES = 4 + + +class PoolGeometry(NamedTuple): + """The pool shapes one rank binds. Head counts and channel widths are this rank's share.""" + + tokens_per_block: int + heads_kv: int # KV heads in this rank's K/V pages + kv_head_start: int # the global KV head the rank's first page head holds + head_dim: int + index_kv_heads: int + index_head_dim: int + conv_dim: int # this rank's GDN conv channels, [q | k | v] of its own heads + conv_kernel: int + num_k_heads: int # this rank's GDN key heads + k_head_start: int # the global key head the rank's first one is + num_v_heads: int # this rank's GDN value heads + v_head_start: int # the global value head the rank's first one is + head_v_dim: int + head_k_dim: int + stream: int # the hyper-connection stream width, the PLE conv channels + ple_state_len: int + ngram_context_len: int + dtype: torch.dtype # the activation dtype: GDN conv, QSA index and PLE conv pools + kv_dtype: torch.dtype # the paged K/V pool's dtype: bf16, e4m3 (FP8) or uint8 (NVFP4) + kv_width: int # its innermost extent: head_dim, or head_dim // 2 for NVFP4's e2m1 pairs + qsa_side_planes: bool # the index and position planes share one packed page per slot + + @property + def conv_sections(self) -> tuple[int, int, int]: + """The q, k and v channel widths of this rank's conv state, in that order.""" + key = self.num_k_heads * self.head_k_dim + return key, key, self.num_v_heads * self.head_v_dim + + @property + def kv_block_scales(self) -> bool: + """Whether the K/V pool is NVFP4, whose e4m3 block scales sit in a second pool.""" + return self.kv_dtype == torch.uint8 + + @property + def index_page_bytes(self) -> int: + per_token = self.index_kv_heads * self.index_head_dim * _INDEX_ELEMENT_BYTES + return self.tokens_per_block * per_token + + @property + def position_page_bytes(self) -> int: + return self.tokens_per_block * QSA_POSITION_AXES * _POSITION_ELEMENT_BYTES + + +def attention_tp_size(mapping) -> int: + """The ranks the KV heads are split over (`KVCacheManagerV2.__init__`).""" + return 1 if mapping.enable_attention_dp else mapping.tp_size + + +def reserved_dummy_slots(mapping) -> int: + """State slots beyond one per request: the graph padding dummy and the attention-DP dummy.""" + return CUDA_GRAPH_PADDING_DUMMY_SLOTS + int(mapping.enable_attention_dp) + + +def draft_kv_cache_is_separate(mapping, spec_config=None) -> bool: + """Whether the one-model MTP draft layer has a KV cache manager of its own. + + Only under attention TP, unless `spec_config` disallows it (disaggregated serving does).""" + if mapping.enable_attention_dp: + return False + return spec_config is None or should_use_separate_draft_kv_cache(spec_config) + + +def _share(total: int, ranks: int, rank: int) -> tuple[int, int]: + """(heads on `rank`, its first global head): an even split, or each head on several ranks.""" + if total % ranks and ranks % total: + raise ValueError(f"{total} heads do not split over {ranks} ranks") + return -(-total // ranks), rank * total // ranks + + +#: K/V pool storage per `kv_cache_quant_algo`: (dtype or None for activation, values per element). +_KV_STORAGE = { + None: (None, 1), + QuantAlgo.FP8: (torch.float8_e4m3fn, 1), + QuantAlgo.NVFP4: (torch.uint8, 2), +} + + +def _side_planes_packed( + num_kv_heads: int, + kv_page_bytes_per_head: int, + scale_page_bytes_per_head: int, + index_page: int, + position_page: int, +) -> bool: + """Whether `_extra_buffers_per_layer` packs the QSA index and position pages into one page.""" + pages = [p for p in (kv_page_bytes_per_head, scale_page_bytes_per_head) if p] + main = {page * heads for page in pages for heads in range(1, num_kv_heads + 1)} + return index_page in main or position_page in main + + +def pool_geometry( + cfg, mapping, *, tokens_per_block: int, kv_cache_quant_algo, dtype: torch.dtype +) -> PoolGeometry: + """The pools rank `mapping.rank` binds, from the engine-normalized `Qwen4ExpTextConfig`.""" + if kv_cache_quant_algo not in _KV_STORAGE: + raise ValueError( + f"kv_cache_quant_algo {kv_cache_quant_algo!r}; the K/V pool stores the checkpoint's " + "bf16, FP8 or NVFP4" + ) + if (mapping.pp_size, mapping.cp_size) != (1, 1): + raise ValueError( + f"pp_size={mapping.pp_size}, cp_size={mapping.cp_size}: the members split heads over " + "the TP group alone, and the manager shards pipeline and helix layouts otherwise" + ) + attn_tp = attention_tp_size(mapping) + heads_kv, kv_head_start = _share( + cfg.num_key_value_heads, attn_tp, mapping.tp_rank if attn_tp > 1 else 0 + ) + state_tp = mamba_effective_tp_size(mapping) + state_rank = mapping.tp_rank if state_tp > 1 else 0 + num_k_heads, k_head_start = _share(cfg.linear_num_key_heads, state_tp, state_rank) + num_v_heads, v_head_start = _share(cfg.linear_num_value_heads, state_tp, state_rank) + kv_dtype, per_element = _KV_STORAGE[kv_cache_quant_algo] + kv_dtype = kv_dtype or dtype + kv_width = cfg.head_dim // per_element + block_scales = kv_dtype == torch.uint8 + index_page = ( + tokens_per_block * cfg.indexer_kv_heads * cfg.indexer_head_dim * _INDEX_ELEMENT_BYTES + ) + position_page = tokens_per_block * QSA_POSITION_AXES * _POSITION_ELEMENT_BYTES + side_planes = _side_planes_packed( + cfg.num_key_value_heads, + tokens_per_block * kv_width * kv_dtype.itemsize, + tokens_per_block * (cfg.head_dim // NVFP4_BLOCK) if block_scales else 0, + index_page, + position_page, + ) + return PoolGeometry( + tokens_per_block=tokens_per_block, + heads_kv=heads_kv, + kv_head_start=kv_head_start, + head_dim=cfg.head_dim, + index_kv_heads=cfg.indexer_kv_heads, + index_head_dim=cfg.indexer_head_dim, + conv_dim=2 * num_k_heads * cfg.linear_key_head_dim + + num_v_heads * cfg.linear_value_head_dim, + conv_kernel=cfg.linear_conv_kernel_dim, + num_k_heads=num_k_heads, + k_head_start=k_head_start, + num_v_heads=num_v_heads, + v_head_start=v_head_start, + head_v_dim=cfg.linear_value_head_dim, + head_k_dim=cfg.linear_key_head_dim, + stream=cfg.hc_count * cfg.hidden_size, + ple_state_len=(cfg.ple_conv_kernel_size - 1) * cfg.ngram_size, + ngram_context_len=cfg.ngram_size - 1, + dtype=dtype, + kv_dtype=kv_dtype, + kv_width=kv_width, + qsa_side_planes=side_planes, + ) + + +def require(cm, names, what: str) -> None: + for name in names: + assert hasattr(cm, name), ( + f"the KV cache manager {type(cm).__name__} does not expose {name}; this target {what}" + ) + + +def _check_pool(t, what, lead, tail, dtype=None) -> None: + """`t` has `lead` leading dims of any size, then exactly `tail`, at `dtype`.""" + assert t is not None, f"{what}: the runtime did not allocate it" + expected = ["*"] * lead + [str(n) for n in tail] + assert t.dim() == lead + len(tail) and tuple(t.shape[lead:]) == tuple(tail), ( + f"{what} is {tuple(t.shape)}, expected [{', '.join(expected)}]" + ) + assert dtype is None or t.dtype == dtype, f"{what} is {t.dtype}, expected {dtype}" + + +def check_position_plane(g: PoolGeometry, position) -> None: + shape = (g.tokens_per_block, QSA_POSITION_AXES) + _check_pool(position, "QSA position plane", 1, shape, torch.int32) + + +def check_attention_pools(g: PoolGeometry, layer: int, index_cache, pool) -> None: + index_shape = (g.tokens_per_block, g.index_kv_heads, g.index_head_dim) + _check_pool(index_cache, f"layer {layer}: QSA index plane", 1, index_shape, g.dtype) + # The sparse path needs adjacent HND K/V role planes: [pages, 2, H, block, D]. + kv_shape = (g.heads_kv, g.tokens_per_block, g.kv_width) + _check_pool(pool, f"layer {layer}: paged K/V", 2, kv_shape, g.kv_dtype) + assert pool.shape[1] == 2, f"layer {layer}: paged K/V has {pool.shape[1]} role planes, not 2" + + +def check_kv_scale_planes(g: PoolGeometry, layer: int, pool, scales) -> None: + """NVFP4 block scales: `[slots, 2, H, block, D / 16]`, as many pages per slot as the data.""" + scale_shape = (g.heads_kv, g.tokens_per_block, g.head_dim // NVFP4_BLOCK) + _check_pool(scales, f"layer {layer}: NVFP4 block scales", 2, scale_shape, torch.uint8) + assert scales.shape[1] == 2 and scales.shape[0] == pool.shape[0], ( + f"layer {layer}: the block scales are {tuple(scales.shape)} against paged K/V " + f"{tuple(pool.shape)}; they must hold the same slots, K then V" + ) + pages = [] + for what, t in (("paged K/V", pool), ("block scales", scales)): + page = t[0, 0].numel() + assert t.stride(0) % page == 0, ( + f"layer {layer}: the {what} slot stride {t.stride(0)} is not a whole number of " + f"{page}-element pages" + ) + pages.append(t.stride(0) // page) + assert pages[0] == pages[1], ( + f"layer {layer}: the paged K/V pool holds {pages[0]} pages per slot and the block-scale " + f"pool {pages[1]}; the attention op would address the scales of other slots" + ) + + +def check_side_planes(g: PoolGeometry, index_caches, position) -> None: + """Packed QSA planes (`g.qsa_side_planes`) all advance by one slot stride covering them.""" + if not g.qsa_side_planes: + return + stride = position.stride(0) * position.element_size() + for layer, index in index_caches.items(): + index_stride = index.stride(0) * index.element_size() + assert index_stride == stride, ( + f"layer {layer}: the QSA index plane advances {index_stride} bytes a slot and the " + f"position plane {stride}; packed side planes share one per-slot page" + ) + need = len(index_caches) * g.index_page_bytes + g.position_page_bytes + assert stride >= need, ( + f"the packed QSA side page is {stride} bytes a slot, short of the {len(index_caches)} " + f"index pages and the position page it holds ({need} bytes)" + ) + + +def check_recurrent_pools(g: PoolGeometry, conv, ssm) -> None: + _check_pool(conv, "conv state pool", 1, (g.conv_dim, g.conv_kernel - 1), g.dtype) + # Key dim innermost. + _check_pool(ssm, "recurrent state pool", 1, (g.num_v_heads, g.head_v_dim, g.head_k_dim)) + assert ssm.dtype in SSM_STATE_DTYPES, ( + f"the recurrent state pool is {ssm.dtype}; this target serves " + f"{' or '.join(str(d) for d in SSM_STATE_DTYPES)} (kv_cache_config.mamba_ssm_cache_dtype)" + ) + + +def check_conv_layout(g: PoolGeometry, section_dims) -> None: + assert tuple(section_dims) == g.conv_sections, ( + f"the manager lays a rank's conv state out as sections {tuple(section_dims)}; this target " + f"writes [q | k | v] = {g.conv_sections} (conv_state_layout 'q_k_v')" + ) + + +def check_ple_pools(g: PoolGeometry, ple_conv, ple_ngram, conv) -> None: + _check_pool(ple_conv, "PLE conv state", 1, (g.stream, g.ple_state_len), g.dtype) + # The id-history entry writes int64 into it; an int32 pool raises. + _check_pool(ple_ngram, "PLE n-gram context pool", 1, (g.ngram_context_len,), torch.int64) + # The forward addresses all three with one scheduler slot vector. + assert ple_conv.shape[0] == conv.shape[0] == ple_ngram.shape[0], ( + "the PLE and Gated-DeltaNet pools must share one slot space" + ) + + +def check_state_slots(mapping, max_batch_size: int, slots: int) -> None: + need = max_batch_size + reserved_dummy_slots(mapping) + assert slots >= need, ( + f"the recurrent-state pools hold {slots} slots; max_batch_size {max_batch_size} plus " + f"{reserved_dummy_slots(mapping)} reserved dummies (attention DP " + f"{mapping.enable_attention_dp}) need {need}" + ) + + +def check_step_buffers(g: PoolGeometry, int_conv, conv, int_idx) -> None: + """MTP verify buffers: per-step conv windows at the pool dtype (promotion does not convert).""" + assert int_conv is not None, ( + "the runtime allocated no per-step conv buffer; a partially accepted draft " + "cannot be rolled back" + ) + _check_pool(int_conv, "per-step conv buffer", 2, (g.conv_dim, g.conv_kernel - 1), conv.dtype) + assert int_idx is not None and int_idx.dtype == torch.int32, ( + "the cache manager exposes no int32 intermediate_state_indices" + ) + + +def check_replay_state(history, width, prev_accepted, cache_buf_idx, ssm) -> None: + """Cached replay, manager-wide: the certified geometry and the per-slot bookkeeping.""" + assert ssm.dtype in SSM_STATE_DTYPES, ( + f"the recurrent pool is {ssm.dtype}; this target runs the replay over a bf16 checkpoint " + "state (kv_cache_config.mamba_ssm_cache_dtype: bfloat16)" + ) + assert history == GDN_REPLAY_HISTORY, ( + f"replay_history_size {history}; the replay entry is certified at {GDN_REPLAY_HISTORY}" + ) + assert 2 <= width <= min(MAX_VERIFY_WIDTH, history), ( + f"replay_step_width {width}; the replay entry is certified at 2..{MAX_VERIFY_WIDTH} " + f"verify rows and never above the history ({history})" + ) + for name, t in (("prev_num_accepted_tokens", prev_accepted), ("cache_buf_idx", cache_buf_idx)): + assert t.dim() == 1 and t.shape[0] >= ssm.shape[0] and t.dtype == torch.int32, ( + f"{name} is {tuple(t.shape)} {t.dtype}; the kernel reads one int32 per pool slot " + f"({ssm.shape[0]})" + ) + + +def check_replay_history( + g: PoolGeometry, layer: int, history, pool_slots, old_x, old_B, old_dt +) -> None: + """One GDN layer's replay history: `[slots, 2 halves, ...]` over this rank's heads, addressed + by `history` and the head geometry, never by the buffer's own shape.""" + for name, t in (("old_x", old_x), ("old_B", old_B), ("old_dt", old_dt)): + assert t is not None, f"layer {layer}: the manager's layer cache carries no {name}" + assert old_x.shape[0] >= pool_slots, ( + f"layer {layer}: the replay history has {old_x.shape[0]} slots for a {pool_slots}-slot pool" + ) + bf16, fp32 = torch.bfloat16, torch.float32 + x_tail, b_tail = (history, g.num_v_heads, g.head_v_dim), (history, g.num_k_heads, g.head_k_dim) + _check_pool(old_x, f"layer {layer}: old_x", 2, x_tail, bf16) + _check_pool(old_B, f"layer {layer}: old_B", 2, b_tail, bf16) + _check_pool(old_dt, f"layer {layer}: old_dt", 2, (g.num_v_heads, history), fp32) + assert old_x.shape[1] == old_B.shape[1] == old_dt.shape[1] == 2, ( + f"layer {layer}: the replay history is not double-buffered" + ) + + +def check_verify_width(rows: int) -> None: + assert 1 <= rows <= MAX_VERIFY_WIDTH, ( + f"predicted_tokens_per_seq {rows} is outside the 1..{MAX_VERIFY_WIDTH} range " + "attention/thop_attention is certified at" + ) + + +def check_replay_width(replay, rows: int) -> None: + """A cached-replay step carries exactly `replay.width` rows per sequence. + + The pool row is a checkpoint whose pending history only the replay kernel applies.""" + assert replay is None or rows == replay.width, ( + f"the batch carries {rows} query rows per generation sequence and the GDN cached " + f"replay runs at step width {replay.width}; a runtime_draft_len of {rows - 1} (a " + "draft-length schedule, or skipped drafting) cannot run under it. Export " + "TRTLLM_USE_GDN_REPLAY=0 for variable draft lengths" + ) + + +def check_shared_selection(shared, slots, layer: int, block_topk: int) -> None: + assert shared is not None, "IndexShare is on but the QSA metadata allocated no shared buffers" + assert layer in slots and slots[layer] < shared.shape[0], ( + f"the KV cache manager gives layer {layer} no row of the {shared.shape[0]}-row shared " + f"selection buffers (qsa_shared_topk_slots={slots})" + ) + assert shared.shape[2] == block_topk, ( + f"the shared selection rows are {shared.shape[2]} wide and this target selects " + f"{block_topk} blocks per row" + ) + + +class KvPlanes(NamedTuple): + """One layer's paged K/V views, and under an NVFP4 pool its e4m3 block-scale views: + `[slots, H_kv, tokens_per_block, kv_width]` and `[slots, H_kv, tokens_per_block, D / 16]`.""" + + k: torch.Tensor + v: torch.Tensor + k_scale: torch.Tensor | None = None + v_scale: torch.Tensor | None = None + + +class RankPools(NamedTuple): + """The trunk's pool views on one rank, from one manager. A view is a `TensorWrapper` over a + raw device address that keeps no memory alive, so it is valid only for that manager.""" + + pool_idx: dict # attention layer -> pool offset (what thop_attention addresses) + index_cache: dict # attention layer -> QSA index plane + position_cache: torch.Tensor # the shared QSA position plane + kv: dict # attention layer -> `KvPlanes` + conv: dict # GDN layer -> conv state pool + ssm: dict # GDN layer -> recurrent state pool + ple_state: tuple # (PLE conv state, PLE n-gram id history) + + +class DraftPools(NamedTuple): + """The MTP draft layer's views in the manager its draft step is handed.""" + + pool_idx: int # the draft layer's pool offset in that manager + index_cache: torch.Tensor # its QSA index plane + position_cache: torch.Tensor # that manager's position plane + kv: KvPlanes + + +def kv_planes(cm, g: PoolGeometry, layer: int, index_cache) -> KvPlanes: + """One layer's checked paged K/V views from `cm`, plus the block-scale views under NVFP4.""" + pool = cm.get_buffers(layer, kv_layout="HND") + check_attention_pools(g, layer, index_cache, pool) + if not g.kv_block_scales: + return KvPlanes(pool[:, 0], pool[:, 1]) + require(cm, ("get_block_scale_buffers",), "reads an NVFP4 pool's e4m3 block-scale planes") + scales = cm.get_block_scale_buffers(layer, kv_layout="HND") + check_kv_scale_planes(g, layer, pool, scales) + return KvPlanes(pool[:, 0], pool[:, 1], scales[:, 0], scales[:, 1]) + + +def take_pools(cm, g: PoolGeometry, attn_layers, gdn_layers, ple_layer: int) -> RankPools: + """Every trunk pool view of `cm` this rank binds, each checked against `g`. + + Step-dependent state (pool ids, windows, the QSA selection width) is the binder's to check.""" + require( + cm, + ("get_index_k_buffer", "get_qsa_position_buffer", "get_buffers", "layer_offsets"), + "reads the QSA index plane, the shared position plane and the paged K/V", + ) + # Only the V2 hybrid manager has `ple_layer_cache`; under V1 the PLE state is silently absent. + require( + cm, + ("get_conv_states", "get_ssm_states", "conv_section_dims", "ple_layer_cache"), + "reads the recurrent, conv and PLE pools", + ) + require(cm, ("max_batch_size",), "sizes the recurrent-state slots it addresses") + # thop_attention addresses a layer by its pool offset, not its ordinal among attention layers. + pool_idx = {i: cm.layer_offsets[i] for i in attn_layers} + position_cache = cm.get_qsa_position_buffer() + check_position_plane(g, position_cache) + index_cache, kv = {}, {} + for i in attn_layers: + index_cache[i] = cm.get_index_k_buffer(i) + kv[i] = kv_planes(cm, g, i, index_cache[i]) + check_side_planes(g, index_cache, position_cache) + + check_conv_layout(g, cm.conv_section_dims) + conv = {i: cm.get_conv_states(i) for i in gdn_layers} + ssm = {i: cm.get_ssm_states(i) for i in gdn_layers} + for i in gdn_layers: + check_recurrent_pools(g, conv[i], ssm[i]) + + ple_cache = cm.ple_layer_cache(ple_layer) + assert ple_cache is not None, ( + f"no PLE lifecycle state for decoder layer {ple_layer}: the runtime allocated neither " + "the dilated conv state nor the n-gram id history" + ) + ple_conv, ple_ngram = ple_cache + check_ple_pools(g, ple_conv, ple_ngram, conv[gdn_layers[0]]) + check_state_slots(cm.mapping, cm.max_batch_size, ple_conv.shape[0]) + return RankPools(pool_idx, index_cache, position_cache, kv, conv, ssm, (ple_conv, ple_ngram)) + + +def take_draft_pools(cm, g: PoolGeometry, layer: int, *, separate: bool) -> DraftPools: + """The draft layer's checked views in `cm`, the manager its draft step is handed.""" + require( + cm, + ("get_index_k_buffer", "get_qsa_position_buffer", "get_buffers", "layer_offsets"), + "reads the draft layer's QSA index plane, position plane and paged K/V", + ) + layers = set(cm.layer_offsets) + assert layer in layers, ( + f"the MTP module's attention is global layer {layer}, which the cache manager " + f"{type(cm).__name__} does not map: the engine allocated no KV layer for the draft module" + ) + if separate: + assert layers == {layer}, ( + f"the separate draft KV cache maps layers {sorted(layers)}; it holds the draft layer " + f"{layer} alone" + ) + else: + assert len(layers) > 1, ( + f"without a separate draft KV cache the draft layer {layer} is a layer of the trunk's " + "manager, which maps the trunk's layers too" + ) + position_cache = cm.get_qsa_position_buffer() + check_position_plane(g, position_cache) + index_cache = cm.get_index_k_buffer(layer) + kv = kv_planes(cm, g, layer, index_cache) + if separate: + check_side_planes(g, {layer: index_cache}, position_cache) + return DraftPools(cm.layer_offsets[layer], index_cache, position_cache, kv) + + +__all__ = [ + "CUDA_GRAPH_PADDING_DUMMY_SLOTS", + "GDN_REPLAY_HISTORY", + "MAX_VERIFY_WIDTH", + "NVFP4_BLOCK", + "QSA_POSITION_AXES", + "SSM_STATE_DTYPES", + "DraftPools", + "KvPlanes", + "PoolGeometry", + "RankPools", + "attention_tp_size", + "check_attention_pools", + "check_conv_layout", + "check_kv_scale_planes", + "check_ple_pools", + "check_position_plane", + "check_recurrent_pools", + "check_replay_history", + "check_replay_state", + "check_replay_width", + "check_shared_selection", + "check_side_planes", + "check_state_slots", + "check_step_buffers", + "check_verify_width", + "draft_kv_cache_is_separate", + "kv_planes", + "pool_geometry", + "require", + "reserved_dummy_slots", + "take_draft_pools", + "take_pools", +] diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py new file mode 100644 index 000000000000..b4d169760aa4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -0,0 +1,617 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The vision tower and the multimodal splice's host bookkeeping. + +The checkpoint's 27-block bf16 ViT and its 2x2 patch merger turn a request's media into one +`[rows, hidden]` row per placeholder token, which the prefill target's `_splice` writes over the +token embedding. Every rank holds the whole tower and runs it eagerly, outside any captured graph, +on the media of the requests it serves (under attention TP the whole batch's, with identical rows +on every rank). The math is HuggingFace's `Qwen3_5VisionModel`. +""" + +from __future__ import annotations + +import math +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import ( + thop_attention, +) +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_erf import ( + gelu_erf, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_tanh import ( + gelu_tanh, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.layer_norm import ( + layer_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.vision.patch_embed_conv3d import ( + patch_embed_conv3d, +) +from tensorrt_llm.inputs.multimodal import get_multimodal_item_metadata + +from .ops.attention.vision_rope_2d_packed_qk import vision_rope_2d_packed_qk +from .ops.vision.triton_pos_embed_interpolate import triton_pos_embed_interpolate + +#: Tower rope base (`vision_config` has no rope field); unrelated to the text 1e7. +VISION_ROPE_THETA = 10000.0 + +#: LayerNorm epsilon of `norm1`, `norm2`, `merger.norm`: a literal in the reference. +VISION_NORM_EPS = 1e-6 + +#: Most patch rows per tower call (8192 placeholder tokens x `spatial_merge_size`**2), the vision +#: ops' certified ceiling. Calls are cut between frames, so a request is bounded per frame. +VISION_CALL_MAX_PATCHES = 32768 + +#: `mask_type` 0: bidirectional within a batch segment (causal would depend on raster order). +_VISION_MASK_PADDING = 0 + +# `thop_attention`'s encoder-only mode, selected by omitting the KV cache pool arguments. +# `update_kv_cache=False` is rejected, so it stays True. +_VISION_CALL_CONSTANTS = dict( + k=None, + v=None, + output_sf=None, + kv_cache_block_offsets=None, + host_kv_cache_pool_pointers=None, + host_kv_cache_pool_mapping=None, + tokens_per_block=None, + use_paged_context_fmha=False, + is_fused_qkv=True, + update_kv_cache=True, + predicted_tokens_per_seq=1, + local_layer_idx=0, + max_context_q_len_override=None, + cache_indirection=None, + kv_scale_orig_quant=None, + kv_scale_quant_orig=None, + out_scale=None, + quant_mode=0, + beam_width=1, + mask_type=_VISION_MASK_PADDING, + q_scaling=1.0, + # Rotation is applied before the op (`vision_rope_2d_packed_qk`); rope_* inert. + position_embedding_type=0, + rotary_inv_freq=None, + rotary_cos_sin=None, + rope_dim=0, + rope_base=10000.0, + rope_scale_type=0, + rope_scale=1.0, + rope_short_m_scale=1.0, + rope_long_m_scale=1.0, + rope_max_positions=1024, + rope_original_max_positions=1024, + attention_input_type=0, + is_mla_enable=False, + chunked_prefill_buffer_batch_size=1, + attention_sinks=None, + block_ids_per_seq=None, + latent_cache=None, + q_pe=None, + q_lora_rank=None, + kv_lora_rank=None, + qk_nope_head_dim=None, + qk_rope_head_dim=None, + v_head_dim=None, + rope_append=None, + attention_chunk_size=None, + softmax_stats_tensor=None, + is_spec_decoding_enabled=False, + use_spec_decoding=False, + is_spec_dec_tree=False, + spec_decoding_generation_lengths=None, + spec_decoding_position_offsets=None, + spec_decoding_packed_mask=None, + spec_decoding_bl_tree_mask_offset=None, + spec_decoding_bl_tree_mask=None, + spec_bl_tree_first_sparse_mask_offset_kv=None, + sparse_kv_indices=None, + sparse_kv_offsets=None, + sparse_attn_indices=None, + sparse_attn_offsets=None, + sparse_attn_indices_block_size=0, + mrope_rotary_cos_sin=None, + mrope_position_deltas=None, + helix_position_offsets=None, + helix_is_inactive_rank=None, + is_cross=False, + cross_kv=None, + relative_attention_bias=None, + relative_attention_max_distance=0, + kv_norm_weight=None, + kv_norm_eps=1e-6, + skip_correction_threshold=0.0, +) + +#: Per modality: the `multimodal_data` keys of the flat patch rows and the host `(t, h, w)` grid. +MEDIA_KEYS = { + "image": ("pixel_values", "image_grid_thw"), + "video": ("pixel_values_videos", "video_grid_thw"), +} + +#: `multimodal_data` tensors moved to the device; the grids stay on the host, read sync-free. +DEVICE_PATHS = ( + "image.pixel_values", + "video.pixel_values_videos", + "mrope_config.mrope_position_ids", + "mrope_config.mrope_position_deltas", +) + +#: The position-ids axes of the mRoPE layout a media request carries. +_POSITION_AXES = 3 + + +class _VisionBlockWeights(NamedTuple): + """One vision block's weights (`.t()` GEMM views); its LayerNorms have biases.""" + + n1_w: torch.Tensor + n1_b: torch.Tensor + qkv: torch.Tensor + qkv_b: torch.Tensor + proj: torch.Tensor + proj_b: torch.Tensor + n2_w: torch.Tensor + n2_b: torch.Tensor + fc1: torch.Tensor + fc1_b: torch.Tensor + fc2: torch.Tensor + fc2_b: torch.Tensor + + +class _VisionWeights(NamedTuple): + """The tower's weights: patch embedding, position table, blocks and patch merger.""" + + patch_w: torch.Tensor + patch_b: torch.Tensor + pos: torch.Tensor + blocks: list + m_n_w: torch.Tensor + m_n_b: torch.Tensor + m_fc1: torch.Tensor + m_fc1_b: torch.Tensor + m_fc2: torch.Tensor + m_fc2_b: torch.Tensor + + +def _vision_views(w: dict, depth: int) -> _VisionWeights: + """The weight table's `vis_*` entries (`weights.py`) as the views the ops read.""" + return _VisionWeights( + # Not transposed: the entry takes the checkpoint's `[out, in, kt, kh, kw]`. + patch_w=w["vis_patch_w"], + patch_b=w["vis_patch_b"], + pos=w["vis_pos"], + blocks=[ + _VisionBlockWeights( + n1_w=w[f"l{b}_vis_n1_w"], + n1_b=w[f"l{b}_vis_n1_b"], + # Row layout (q heads, k, v) is already thop_attention's packed form. + qkv=w[f"l{b}_vis_qkv_w"].t(), + qkv_b=w[f"l{b}_vis_qkv_b"], + proj=w[f"l{b}_vis_proj_w"].t(), + proj_b=w[f"l{b}_vis_proj_b"], + n2_w=w[f"l{b}_vis_n2_w"], + n2_b=w[f"l{b}_vis_n2_b"], + fc1=w[f"l{b}_vis_fc1_w"].t(), + fc1_b=w[f"l{b}_vis_fc1_b"], + fc2=w[f"l{b}_vis_fc2_w"].t(), + fc2_b=w[f"l{b}_vis_fc2_b"], + ) + for b in range(depth) + ], + m_n_w=w["vis_m_n_w"], + m_n_b=w["vis_m_n_b"], + m_fc1=w["vis_m_fc1_w"].t(), + m_fc1_b=w["vis_m_fc1_b"], + m_fc2=w["vis_m_fc2_w"].t(), + m_fc2_b=w["vis_m_fc2_b"], + ) + + +def _frame_calls( + grid: list[tuple[int, int, int]], max_rows: int +) -> list[list[tuple[int, int, int]]]: + """Split `grid` into calls of whole frames, at most `max_rows` rows each, as grid rows.""" + calls: list[list[tuple[int, int, int]]] = [[]] + room = max_rows + for frames, height, width in grid: + rows = height * width + assert rows <= max_rows, ( + f"one frame is {height}x{width} = {rows} patch rows; the vision ops are " + f"certified up to {max_rows} rows in one call" + ) + while frames: + take = min(frames, room // rows) + if take == 0: + calls.append([]) + room = max_rows + continue + calls[-1].append((take, height, width)) + room -= take * rows + frames -= take + return calls + + +def _merge_order_positions(grid: list[tuple[int, int, int]], merge: int) -> torch.Tensor: + """Each patch row's `(row, col)` in its frame, int64 `[rows, 2]` on the host, in + spatial-merge order: whole `merge x merge` blocks row-major, row-major inside a block, + the order the patch merger's 4-to-1 reshape consumes. A frame group repeats its frame.""" + intra = torch.arange(merge) + parts = [] + for frames, height, width in grid: + blocks_h, blocks_w = height // merge, width // merge + shape = (blocks_h, blocks_w, merge, merge) + rows = torch.arange(blocks_h)[:, None, None, None] * merge + intra[None, None, :, None] + cols = torch.arange(blocks_w)[None, :, None, None] * merge + intra[None, None, None, :] + coords = torch.stack((rows.expand(shape).reshape(-1), cols.expand(shape).reshape(-1)), 1) + parts.append(coords.repeat(frames, 1) if frames > 1 else coords) + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _rope_tables( + grid: list[tuple[int, int, int]], merge: int, theta: float, head_dim: int, device +) -> tuple[torch.Tensor, torch.Tensor]: + """Per-row fp32 2-D rotary `cos` / `sin`, contiguous `[rows, head_dim // 2]` each. + + Height angles, then width angles; formed in fp64 and rounded once (fp32 math rounds twice).""" + rope_dim = head_dim // 2 + inv_freq = 1.0 / (theta ** (torch.arange(0, rope_dim, 2, dtype=torch.float64) / rope_dim)) + pos = _merge_order_positions(grid, merge).to(torch.float64) + angles = (pos[:, :, None] * inv_freq).flatten(1).to(device=device, dtype=torch.float32) + return angles.cos().contiguous(), angles.sin().contiguous() + + +class VisionTower: + """One request's images or videos through the 27-block ViT and its patch merger. + + Attention is bidirectional per frame (a video's frame group is a frame): one context + "sequence" per frame for `thop_attention`. The position embedding and the 2-D rope emit in + spatial-merge order, which the merger's 4-to-1 reshape needs; raster order gives a correctly + shaped, silently wrong tower. Calls of at most `VISION_CALL_MAX_PATCHES` rows are cut only + between frames: every op is per frame or per row, and merger blocks stay whole.""" + + def __init__(self, w: dict, vision_config) -> None: + """`w` is the core's loaded weight table; `vision_config` the vision half of the config.""" + vcfg = vision_config + self.depth = vcfg.depth + self.hidden = vcfg.hidden_size + self.heads = vcfg.num_heads + self.head_dim = self.hidden // self.heads + self.merge = vcfg.spatial_merge_size + self.patch = vcfg.patch_size + # One flat pixel row per patch, `(in_channels, kt, kh, kw)` channel-slowest. + self.pixel_width = vcfg.in_channels * vcfg.temporal_patch_size * self.patch * self.patch + # `spatial_merge_size ** 2` patches become one token at the merger. + self.merge_dim = self.hidden * self.merge * self.merge + self.grid_side = math.isqrt(vcfg.num_position_embeddings) + self.dtype = w["embed"].dtype + self._w = _vision_views(w, self.depth) + # Persistent scratch `thop_attention` grows in place; not the engine's own workspace. + self._ws: torch.Tensor | None = None + + def __call__(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """Return the `[sum(t*h*w) / merge**2, out_hidden_size]` placeholder rows, in grid order. + + `pixel_values` is `[sum(t*h*w), in_channels*kt*kh*kw]`; `grid_thw` stays on the host.""" + assert pixel_values.dtype == self.dtype, ( + f"the tower runs at the activation dtype {self.dtype}; the runtime handed " + f"pixel_values as {pixel_values.dtype}. The input processor's `dtype` property is " + "what decides this, and no cast happens here" + ) + assert pixel_values.dim() == 2 and pixel_values.shape[1] == self.pixel_width, ( + f"pixel_values must be [patches, {self.pixel_width}] flat patch rows; got " + f"{tuple(pixel_values.shape)}" + ) + grid: list[tuple[int, int, int]] = [] + for row in grid_thw.tolist(): + assert len(row) == 3, f"grid_thw rows are (t, h, w); got {row}" + frames, height, width = (int(v) for v in row) + assert frames >= 1 and height >= 1 and width >= 1, row + assert height % self.merge == 0 and width % self.merge == 0, ( + f"grid sides must divide by spatial_merge_size {self.merge}; got " + f"height={height}, width={width}" + ) + grid.append((frames, height, width)) + assert grid, "grid_thw describes no image or video" + described = sum(t * h * w for t, h, w in grid) + assert described == pixel_values.shape[0], ( + f"grid_thw describes {described} patch rows and pixel_values carries " + f"{pixel_values.shape[0]}" + ) + + outputs = [] + start = 0 + for call in _frame_calls(grid, VISION_CALL_MAX_PATCHES): + rows = sum(t * h * w for t, h, w in call) + outputs.append(self._run(pixel_values[start : start + rows], call)) + start += rows + return outputs[0] if len(outputs) == 1 else torch.cat(outputs, 0) + + def _run(self, pixel_values: torch.Tensor, grid: list[tuple[int, int, int]]) -> torch.Tensor: + """One tower call over whole frames; `grid` describes exactly the rows of `pixel_values`.""" + dt, dev = pixel_values.dtype, pixel_values.device + assert pixel_values.is_cuda, "pixel_values must reach the tower on the device" + + # One attention segment per frame: a video's frames do not attend to each other. + seg_lens: list[int] = [] + for frames, height, width in grid: + seg_lens.extend([height * width] * frames) + num_patches = pixel_values.shape[0] + num_segments = len(seg_lens) + max_seg = max(seg_lens) + + host_lens = torch.tensor(seg_lens, dtype=torch.int32) + host_types = torch.zeros([num_segments], dtype=torch.int32) + host_total = torch.tensor([num_patches, 0], dtype=torch.int32) + dev_lens = host_lens.to(dev) + if self._ws is None: + self._ws = torch.empty([0], dtype=torch.int8, device=dev) + cos, sin = _rope_tables(grid, self.merge, VISION_ROPE_THETA, self.head_dim, dev) + + V = self._w + pos = [ + triton_pos_embed_interpolate(V.pos, t, h, w, self.grid_side, self.merge, dt) + for t, h, w in grid + ] + h = torch.add( + patch_embed_conv3d(pixel_values, V.patch_w, V.patch_b), + pos[0] if len(pos) == 1 else torch.cat(pos, 0), + ) + attn_out = torch.empty([num_patches, self.hidden], dtype=dt, device=dev) + + for B in V.blocks: + qkv = cublas_mm(layer_norm(h, B.n1_w, B.n1_b, VISION_NORM_EPS), B.qkv, B.qkv_b) + # The q|k prefix of each packed row, rotated in place; v is never addressed. + qkv_heads = torch.reshape(qkv, [num_patches, 3 * self.heads, self.head_dim]) + vision_rope_2d_packed_qk(qkv_heads[:, : 2 * self.heads], cos, sin) + attn_out = thop_attention( + q=qkv, + output=attn_out, + workspace=self._ws, + sequence_length=dev_lens, + host_past_key_value_lengths=host_lens, + host_total_kv_lens=host_total, + context_lengths=dev_lens, + host_context_lengths=host_lens, + host_request_types=host_types, + max_num_requests=num_segments, + max_context_length=max_seg, + max_seq_len=max_seg, + attention_window_size=max_seg, + num_contexts=num_segments, + num_ctx_tokens=num_patches, + num_heads=self.heads, + num_kv_heads=self.heads, + head_size=self.head_dim, + **_VISION_CALL_CONSTANTS, + ) + h = torch.add(h, cublas_mm(attn_out, B.proj, B.proj_b)) + mlp = cublas_mm(layer_norm(h, B.n2_w, B.n2_b, VISION_NORM_EPS), B.fc1, B.fc1_b) + # Non-gated MLP with the tanh GELU (not the merger's erf). + h = torch.add(h, cublas_mm(gelu_tanh(mlp), B.fc2, B.fc2_b)) + + # Patch merger: norm over `hidden`, then the 4-to-1 reshape (rows in spatial-merge order). + merged = torch.reshape( + layer_norm(h, V.m_n_w, V.m_n_b, VISION_NORM_EPS), [-1, self.merge_dim] + ) + # The erf GELU (`nn.GELU()`); within one bf16 ulp of tanh, hence a separate entry. + return cublas_mm(gelu_erf(cublas_mm(merged, V.m_fc1, V.m_fc1_b)), V.m_fc2, V.m_fc2_b) + + +def multimodal_embeds(tower: VisionTower | None, kwargs: dict): + """Run the tower on this batch's media: `(rows, mm_token_indices)`, or None without media. + + `rows` is `[R, hidden]` in packed-batch order, `mm_token_indices` the `[R]` rows they replace. + The shell calls it: the inherited `DecoderModelForCausalLM.forward` drops `multimodal_params`, + which covers a subset of the context requests (never zip it positionally). + `multimodal_runtime` counts rows over a request's media together, in prompt order.""" + mm_params = kwargs.get("multimodal_params") or [] + if not mm_params: + return None + rows = [] + for params in mm_params: + data = getattr(params, "multimodal_data", None) or {} + image, video = data.get("image"), data.get("video") + if image is None and video is None: + continue + mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") + assert ( + mrope_ids is not None and mrope_ids.dim() == 3 and mrope_ids.shape[0] == _POSITION_AXES + ), ( + f"a media request needs mrope_config.mrope_position_ids of shape " + f"[{_POSITION_AXES}, 1, L]; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" + ) + runtime = getattr(params, "multimodal_runtime", None) + if runtime is not None and int(runtime.num_mm_tokens_in_chunk) == 0: + # Every placeholder row of this request lies in another chunk. + continue + assert tower is not None, "post_load_weights builds the vision tower before any step" + towers = {} + if image is not None: + towers["image"] = run_tower(tower, image, *MEDIA_KEYS["image"]) + if video is not None: + towers["video"] = run_tower(tower, video, *MEDIA_KEYS["video"]) + embeds = in_prompt_order(tower.merge, params, towers) + if runtime is not None: + # Chunked prefill or reused prefix: splice only this chunk's slice. + start = int(runtime.num_cached_mm_tokens) + count = int(runtime.num_mm_tokens_in_chunk) + total = embeds.shape[0] + assert int(runtime.total_embeds_in_request) == total, ( + f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " + f"this request and the tower produced {total} over its {sorted(towers)}; the " + "two are derived from the same grids and a disagreement means the merger's " + "row count is wrong" + ) + assert 0 <= start <= start + count <= total, (start, count, total) + embeds = embeds[start : start + count] + rows.append(embeds) + if not rows: + return None + vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) + mm_token_indices = kwargs.get("mm_token_indices") + assert mm_token_indices is not None, ( + "the executor shipped multimodal params but no mm_token_indices; that pairing is what " + "the shell's `multimodal_token_ids` buys, and without it there is no sync-free way to " + "find the placeholder rows" + ) + assert mm_token_indices.shape[0] == vision.shape[0], ( + f"the batch has {mm_token_indices.shape[0]} placeholder rows and the tower produced " + f"{vision.shape[0]} embedding rows" + ) + return vision, mm_token_indices + + +def run_tower(tower, media: dict, pixel_key: str, grid_key: str): + """Run the tower on one modality's payload: `(rows, grid_thw)`, items in processor order.""" + pixel_values, grid_thw = media.get(pixel_key), media.get(grid_key) + assert pixel_values is not None and grid_thw is not None, ( + f"the multimodal payload carries {sorted(media)}; this target reads {pixel_key!r} and " + f"{grid_key!r} from it" + ) + assert not grid_thw.is_cuda, f"{grid_key} must stay on the host; see DEVICE_PATHS" + return tower(pixel_values, grid_thw), grid_thw + + +def in_prompt_order(merge: int, params, towers: dict) -> torch.Tensor: + """One request's tower rows (`towers`: modality -> `(rows, grid_thw)`) in placeholder order. + + Two modalities interleave by `item_metadata.item_refs`, the only host-side record of order.""" + if len(towers) == 1: + return next(iter(towers.values()))[0] + metadata = params.item_metadata + order = metadata.item_refs if metadata is not None else None + assert order, ( + f"a request carrying {sorted(towers)} needs item metadata to splice them in prompt " + "order; the payload holds each modality's rows apart" + ) + merge_unit = merge * merge + items = {} + for modality, (embeds, grid_thw) in towers.items(): + counts = [t * h * w // merge_unit for t, h, w in grid_thw.tolist()] + assert sum(counts) == embeds.shape[0], ( + f"the {modality} grid describes {sum(counts)} rows and the tower produced " + f"{embeds.shape[0]}" + ) + items[modality] = torch.split(embeds, counts, 0) + listed = {modality: [] for modality in items} + for modality, index in order: + assert modality in items, ( + f"item_refs lists a {modality!r} item; this request carries {sorted(items)}" + ) + listed[modality].append(int(index)) + for modality, parts in items.items(): + assert sorted(listed[modality]) == list(range(len(parts))), ( + f"item_refs lists {modality} items {listed[modality]}; the payload carries {len(parts)}" + ) + return torch.cat([items[modality][int(index)] for modality, index in order], 0) + + +def validate_request(tower, request) -> None: + """Admission check the executor runs on every new request, from its host-side payload only. + + It refuses what the forward would assert on: an assert inside the step ends the executor.""" + data = request.py_multimodal_data + if not data: + return + grids = {} + for modality, (pixel_key, grid_key) in MEDIA_KEYS.items(): + media = data.get(modality) + if media is not None: + grids[modality] = _media_grid(tower, modality, media, pixel_key, grid_key) + if not grids: + return + + mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") + if ( + mrope_ids is None + or mrope_ids.dim() != 3 + or mrope_ids.shape[0] != _POSITION_AXES + or mrope_ids.shape[2] != request.prompt_len + ): + raise ValueError( + f"a media request needs the [{_POSITION_AXES}, 1, {request.prompt_len}] " + "mrope_config.mrope_position_ids from the input processor, one position per prompt " + f"token; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" + ) + + if len(grids) > 1: + metadata = get_multimodal_item_metadata(data) + _check_item_order(grids, metadata.item_refs if metadata is not None else None) + + cumsum = data.get("multimodal_embed_mask_cumsum") + if cumsum is not None: + merge_unit = tower.merge * tower.merge + rows = sum(t * h * w for grid in grids.values() for t, h, w in grid) // merge_unit + placeholders = int(cumsum[-1]) + if placeholders != rows: + raise ValueError( + f"the prompt holds {placeholders} image/video placeholder tokens and its " + f"{' and '.join(sorted(grids))} payload produces {rows} embedding rows" + ) + + +def _media_grid(tower, modality, media, pixel_key, grid_key): + """One modality's `(frames, height, width)` rows, checked on the host.""" + pixel_values, grid_thw = media.get(pixel_key), media.get(grid_key) + if pixel_values is None or grid_thw is None: + raise ValueError( + f"the {modality} payload carries {sorted(media)}; this target reads {pixel_key!r} " + f"and {grid_key!r} from it" + ) + if grid_thw.dim() != 2 or grid_thw.shape[0] == 0 or grid_thw.shape[1] != 3: + raise ValueError( + f"{grid_key} must hold one (t, h, w) row per {modality}; got shape " + f"{tuple(grid_thw.shape)}" + ) + merge = tower.merge + grid = [tuple(int(v) for v in row) for row in grid_thw.tolist()] + for item, (frames, height, width) in enumerate(grid): + if min(frames, height, width) < 1 or height % merge or width % merge: + raise ValueError( + f"{modality} {item} has grid {(frames, height, width)}; every entry must be " + f"positive and the sides divisible by spatial_merge_size {merge}" + ) + if height * width > VISION_CALL_MAX_PATCHES: + raise ValueError( + f"{modality} {item} has frames of {height}x{width} = {height * width} patches " + f"({height * tower.patch}x{width * tower.patch} pixels); the vision tower " + f"serves frames of at most {VISION_CALL_MAX_PATCHES} patches, so send it at a " + "lower resolution" + ) + patches = sum(t * h * w for t, h, w in grid) + if tuple(pixel_values.shape) != (patches, tower.pixel_width): + raise ValueError( + f"{pixel_key} is {tuple(pixel_values.shape)}; its grids describe {patches} patch " + f"rows of {tower.pixel_width}" + ) + return grid + + +def _check_item_order(grids, order) -> None: + """A request with several modalities lists its items in prompt order.""" + if not order: + raise ValueError( + f"a request carrying {sorted(grids)} needs item metadata to splice them in prompt " + "order; the payload holds each modality's rows apart" + ) + listed = {modality: [] for modality in grids} + for entry in order: + try: + modality, index = entry + listed[modality].append(int(index)) + except (KeyError, TypeError, ValueError) as error: + raise ValueError( + f"item_refs entry {entry!r} is not an item of this request, which carries " + f"{sorted(grids)}; entries are (modality, index) pairs" + ) from error + for modality, grid in grids.items(): + if sorted(listed[modality]) != list(range(len(grid))): + raise ValueError( + f"item_refs lists {modality} items {listed[modality]}; the payload carries " + f"{len(grid)}" + ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py new file mode 100644 index 000000000000..4a69b5a2db3d --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -0,0 +1,1747 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Weight table and loader: Qwen3.8-Flash-Next-FP8 (hf-236dfdf_orig) / sm_103 / unified. + +One `W` per weight role (shape, checkpoint source, the rank's cut, dtype, load-time +transform); `Qwen4ExpFp8Weights` unrolls the table into declaration, manifest and load. +Storage is HF `[out, in]`; GEMM views, grouped-norm slices and fp32 gating params are +derived in `modeling.py` after load. + +**Checkpoint.** Block FP8 (`weight_block_size: [128, 128]`, dynamic activations; the engine's +`FP8_BLOCK_SCALES`). Only the routed experts (trunk 48 x 512, MTP 512) are quantized: per +expert and unfused, e4m3 `gate_proj` / `up_proj` / `down_proj` with one bf16 +`weight_scale_inv` per 128x128 block and no per-tensor scalars. Everything else is bf16, +except the PLE n-gram table (e4m3, one bf16 per-tensor `weight_scale`) and its three int64 +id-space tensors. There is no `model.language_model.norm.weight`: the stack-final +hyper-connection mixer is the final norm. `mtp.*` is loaded only with a `speculative_config`. + +**Shares** (`rank_dims`, from the engine's `Mapping`: tp1, tp2, tep2, dep2, tp4, tep4, dep4). +Rank `r` of `t` holds: + +* attention (the attention TP group; one rank under attention DP): q heads + `[24r / t, 24(r + 1) / t)`, i.e. their `q_proj` rows (q and output gate interleave per + head) and `o_proj` input columns, and KV head `r * 2 // t` (split at two ranks, repeated at + four, as `duplicate_kv_weight` and the K/V pool do). The QSA indexer and q/k norms are + replicated; +* Gated-DeltaNet (`mamba_effective_tp_size`, as the recurrent pools): key heads + `[16r / t, 16(r + 1) / t)` and value heads `[48r / t, 48(r + 1) / t)`; +* routed experts, trunk and draft: experts `[512r / t, 512(r + 1) / t)` over the whole TP + group, each at its whole 640-row intermediate. The cut ignores `moe_tp_size` / `moe_ep_size`: + 640 / 2 and 640 / 4 rows do not tile the 128-row FP8 block, so nothing is requantized. An + off-window slot adds nothing, so the windows' partial outputs sum to the layer's. + +The MTP layer is cut like a trunk attention layer. Everything else is replicated; `lm_head` +is the shell's (vocab-parallel under attention TP, whole under attention DP and at tp1). + +A cut is applied to the checkpoint entry before it is materialized, so a rank reads only its +share (a lazy safetensors slice reads only those bytes) and never an off-window expert. + +Parameters (`core.w`, an `nn.ParameterDict`) are keyed `l{i}_` for decoder layer `i`, +`l{num_hidden_layers}_mtp_` for the MTP module, `l{b}_vis_` for vision block `b`, +and bare names otherwise. + +`load` asserts coverage both ways: every parameter written once, and every checkpoint key +consumed or predicted by `expected_unconsumed`. +""" + +import os +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Callable, NamedTuple + +import torch +from cuda.bindings import driver as cuda_driver +from cuda.bindings import runtime as cudart +from torch import nn + +from tensorrt_llm._dlpack_utils import pack_strided_memory +from tensorrt_llm._torch.pyexecutor.config_utils import mamba_effective_tp_size +from tensorrt_llm._utils import CUASSERT +from tensorrt_llm.logger import logger + +from . import host_share as _host_share + +ALL_LAYERS = object() + +Cut = tuple[slice, ...] + +#: Opt-in (`rr`; default off) block-major copy of every MoE layer's routed experts for +#: `ops/moe/moe_tpb_decode` (attention TP4 only): expert e's 128-row intermediate block b on rank +#: (e + b) % 4, plus this rank's slice of the shared expert. The whole-expert window stays for +#: prefill and the runner, so the copy costs one more window per layer (~31 GB / rank at tep4). +MOE_TPB_ENV = "TRTLLM_MODELING_V2_DECODE_MOE_TPB" + + +def moe_tpb_enabled(mapping) -> bool: + raw = os.environ.get(MOE_TPB_ENV, "").strip().lower() + if raw in ("", "0", "false", "off", "no"): + return False + assert raw == "rr", f"{MOE_TPB_ENV}={raw!r}: expected off or rr" + return mapping.tp_size == 4 and not mapping.enable_attention_dp + + +@dataclass(frozen=True) +class W: + """One weight role; `d` below is the `rank_dims` bundle. + + `shape(d)` is the declared shape at this rank's share. `src` is a key template (`{p}` is + the layer prefix) or `(d, i) -> [(key, index[, cut]), ...]`: `index` addresses the + parameter (None for all of it), a tuple `key` hands its tensors to `transform` together. + `cut(d)` is this rank's rows or columns of the checkpoint tensor (one per key for a tuple + key; None reads it whole). `layers` is `ALL_LAYERS`, None (not per-layer) or + `d -> iterable[int]`. `transform(t, d)` relayouts the cut tensor(s) on the target device. + """ + + name: str + shape: Callable[[Any], tuple[int, ...]] + src: str | Callable + dtype: torch.dtype | None = None + cut: Callable[[Any], Any] | None = None + transform: Callable | None = None + layers: object | None = ALL_LAYERS + + def layer_indices(self, d: SimpleNamespace) -> list[int] | None: + if self.layers is None: + return None + if self.layers is ALL_LAYERS: + return list(range(d.num_layers)) + return list(self.layers(d)) + + def sources(self, d: SimpleNamespace, i: int | None, prefix: str) -> list[tuple]: + """`[(checkpoint key or key tuple, index into the param | None, cut | None), ...]`.""" + rows = self.src(d, i) if callable(self.src) else [(self.src.format(p=prefix), None)] + cut = self.cut(d) if self.cut is not None else None + out = [] + for row in rows: + if len(row) == 3: + assert self.cut is None, f"{self.name}: a source with its own cut takes no W.cut" + out.append(tuple(row)) + else: + out.append((*row, cut)) + return out + + +# ----------------------------------------------------------- host residency -- +# The PLE n-gram table is a `cuda` tensor over registered, device-mapped host pages (not a +# host-side gather): the gather's row indices are produced on device inside the decode graph. + +#: `cudaMemoryType` of host memory; the core asserts on it after load. +CUDA_MEMORY_TYPE_HOST = 1 + + +def memory_kind(ptr: int) -> int: + (attributes,) = CUASSERT(cudart.cudaPointerGetAttributes(ptr)) + return int(attributes.type) + + +#: `cudaHostRegister` flags: mapped (kernels may dereference the pages), portable (all contexts). +_CUDA_HOST_REGISTER_PORTABLE = 0x01 +_CUDA_HOST_REGISTER_MAPPED = 0x02 + + +def _local_cpus(device) -> set | None: + """CPUs local to `device`'s PCI function that this process may run on, or None.""" + try: + props = torch.cuda.get_device_properties(device) + path = ( + f"/sys/bus/pci/devices/{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:" + f"{props.pci_device_id:02x}.0/local_cpulist" + ) + with open(path) as fh: + text = fh.read().strip() + except (OSError, AttributeError, RuntimeError): + return None + cpus = set() + for part in filter(None, text.split(",")): + lo, _, hi = part.partition("-") + cpus.update(range(int(lo), int(hi or lo) + 1)) + return (cpus & os.sched_getaffinity(0)) or None + + +class _RegisteredHostBuffer: + """Pageable CPU memory registered (pinned, mapped) for the device. + + Unregistered before the storage is freed, since glibc returns this size to the OS with + `munmap`. Not torch's pinned allocator, which rounds the 47.7 GiB table up to 64 GiB. + """ + + def __init__(self, nbytes: int, device=None) -> None: + # Registration faults the pages in on this thread, so they land on its NUMA node: + # run on `device`'s local CPUs meanwhile to put the table next to its GPU. + local = _local_cpus(device) if device is not None else None + previous = os.sched_getaffinity(0) if local else None + if local: + os.sched_setaffinity(0, local) + try: + self._storage = torch.empty(nbytes, dtype=torch.uint8) + self.pointer = self._storage.data_ptr() + CUASSERT( + cudart.cudaHostRegister( + self.pointer, nbytes, _CUDA_HOST_REGISTER_PORTABLE | _CUDA_HOST_REGISTER_MAPPED + ) + ) + finally: + if previous: + os.sched_setaffinity(0, previous) + # Bound now: at interpreter shutdown the module may already be torn down. + self._unregister = cudart.cudaHostUnregister + + def __del__(self) -> None: + pointer, self.pointer = self.pointer, 0 + if pointer: + self._unregister(pointer) + + +def _host_resident_empty( + rows: int, cols: int, dtype, device +) -> tuple[torch.Tensor, _RegisteredHostBuffer]: + """`(table, owner)`: `[rows, cols]` on `device` in host memory; keep `owner` alive.""" + itemsize = torch.empty(0, dtype=dtype).element_size() + assert itemsize == 1, ( + f"the host-resident allocation reinterprets a `|u1` buffer, so its " + f"element must be one byte; {dtype} is {itemsize}" + ) + assert device.type == "cuda", f"host offload needs a cuda device to map into; got {device}" + + host = _RegisteredHostBuffer(rows * cols, device) + return _device_view(host.pointer, rows, cols, dtype, device), host + + +def _device_view(host_pointer: int, rows: int, cols: int, dtype, device) -> torch.Tensor: + """`[rows, cols]` one-byte `dtype` view of mapped host pages at `host_pointer`.""" + (device_pointer,) = CUASSERT(cudart.cudaHostGetDevicePointer(host_pointer, 0)) + + # No public torch API wraps a pointer the caching allocator did not hand out. `|u1` + # because `__cuda_array_interface__` has no float8 typestr; reinterpreted below. + interface = SimpleNamespace( + __cuda_array_interface__={ + "shape": (rows, cols), + "typestr": "|u1", + "data": (int(device_pointer), False), + "version": 3, + "strides": None, + } + ) + table = torch.as_tensor(interface, device=device).view(dtype) + # A copy here (`as_tensor` may copy instead of view) would be a silent 48 GiB allocation. + assert table.data_ptr() == int(device_pointer), ( + "torch.as_tensor copied the host-mapped buffer instead of viewing it, " + "so the table would be device-resident after all" + ) + assert table.shape == (rows, cols) and table.dtype == dtype, (table.shape, table.dtype) + return table + + +#: One host copy of the n-gram table per host (`node`), mapped by every rank there; default off. +PLE_SHARED_HOST_ENV = _host_share.PLE_SHARED_HOST_ENV + + +def ple_shared_host_scope() -> str | None: + """`node` or None, from `PLE_SHARED_HOST_ENV`; read once into `core.ple_shared_host`.""" + return _host_share.shared_host_scope() + + +def _device_numa_node(device) -> int | None: + """NUMA node of `device`'s local CPUs (ignoring this process's affinity), or None.""" + try: + props = torch.cuda.get_device_properties(device) + base = ( + f"/sys/bus/pci/devices/{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:" + f"{props.pci_device_id:02x}.0" + ) + with open(f"{base}/local_cpulist") as fh: + cpus = _host_share._parse_cpulist(fh.read()) + except (OSError, AttributeError, RuntimeError, ValueError): + return None + return _host_share.numa_node_of_cpus(cpus) + + +def _cu(result): + err, *values = result + if err != cuda_driver.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"CUDA driver call failed: {err}") + return values[0] if len(values) == 1 else tuple(values) + + +class _VmmHostBacking: + """`host_share.Backing` on CUDA VMM memory on host NUMA node `node`, shared by POSIX fd. + + Not `cudaHostRegister` on shared memory: device gathers from such pages are over 10x + slower than from a private registered buffer. + """ + + def __init__(self, device_index: int, node: int) -> None: + self.device_index = device_index + self.node = node + self._maps: dict[int, tuple[int, int]] = {} + + def _prop(self): + D = cuda_driver + prop = D.CUmemAllocationProp() + prop.type = D.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + prop.location.id = self.node + prop.requestedHandleTypes = ( + D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + return prop + + def granularity(self) -> int: + option = cuda_driver.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED + return int(_cu(cuda_driver.cuMemGetAllocationGranularity(self._prop(), option))) + + def _map(self, handle, size: int, read_only: bool) -> int: + D = cuda_driver + va = _cu(D.cuMemAddressReserve(size, self.granularity(), 0, 0)) + try: + _cu(D.cuMemMap(va, size, 0, handle, 0)) + desc = D.CUmemAccessDesc() + desc.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = self.device_index + desc.flags = ( + D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READ + if read_only + else D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + ) + _cu(D.cuMemSetAccess(va, size, [desc], 1)) + except BaseException: + D.cuMemUnmap(va, size) + D.cuMemAddressFree(va, size) + raise + self._maps[int(va)] = (handle, size) + return int(va) + + def _size(self, nbytes: int) -> int: + granularity = self.granularity() + return -(-nbytes // granularity) * granularity + + def create(self, nbytes: int) -> tuple[int, int]: + D = cuda_driver + size = self._size(nbytes) + handle = _cu(D.cuMemCreate(size, self._prop(), 0)) + try: + va = self._map(handle, size, read_only=False) + except BaseException: + D.cuMemRelease(handle) + raise + try: + fd = _cu( + D.cuMemExportToShareableHandle( + handle, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0 + ) + ) + except BaseException: + self.release(va, nbytes) + raise + return va, int(fd) + + def attach(self, fd: int, nbytes: int) -> int: + D = cuda_driver + handle = _cu( + D.cuMemImportFromShareableHandle( + fd, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + ) + size = self._size(nbytes) + try: + # The owner's node, not this device's: under the `node` scope they may differ. + self.node = int(_cu(D.cuMemGetAllocationPropertiesFromHandle(handle)).location.id) + try: + return self._map(handle, size, read_only=True) + except RuntimeError: + return self._map(handle, size, read_only=False) + except BaseException: + D.cuMemRelease(handle) + raise + + def release(self, address: int, nbytes: int) -> None: + handle, size = self._maps.pop(address) + D = cuda_driver + D.cuMemUnmap(address, size) + D.cuMemAddressFree(address, size) + D.cuMemRelease(handle) + + def resident_on_host(self, address: int) -> bool: + prop = _cu(cuda_driver.cuMemGetAllocationPropertiesFromHandle(self._maps[address][0])) + return prop.location.type == cuda_driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + + def describe(self) -> str: + return f"vmm host numa {self.node}" + + +def _view_at(pointer: int, rows: int, cols: int, dtype, device) -> torch.Tensor: + """`[rows, cols]` view of a VMM mapping via DLPack, which names the device explicitly.""" + index = device.index if device.index is not None else torch.cuda.current_device() + flat = pack_strided_memory(pointer, rows * cols, rows * cols, 1, torch.uint8, index) + table = flat.view(rows, cols).view(dtype) + assert table.data_ptr() == pointer and table.device == torch.device("cuda", index) + return table + + +#: Rows sampled at each shard's head and tail: the shared segment's key and an attacher's check. +_SAMPLE_ROWS = 2 + + +def _ngram_samples(weights, keys, rows_per_shard: int) -> list[tuple[int, bytes]]: + samples = [] + for s, key in enumerate(keys): + for local in (0, rows_per_shard - _SAMPLE_ROWS): + rows = _read(weights[key], (slice(local, local + _SAMPLE_ROWS),)) + data = rows.contiguous().view(torch.uint8).cpu().numpy().tobytes() + width = len(data) // _SAMPLE_ROWS + for i in range(_SAMPLE_ROWS): + samples.append((s * rows_per_shard + local + i, data[i * width : (i + 1) * width])) + return samples + + +def _shared_host_table(rows: int, cols: int, dtype, device, weights, keys, rows_per_shard): + """`(table, segment)` over the scope's `host_share.SharedHostSegment`. + + Owner: mapped, empty until load fills and publishes it. Attached: ready. Pending: a meta + placeholder until `_fill_shared_table`. Without VMM host NUMA: `_host_resident_empty`. + """ + index = device.index if device.index is not None else torch.cuda.current_device() + node = _device_numa_node(device) + backing = _VmmHostBacking(index, 0 if node is None else node) + try: + backing.granularity() + except RuntimeError as error: + logger.warning( + f"ple shared host: no VMM host NUMA allocation on this device ({error}); " + "keeping one copy per rank" + ) + return _host_resident_empty(rows, cols, dtype, device) + samples = _ngram_samples(weights, keys, rows_per_shard) + key = _host_share.table_key( + "qwen4_exp ple n-gram table", rows, cols, dtype, *(b for _, b in samples) + ) + segment = _host_share.SharedHostSegment(key, rows * cols, "host", backing) + segment.samples = samples + if segment.role == _host_share.PENDING: + return torch.empty(rows, cols, dtype=dtype, device="meta"), segment + return _view_at(segment.pointer, rows, cols, dtype, device), segment + + +def _check_samples(segment, table: torch.Tensor) -> None: + """An attached segment's sampled rows are the checkpoint's: shard order and content.""" + rows = torch.tensor([row for row, _ in segment.samples], device=table.device) + got = table.view(torch.uint8)[rows].cpu() + for (row, expected), bytes_ in zip(segment.samples, got): + assert bytes(bytes_.numpy()) == expected, ( + f"the shared n-gram table ({segment.describe()}) differs from the checkpoint at " + f"row {row}" + ) + + +# ------------------------------------------------------------------ the table -- + +_LM = "model.language_model" +_VISUAL = "model.visual" +_MTP = "mtp" +_MTP_LAYER = "mtp.layers.0" + +#: FP8 block edge: 128x128 expert weight-scale blocks, 1x128 activation quantization. +FP8_BLOCK = 128 + +#: The n-gram table is kept as stored (per-tensor FP8); the gather dequantizes the rows it reads. +PLE_TABLE_DTYPE = torch.float8_e4m3fn + +#: Expert block-scale dtype the runner takes; the checkpoint's bf16 converts exactly. +EXPERT_SCALE_DTYPE = torch.float32 + +# Parameter-key suffixes after which `load` empties the CUDA cache, so peak load memory +# stays near one parameter (sources are staged one expert or shard at a time). +_RELEASE_AFTER = ( + "_fc1_w", + "_fc1_s", + "_fc2_w", + "_fc2_s", + "ple_table", +) + + +def _read(entry, cut: Cut | None) -> torch.Tensor: + """Materialize `entry`, reading only `cut` of it (a 0-dim entry rejects `[:]`).""" + if cut is None: + return entry if getattr(entry, "ndim", 1) == 0 else entry[:] + return entry[cut[0]] if len(cut) == 1 else entry[cut] + + +# --- cuts: this rank's block of rows or columns of a checkpoint tensor --- + + +def _span(start: int, width: int) -> slice: + return slice(start, start + width) + + +def _q_rows(d) -> Cut: + """`q_proj` rows of this rank's q heads: each head's q rows, then its gate rows.""" + return (_span(d.q_head_start * 2 * d.head_dim, 2 * d.q_width),) + + +def _kv_rows(d) -> Cut: + return (_span(d.kv_head_start * d.head_dim, d.kv_width),) + + +def _o_cols(d) -> Cut: + return (slice(None), _span(d.q_head_start * d.head_dim, d.q_width)) + + +def _key_rows(d, section: int) -> Cut: + """Rows of this rank's key heads in section 0 (q) or 1 (k) of a `[q | k | v]` GDN tensor.""" + return (_span(section * d.key_dim_total + d.k_head_start * d.head_k_dim, d.key_dim),) + + +def _value_rows(d, base: int = 0) -> Cut: + return (_span(base + d.v_head_start * d.head_v_dim, d.value_dim),) + + +def _value_head_rows(d) -> Cut: + """One row per value head (`in_proj_a`, `in_proj_b`, `A_log`, `dt_bias`).""" + return (_span(d.v_head_start, d.num_v_heads),) + + +def _value_cols(d) -> Cut: + return (slice(None), _span(d.v_head_start * d.head_v_dim, d.value_dim)) + + +def _qkv_blocks(d) -> tuple[tuple[slice, Cut], ...]: + """(destination rows, checkpoint cut) of this rank's q, k and v blocks, in conv-pool order.""" + return ( + (slice(0, d.key_dim), _key_rows(d, 0)), + (slice(d.key_dim, 2 * d.key_dim), _key_rows(d, 1)), + (slice(2 * d.key_dim, d.conv_dim), _value_rows(d, 2 * d.key_dim_total)), + ) + + +# --- relayouts, on the rank's cut --- + + +#: Rows after the shared expert's gate|up rows in its fused operand: the per-token gate's +#: one row, then zeros, so the GEMM's output row stays a whole number of 16-byte vectors. +SH_GATE_ROWS = 8 + +#: `in_proj_w` rows are padded to a multiple of this (`d.in_proj_rows`): FlashInfer's low-M warp +#: split-K GEMM needs 16 | N (tp4/tep4: 4120 -> 4128). The checkpoint fills +#: `sum(d.in_proj_split)` rows; the core zeroes the rest after load. +IN_PROJ_ROW_ALIGN = 16 + + +def _cat_rows(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(gate, up, gate logit) -> `[gate ; up ; logit ; zeros]`, one GEMM for gate|up and the logit. + + Gate first: the half order `flashinfer_silu_and_mul` reads. + """ + gate, up, sh_gate = t + pad = sh_gate.new_zeros(SH_GATE_ROWS - 1, sh_gate.shape[-1]) + return torch.cat((gate, up, sh_gate, pad), dim=0).contiguous() + + +def _attn_in_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(q_proj, k_proj, v_proj, index_qk_proj) at this rank's cut -> `[q|k|v|gate|index|pad]`. + + `q_proj` rows are `[heads, 2, head_dim]` (q, then gate). q goes first so the GEMM output + leads with the `[q | k | v]` the fused norm+rope rewrites in place. + """ + q, k, v, idx = t + heads, head_dim, hidden = d.heads_q, d.head_dim, q.shape[-1] + qg = q.reshape(heads, 2, head_dim, hidden) + pad = q.new_zeros(d.index_head_dim, hidden) + rows = (qg[:, 0].reshape(heads * head_dim, hidden), k, v) + rows += (qg[:, 1].reshape(heads * head_dim, hidden), idx, pad) + return torch.cat(rows, dim=0).contiguous() + + +def _expert_fc1_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) e4m3 codes -> one expert's FC1 operand `[up ; gate]`, plain row-major.""" + up, gate = t + return torch.cat([up, gate], dim=0).contiguous() + + +def _expert_fc1_s(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) bf16 block scales -> one expert's `[up ; gate]` FC1 scales in fp32.""" + up, gate = t + return torch.cat([up, gate], dim=0).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _expert_fc2_s(t: torch.Tensor, d) -> torch.Tensor: + """`down_proj` bf16 block scales -> fp32; FC2's e4m3 codes are used as stored.""" + return t.to(EXPERT_SCALE_DTYPE).contiguous() + + +# --- which layers carry a weight --- + + +def _attn_layers(d): + """The full-attention layers (`i % 4 == 3` on this checkpoint).""" + return [i for i, a in enumerate(d.is_attn) if a] + + +def _gdn_layers(d): + return [i for i, a in enumerate(d.is_attn) if not a] + + +def _vision_blocks(d): + """The vision tower's blocks. Keyed `l{b}_vis_*`: block `b`, not decoder layer `b`.""" + return range(d.v_depth) + + +def _mtp_only(d): + """The MTP module, at its own layer index `num_hidden_layers`, only when it is live.""" + return [d.num_layers] if d.mtp_enabled else [] + + +# --- where the bytes come from --- + + +def _ple(suffix): + return lambda d, i: [(f"{_LM}.layers.{d.ple_layer}.ple.{suffix}", None)] + + +def _ngram_prefix(ple_layer: int) -> str: + """The quantized n-gram submodule, holding the shards and their per-tensor scale.""" + return f"{_LM}.layers.{ple_layer}.ple.ple_embedding.ngram_embedding" + + +def _visual(suffix): + return lambda d, i: [(f"{_VISUAL}.{suffix}", None)] + + +def _vision_block(suffix): + return lambda d, i: [(f"{_VISUAL}.blocks.{i}.{suffix}", None)] + + +def _window(d) -> tuple[int, int]: + """(first expert, count) of this rank's routed-expert window, trunk and draft alike.""" + return d.expert_offset, d.local_experts + + +def _experts(prefix, *suffixes, window): + """One source per expert of `window(d)`, at its local index.""" + + def build(d, i): + start, count = window(d) + base = prefix(i) + out = [] + for local in range(count): + keys = tuple(f"{base}.experts.{start + local}.{s}" for s in suffixes) + out.append((keys if len(keys) > 1 else keys[0], (local,))) + return out + + return build + + +def tpb_blocks(d) -> list[tuple[int, int]]: + """(expert, intermediate block) of the 640 blocks this rank's block-major copy holds + (`MOE_TPB_ENV`): j < 512 is expert j's block (rank - j) % 4, j >= 512 is expert + 4 (j - 512) + rank's block 4.""" + blocks = d.moe_inter // FP8_BLOCK + r, n = d.tp_rank, d.num_experts + return [ + (j, (r - j) % 4) if j < n else (4 * (j - n) + r, blocks - 1) for j in range(n * blocks // 4) + ] + + +def _tpb_experts(prefix, suffixes, cut): + """One source per block of `tpb_blocks(d)`, at its block index, read through `cut(b)`.""" + + def build(d, i): + base = prefix(i) + out = [] + for j, (e, b) in enumerate(tpb_blocks(d)): + keys = tuple(f"{base}.experts.{e}.{s}" for s in suffixes) + c = cut(b) + out.append((keys, (j,), (c,) * len(keys)) if len(keys) > 1 else (keys[0], (j,), c)) + return out + + return build + + +def _tpb_fc1_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + up, gate = t + return torch.cat([up, gate], dim=0).contiguous() + + +def _tpb_fc1_s(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) block-scale rows of one block -> `[2, hidden / 128]` fp32.""" + up, gate = t + return torch.cat([up, gate], dim=0).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _tpb_fc2_s(t: torch.Tensor, d) -> torch.Tensor: + """down_proj's block-scale column of one block -> `[hidden / 128]` fp32.""" + return t.reshape(-1).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _tpb_sh_gu(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(gate, up) rows of one 32-column shared-expert item -> `[64, hidden]`. + + 8 groups of 4 columns, each `[gate 4 ; up 4]` (`moe_tpb_decode.shared_blocks`). + """ + gate, up = t + return ( + torch.stack([gate.reshape(8, 4, -1), up.reshape(8, 4, -1)], dim=1) + .reshape(64, -1) + .contiguous() + ) + + +def _block_shared(tag, prefix, layers): + """This rank's TP slice of a MoE layer's bf16 shared expert (`moe_tpb_decode.SharedBlocks`). + + Columns `[rank * w, (rank + 1) * w)` (w = shared_inter / 4) in 5 items of 32: gate / up + rows interleaved by 4 (`tpb_sh_gu [5, 64, hidden]`) and down_proj columns + (`tpb_sh_dn [5, hidden, 32]`). + """ + items, cols = 5, 32 + + def lo(d, sb): + return d.tp_rank * (d.shared_inter // 4) + cols * sb + + return ( + W( + f"{tag}tpb_sh_gu", + shape=lambda d: (items, 64, d.hidden), + src=lambda d, i: [ + ( + ( + f"{prefix(i)}.shared_expert.gate_proj.weight", + f"{prefix(i)}.shared_expert.up_proj.weight", + ), + (sb,), + ((slice(lo(d, sb), lo(d, sb) + cols),), (slice(lo(d, sb), lo(d, sb) + cols),)), + ) + for sb in range(items) + ], + transform=_tpb_sh_gu, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_sh_dn", + shape=lambda d: (items, d.hidden, cols), + src=lambda d, i: [ + ( + f"{prefix(i)}.shared_expert.down_proj.weight", + (sb,), + (slice(None), slice(lo(d, sb), lo(d, sb) + cols)), + ) + for sb in range(items) + ], + layers=_tpb_layers(layers), + ), + ) + + +def _tpb_layers(layers): + """`layers` when the block-major copy is on (`d.moe_tpb`), else none.""" + + def pick(d): + if not getattr(d, "moe_tpb", False): + return [] + return range(d.num_layers) if layers is ALL_LAYERS else list(layers(d)) + + return pick + + +def _block_experts(tag, prefix, layers): + """One MoE layer's routed experts block-major (`MOE_TPB_ENV`), one entry per `tpb_blocks(d)`.""" + rows = lambda b: (slice(FP8_BLOCK * b, FP8_BLOCK * (b + 1)),) # noqa: E731 + srow = lambda b: (slice(b, b + 1),) # noqa: E731 + cols = lambda b: (slice(None), slice(FP8_BLOCK * b, FP8_BLOCK * (b + 1))) # noqa: E731 + scol = lambda b: (slice(None), slice(b, b + 1)) # noqa: E731 + n = lambda d: len(tpb_blocks(d)) # noqa: E731 + return ( + W( + f"{tag}tpb_w1", + shape=lambda d: (n(d), 2 * FP8_BLOCK, d.hidden), + dtype=f8, + src=_tpb_experts(prefix, ("up_proj.weight", "gate_proj.weight"), rows), + transform=_tpb_fc1_w, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_s1", + shape=lambda d: (n(d), 2, d.hidden // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_tpb_experts( + prefix, ("up_proj.weight_scale_inv", "gate_proj.weight_scale_inv"), srow + ), + transform=_tpb_fc1_s, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_w2", + shape=lambda d: (n(d), d.hidden, FP8_BLOCK), + dtype=f8, + src=_tpb_experts(prefix, ("down_proj.weight",), cols), + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_s2", + shape=lambda d: (n(d), d.hidden // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_tpb_experts(prefix, ("down_proj.weight_scale_inv",), scol), + transform=_tpb_fc2_s, + layers=_tpb_layers(layers), + ), + ) + + +def _trunk_mlp(i): + return f"{_LM}.layers.{i}.mlp" + + +def _mtp_mlp(i): + return f"{_MTP_LAYER}.mlp" + + +def _in_proj(d, i): + """GDN input projections at this rank's heads, in `in_proj_w`'s row order + `[q | k | v (conv_dim) | z (value_dim) | a | b]` (`d.in_proj_split`).""" + p = f"{_LM}.layers.{i}.linear_attn" + out = [(f"{p}.in_proj_qkv.weight", (rows,), cut) for rows, cut in _qkv_blocks(d)] + row = d.conv_dim + out.append((f"{p}.in_proj_z.weight", (_span(row, d.value_dim),), _value_rows(d))) + row += d.value_dim + for name in ("in_proj_a", "in_proj_b"): + out.append((f"{p}.{name}.weight", (_span(row, d.num_v_heads),), _value_head_rows(d))) + row += d.num_v_heads + return out + + +def _conv(d, i): + """The depthwise conv's `[q | k | v]` channels of this rank's heads.""" + key = f"{_LM}.layers.{i}.linear_attn.conv1d.weight" + return [(key, (rows,), cut) for rows, cut in _qkv_blocks(d)] + + +#: One routed expert's checkpoint tensors, left unread by a rank outside its window. +_PROJECTIONS = ("gate_proj", "up_proj", "down_proj") +_EXPERT_TENSORS = tuple(f"{p}.{s}" for p in _PROJECTIONS for s in ("weight", "weight_scale_inv")) + + +#: (parameter tag, checkpoint module) of the two per-layer hyper-connection gates. +_HC_MODULES = (("ahc", "attn_hyper_connection"), ("mhc", "mlp_hyper_connection")) + + +def _hyper_connection(tag, src, layers): + """The four weights of one gated-residual hyper-connection; `src(name)` names the source.""" + return ( + W(f"{tag}_norm", shape=lambda d: (d.stream,), src=src("hc_norm"), layers=layers), + W( + f"{tag}_down", + shape=lambda d: (d.hc_lowrank, d.stream), + src=src("input_mix_weight_down"), + layers=layers, + ), + W( + f"{tag}_up", + shape=lambda d: (d.stream, d.hc_lowrank), + src=src("input_mix_weight_up"), + layers=layers, + ), + W( + f"{tag}_inject", + shape=lambda d: (d.hc_count, d.stream), + src=src("block_inject_weight"), + layers=layers, + ), + ) + + +def _mixer(tag, prefix, layers): + """A stack-final hyper-connection mixer: a layer's gate minus the injection weight.""" + return ( + W( + f"{tag}_norm", + shape=lambda d: (d.stream,), + src=f"{prefix}.hc_norm.weight", + layers=layers, + ), + W( + f"{tag}_down", + shape=lambda d: (d.hc_lowrank, d.stream), + src=f"{prefix}.input_mix_weight_down.weight", + layers=layers, + ), + W( + f"{tag}_up", + shape=lambda d: (d.stream, d.hc_lowrank), + src=f"{prefix}.input_mix_weight_up.weight", + layers=layers, + ), + ) + + +def _attn_in_fused(d, i): + """Trunk attention input sources: q/k/v_proj at this rank's heads, replicated index_qk_proj.""" + p = f"{_LM}.layers.{i}.self_attn" + keys = ( + f"{p}.q_proj.weight", + f"{p}.k_proj.weight", + f"{p}.v_proj.weight", + f"{p}.indexer.index_qk_proj.weight", + ) + return [(keys, None, (_q_rows(d), _kv_rows(d), _kv_rows(d), None))] + + +def _mtp_attn_in_fused(d, i): + """`_attn_in_fused` for the draft layer.""" + p = f"{_MTP_LAYER}.self_attn" + keys = ( + f"{p}.q_proj.weight", + f"{p}.k_proj.weight", + f"{p}.v_proj.weight", + f"{p}.indexer.index_qk_proj.weight", + ) + return [(keys, None, (_q_rows(d), _kv_rows(d), _kv_rows(d), None))] + + +def _attn_out_and_norms(tag, prefix, layers): + """o_proj at this rank's q heads, and the replicated per-head q/k norms.""" + return ( + W( + f"{tag}o_w", + shape=lambda d: (d.hidden, d.q_width), + src=f"{prefix}.o_proj.weight", + cut=_o_cols, + layers=layers, + ), + W( + f"{tag}q_norm", + shape=lambda d: (d.head_dim,), + src=f"{prefix}.q_norm.weight", + layers=layers, + ), + W( + f"{tag}k_norm", + shape=lambda d: (d.head_dim,), + src=f"{prefix}.k_norm.weight", + layers=layers, + ), + ) + + +def _index_branch(tag, prefix, layers): + """The QSA index branch's two per-head norms; its projection is part of `attn_in_w`.""" + return ( + W( + f"{tag}idx_qn", + shape=lambda d: (d.index_head_dim,), + src=f"{prefix}.q_layernorm.weight", + layers=layers, + ), + W( + f"{tag}idx_kn", + shape=lambda d: (d.index_head_dim,), + src=f"{prefix}.k_layernorm.weight", + layers=layers, + ), + ) + + +def _shared_expert(tag, prefix, layers): + """The replicated bf16 shared expert; its gate logit shares the gate|up operand.""" + return ( + W( + f"{tag}sh_gu", + shape=lambda d: (2 * d.shared_inter + SH_GATE_ROWS, d.hidden), + src=lambda d, i: [ + ( + ( + f"{prefix(i)}.shared_expert.gate_proj.weight", + f"{prefix(i)}.shared_expert.up_proj.weight", + f"{prefix(i)}.shared_expert_gate.weight", + ), + None, + ) + ], + transform=_cat_rows, + layers=layers, + ), + W( + f"{tag}sh_dn", + shape=lambda d: (d.hidden, d.shared_inter), + src=lambda d, i: [(f"{prefix(i)}.shared_expert.down_proj.weight", None)], + layers=layers, + ), + ) + + +def _routed_experts(tag, prefix, layers): + """One MoE layer's routed experts at this rank's window, as `fp8_block_scale_moe_runner` + takes them. + + Plain row-major e4m3 (no shuffle, swizzle or padding) with one fp32 scale per 128x128 + block; whole experts only, no cut reaches the intermediate. + """ + return ( + W( + f"{tag}fc1_w", + shape=lambda d: (d.local_experts, 2 * d.moe_inter, d.hidden), + dtype=f8, + src=_experts(prefix, "up_proj.weight", "gate_proj.weight", window=_window), + transform=_expert_fc1_w, + layers=layers, + ), + W( + f"{tag}fc1_s", + shape=lambda d: ( + d.local_experts, + (2 * d.moe_inter) // FP8_BLOCK, + d.hidden // FP8_BLOCK, + ), + dtype=EXPERT_SCALE_DTYPE, + src=_experts( + prefix, "up_proj.weight_scale_inv", "gate_proj.weight_scale_inv", window=_window + ), + transform=_expert_fc1_s, + layers=layers, + ), + W( + f"{tag}fc2_w", + shape=lambda d: (d.local_experts, d.hidden, d.moe_inter), + dtype=f8, + src=_experts(prefix, "down_proj.weight", window=_window), + layers=layers, + ), + W( + f"{tag}fc2_s", + shape=lambda d: (d.local_experts, d.hidden // FP8_BLOCK, d.moe_inter // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_experts(prefix, "down_proj.weight_scale_inv", window=_window), + transform=_expert_fc2_s, + layers=layers, + ), + ) + + +f8, i64 = torch.float8_e4m3fn, torch.int64 + + +class Qwen4ExpFp8Weights: + """`declare`, `manifest` and `load` over `WEIGHTS`, at one rank's share. + + `declare` and `manifest` share `_targets`, so they agree on which parameters exist. + """ + + WEIGHTS: tuple[W, ...] = ( + # --- the two per-layer hyper-connection gates: every trunk layer, replicated --- + *( + w + for tag, module in _HC_MODULES + for w in _hyper_connection( + tag, lambda name, m=module: f"{{p}}.{m}.{name}.weight", ALL_LAYERS + ) + ), + # --- the gated GQA layers and their QSA index branch --- + W( + "attn_in_w", + shape=lambda d: (d.attn_in_rows, d.hidden), + src=_attn_in_fused, + transform=_attn_in_w, + layers=_attn_layers, + ), + *_attn_out_and_norms("", "{p}.self_attn", _attn_layers), + *_index_branch("", "{p}.self_attn.indexer", _attn_layers), + # --- the Gated-DeltaNet layers, at this rank's key and value heads --- + W( + "in_proj_w", + shape=lambda d: (d.in_proj_rows, d.hidden), + src=_in_proj, + layers=_gdn_layers, + ), + W( + "A_log", + shape=lambda d: (d.num_v_heads,), + src="{p}.linear_attn.A_log", + cut=_value_head_rows, + layers=_gdn_layers, + ), + W( + "dt_bias", + shape=lambda d: (d.num_v_heads,), + src="{p}.linear_attn.dt_bias", + cut=_value_head_rows, + layers=_gdn_layers, + ), + # Stored `[conv_dim, 1, width]`; the core takes the `[conv_dim, width]` view after load. + W( + "conv_w", + shape=lambda d: (d.conv_dim, 1, d.conv_kernel), + src=_conv, + layers=_gdn_layers, + ), + W( + "gnorm", + shape=lambda d: (d.head_v_dim,), + src="{p}.linear_attn.norm.weight", + layers=_gdn_layers, + ), + W( + "out_w", + shape=lambda d: (d.hidden, d.value_dim), + src="{p}.linear_attn.out_proj.weight", + cut=_value_cols, + layers=_gdn_layers, + ), + # --- MoE on every layer: routed-expert window, replicated router and shared expert --- + W("router", shape=lambda d: (d.num_experts, d.hidden), src="{p}.mlp.gate.weight"), + *_routed_experts("", _trunk_mlp, ALL_LAYERS), + *_block_experts("", _trunk_mlp, ALL_LAYERS), + *_block_shared("", _trunk_mlp, ALL_LAYERS), + *_shared_expert("", _trunk_mlp, ALL_LAYERS), + # --- not per-layer: the stack-final mixer, which is also the final norm --- + *_mixer("fhc", f"{_LM}.hyper_connection_mixer", None), + W( + "embed", + shape=lambda d: (d.vocab, d.hidden), + src=f"{_LM}.embed_tokens.weight", + layers=None, + ), + # --- PLE, on its one decoder layer (the n-gram table itself is declared at load) --- + W( + "ple_key_w", + shape=lambda d: (d.stream, d.hidden), + src=_ple("key_proj.weight"), + layers=None, + ), + W( + "ple_val_w", + shape=lambda d: (d.ple_embed_dim, d.hidden), + src=_ple("value_proj.weight"), + layers=None, + ), + W("ple_nk", shape=lambda d: (d.stream,), src=_ple("norm_key.weight"), layers=None), + W("ple_nq", shape=lambda d: (d.stream,), src=_ple("norm_query.weight"), layers=None), + W("ple_nc", shape=lambda d: (d.stream,), src=_ple("norm_conv.weight"), layers=None), + W( + "ple_conv_w", + shape=lambda d: (d.stream, 1, d.ple_conv_kernel), + src=_ple("conv1d.weight"), + layers=None, + ), + W( + "ple_mult", + shape=lambda d: (d.ngram_size,), + dtype=i64, + src=_ple("ple_embedding.layer_multipliers"), + layers=None, + ), + W( + "ple_vocab", + shape=lambda d: (d.ngram_heads,), + dtype=i64, + src=_ple("ple_embedding.ngram_heads_vocab_sizes"), + layers=None, + ), + W( + "ple_offs", + shape=lambda d: (d.ngram_heads,), + dtype=i64, + src=_ple("ple_embedding.ngram_heads_offsets"), + layers=None, + ), + # The table's scale is on `ngram_embedding` with the shards, not on `ple_embedding`. + W( + "ple_tscale", + shape=lambda d: (1,), + src=lambda d, i: [(f"{_ngram_prefix(d.ple_layer)}.weight_scale", None)], + layers=None, + ), + # --- the vision tower: all bf16, replicated, one-to-one, no transform --- + W( + "vis_patch_w", + shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), + src=_visual("patch_embed.proj.weight"), + layers=None, + ), + W( + "vis_patch_b", + shape=lambda d: (d.v_hidden,), + src=_visual("patch_embed.proj.bias"), + layers=None, + ), + W( + "vis_pos", + shape=lambda d: (d.v_pos_rows, d.v_hidden), + src=_visual("pos_embed.weight"), + layers=None, + ), + W( + "vis_n1_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.weight"), + layers=_vision_blocks, + ), + W( + "vis_n1_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.bias"), + layers=_vision_blocks, + ), + W( + "vis_qkv_w", + shape=lambda d: (3 * d.v_hidden, d.v_hidden), + src=_vision_block("attn.qkv.weight"), + layers=_vision_blocks, + ), + W( + "vis_qkv_b", + shape=lambda d: (3 * d.v_hidden,), + src=_vision_block("attn.qkv.bias"), + layers=_vision_blocks, + ), + W( + "vis_proj_w", + shape=lambda d: (d.v_hidden, d.v_hidden), + src=_vision_block("attn.proj.weight"), + layers=_vision_blocks, + ), + W( + "vis_proj_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("attn.proj.bias"), + layers=_vision_blocks, + ), + W( + "vis_n2_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.weight"), + layers=_vision_blocks, + ), + W( + "vis_n2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc1_w", + shape=lambda d: (d.v_inter, d.v_hidden), + src=_vision_block("mlp.linear_fc1.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc1_b", + shape=lambda d: (d.v_inter,), + src=_vision_block("mlp.linear_fc1.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc2_w", + shape=lambda d: (d.v_hidden, d.v_inter), + src=_vision_block("mlp.linear_fc2.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("mlp.linear_fc2.bias"), + layers=_vision_blocks, + ), + # The merger's norm runs before the 4-to-1 shuffle, so it is `hidden` wide. + W( + "vis_m_n_w", + shape=lambda d: (d.v_hidden,), + src=_visual("merger.norm.weight"), + layers=None, + ), + W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), + W( + "vis_m_fc1_w", + shape=lambda d: (d.v_merge_dim, d.v_merge_dim), + src=_visual("merger.linear_fc1.weight"), + layers=None, + ), + W( + "vis_m_fc1_b", + shape=lambda d: (d.v_merge_dim,), + src=_visual("merger.linear_fc1.bias"), + layers=None, + ), + W( + "vis_m_fc2_w", + shape=lambda d: (d.hidden, d.v_merge_dim), + src=_visual("merger.linear_fc2.weight"), + layers=None, + ), + W( + "vis_m_fc2_b", + shape=lambda d: (d.hidden,), + src=_visual("merger.linear_fc2.bias"), + layers=None, + ), + # --- the MTP draft module, at layer index num_hidden_layers, only when live --- + # A trunk full-attention layer plus a two-branch front end; keys carry no `{p}`. + W( + "mtp_pre_e_norm", + shape=lambda d: (d.hidden,), + src=f"{_MTP}.pre_fc_norm_embedding.weight", + layers=_mtp_only, + ), + # `[hc_count * hidden]`: the module's input is the trunk's four-stream residual. + W( + "mtp_pre_h_norm", + shape=lambda d: (d.stream,), + src=f"{_MTP}.pre_fc_norm_hidden.weight", + layers=_mtp_only, + ), + W( + "mtp_fc_e", + shape=lambda d: (d.hidden, d.hidden), + src=f"{_MTP}.fc_embedding.weight", + layers=_mtp_only, + ), + W( + "mtp_fc_h", + shape=lambda d: (d.hidden, d.hidden), + src=f"{_MTP}.fc_hidden.weight", + layers=_mtp_only, + ), + *( + w + for tag, module in _HC_MODULES + for w in _hyper_connection( + f"mtp_{tag}", + lambda name, m=module: f"{_MTP_LAYER}.{m}.{name}.weight", + _mtp_only, + ) + ), + W( + "mtp_attn_in_w", + shape=lambda d: (d.attn_in_rows, d.hidden), + src=_mtp_attn_in_fused, + transform=_attn_in_w, + layers=_mtp_only, + ), + *_attn_out_and_norms("mtp_", f"{_MTP_LAYER}.self_attn", _mtp_only), + *_index_branch("mtp_", f"{_MTP_LAYER}.self_attn.indexer", _mtp_only), + W( + "mtp_router", + shape=lambda d: (d.num_experts, d.hidden), + src=f"{_MTP_LAYER}.mlp.gate.weight", + layers=_mtp_only, + ), + *_routed_experts("mtp_", _mtp_mlp, _mtp_only), + *_block_experts("mtp_", _mtp_mlp, _mtp_only), + *_block_shared("mtp_", _mtp_mlp, _mtp_only), + *_shared_expert("mtp_", _mtp_mlp, _mtp_only), + # The module's own stack-final mixer, which is its `shared_head`. + *_mixer("mtp_fhc", f"{_MTP}.hyper_connection_mixer", _mtp_only), + ) + + def dims(self, core) -> SimpleNamespace: + """This rank's `rank_dims` bundle; `core.vision_config` must be set before `declare`.""" + return rank_dims( + core.model_config.pretrained_config, + core.vision_config, + core.model_config.mapping, + mtp_enabled=getattr(core.model_config, "spec_config", None) is not None, + dtype=core.model_config.torch_dtype, + ) + + def expected_unconsumed(self, d: SimpleNamespace, keys) -> set: + """Keys this rank leaves unread: off-window experts, and `mtp.*` without MTP. + + Computed from `d`, not the checkpoint, so a real weight's key can never land here. + """ + start, count = _window(d) + + def off_window(i): + if getattr(d, "moe_tpb", False): + return [] # the block-major copy reads a block of every expert + return [e for e in range(d.num_experts) if not start <= e < start + count] + + out = { + f"{_trunk_mlp(i)}.experts.{e}.{t}" + for i in range(d.num_layers) + for e in off_window(i) + for t in _EXPERT_TENSORS + } + if not d.mtp_enabled: + return out | {k for k in keys if k.startswith(f"{_MTP}.")} + out.update( + f"{_mtp_mlp(None)}.experts.{e}.{t}" + for e in off_window(d.num_layers) + for t in _EXPERT_TENSORS + ) + return out + + def declare(self, d: SimpleNamespace) -> nn.ParameterDict: + """Allocate every weight at this rank's share (meta-init intercepts `torch.empty`).""" + w = nn.ParameterDict() + for entry in self.WEIGHTS: + dtype = entry.dtype or d.dtype + for key, _, _ in self._targets(entry, d): + w[key] = nn.Parameter( + torch.empty(*entry.shape(d), dtype=dtype), requires_grad=False + ) + return w + + def _targets(self, entry: W, d: SimpleNamespace) -> list[tuple[str, int | None, str]]: + """`[(param key, layer index | None, checkpoint layer prefix), ...]` of one entry.""" + layers = entry.layer_indices(d) + if layers is None: + return [(entry.name, None, "")] + return [(f"l{i}_{entry.name}", i, f"{_LM}.layers.{i}") for i in layers] + + def manifest(self, d: SimpleNamespace) -> dict[str, list[tuple]]: + """Param key -> `[(checkpoint key, index | None, cut | None, transform | None), ...]`.""" + rows: dict[str, list[tuple]] = {} + for entry in self.WEIGHTS: + for key, i, prefix in self._targets(entry, d): + rows[key] = [ + (ckpt, index, cut, entry.transform) + for ckpt, index, cut in entry.sources(d, i, prefix) + ] + return rows + + def read(self, weights, ckpt_key, cut, transform, d, device=None) -> torch.Tensor: + """One manifest source -> its destination's tensor: keys read through cuts, transformed.""" + multi = isinstance(ckpt_key, tuple) + keys = ckpt_key if multi else (ckpt_key,) + cuts = (cut if cut is not None else (None,) * len(keys)) if multi else (cut,) + assert len(cuts) == len(keys), (ckpt_key, cut) + for key in keys: + assert key in weights, f"checkpoint key missing: {key}" + src_all = tuple(_read(weights[key], c) for key, c in zip(keys, cuts)) + if device is not None: + src_all = tuple(t.to(device, non_blocking=True) for t in src_all) + if transform is not None: + return transform(src_all if multi else src_all[0], d) + assert len(src_all) == 1, "a multi-key row needs a transform" + return src_all[0] + + def fill(self, param: nn.Parameter, sources, weights, d: SimpleNamespace) -> set: + """Fill `param` from its manifest rows on the parameter's device; return the keys read.""" + consumed = set() + for ckpt_key, index, cut, transform in sources: + dst = param.data if index is None else param.data[index] + src = self.read(weights, ckpt_key, cut, transform, d, dst.device) + assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) + assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + return consumed + + def _fill_shared_table(self, core, sources, weights, d: SimpleNamespace) -> set: + """Fill (owner), wait for (pending) or check (attached) the shared `ple_table`.""" + segment = core._ple_table_host + table = core.w["ple_table"] + rows, cols = table.shape + if segment.role == _host_share.PENDING: + segment.wait() + view = _view_at(segment.pointer, rows, cols, table.dtype, core.w["embed"].device) + core.w["ple_table"] = nn.Parameter(view, requires_grad=False) + assert core.w["ple_table"].data_ptr() == view.data_ptr() + if segment.role == _host_share.OWNER: + consumed = self.fill(core.w["ple_table"], sources, weights, d) + torch.cuda.synchronize() + segment.publish() + return consumed + _check_samples(segment, core.w["ple_table"]) + return {ckpt for ckpt, _, _, _ in sources} + + def load(self, model, weights) -> None: + """Fill every declared parameter and assert checkpoint coverage both ways.""" + core = model.model + d = self.dims(core) + # Declared before the manifest: its height comes from the shards. + rows_per_shard = _declare_ngram_table(core, d, weights) + manifest = self.manifest(d) + manifest["ple_table"] = ngram_table_sources(d, rows_per_shard) + consumed: set = set() + + assert set(manifest.keys()) == set(core.w.keys()), ( + "manifest/parameter drift", + set(manifest.keys()) ^ set(core.w.keys()), + ) + shared = isinstance(getattr(core, "_ple_table_host", None), _host_share.SharedHostSegment) + for param_key, sources in manifest.items(): + if shared and param_key == "ple_table": + consumed |= self._fill_shared_table(core, sources, weights, d) + else: + consumed |= self.fill(core.w[param_key], sources, weights, d) + if param_key.endswith(_RELEASE_AFTER): + torch.cuda.empty_cache() + + # The shell owns lm_head (untied, bf16); read only the rows it built. + cut, pad = lm_head_share(model.lm_head.weight.shape[0], d.vocab, d.tp_rank) + assert tuple(model.lm_head.weight.shape[1:]) == (d.hidden,), ( + f"the shell built lm_head {tuple(model.lm_head.weight.shape)}; this checkpoint's " + f"is a dense [{d.vocab}, {d.hidden}] weight" + ) + consumed |= self.fill( + model.lm_head.weight, [("lm_head.weight", None, cut, pad)], weights, d + ) + # Every parameter outside `core.w` must be lm_head. + shell = {n for n, _ in model.named_parameters() if not n.startswith("model.w.")} + assert shell == {"lm_head.weight"}, f"unfed shell parameters: {sorted(shell)}" + + torch.cuda.synchronize() + leftover = set(weights.keys()) - consumed + expected = self.expected_unconsumed(d, weights.keys()) + assert leftover == expected, ( + "checkpoint coverage: leftover keys are not exactly this rank's off-window " + f"experts{'' if d.mtp_enabled else ' and the MTP module'}; unexpected " + f"{sorted(leftover - expected)[:8]}, missed {sorted(expected - leftover)[:8]}" + ) + + +MODEL_WEIGHTS = Qwen4ExpFp8Weights() + + +class ExpertWindow(NamedTuple): + """`fp8_block_scale_moe_runner` operands of one MoE layer at this rank's window. + + `offset` / `count` are `local_expert_offset` / `local_num_experts`. + """ + + fc1_w: torch.Tensor # [count, 2 * moe_inter, hidden] e4m3, rows `[up ; gate]` + fc1_s: torch.Tensor # [count, 2 * moe_inter / 128, hidden / 128] fp32 + fc2_w: torch.Tensor # [count, hidden, moe_inter] e4m3 + fc2_s: torch.Tensor # [count, hidden / 128, moe_inter / 128] fp32 + offset: int + count: int + #: the layer's block-major copy (`MOE_TPB_ENV`), or None + blocks: "BlockWindow | None" = None + + +class BlockWindow(NamedTuple): + """One MoE layer's block-major routed experts and shared-expert slice at this rank.""" + + w1: torch.Tensor # [640, 256, hidden] e4m3, a block's rows `[up ; gate]` + s1: torch.Tensor # [640, 2, hidden / 128] fp32 + w2: torch.Tensor # [640, hidden, 128] e4m3 + s2: torch.Tensor # [640, hidden / 128] fp32 + rank: int + sh_gu: torch.Tensor # [5, 64, hidden] bf16: the slice's gate / up items + sh_gl: torch.Tensor # [hidden] bf16: the gate-logit row + sh_dn: torch.Tensor # [5, hidden, 32] bf16: the slice's down items + + +def _block_window(w, d: SimpleNamespace, layer: int) -> BlockWindow | None: + if not getattr(d, "moe_tpb", False): + return None + p = f"l{layer}_mtp_" if layer == d.num_layers else f"l{layer}_" + return BlockWindow( + w[f"{p}tpb_w1"], + w[f"{p}tpb_s1"], + w[f"{p}tpb_w2"], + w[f"{p}tpb_s2"], + d.tp_rank, + w[f"{p}tpb_sh_gu"], + w[f"{p}sh_gu"][2 * d.shared_inter], + w[f"{p}tpb_sh_dn"], + ) + + +def expert_window(w, d: SimpleNamespace, layer: int) -> ExpertWindow: + """Routed-expert operands of decoder layer `layer`, or of the MTP module at `d.num_layers`.""" + assert 0 <= layer < d.num_layers or (layer == d.num_layers and d.mtp_enabled), ( + f"layer {layer}: the trunk has {d.num_layers} MoE layers" + f"{' and the MTP module is not loaded' if layer == d.num_layers else ''}" + ) + p = f"l{layer}_mtp_" if layer == d.num_layers else f"l{layer}_" + start, count = _window(d) + return ExpertWindow( + w[f"{p}fc1_w"], + w[f"{p}fc1_s"], + w[f"{p}fc2_w"], + w[f"{p}fc2_s"], + start, + count, + _block_window(w, d, layer), + ) + + +def attention_tp_size(mapping) -> int: + """Ranks attention and K/V pool heads split over: the TP group, or 1 under attention DP.""" + return 1 if mapping.enable_attention_dp else mapping.tp_size + + +def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: + """(heads on `rank`, first head); with more ranks than heads, neighbours repeat a head.""" + assert total % ranks == 0 or ranks % total == 0, (total, ranks) + return -(-total // ranks), rank * total // ranks + + +def rank_dims(cfg, vcfg, mapping, *, mtp_enabled: bool, dtype: torch.dtype) -> SimpleNamespace: + """Every width the table declares at rank `mapping.rank`, and the shares behind them.""" + hidden = cfg.hidden_size + head_dim = cfg.head_dim + index_heads, index_kv_heads = cfg.indexer_n_heads, cfg.indexer_kv_heads + + # Every member spans its whole world in one TP group (routing admits no other). + assert mapping.tp_size == mapping.world_size, (mapping.tp_size, mapping.world_size) + + attn_tp = attention_tp_size(mapping) + attn_rank = mapping.tp_rank if attn_tp > 1 else 0 + heads_q, q_head_start = _heads(cfg.num_attention_heads, attn_tp, attn_rank) + heads_kv, kv_head_start = _heads(cfg.num_key_value_heads, attn_tp, attn_rank) + + state_tp = mamba_effective_tp_size(mapping) + state_rank = mapping.tp_rank if state_tp > 1 else 0 + num_k_heads, k_head_start = _heads(cfg.linear_num_key_heads, state_tp, state_rank) + num_v_heads, v_head_start = _heads(cfg.linear_num_value_heads, state_tp, state_rank) + key_dim = num_k_heads * cfg.linear_key_head_dim + value_dim = num_v_heads * cfg.linear_value_head_dim + + # Routed experts: whole-expert windows over the TP group, whatever the MoE TP / EP split. + num_experts = cfg.num_experts + moe_inter = cfg.moe_intermediate_size + assert mapping.moe_tp_size * mapping.moe_ep_size == mapping.tp_size, ( + mapping.moe_tp_size, + mapping.moe_ep_size, + mapping.tp_size, + ) + assert num_experts % mapping.tp_size == 0, (num_experts, mapping.tp_size) + assert moe_inter % FP8_BLOCK == 0 and hidden % FP8_BLOCK == 0, (moe_inter, hidden) + local_experts = num_experts // mapping.tp_size + + ngram_heads = cfg.heads_per_ngram * (cfg.ngram_size - 1) + v_hidden = vcfg.hidden_size + return SimpleNamespace( + num_layers=cfg.num_hidden_layers, + is_attn=[t == "full_attention" for t in cfg.layer_types], + hidden=hidden, + hc_count=cfg.hc_count, + hc_lowrank=cfg.hc_lowrank, + stream=cfg.hc_count * hidden, + tp_rank=mapping.tp_rank, + # --- attention, at this rank's heads --- + head_dim=head_dim, + heads_q=heads_q, + q_head_start=q_head_start, + heads_kv=heads_kv, + kv_head_start=kv_head_start, + q_width=heads_q * head_dim, + kv_width=heads_kv * head_dim, + index_head_dim=cfg.indexer_head_dim, + # [q | k | v | gate | index q, k | pad of one index head] + attn_in_rows=2 * heads_q * head_dim + + 2 * heads_kv * head_dim + + (index_heads + index_kv_heads + 1) * cfg.indexer_head_dim, + # --- Gated-DeltaNet, at this rank's key and value heads --- + num_k_heads=num_k_heads, + k_head_start=k_head_start, + num_v_heads=num_v_heads, + v_head_start=v_head_start, + head_k_dim=cfg.linear_key_head_dim, + head_v_dim=cfg.linear_value_head_dim, + key_dim=key_dim, + key_dim_total=cfg.linear_num_key_heads * cfg.linear_key_head_dim, + value_dim=value_dim, + conv_dim=2 * key_dim + value_dim, + in_proj_split=(2 * key_dim + value_dim, value_dim, num_v_heads, num_v_heads), + in_proj_rows=-(-(2 * key_dim + 2 * value_dim + 2 * num_v_heads) // IN_PROJ_ROW_ALIGN) + * IN_PROJ_ROW_ALIGN, + conv_kernel=cfg.linear_conv_kernel_dim, + # --- MoE: window `[expert_offset, expert_offset + local_experts)`, whole `moe_inter` --- + num_experts=num_experts, + local_experts=local_experts, + expert_offset=local_experts * mapping.tp_rank, + moe_tpb=moe_tpb_enabled(mapping), + moe_inter=moe_inter, + shared_inter=cfg.shared_expert_intermediate_size, + vocab=cfg.vocab_size, + ple_layer=list(cfg.ple_layer_ids)[0] - 1, # one-based in the config + ple_embed_dim=cfg.ple_embed_dim, + ple_conv_kernel=cfg.ple_conv_kernel_size, + ngram_size=cfg.ngram_size, + ngram_heads=ngram_heads, + ple_head_dim=cfg.ple_embed_dim // ngram_heads, + ple_shards=cfg.split_ngram_parts, + ple_table_dtype=PLE_TABLE_DTYPE, + v_depth=vcfg.depth, + v_hidden=v_hidden, + v_inter=vcfg.intermediate_size, + v_channels=vcfg.in_channels, + v_tpatch=vcfg.temporal_patch_size, + v_patch=vcfg.patch_size, + v_pos_rows=vcfg.num_position_embeddings, + v_merge_dim=v_hidden * vcfg.spatial_merge_size * vcfg.spatial_merge_size, + mtp_enabled=mtp_enabled, + dtype=dtype, + ) + + +def lm_head_share(rows: int, vocab: int, tp_rank: int) -> tuple[Cut, Callable]: + """The cut and the pad that fill an lm_head the shell built with `rows` rows. + + Whole under attention DP (`rows == vocab`), else a vocab-parallel shard; an uneven vocab + leaves the last shard short, zero-padded as the built-in LMHead does. + """ + start = 0 if rows == vocab else tp_rank * rows + stop = min(start + rows, vocab) + assert start < stop, (rows, vocab, tp_rank) + + def pad(t: torch.Tensor, d) -> torch.Tensor: + if t.shape[0] == rows: + return t.contiguous() + out = torch.zeros(rows, t.shape[1], dtype=t.dtype, device=t.device) + out[: t.shape[0]] = t + return out + + return (slice(start, stop),), pad + + +def ngram_shard_keys(d: SimpleNamespace) -> list[str]: + """The n-gram table's shard keys, in concatenation order.""" + prefix = _ngram_prefix(d.ple_layer) + return [f"{prefix}.shard_{s}.weight" for s in range(d.ple_shards)] + + +def ngram_table_sources(d: SimpleNamespace, rows_per_shard: int) -> list[tuple]: + """`ple_table` manifest rows: shard `s`, read whole, fills rows from `s * rows_per_shard`.""" + return [ + (key, (_span(s * rows_per_shard, rows_per_shard),), None, None) + for s, key in enumerate(ngram_shard_keys(d)) + ] + + +def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: + """Register the n-gram table at the height its shards concatenate to; return rows per shard. + + The shards are contiguous row chunks (global row `r` is shard `r // rows_per_shard`), so + the flat buffer takes the hash's global rows. On CUDA it is host memory the device + addresses (the scope's shared segment with `core.ple_shared_host`); otherwise an ordinary + allocation on the weights' device. + """ + keys = ngram_shard_keys(d) + for key in keys: + assert key in weights, f"checkpoint key missing: {key}" + shapes = {tuple(weights[key].shape) for key in keys} + assert len(shapes) == 1, ( + f"the n-gram table's {len(keys)} shards do not share one shape: " + f"{sorted(shapes)}; they are concatenated as equal contiguous row " + f"chunks and an uneven one would shift every row above it" + ) + rows_per_shard, head_dim = shapes.pop() + assert head_dim == d.ple_head_dim, ( + f"the n-gram table is {head_dim} wide but {d.ngram_heads} heads must " + f"tile ple_embed_dim {d.ple_embed_dim}, i.e. {d.ple_head_dim}" + ) + dtype = weights[keys[0]].dtype + assert dtype == d.ple_table_dtype, ( + f"the n-gram table is stored {dtype}; this target declares " + f"{d.ple_table_dtype} and does not convert it" + ) + device = core.w["embed"].device + rows = d.ple_shards * rows_per_shard + if device.type != "cuda": + table = torch.empty(rows, head_dim, dtype=dtype, device=device) + core._ple_table_host = None + elif getattr(core, "ple_shared_host", None): + table, core._ple_table_host = _shared_host_table( + rows, head_dim, dtype, device, weights, keys, rows_per_shard + ) + else: + table, core._ple_table_host = _host_resident_empty(rows, head_dim, dtype, device) + core.w["ple_table"] = nn.Parameter(table, requires_grad=False) + # Host offload relies on `nn.Parameter` sharing storage rather than copying. + assert core.w["ple_table"].data_ptr() == table.data_ptr(), ( + "wrapping the n-gram table in nn.Parameter moved its storage" + ) + return rows_per_shard diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py index f76bf6a20204..2eff81e160ce 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py @@ -3,10 +3,11 @@ """Routing for Qwen4ExpForConditionalGeneration. Qwen3.8-Flash-Next-NVFP4 on sm103 lands on `qwen3_8_flash_next_nvfp4__sm_103__default` at one -of the layouts in `_LAYOUTS`, with one-model MTP-Eagle (not at dep2) or no speculative -decoding. The target's `check_deployment` then declines the checkpoints and engine choices it -does not run, so `auto` falls back to the built-in implementation instead of failing in the -build. +of the layouts in `_LAYOUTS`, and Qwen3.8-Flash-Next-FP8 (FP8_BLOCK_SCALES) on +`qwen3_8_flash_next_fp8__sm_103__unified` at one of `_FP8_LAYOUTS`, each with one-model +MTP-Eagle (not at dep2) or no speculative decoding. The target's `check_deployment` then +declines the checkpoints and engine choices it does not run, so `auto` falls back to the +built-in implementation instead of failing in the build. """ from __future__ import annotations @@ -14,6 +15,7 @@ from typing import TYPE_CHECKING, Optional, Tuple from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig +from tensorrt_llm.quantization.mode import QuantAlgo from ..._router_index import NoTarget, routes @@ -22,7 +24,7 @@ from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from tensorrt_llm.mapping import Mapping -# The GPU the target is certified on. +# The GPU the targets are certified on. _SM = (10, 3) # (world size, attention DP, MoE TP, MoE EP) -> the target's layout, each in one node. tp1 holds @@ -37,9 +39,12 @@ (2, True, 1, 2): "dep2", } +# The FP8 target also runs tp2 and tep2. +_FP8_LAYOUTS = {**_LAYOUTS, (2, False, 2, 1): "tp2", (2, False, 1, 2): "tep2"} -def layout(mapping: Mapping) -> Optional[str]: - """The layout ``mapping`` describes, or None. + +def layout(mapping: Mapping, layouts: dict = _LAYOUTS) -> Optional[str]: + """The layout ``mapping`` describes in ``layouts``, or None. Attention runs TP over the whole world, or DP with the LM head whole. Pipeline and context parallelism, MoE clusters and a world spanning nodes are other forwards. @@ -55,7 +60,7 @@ def layout(mapping: Mapping) -> Optional[str]: or mapping.is_multi_node() ): return None - return _LAYOUTS.get( + return layouts.get( (mapping.world_size, mapping.enable_attention_dp, mapping.moe_tp_size, mapping.moe_ep_size) ) @@ -68,7 +73,8 @@ def route( ) -> type: """Return the target class for this deployment, or raise ``NoTarget`` saying why none fits.""" mapping, spec = model_config.mapping, model_config.spec_config - member = layout(mapping) + fp8 = getattr(model_config.quant_config, "quant_algo", None) == QuantAlgo.FP8_BLOCK_SCALES + member = layout(mapping, _FP8_LAYOUTS if fp8 else _LAYOUTS) if compute_capability != _SM or member is None: raise NoTarget( f"compute capability {compute_capability}, world_size={mapping.world_size} " @@ -76,7 +82,8 @@ def route( f"attention_dp={mapping.enable_attention_dp}, moe_tp={mapping.moe_tp_size}, " f"moe_ep={mapping.moe_ep_size}); the qwen4_exp target is Qwen3.8-Flash-Next-NVFP4 " "on sm103 in one node: tp1, tp4 (experts split by intermediate), tep4, dep4 " - "(attention DP) or dep2 (attention DP, experts EP2)" + "(attention DP) or dep2 (attention DP, experts EP2); Qwen3.8-Flash-Next-FP8's also " + "tp2 or tep2" ) if spec is not None and not ( isinstance(spec, MTPDecodingConfig) and spec.spec_dec_mode.is_mtp_eagle_one_model() @@ -90,6 +97,13 @@ def route( "speculative decoding with attention DP (dep2): no draft acceptance was measured at " "this layout; MTP under attention DP is admitted at dep4" ) + if fp8: + from .qwen3_8_flash_next_fp8__sm_103__unified.modeling import ( + ModelingV2Qwen38FlashNextFp8Sm103Unified, + ) + + ModelingV2Qwen38FlashNextFp8Sm103Unified.check_deployment(model_config, llm_args, member) + return ModelingV2Qwen38FlashNextFp8Sm103Unified from .qwen3_8_flash_next_nvfp4__sm_103__default.modeling import ( ModelingV2Qwen38FlashNextNvfp4Sm103Default, ) diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..467079831e16 --- /dev/null +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py new file mode 100644 index 000000000000..0c589671d855 --- /dev/null +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Whole-model GSM8K gates for the qwen3_8_flash_next_fp8__sm_103__unified target on sm_103.""" + +import pytest + +from tensorrt_llm import LLM +from tensorrt_llm._torch.modeling_v2 import MODELING_V2_VALIDATE_ENV +from tensorrt_llm._utils import get_sm_version +from tensorrt_llm.llmapi import ( + CudaGraphConfig, + KvCacheConfig, + MambaStateConfig, + MoeConfig, + MTPDecodingConfig, +) +from tensorrt_llm.quantization import QuantAlgo + +from .....conftest import llm_models_root +from ....accuracy_core import GSM8K, LlmapiAccuracyTestHarness, compute_acceptance_length +from ....test_disaggregated_serving import launch_disaggregated_llm, run_accuracy_test +from ..._env import scoped_env_overrides + +skip_not_sm103 = pytest.mark.skipif( + get_sm_version() != 103, reason="modeling_v2 targets in this batch are certified on sm_103 only" +) + +# Check the step contract on the first forward; the flag has to reach the worker ranks. +_VALIDATE = {MODELING_V2_VALIDATE_ENV: "1"} + +# Workers spawn their second rank over MPI in a root container (docker/Dockerfile.multi). +_DISAGG_WORKER_ENV = { + **_VALIDATE, + "PRTE_ALLOW_RUN_AS_ROOT": "1", + "PRTE_ALLOW_RUN_AS_ROOT_CONFIRM": "1", +} + + +def _disagg_server_config() -> dict: + """One context and one generation server on this host.""" + return { + "hostname": "localhost", + "port": 8000, + "backend": "pytorch", + "context_servers": {"num_instances": 1, "urls": ["localhost:8001"]}, + "generation_servers": {"num_instances": 1, "urls": ["localhost:8002"]}, + } + + +_TEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=False) +_DEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=True) + +# Only tep4 runs the breakable prefill graph: routing admits it under attention TP alone. +_MTP3_ENGINES = [ + pytest.param(_TEP4, "fp8", True, {}, id="tep4_fp8_kv_breakable"), + pytest.param(_DEP4, "auto", False, {}, id="dep4"), + pytest.param( + _DEP4, "auto", False, {"TRTLLM_MODELING_V2_MOE_ALLTOALL": "1"}, id="dep4_alltoall" + ), +] + + +class TestModelingV2Qwen38FlashNextFp8Sm103Unified(LlmapiAccuracyTestHarness): + """Every test requires modeling_v2, so a missed route fails rather than scoring the built-in.""" + + # References are keyed by this name plus the resolved quantization (FP8_BLOCK_SCALES). + MODEL_NAME = "Qwen/Qwen3.8-Flash-Next" + MODEL_PATH = f"{llm_models_root()}/Qwen3.8-Flash-Next-FP8" + + # The anchors' protocol: the chat template thinks, so the system prompt caps the reasoning + # and asks for the bare answer, which then fits GSM8K_MAX_OUTPUT_LEN. + EXTRA_EVALUATOR_KWARGS = dict( + apply_chat_template=True, + fewshot_as_multiturn=True, + system_prompt=( + "Use at most three short reasoning sentences, then end with `#### NUMBER`. " + "Do not restate the problem." + ), + chat_template_kwargs=dict(enable_thinking=True, reasoning_effort="xhigh"), + ) + + # The anchor's `max_tokens`; the class default (256) truncates this checkpoint's answers. + GSM8K_MAX_OUTPUT_LEN = 512 + + # The checkpoint's bf16 K/V and a bf16 recurrent state, the one the target serves. + KV_CACHE = KvCacheConfig( + free_gpu_memory_fraction=0.6, + enable_block_reuse=False, + mamba_ssm_cache_dtype="bfloat16", + ) + + # The trtllm-gen FP8 block-scale runner is the one MoE backend the target implements. + MOE = MoeConfig(backend="TRTLLM") + + ENGINE = dict( + max_seq_len=9216, + max_batch_size=128, + max_num_tokens=8192, + enable_chunked_prefill=False, + kv_cache_config=KV_CACHE, + cuda_graph_config=CudaGraphConfig(enable_padding=True, max_batch_size=128), + moe_config=MOE, + ) + + # One-model MTP-Eagle with the checkpoint's single MTP module replayed three times. + MTP3 = MTPDecodingConfig(max_draft_len=3) + + # Acceptance length reads 1.0 for a collapsed draft path and 1 + max_draft_len when every + # draft is accepted; at or above the midpoint, more than half the drafts are accepted. + MTP3_COLLAPSE_FLOOR = 1.0 + MTP3.max_draft_len / 2 + + # The breakable prefill CUDA graph, up to max_num_tokens. + BREAKABLE_PREFILL = dict( + prefill_cuda_graph_backend="breakable", + prefill_capture_num_tokens=[512, 1024, 2048, 4096, 8192], + ) + + def _gsm8k(self, mocker, llm, label: str) -> None: + """gsm8k against the engine's FP8_BLOCK_SCALES row of references/gsm8k.yaml. + + Under speculation also the acceptance length: rejection sampling keeps the score of a + miscomputed draft layer, so only the acceptance length sees it. + """ + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES, ( + f"{label}: the engine resolved quant_algo {llm.args.quant_config.quant_algo}; the " + "references are this checkpoint's FP8_BLOCK_SCALES rows" + ) + mocker.patch.object(GSM8K, "MAX_OUTPUT_LEN", self.GSM8K_MAX_OUTPUT_LEN) + GSM8K(self.MODEL_NAME).evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + if llm.args.speculative_config is None: + return + acceptance_length = compute_acceptance_length(llm) + print(f"[AL] {label} acceptance_length = {acceptance_length:.3f}") + assert acceptance_length >= self.MTP3_COLLAPSE_FLOOR, ( + f"{label}: acceptance length {acceptance_length:.3f} is below " + f"{self.MTP3_COLLAPSE_FLOOR}: the draft path is rejected more often than accepted" + ) + + @skip_not_sm103 + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + @pytest.mark.parametrize("parallel, kv_dtype, breakable, env", _MTP3_ENGINES) + def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): + """Three drafts with block reuse and chunked prefill, against the FP8_BLOCK_SCALES MTP row. + + Block reuse needs the Gated-DeltaNet and PLE snapshots (every 256 tokens); without them the + runtime turns reuse off. No FP8_BLOCK_SCALES row names a KV-cache algorithm, so the FP8 + pool is held to the bf16 pool's row. + """ + kv_cache_config = KvCacheConfig( + dtype=kv_dtype, + free_gpu_memory_fraction=0.6, + enable_block_reuse=True, + mamba_ssm_cache_dtype="bfloat16", + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + ) + engine = dict(self.ENGINE, enable_chunked_prefill=True, kv_cache_config=kv_cache_config) + if breakable: + engine.update(self.BREAKABLE_PREFILL) + with ( + scoped_env_overrides({**_VALIDATE, **env}) as rank_env, + LLM( + self.MODEL_PATH, + speculative_config=self.MTP3, + enable_iter_perf_stats=True, + **engine, + **parallel, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + if kv_dtype != "auto": + assert llm.args.quant_config.kv_cache_quant_algo == QuantAlgo(kv_dtype.upper()), ( + f"kv_cache_config.dtype={kv_dtype} resolved kv_cache_quant_algo " + f"{llm.args.quant_config.kv_cache_quant_algo}" + ) + # Only the reference lookup reads this; the engine is built over the FP8 pool. + llm.args.quant_config.kv_cache_quant_algo = None + self._gsm8k(mocker, llm, request.node.callspec.id) + + @skip_not_sm103 + @pytest.mark.timeout(5400) + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + @pytest.mark.skip_less_host_memory(204800) + def test_disaggregated_dep2(self, mocker): + """dep2 on both engines, one context and one generation server of two GPUs each. + + Both engines run one layout, so every pool matches on both sides of the NIXL transceiver. + """ + common_config = { + "modeling_v2": "require", + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_attention_dp": True, + "max_seq_len": 9216, + "max_batch_size": 128, + "max_num_tokens": 8192, + "enable_chunked_prefill": False, + "moe_config": {"backend": "TRTLLM"}, + "kv_cache_config": { + "enable_block_reuse": False, + "mamba_ssm_cache_dtype": "bfloat16", + "free_gpu_memory_fraction": 0.6, + }, + # The V2 hybrid manager's pools travel over the Python transceiver. + "cache_transceiver_config": {"backend": "NIXL", "transceiver_runtime": "PYTHON"}, + } + ctx_server_config = { + **common_config, + "disable_overlap_scheduler": True, + "cuda_graph_config": None, + } + gen_server_config = { + **common_config, + "disable_overlap_scheduler": False, + "cuda_graph_config": {"enable_padding": True, "max_batch_size": 128}, + } + + mocker.patch.object(GSM8K, "MAX_OUTPUT_LEN", self.GSM8K_MAX_OUTPUT_LEN) + with launch_disaggregated_llm( + _disagg_server_config(), + ctx_server_config, + gen_server_config, + self.MODEL_PATH, + max_workers=128, + extra_env=_DISAGG_WORKER_ENV, + # What every rank of both engines prints once it has bound the manager's pools. + assert_worker_log_contains="modeling_v2 qwen4_exp fp8 unified target: rank", + ) as llm: + # The reference is keyed by this quantization; the launcher reads none off the name. + llm.args.quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES + run_accuracy_test( + llm, + self.MODEL_NAME, + ["GSM8K"], + extra_evaluator_kwargs={GSM8K: self.EXTRA_EVALUATOR_KWARGS}, + timeout=3000, + ) diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index fc51a8ccd73a..8981defa6c5c 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -310,6 +310,11 @@ Qwen/Qwen3.8-2.4T-A95B: accuracy: 93.80 Qwen/Qwen3.8-Flash-Next: - accuracy: 95.72 + # Block FP8 without speculative decoding, under the protocol of the FP8 MTP row (chat + # template, thinking with a short-reasoning system prompt, 512 output tokens), measured on + # the built-in implementation: TEP4 on GB300, TRTLLM MoE, bf16 Mamba SSM cache. + - quant_algo: FP8_BLOCK_SCALES + accuracy: 95.34 - quant_algo: FP8_BLOCK_SCALES spec_dec_algo: MTP accuracy: 95.11 diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 1bcefa55770a..008afc6b6e8d 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -179,6 +179,7 @@ l0_b300: - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default TIMEOUT (30) + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/test_shared_kernel_contracts.py --import-mode=importlib TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 158acee33ce7..d338b53b88a8 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -104,6 +104,8 @@ l0_cpu: # The qwen4_exp default target's cpu_only tests (its routing among them); this stage runs # -m cpu_only, so the directory's GPU tests are left to l0_b300's modeling_v2 entry. - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default + # The qwen4_exp FP8 target's cpu_only tests, likewise. + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified - unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py - unittest/_torch/pyexecutor - unittest/_torch/executor/test_handle_responses_fast_path.py diff --git a/tests/integration/test_lists/test-db/l0_gb300.yml b/tests/integration/test_lists/test-db/l0_gb300.yml index 2795556365d1..1cec7166955f 100644 --- a/tests/integration/test_lists/test-db/l0_gb300.yml +++ b/tests/integration/test_lists/test-db/l0_gb300.yml @@ -165,6 +165,10 @@ l0_gb300: - unittest/_torch/attention/multi_gpu/test_gqa_kv_replica.py - unittest/_torch/multi_gpu/test_mnnvl_fused_allreduce_norm.py - unittest/_torch/multi_gpu/test_ll_exchange.py + # qwen4_exp FP8 target, pre-merge: its 2- and 4-rank collective matrices (l0_b300 skips them) + # and tep4 MTP3 over an FP8 KV pool. dep4 and dep2 disaggregated run post-merge. + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) # K3 collectives and decode-path exchanges, four ranks in one NVLink domain. - condition: diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml index 2972172dcbdb..887e69b38423 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml @@ -43,6 +43,10 @@ l0_gb300_multi_gpus: - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_fp8_kv_flashinfer_ring] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_nvfp4_kv_no_reuse] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2 TIMEOUT (90) + # qwen4_exp FP8 target, post-merge: dep4 MTP3 with each MoE exchange, dep2 disaggregated. + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4_alltoall] TIMEOUT (60) + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) # qwen4_exp target at tp1, post-merge: its AgentX engines (NVFP4 KV, MTP2 with rejection # sampling, the breakable prefill graph, every opt-in projection) on one context and one # generation GPU. diff --git a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py index 83ffa8f68833..948de78ced99 100644 --- a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py +++ b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py @@ -160,6 +160,11 @@ def _deviation(out: torch.Tensor, ref: torch.Tensor) -> str: _QWEN_INDEX_HEADS = dict(num_heads_q=4, num_heads_k=1, num_heads_v=1, head_dim=128) _QWEN_INDEX_MROPE = dict(**_QWEN_INDEX_HEADS, **_QWEN_MROPE) _QWEN_INDEX_PLAIN = dict(**_QWEN_INDEX_HEADS, **_QWEN_ROPE) +# One rank's main heads under attention TP2 / TP4 on the fused input projection row (the heads +# after k pass as v and only set the row stride), and the MTP draft layer's compact TP4 row. +_QWEN_TP2_ROW = dict(_QWEN_MAIN, num_heads_q=12, num_heads_k=1, num_heads_v=16) +_QWEN_TP4_ROW = dict(_QWEN_MAIN, num_heads_q=6, num_heads_k=1, num_heads_v=10) +_QWEN_TP4_MTP = dict(_QWEN_MAIN, num_heads_q=6, num_heads_k=1, num_heads_v=1) # The shapes each shipped target calls this entry with, by target directory # under `models/`. The entry's own `CELLS` are the op's whoever calls it. @@ -258,6 +263,29 @@ def _deviation(out: torch.Tensor, ref: torch.Tensor) -> str: spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_INDEX_PLAIN), ), ), + # Its tp1 / dep2 / dep4 calls are the NVFP4 target's shapes above. + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="qwen4_exp tp2 fused-row main rope, 1 token: 12 q / 1 k, 16 tail heads", + spec=dict(tokens=1, **_QWEN_TP2_ROW), + ), + Cell( + why="qwen4_exp tp2 fused-row main rope, 8192-token prefill, under GEMMA_PREFILL_ATOL", + spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_TP2_ROW), + ), + Cell( + why="qwen4_exp tp4 fused-row main rope, 1 token: 6 q / 1 k, 10 tail heads", + spec=dict(tokens=1, **_QWEN_TP4_ROW), + ), + Cell( + why="qwen4_exp tp4 fused-row main rope, 8192-token prefill, under GEMMA_PREFILL_ATOL", + spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_TP4_ROW), + ), + Cell( + why="qwen4_exp tep4 MTP draft compact qkv 6 q / 1 k / 1 v, 1024 rows (128 x 8 verify)", + spec=dict(tokens=1024, **_QWEN_TP4_MTP), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py index 171e3358a88c..3552749064d4 100644 --- a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py +++ b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py @@ -7303,6 +7303,8 @@ def _cell_paged(cell, arm) -> None: _seed(cell) fp8 = bool(spec["quant_mode"] & QUANT_MODE_FP8_KV_CACHE) nvfp4 = bool(spec["quant_mode"] & QUANT_MODE_NVFP4_KV_CACHE) + cached, new, nc = spec["cached"], spec["new"], spec["num_contexts"] + longest = max(c + n for c, n in zip(cached, new)) env = _PagedAttnEnv( num_heads=spec["num_heads"], num_kv_heads=spec["num_kv_heads"], @@ -7310,21 +7312,27 @@ def _cell_paged(cell, arm) -> None: tokens_per_block=spec["tokens_per_block"], num_blocks=spec["num_blocks"], max_batch=spec["max_num_requests"], - max_blocks_per_seq=spec.get("max_blocks_per_seq", 16), + max_blocks_per_seq=spec.get( + "max_blocks_per_seq", max(16, -(-longest // spec["tokens_per_block"])) + ), pool_dtype=torch.float8_e4m3fn if fp8 else torch.bfloat16, quant_mode=spec["quant_mode"], kv_scaling_factor=spec["kv_scale"] if fp8 or nvfp4 else None, attention_window_size=spec["window"], ) - cached, new, nc = spec["cached"], spec["new"], spec["num_contexts"] p, paged = spec["predicted_tokens_per_seq"], spec["use_paged_context_fmha"] assert all(n == p for n in new[nc:]), "a generation row carries P q rows" assert paged or not any(cached[:nc]), "a cached prefix needs the paged-context path" rids = list(range(len(new))) for rid in rids: env.add_request(rid, cached[rid] + (new[rid] if rid < nc else 0)) - prior = [rid for rid in rids if cached[rid]] - if prior: + # The histories go in through context calls of at most `_FILL_TOKENS` rows each. + groups: List[List[int]] = [] + for rid in (rid for rid in rids if cached[rid]): + if not groups or sum(cached[r] for r in groups[-1]) + cached[rid] > _FILL_TOKENS: + groups.append([]) + groups[-1].append(rid) + for prior in groups: hist = [cached[rid] for rid in prior] env.call_op( env.random_qkv(sum(hist)), @@ -7488,6 +7496,40 @@ def _spread(batch: int, longest: int) -> tuple[int, ...]: return tuple(1 + (longest - 1) * i // (batch - 1) for i in range(batch)) +#: The most rows one uncertified history-filling context call of `_cell_paged` takes. +_FILL_TOKENS = 8192 +#: One rank of an attention-TP member of qwen4_exp: 12 q / 1 kv heads (TP2), 6 / 1 (TP4). +_QWEN4_TP2 = dict(QWEN4_DENSE, num_heads=12, num_kv_heads=1) +_QWEN4_TP4 = dict(QWEN4_DENSE, num_heads=6, num_kv_heads=1) + + +def _qwen4_batch(base: dict, requests: int, longest: int, p: int) -> dict: + """A generation step of `requests` `_spread` histories at P = `p`, plus 32 unwritten pages.""" + cached = _spread(requests, longest) + pages = sum(-(-(c + p) // base["tokens_per_block"]) for c in cached) + return dict( + base, + max_num_requests=requests, + num_blocks=pages + 32, + cached=cached, + new=(p,) * requests, + num_contexts=0, + predicted_tokens_per_seq=p, + ) + + +def _qwen4_prefill_chunk(base: dict) -> dict: + """max_num_tokens of fresh prefill: four 2048-token contexts, 8192 q rows.""" + return dict( + base, + num_blocks=4 * 2048 // base["tokens_per_block"] + 32, + cached=(0,) * 4, + new=(2048,) * 4, + num_contexts=4, + predicted_tokens_per_seq=1, + ) + + # The shapes each shipped target calls this entry with, by target directory # under `models/`. The entry's own `CELLS` are the op's whoever calls it. TARGET_CELLS: dict[str, tuple[Cell, ...]] = { @@ -7970,6 +8012,56 @@ def _spread(batch: int, longest: int) -> tuple[int, ...]: ), ), ), + # One rank of each attention layout at the target's largest batches; its other shapes are + # the NVFP4 target's above. + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="tp2 decode 12q/1kv at the dense ceiling: one row over a 2047-token history", + spec=dict( + _QWEN4_TP2, + cached=(2047,), + new=(1,), + num_contexts=0, + predicted_tokens_per_seq=1, + ), + ), + Cell( + why="tp2 decode 12q/1kv at 128 requests over histories 1..2047", + spec=_qwen4_batch(_QWEN4_TP2, 128, 2047, 1), + ), + Cell( + why="tp2 prefill chunk 12q/1kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(_QWEN4_TP2), + ), + Cell( + why="tp4 decode 6q/1kv at 128 requests over histories 1..2047", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2047, 1), + ), + Cell( + why="tep4 MTP3 verify P=4 6q/1kv: 128 blocks over histories 1..2040, 512 rows", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2040, 4), + ), + Cell( + why="tep4 MTP7 verify P=8 6q/1kv: 128 blocks over histories 1..2040, 1024 rows", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2040, 8), + ), + Cell( + why="tp4 prefill chunk 6q/1kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(_QWEN4_TP4), + ), + Cell( + why="dep4 decode 24q/2kv at 128 requests per rank over histories 1..2047", + spec=_qwen4_batch(QWEN4_DENSE, 128, 2047, 1), + ), + Cell( + why="dep4 MTP3 verify P=4 24q/2kv: 128 blocks over histories 1..2040, 512 rows", + spec=_qwen4_batch(QWEN4_DENSE, 128, 2040, 4), + ), + Cell( + why="tp1 prefill chunk 24q/2kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(QWEN4_DENSE), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py b/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py index 2d8d25a3f402..e5d4d9bbe225 100644 --- a/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py +++ b/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py @@ -48,12 +48,17 @@ def _build(spec, seed): `mat_b` is built as [N, K] and transposed, because that is the one thing a spec cannot say: the op reads it column-major, and a stored weight's `.t()` - already is that, which is why no transpose kernel runs in a target. + already is that, which is why no transpose kernel runs in a target. With + `rows`, the weight is `[rows, K]` and `mat_b` the first N columns of its + `.t()` (column stride still K); rows from `live` on are zero. """ g = torch.Generator(device="cuda").manual_seed(seed) m, k, n, dt = spec["m"], spec["k"], spec["n"], spec["dtype"] mat_a = torch.randn(m, k, generator=g, device="cuda").to(dt) - mat_b = torch.randn(n, k, generator=g, device="cuda").to(dt).t() + w = torch.randn(spec.get("rows", n), k, generator=g, device="cuda").to(dt) + if "live" in spec: + w[spec["live"] :] = 0 + mat_b = w.t()[:, :n] out_dtype = spec["out_dtype"] bias = None if spec["bias"]: @@ -114,6 +119,45 @@ def _build(spec, seed): spec=dict(m=16, k=2880, n=5120, dtype=torch.bfloat16, bias=True, out_dtype=None), ), ), + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="qwen fp8 attn in-proj tp2 7424 wide, the largest captured decode batch 128", + spec=dict(m=128, k=2560, n=7424, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 o_proj / GDN out_proj tp1, K 6144 -> hidden 2560, 128 rows", + spec=dict(m=128, k=6144, n=2560, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 GDN in_proj tp4 4120 wide, MTP-7 verify of 128 requests: 1024 rows", + spec=dict(m=1024, k=2560, n=4120, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 attn in-proj tp2 7424 wide, a max_num_tokens prefill chunk: 8192 rows", + spec=dict(m=8192, k=2560, n=7424, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 draft fc_h over 4 streams of an 8192-row chunk: 32768 rows", + spec=dict(m=32768, k=2560, n=2560, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 HC first fp32 logits: 352 columns of a padded [384, 10240] weight", + spec=dict( + m=128, + k=10240, + n=352, + dtype=torch.bfloat16, + bias=False, + out_dtype=torch.float32, + rows=384, + live=324, + ), + ), + Cell( + why="qwen fp8 vision qkv 1152 -> 3456 with bias, 5580 patches", + spec=dict(m=5580, k=1152, n=3456, dtype=torch.bfloat16, bias=True, out_dtype=None), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..8d0ea72120d1 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Package marker: without it pytest imports this `test_vision.py` and the NVFP4 target's under +one module name and refuses the second.""" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py new file mode 100644 index 000000000000..117253966a5d --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's `ce_exchange__sp_pair` copy-engine exchange at the +tep2 / tp2 pair under mpirun: eager exchanges and graph replays with new inputs, byte-exact. +Run as `CUDA_VISIBLE_DEVICES=0,1 python _ce_exchange__sp_pair_op_matrix.py --world-size 2`.""" + +import argparse +import sys + +DEADLINE_S = 600 +CERTIFIED_WORLDS = (2,) +_WORKER_FLAG = "--rank-worker" +HIDDEN = 2560 +CHAIN = 6 +_PKG = ( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." + "qwen3_8_flash_next_fp8__sm_103__unified.ops.comm." +) + + +def _worker() -> None: + import torch + from mpi4py import MPI + + from tensorrt_llm.mapping import Mapping + + comm = MPI.COMM_WORLD + rank, world = comm.Get_rank(), comm.Get_size() + assert world == 2 + torch.cuda.set_device(rank % torch.cuda.device_count()) + dev = torch.device("cuda") + ce_mod = __import__(_PKG + "ce_exchange__sp_pair", fromlist=["x"]) + mapping = Mapping(world_size=2, rank=rank, gpus_per_node=2, tp_size=2) + peer = 1 - rank + + def fill(t, r, it): + g = torch.Generator(device=dev).manual_seed(1000 * it + r) + if t.dtype == torch.int32: + t.copy_( + torch.randint( + -(2**31), 2**31 - 1, t.shape, generator=g, device=dev, dtype=torch.int32 + ) + ) + else: + t.copy_(torch.randn(t.shape, generator=g, device=dev).to(t.dtype)) + + # check_create: the buffer holds the largest exchange, 8192 int32 rows of 2051. + capacity = 72 << 20 + ce = ce_mod.CopyEngineExchange.create(mapping, capacity) + assert ce is not None, f"rank {rank}: the exchange was not built" + + # check_eager + for it in range(24): + rows = (8192, 1024, 77, 4096)[it % 4] + dtype = torch.int32 if it % 3 == 2 else torch.bfloat16 + width = 2051 if dtype == torch.int32 else HIDDEN + x = torch.empty(rows, width, dtype=dtype, device=dev) + fill(x, rank, it) + got = ce.exchange(x).clone() + want = torch.empty_like(x) + fill(want, peer, it) + torch.cuda.synchronize() + assert torch.equal(got, want), f"rank {rank}: eager exchange {it} differs" + comm.Barrier() + + # check_graph + xs = [torch.empty(4096, HIDDEN, dtype=torch.bfloat16, device=dev) for _ in range(CHAIN)] + outs = [torch.empty_like(x) for x in xs] + + def body(): + for k in range(CHAIN): + outs[k].copy_(ce.exchange(xs[k])) + + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + body() + torch.cuda.current_stream().wait_stream(side) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + body() + for rep in range(30): + for k in range(CHAIN): + fill(xs[k], rank, 100 + rep * CHAIN + k) + graph.replay() + torch.cuda.synchronize() + for k in range(CHAIN): + want = torch.empty_like(xs[k]) + fill(want, peer, 100 + rep * CHAIN + k) + assert torch.equal(outs[k], want), f"rank {rank}: replay {rep} exchange {k} differs" + comm.Barrier() + + if rank == 0: + print("ce_exchange__sp_pair: all checks passed", flush=True) + MPI.Finalize() + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--world-size", type=int, default=2) + a, _ = p.parse_known_args() + if _WORKER_FLAG in sys.argv: + _worker() + return + assert a.world_size in CERTIFIED_WORLDS, a.world_size + import _launch # beside this file + + _launch.mpirun_ranks(__file__, a.world_size, [_WORKER_FLAG], DEADLINE_S) + + +if __name__ == "__main__": + main() diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py new file mode 100644 index 000000000000..b6588f80923d --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's fused TP sum + injection at 2 and 4 ranks: `hc_out` +and `normed` bitwise the IPC all-reduce then the Triton injection, with and without the MoE +combine, interleaved with plain all-reduces, and in a CUDA graph chain.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import torch + +assert torch.cuda.is_available(), "the fused TP sum + injection requires CUDA devices" + +DEADLINE_S = 1500 +HC, HIDDEN = 4, 2560 +WIDTH = HC * HIDDEN +EPS = 1e-6 +#: Decode-site row counts on both sides of the switch to 2-warp programs at 512 rows. +ROWS = (33, 96, 168, 192, 216, 288, 384, 480, 511, 512, 576, 640, 1024) +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +modeling = fused_mod = tp_sum = MPI_RANK = WS = None +RANK = WORLD = 0 + + +def _gen(seed: int) -> torch.Generator: + return torch.Generator(device="cuda").manual_seed(seed) + + +def _inputs(rows: int, seed: int, combine: bool): + g = _gen(seed) + hc = (torch.randn(rows, WIDTH, generator=g, device="cuda") * 2.0).to(torch.bfloat16) + # The injection weights as the mixer leaves them: a column slice of a wider row. + inject = (torch.randn(rows, 336, generator=g, device="cuda") * 0.5 + 1.0).to(torch.bfloat16)[ + :, 320:324 + ] + norm = (torch.randn(WIDTH, generator=g, device="cuda") * 0.477).to(torch.bfloat16) + shared = gate = None + if combine: + shared = (torch.randn(rows, HIDDEN, generator=g, device="cuda") * 0.5).to(torch.bfloat16) + gate = torch.randn(rows, 1288, generator=g, device="cuda").to(torch.bfloat16)[:, 1280:1281] + pg = _gen(seed * 977 + RANK + 1) + partial = (torch.randn(rows, HIDDEN, generator=pg, device="cuda") * 0.25).to(torch.bfloat16) + # Signed zeros: a (+0, -0) bf16 pair is the -0.0f word, the not-arrived marker; every rank + # -0.0 in columns 7..16 (the sanitized push makes their sum +0.0, as the all-reduce does). + partial[:, :7] = 0.0 + partial[:, 7:17] = -0.0 + # A few large rows (residual-stream outliers) and an all-zero residual row (rstd of eps). + if rows > 40: + hc[3] *= 300.0 + hc[5].zero_() + return hc, inject, norm, shared, gate, partial + + +def _warps(rows): + return 2 if rows >= modeling._HC_INJECT_FEW_WARPS_MIN_ROWS else modeling._HC_INJECT_WARPS + + +def _plain(hc, inject, norm, shared, gate, partial): + blk = tp_sum(partial) + return modeling._hc_inject_norm_launch( + hc, blk, inject, norm, EPS, HC, shared, gate, _warps(hc.shape[0]) + ) + + +def _fused(hc, inject, norm, shared, gate, partial): + assert fused_mod.fits(hc, partial, inject, shared, gate) + return fused_mod.hc_allreduce_inject_norm( + partial, WS, RANK, WORLD, hc, inject, norm, EPS, shared, gate, _warps(hc.shape[0]) + ) + + +def _same(a, b, what: str) -> None: + for name, x, y in zip(("hc_out", "normed"), a, b): + if not torch.equal(x, y): + idx = (x != y).nonzero()[:8].tolist() + vals = [(i, float(x[i[0], i[1]]), float(y[i[0], i[1]])) for i in idx] + raise AssertionError( + f"[rank {RANK}] {what}: {name} differs ({(x != y).sum().item()} elements), first {vals}" + ) + + +def check_fused_is_bitwise_the_all_reduce_then_the_injection() -> None: + n = 0 + for rows in ROWS: + for combine in (False, True): + for rep in range(2): + args = _inputs(rows, 7000 + 10 * rows + 2 * combine + rep, combine) + want = _plain(*args) + got = _fused(*args) + again = _plain(*args) + torch.cuda.synchronize() + what = f"{rows} rows combine {combine} rep {rep}" + _same(got, want, what) + _same(again, want, what + " (plain after fused)") + n += 1 + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] {n} fused calls bitwise the all-reduce + injection", flush=True) + + +def check_cuda_graph_chain_replays_bitwise() -> None: + """A captured chain of mixed plain and fused sites (an odd call count, so replays start on + every slot), replayed with fresh partials, is bitwise the plain chain run eagerly.""" + for rows in (96, 192, 480): + n_sites = 7 + g = _gen(60 + rows) + hc0 = (torch.randn(rows, WIDTH, generator=g, device="cuda") * 2.0).to(torch.bfloat16) + inj = (torch.randn(rows, HC, generator=g, device="cuda") * 0.5 + 1.0).to(torch.bfloat16) + norms = [ + (torch.randn(WIDTH, generator=g, device="cuda") * 0.4).to(torch.bfloat16) + for _ in range(n_sites) + ] + shared = (torch.randn(rows, HIDDEN, generator=g, device="cuda") * 0.5).to(torch.bfloat16) + gate = torch.randn(rows, 1, generator=g, device="cuda").to(torch.bfloat16) + parts = [ + torch.empty(rows, HIDDEN, dtype=torch.bfloat16, device="cuda") for _ in range(n_sites) + ] + + def chain(kinds): + hc, outs = hc0, [] + for k in range(n_sites): + fn = _fused if kinds[k] else _plain + sh, gt = (shared, gate) if k % 2 else (None, None) + hc, normed = fn(hc, inj, norms[k], sh, gt, parts[k]) + outs.append(normed) + return hc, outs + + kinds = [k % 3 != 1 for k in range(n_sites)] + for p in parts: + p.copy_( + torch.randn(rows, HIDDEN, generator=_gen(1 + RANK), device="cuda").to( + torch.bfloat16 + ) + ) + chain(kinds) # warm the compiles + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + g_hc, g_outs = chain(kinds) + torch.cuda.synchronize() + MPI_RANK.barrier() + for rep in range(5): + pg = _gen(1000 * rep + 31 * rows + RANK) + for p in parts: + p.copy_( + (torch.randn(rows, HIDDEN, generator=pg, device="cuda") * 0.3).to( + torch.bfloat16 + ) + ) + graph.replay() + torch.cuda.synchronize() + got = (g_hc.clone(), [o.clone() for o in g_outs]) + MPI_RANK.barrier() + want = chain([False] * n_sites) + torch.cuda.synchronize() + assert torch.equal(got[0], want[0]), f"{rows} rows replay {rep}: hc differs" + for k in range(n_sites): + assert torch.equal(got[1][k], want[1][k]), ( + f"{rows} rows replay {rep}: site {k} normed differs" + ) + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] CUDA graph chains replay bitwise", flush=True) + + +CHECKS = ( + check_fused_is_bitwise_the_all_reduce_then_the_injection, + check_cuda_graph_chain_replays_bitwise, +) + + +def _run_one_rank() -> int: + global modeling, fused_mod, tp_sum, MPI_RANK, RANK, WORLD, WS + os.environ.pop("TLLM_DISABLE_MPI", None) + os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + assert WORLD in (2, 4), "the fused TP sum + injection is certified at 2 and 4 ranks" + from tensorrt_llm._torch.modeling_v2.catalog.comm import allreduce as entry + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _modeling, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + hc_allreduce_inject_norm__tp_sum as _fused_mod, + ) + from tensorrt_llm.functional import AllReduceFusionOp + + modeling, fused_mod = _modeling, _fused_mod + group = entry.allreduce_group_args(MPI_RANK.mapping) + WS = group["workspace"] + + def _tp_sum(x): + return entry.allreduce( + x, + None, + None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=EPS, + trigger_completion_at_end=False, + **group, + ) + + tp_sum = _tp_sum + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + import torch.distributed as dist + + if dist.is_initialized(): + dist.barrier() + dist.destroy_process_group() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + args = _lockstep.parse_args(sys.argv[1:]) + if args.rank_worker: + sys.exit(_run_one_rank()) + import _launch # beside this file + + world = args.world_size or len(_launch.visible_devices()) + assert world in (2, 4), "the fused sum runs at 2 or 4 ranks" + worker = [_lockstep.WORKER_FLAG, "--world-size", str(world)] + worker += ["--fabric-handle", args.fabric_handle] + _launch.mpirun_ranks(__file__, world, worker, DEADLINE_S) + print("OK: fused TP sum + injection") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py new file mode 100644 index 000000000000..a93e4965d9e3 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's gather of a decode step's QSA selection rows over the +TP group (`lamport_allgather__index_split`): every output bitwise the NCCL all-gather, over mixed +trunk and draft calls and with the split's row map, eagerly and replayed from a CUDA graph.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import torch + +assert torch.cuda.is_available(), "the selection gather requires CUDA devices" + +DEADLINE_S = 900 +WIDTH = 2051 +#: Requests a step (each rank scores `requests / world` of them): tep2 / tep4 HT batches. +REQUESTS = (16, 32, 64, 128, 192) +MAX_ROWS = 3 * max(REQUESTS) +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +lam = gather = MPI_RANK = None +RANK = WORLD = 0 + + +def _share(rows: int, seed: int) -> torch.Tensor: + """This rank's `[rows / WORLD, WIDTH]` selection rows: token indices, -1 padding, and the + int32 extremes next to the not-arrived marker (int32 min itself never occurs).""" + g = torch.Generator(device="cuda").manual_seed(seed * 131 + RANK) + share = torch.randint(-1, 1 << 24, (rows // WORLD, WIDTH), generator=g, device="cuda") + share = share.to(torch.int32) + share[:, :3] = -1 + share[:, 3] = 2**31 - 1 + share[:, 4] = -(2**31) + 1 + return share + + +def _nccl(share: torch.Tensor) -> torch.Tensor: + import torch.distributed as dist + + out = torch.empty((WORLD * share.shape[0], WIDTH), dtype=torch.int32, device="cuda") + dist.all_gather_into_tensor(out, share) + return out + + +def _ensure_process_group() -> None: + """A process group for the NCCL reference, from the environment or a port rank 0 broadcasts.""" + import torch.distributed as dist + + if dist.is_initialized(): + return + if not all(os.environ.get(k) for k in ("MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE")): + addr = None + if RANK == 0: + import socket + + with socket.socket() as sock: + sock.bind(("", 0)) + addr = (socket.gethostbyname(socket.gethostname()), sock.getsockname()[1]) + host, port = MPI_RANK.comm.bcast(addr, root=0) + os.environ["MASTER_ADDR"] = host + os.environ["MASTER_PORT"] = str(port) + dist.init_process_group( + backend="nccl", + rank=RANK, + world_size=WORLD, + device_id=torch.device("cuda", torch.cuda.current_device()), + ) + + +def _same(got: torch.Tensor, want: torch.Tensor, what: str) -> None: + assert got.shape == want.shape and torch.equal(got, want), f"{what}: not bitwise NCCL's" + + +def _step_rows() -> list[int]: + """One served step per batch: 12 trunk layers at 3 rows a request, 2 draft steps at 1.""" + seq = [] + for requests in REQUESTS: + seq += [3 * requests] * 12 + [requests] * 2 + return seq + + +def check_eager_mixed_sequence_is_bitwise_nccl() -> None: + """Every call of a long mixed sequence equals NCCL's while the buffers' three slots rotate.""" + for i, rows in enumerate(_step_rows() * 2): + share = _share(rows, 100 + i) + got = gather(share, lam) + _same(got, _nccl(share), f"call {i} ({rows} rows)") + torch.cuda.synchronize() + assert MPI_RANK.same_on_ranks(lam.state[:2]) + + +def _row_map(rows: int, seed: int) -> torch.Tensor: + """A permutation of `rows` rows, the same on every rank (the split plan's `perm_rows`).""" + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randperm(rows, generator=g, device="cuda").to(torch.int32) + + +def check_row_map_stores_each_gathered_row_at_its_row() -> None: + """Gathered row `j` lands at row `dst_rows[j]`, bitwise NCCL's gather permuted the same way, + interleaved with plain calls, eagerly and in a CUDA graph replayed with new rows.""" + for i, rows in enumerate([3 * 64, 64, 3 * 192, 3 * 16, 192, 3 * 128]): + share = _share(rows, 7000 + i) + row_map = _row_map(rows, 7100 + i) + want = torch.empty_like(_nccl(share)) + want[row_map.long()] = _nccl(share) + _same(gather(share, lam, row_map), want, f"mapped call {i} ({rows} rows)") + plain = _share(rows, 7200 + i) + _same(gather(plain, lam), _nccl(plain), f"plain call after mapped call {i}") + torch.cuda.synchronize() + for requests in (32, 96): + rows_seq = [3 * requests] * 12 + [requests] * 2 + maps = [ + _row_map(r, 7300 + j + requests) if j < 12 else None for j, r in enumerate(rows_seq) + ] + shares = [ + torch.zeros((r // WORLD, WIDTH), dtype=torch.int32, device="cuda") for r in rows_seq + ] + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for s_, m_ in zip(shares, maps): + gather(s_, lam, m_) + torch.cuda.current_stream().wait_stream(stream) + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = [gather(s_, lam, m_) for s_, m_ in zip(shares, maps)] + torch.cuda.synchronize() + MPI_RANK.barrier() + for rep in range(6): + for j, s_ in enumerate(shares): + s_.copy_(_share(s_.shape[0] * WORLD, 7500 + 100 * rep + j + requests)) + torch.cuda.synchronize() + MPI_RANK.barrier() + graph.replay() + torch.cuda.synchronize() + for j, (s_, m_, o) in enumerate(zip(shares, maps, outs)): + want = _nccl(s_) + if m_ is not None: + w2 = torch.empty_like(want) + w2[m_.long()] = want + want = w2 + _same(o, want, f"{requests} requests, replay {rep}, call {j}") + MPI_RANK.barrier() + del graph + + +CHECKS = ( + check_eager_mixed_sequence_is_bitwise_nccl, + check_row_map_stores_each_gathered_row_at_its_row, +) + + +def _run_one_rank() -> int: + global lam, gather, MPI_RANK, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + lamport_allgather__index_split as entry, + ) + + # The buffers' handles travel over MPI (the engine passes its TP communicator); the NCCL + # reference needs a torch process group. + lam = entry.build(RANK, WORLD, MAX_ROWS * WIDTH, MPI_RANK.comm.allgather) + _ensure_process_group() + gather = entry.lamport_allgather + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + args = _lockstep.parse_args(sys.argv[1:]) + if args.rank_worker: + sys.exit(_run_one_rank()) + import _launch # beside this file + + world = args.world_size or len(_launch.visible_devices()) + worker = [_lockstep.WORKER_FLAG, "--world-size", str(world)] + _launch.mpirun_ranks( + __file__, world, worker + ["--fabric-handle", args.fabric_handle], DEADLINE_S + ) + print("OK: lamport selection gather") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py new file mode 100644 index 000000000000..7af7e3bf0c52 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Rank launchers for the collective matrices beside it, which import it from launcher code only. +Module scope imports only the standard library, since `_rank_job` runs a matrix's module scope +from pytest. The deadlines are there because a broken collective hangs rather than raising.""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import Sequence + +#: Slurm step variables an mpirun inside a one-task step must not inherit: with them OpenMPI +#: sizes its slots from the step (one) and every rank reads the step's own rank. +SLURM_STEP_VARS = ( + "SLURM_PROCID", + "SLURM_NTASKS", + "SLURM_LOCALID", + "SLURM_NODEID", + "SLURM_STEP_ID", + "SLURM_TASK_PID", + "SLURM_STEP_NUM_TASKS", +) + + +def visible_devices() -> list[str]: + """The devices this run owns: CUDA_VISIBLE_DEVICES, which must name them.""" + visible = [d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()] + assert visible, "set CUDA_VISIBLE_DEVICES to the devices this run owns, e.g. 0,1,2,3" + return visible + + +def _claim(world: int) -> str: + devices = visible_devices() + assert 2 <= world <= len(devices), ( + f"world size {world} needs 2..{len(devices)} visible devices (one rank per device)" + ) + return ",".join(devices[:world]) + + +def mpirun_ranks(entry_file: str, world: int, args: Sequence[str], deadline_s: int) -> None: + """Re-execute `entry_file` as `world` MPI ranks on the first `world` visible devices, without + `TLLM_DISABLE_MPI` (the ranks need tensorrt_llm's MPI session); raise on a nonzero exit or + past `deadline_s`, killing the process group.""" + command = [ + "mpirun", + "--allow-run-as-root", + "--oversubscribe", + "-n", + str(world), + sys.executable, + str(Path(entry_file).resolve()), + *args, + ] + env = { + k: v for k, v in os.environ.items() if k not in SLURM_STEP_VARS and k != "TLLM_DISABLE_MPI" + } + env["CUDA_VISIBLE_DEVICES"] = _claim(world) + print(f"[launcher] {' '.join(command)}", flush=True) + process = subprocess.Popen(command, env=env, start_new_session=True) + try: + code = process.wait(timeout=deadline_s) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise AssertionError( + f"the {world}-rank MPI run did not finish in {deadline_s}s (wedged)" + ) from None + assert code == 0, f"the {world}-rank MPI run exited {code}" + + +def spawn_pg_ranks(entry_file: str, world: int, args: Sequence[str], deadline_s: int) -> None: + """Start `entry_file` as `world` torch.distributed ranks (`env://` on a free local port), each + in its own session; kill them all and raise on the first nonzero exit or past `deadline_s`.""" + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + env = dict( + os.environ, + CUDA_VISIBLE_DEVICES=_claim(world), + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + WORLD_SIZE=str(world), + TLLM_DISABLE_MPI="1", + ) + # Drop the srun step's PMIx/PMI/OMPI namespace: a child that inherits it joins the step, and + # its hard exit makes srun SIGKILL the whole step before any error is reported. + env = {k: v for k, v in env.items() if not k.startswith(("PMIX_", "PMI_", "OMPI_", "SLURM_"))} + print(f"[launcher] spawning {world} torch.distributed ranks, rendezvous on :{port}", flush=True) + children = [ + subprocess.Popen( + [sys.executable, str(Path(entry_file).resolve()), *args], + env=dict(env, RANK=str(r)), + start_new_session=True, + ) + for r in range(world) + ] + deadline = time.monotonic() + deadline_s + live = set(range(world)) + failure = None + while live and failure is None: + if time.monotonic() > deadline: + failure = f"did not finish in {deadline_s}s (wedged)" + break + for r in sorted(live): + code = children[r].poll() + if code is None: + continue + live.discard(r) + if code != 0: + failure = f"rank {r} exited {code}" + break + time.sleep(0.5) + if failure is not None: + for p in children: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for p in children: + p.wait() + raise AssertionError(f"the {world}-rank torch.distributed run failed: {failure}") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py new file mode 100644 index 000000000000..fef3add8edde --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's `mnnvl_allreduce__tp_sum` under mpirun: each sum is +bitwise the IPC op's and within `_compare`'s band of the fp32 sum, eagerly and graph-replayed. +The MPI-disabled launch (`TLLM_DISABLE_MPI=1`) is not certified here.""" + +import argparse +import functools +import os +import sys +from pathlib import Path +from typing import Any + +import torch + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: tests/unittest/_torch/modeling_v2: `_validating` and the shared `comm/` helpers. +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +# Bound inside the rank body: module scope imports only torch and the standard library, since +# `_rank_job` executes it to read `DEADLINE_S`. +entry: Any = None +ipc: Any = None +mnnvl: Any = None +modeling: Any = None +validating: Any = None +WORKSPACE: dict = {} +MPI_RANK: Any = None +RANK = 0 +WORLD = 1 + +#: Row counts a captured `tp_sum` meets inside the budget: decode batches 1..8 and the MTP-3 +#: verify steps of 1..8 requests (4..32 rows). +GRAPH_ROWS = (1, 2, 3, 4, 8, 12, 16, 24, 32) + + +def _compare(out, ref) -> None: + """`catalog/comm/allreduce`'s band: 5 ulp per element against the row scale, 1.25 ulp RMS.""" + from _validating import assert_within_ulp + + assert_within_ulp(out, ref, element_ulp=5.0, rms_ulp=1.25) + + +def _payload(rank: int, rows: int, seed: int) -> torch.Tensor: + """One rank's partial: every rank can regenerate every other rank's from the seed.""" + gen = torch.Generator(device="cuda").manual_seed(seed * 977 + rank + 1) + return torch.randn(rows, entry.HIDDEN, generator=gen, device="cuda").to(torch.bfloat16) + + +def _reference(rows: int, seed: int) -> torch.Tensor: + return entry.mnnvl_tp_sum.reference([_payload(r, rows, seed) for r in range(WORLD)]) + + +def _exact(out, ref) -> float: + return (out == ref).float().mean().item() + + +def check_certified_cells() -> None: + cells = [c for c in entry.mnnvl_tp_sum.CELLS if c.spec["world"] == WORLD] + assert cells, f"no cell is certified at world size {WORLD}" + for i, cell in enumerate(cells): + rows = cell.spec["rows"] + for s in range(3): + seed = 4000 + 31 * i + 1000 * s + x = _payload(RANK, rows, seed) + with validating(entry.mnnvl_tp_sum): + out = mnnvl(x) + twin = ipc(x) + ref = _reference(rows, seed) + torch.cuda.synchronize() + assert out.shape == x.shape and out.dtype == x.dtype, (out.shape, out.dtype) + assert torch.equal(out, twin), f"{cell.why}: not the IPC op's bits (seed {seed})" + _compare(out, ref) + if RANK == 0 and s == 0: + print( + f"[world {WORLD}] {cell.why}: bitwise the IPC op; " + f"{100 * _exact(out, ref):.4f}% bit-exact to the fp32 sum rounded once", + flush=True, + ) + MPI_RANK.barrier() + + +def check_cuda_graph_replay_matches_eager() -> None: + """Captured once per row count, refilled with fresh partials and replayed: bitwise the + eager call and the IPC op on those partials (the Lamport buffers rotate per call).""" + for rows in GRAPH_ROWS: + static = _payload(RANK, rows, 900 + rows) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + for _ in range(3): + mnnvl(static) + torch.cuda.current_stream().wait_stream(side) + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = mnnvl(static) + MPI_RANK.barrier() + for step in range(3): + fresh = _payload(RANK, rows, 1900 + 10 * rows + step) + static.copy_(fresh) + graph.replay() + eager = mnnvl(fresh) + twin = ipc(fresh) + torch.cuda.synchronize() + assert torch.equal(captured, eager), f"{rows} rows step {step}: replay != eager" + assert torch.equal(captured, twin), f"{rows} rows step {step}: replay != IPC op" + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] captured {rows} rows: replay == eager == IPC op", flush=True) + del graph + + +CHECKS = (check_certified_cells, check_cuda_graph_replay_matches_eager) + + +def _run_one_mpi_rank() -> int: + global entry, ipc, mnnvl, modeling, validating, WORKSPACE, MPI_RANK, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm._torch.modeling_v2.catalog.comm import ( + allreduce as allreduce_mod, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _modeling, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + mnnvl_allreduce__tp_sum as _entry, + ) + from tensorrt_llm.functional import AllReduceFusionOp + + entry, modeling, validating = _entry, _modeling, _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + group = allreduce_mod.allreduce_group_args(MPI_RANK.mapping) + ipc = functools.partial( + allreduce_mod.allreduce, + residual=None, + norm_weight=None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=1e-6, + trigger_completion_at_end=False, + **group, + ) + WORKSPACE = entry.mnnvl_workspace_args( + MPI_RANK.mapping, modeling._TP_SUM_MNNVL_MAX_ROWS, entry.HIDDEN, torch.bfloat16 + ) + mnnvl = functools.partial(entry.mnnvl_tp_sum, **WORKSPACE) + + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + parser = argparse.ArgumentParser() + parser.add_argument("--world-size", type=int, default=None) + world_size = parser.parse_args().world_size or len(_launch.visible_devices()) + _launch.mpirun_ranks( + __file__, world_size, [_MPI_WORKER_FLAG, "--world-size", str(world_size)], DEADLINE_S + ) + print("OK: MPI session (mnnvl_fusion_allreduce, fusion NONE)", flush=True) + print("OK") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py new file mode 100644 index 000000000000..4a7dd6a69dbd --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's MoE all-to-all round trip (`moe_a2a_dispatch` and +`moe_a2a_combine` on the stock NVLink one-sided all-to-all) at dep2 and dep4 under mpirun, eagerly +and graph-replayed. Passes without checking where the platform has no MNNVL.""" + +import os +import sys +from pathlib import Path +from typing import Any, List, Tuple + +import torch + +assert torch.cuda.is_available(), "the MoE all-to-all requires CUDA devices" + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: The attention-DP group sizes this target serves (dep2, dep4). +CERTIFIED_WORLDS: Tuple[int, ...] = (2, 4) + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] +_TARGET = ( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." + "qwen3_8_flash_next_fp8__sm_103__unified" +) + +# Bound inside the rank body: importing the entry pulls in tensorrt_llm, which calls MPI_Init at +# import, and the launcher must stay free of MPI and CUDA state. +entry: Any = None +validating: Any = None +MPI_RANK: Any = None +STRATEGY: Any = None +RANK = 0 +WORLD = 1 + + +def _gen(*key: int) -> torch.Generator: + """Seeded alike on every rank for a key, so each rank can rebuild what any other rank holds.""" + seed = 0 + for k in key: + seed = seed * 1000003 + int(k) + return torch.Generator(device="cuda").manual_seed(seed % (2**62)) + + +def _inputs(rows: int, src: int, salt: int) -> tuple: + """Rank `src`'s rows and routing: bf16 rows, ten distinct experts a row, softmax weights.""" + g = _gen(salt, src) + x = torch.randn(rows, HIDDEN, generator=g, device="cuda").to(torch.bfloat16) + ids = torch.rand(rows, NUM_EXPERTS, generator=g, device="cuda").topk(TOPK, dim=1).indices + w = torch.softmax(torch.randn(rows, TOPK, generator=g, device="cuda"), dim=1) + return x, ids.to(torch.int32), w.to(torch.bfloat16) + + +def _window(rows, ids, w, rank: int) -> torch.Tensor: + """Rank `rank`'s stand-in expert window: each row scaled by its in-window experts' weights times + their ids, in fp32, rounded once; a row with no expert there (an empty slot too) is zero.""" + epr = NUM_EXPERTS // WORLD + keep = (ids >= rank * epr) & (ids < (rank + 1) * epr) + scale = (w.float() * keep * (ids.float() + 1) / NUM_EXPERTS).sum(-1, keepdim=True) + out = scale * rows.float() + return torch.where(keep.any(-1, keepdim=True), out, torch.zeros_like(out)).to(torch.bfloat16) + + +def _cells() -> list: + return [c for c in entry.moe_a2a_dispatch.CELLS if c.spec["world"] == WORLD] + + +def _check_received(got, everyone, counts, what: str) -> None: + """Received token slots equal the reference rows as a set: the kernel orders a block itself.""" + want = entry.moe_a2a_dispatch.reference(everyone, counts, NUM_EXPERTS // WORLD, rank=RANK) + by_key = {} + for x, ids, w in want: + for i in range(ids.shape[0]): + by_key[tuple(ids[i].tolist())] = (x[i], w[i]) + valid = (got.topk_ids != entry.NO_EXPERT).any(1) + assert got.rows.shape[0] == WORLD * max(counts), (what, got.rows.shape, counts) + assert int(valid.sum()) == len(by_key), (what, int(valid.sum()), len(by_key)) + for i in torch.nonzero(valid).flatten().tolist(): + key = tuple(got.topk_ids[i].tolist()) + assert key in by_key, f"{what}: slot {i} holds a row no source routed here" + x, w = by_key[key] + assert torch.equal(got.rows[i], x) and torch.equal(got.topk_weights[i], w), (what, i) + + +def _check_combined(out, mine, what: str) -> None: + x, ids, w = mine + want = entry.moe_a2a_combine.reference([_window(x, ids, w, r) for r in range(WORLD)]) + assert out.shape == want.shape, (what, out.shape, want.shape) + torch.testing.assert_close(out.float(), want.float(), rtol=2 * 2**-8, atol=1e-3, msg=what) + + +def _round_trip(x, ids, w, counts): + got = entry.moe_a2a_dispatch(STRATEGY, x, ids, w, counts) + partials = _window(got.rows, got.topk_ids, got.topk_weights, RANK) + return got, entry.moe_a2a_combine(STRATEGY, partials) + + +def check_cells() -> None: + """Every cell of this world size: dispatch as a set against its reference, then the round + trip through the stand-in window against the combine reference.""" + cells = _cells() + assert cells, f"no cell at world {WORLD}" + for i, cell in enumerate(cells): + counts = list(cell.spec["counts"]) + everyone = [_inputs(n, r, 10 + i) for r, n in enumerate(counts)] + with validating(entry.moe_a2a_dispatch): + got, out = _round_trip(*everyone[RANK], counts) + torch.cuda.synchronize() + _check_received(got, everyone, counts, cell.why) + _check_combined(out, everyone[RANK], cell.why) + MPI_RANK.barrier() + + +def check_graph() -> None: + """The round trip captured once at a uniform cell and replayed with fresh rows and routing.""" + cell = max( + (c for c in _cells() if len(set(c.spec["counts"])) == 1), key=lambda c: c.spec["counts"][0] + ) + counts = list(cell.spec["counts"]) + static = [t.clone() for t in _inputs(counts[RANK], RANK, 50)] + _round_trip(*static, counts) # eager first: the graph replays what the warm call set up + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got, out = _round_trip(*static, counts) + for salt in (51, 52): + everyone = [_inputs(n, r, salt) for r, n in enumerate(counts)] + for dst, src in zip(static, everyone[RANK]): + dst.copy_(src) + MPI_RANK.barrier() + graph.replay() + torch.cuda.synchronize() + _check_received(got, everyone, counts, f"replay {salt}") + _check_combined(out, everyone[RANK], f"replay {salt}") + MPI_RANK.barrier() + del graph + torch.cuda.synchronize() + MPI_RANK.barrier() + + +CHECKS = (check_cells, check_graph) + + +def _run_one_mpi_rank() -> int: + """One MPI rank's checks; a failing check aborts the MPI job, which ends its siblings.""" + global entry, validating, MPI_RANK, STRATEGY, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import importlib + + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm.mapping import Mapping + + entry = importlib.import_module(f"{_TARGET}.ops.comm.moe_a2a__dp_dispatch_combine") + validating = _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + mapping = Mapping( + world_size=WORLD, + rank=RANK, + gpus_per_node=WORLD, + tp_size=WORLD, + moe_tp_size=1, + moe_ep_size=WORLD, + enable_attention_dp=True, + ) + max_rows = max(max(c.spec["counts"]) for c in entry.moe_a2a_dispatch.CELLS) + STRATEGY = entry.build_strategy(mapping, max_rows) + if STRATEGY is None: + print(f"[rank {RANK}] no MNNVL on this platform: nothing to certify", flush=True) + MPI_RANK.MPI.Finalize() + os._exit(0) + _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + print(f"[rank {RANK}] {len(CHECKS)} checks passed over the MPI session", flush=True) + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + # Skip static destructors: the symmetric workspace's teardown adds nothing to the verdict. + os._exit(0) + + +def _world_size(argv: List[str]) -> int: + if "--world-size" in argv: + return int(argv[argv.index("--world-size") + 1]) + return len([d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()]) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + WORLD_SIZE = _world_size(sys.argv) + assert WORLD_SIZE in CERTIFIED_WORLDS, (WORLD_SIZE, CERTIFIED_WORLDS) + _launch.mpirun_ranks( + __file__, WORLD_SIZE, [_MPI_WORKER_FLAG, "--world-size", str(WORLD_SIZE)], DEADLINE_S + ) + print(f"OK: {WORLD_SIZE} ranks over the MPI session", flush=True) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py new file mode 100644 index 000000000000..e602819089f9 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Collected entry point for the FP8 target's collective matrices: each `__op_matrix.py` +beside this file is its own launcher, which `_rank_job` runs in a fresh interpreter on `world` +sm_103 GPUs of one node.""" + +from pathlib import Path + +import pytest +import torch + +__extra_import_path__ = ["../../../../comm"] +import _rank_job # noqa: E402 — needs the path declared above + +_HERE = Path(__file__).resolve().parent + +#: The group sizes the target runs each entry at (each launcher's `CERTIFIED_WORLDS`, restated +#: so that collecting this module on a machine without GPUs loads no launcher). +_WORLDS = { + "mnnvl_allreduce__tp_sum": (2, 4), + "ce_exchange__sp_pair": (2,), + "lamport_allgather__index_split": (2, 4), + "hc_ar_inject_norm": (2, 4), + "moe_a2a__dp_dispatch_combine": (2, 4), +} +_CASES = [(entry, world) for entry, worlds in _WORLDS.items() for world in worlds] + + +def _sm103_devices(world: int) -> int: + """How many of the devices a `world`-rank job would claim are sm_103.""" + if not torch.cuda.is_available(): + return 0 + claimed = range(min(torch.cuda.device_count(), world)) + return sum(torch.cuda.get_device_capability(i) == (10, 3) for i in claimed) + + +# Each case spawns its own ranks over the visible devices, so xdist workers would share GPUs. +@pytest.mark.no_xdist +@pytest.mark.parametrize("entry, world", _CASES, ids=[f"{e}-world{w}" for e, w in _CASES]) +def test_op_matrix(entry: str, world: int) -> None: + if _sm103_devices(world) < world: + pytest.skip(f"the {entry} matrix at world {world} runs on {world} sm_103 GPUs in one node") + _rank_job.run(entry, world_size=world, directory=_HERE) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py new file mode 100644 index 000000000000..5aedc026a4ca --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -0,0 +1,2060 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's attention entries other than `thop_attention`: every +certified cell against the entry's `reference` over the engine's paged state, plus the TP index +split, captured stores, kernels without cells and the call sites.""" + +import math +import types +import zlib +from types import SimpleNamespace + +import cuda.bindings.driver as cuda_drv +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( # noqa: E501 + qsa_topk_expand_decode as family_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + pool as P, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_index_scores_cute__prefill as _isc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_index_scores_groups_cute__decode as _igc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_sparse_gqa_rowtile__prefill as _module, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_sparse_gqa_splitk_cute__decode as mod, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_triton_kernels__stable_jit as _stable, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.draft_continue_rows__decode import ( # noqa: E501 + draft_continue_select, + draft_rows_gather, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_fp8_pages_to_bf16__prefill import ( # noqa: E501 + qsa_fp8_pages_to_bf16 as _f2b_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_index_split_plan__decode import ( # noqa: E501 + index_split_plan, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_index_store import ( # noqa: E501 + qsa_paged_index_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_kv_store import ( # noqa: E501 + qsa_paged_kv_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_sparse_gqa import ( # noqa: E501 + qsa_paged_sparse_gqa, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_grouped_cute__decode import ( # noqa: E501 + qsa_sparse_gqa_grouped_cute as _sgg_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_rowtile__prefill import ( # noqa: E501 + qsa_sparse_gqa_rowtile as _rtl_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_rowtile__prefill import ( # noqa: E501 + rows_per_group, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_splitk_cute__decode import ( # noqa: E501 + qsa_sparse_gqa_splitk_cute as _sgc_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_spec_step_metadata__decode import ( # noqa: E501 + qsa_spec_step_metadata, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_expand_cluster__decode import ( # noqa: E501 + qsa_topk_expand_cluster as _tkc_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_expand_cluster__decode import ( # noqa: E501 + supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_rows_cute__prefill import ( # noqa: E501 + qsa_topk_rows as _tkr_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_rows_cute__prefill import ( # noqa: E501 + supports as _tkr_supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qwen4_exp_attn_decode_prologue__decode import ( # noqa: E501 + qwen4_exp_attn_decode_prologue as _pro_op, +) +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_generation_token_mapping, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — needs the path declared above; fixture by name + assert_within_ulp, + certifying, +) + +assert torch.cuda.is_available(), "the attention entries require a CUDA device" + +DEV = "cuda" +#: The element offset a straddling cell's pages lie on both sides of: a 32-bit offset breaks. +_OFFSET_EDGE = 2**31 + + +def _seed(text: str) -> int: + """A seed that is the same in every process; `hash()` of a str is not.""" + return zlib.crc32(text.encode()) + + +def _bits(tensor: torch.Tensor) -> torch.Tensor: + as_int = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[tensor.element_size()] + return tensor.contiguous().view(as_int) + + +def _hole(spec, table, allocated_cols): + """One row that resolves to no slot: its (request, position).""" + assert spec["hole"] == "unallocated_page", spec["hole"] + assert int(table[0, allocated_cols[0]]) == -1 + return 0, allocated_cols[0] * spec["tokens_per_block"] + + +def _index_store_compare(out, ref) -> None: + """Bitwise over both whole planes, as raw bits: a store is a move with one conversion that + `reference` performs too, and comparing every slot is the stray-write check.""" + for name, got, want in zip(("index_cache", "position_cache"), out, ref): + assert got.dtype == want.dtype, (name, got.dtype, want.dtype) + assert got.shape == want.shape, (name, got.shape, want.shape) + torch.testing.assert_close( + _bits(got), _bits(want), rtol=0, atol=0, msg=lambda m, n=name: f"{n}: {m}" + ) + + +KEY_FILL = -7.0 +POS_FILL = -7 +ENGINE_MAX_NUM_TOKENS = 16384 + + +def _key_fill_bits() -> int: + return int(_bits(torch.tensor([KEY_FILL], dtype=torch.bfloat16))[0]) + + +def _shared_page_planes(spec, extras: dict): + """The NVFP4-KV pool's side planes: a page holds the `layers` index-K pages then the position + page, so both caches are strided views of one allocation.""" + layers, tpb, hkv, head_dim, axes, pages = ( + spec[k] for k in ("layers", "tokens_per_block", "hkv", "head_dim", "axes", "pages") + ) + index_bytes = tpb * hkv * head_dim * 2 + page = torch.empty(pages, layers * index_bytes + tpb * axes * 4, dtype=torch.uint8, device=DEV) + page[:, : layers * index_bytes].view(torch.bfloat16).fill_(KEY_FILL) + page[:, layers * index_bytes :].view(torch.int32).fill_(POS_FILL) + index_cache = page[:, (layers - 1) * index_bytes : layers * index_bytes].view(torch.bfloat16) + index_cache = index_cache.view(pages, tpb, hkv, head_dim) + position_cache = page[:, layers * index_bytes :].view(torch.int32).view(pages, tpb, axes) + assert index_cache.stride() == (spec["page_stride"], hkv * head_dim, head_dim, 1) + assert position_cache.stride() == spec["position_strides"] + extras["roles"] = page[:, : (layers - 1) * index_bytes] + return index_cache, position_cache + + +def _coalesced_planes(spec, extras: dict): + """The default target's bf16 index pool: the sparse layers' INDEX_KEY buffers coalesced into + one allocation, the cell's plane a strided view; positions in their own pool.""" + layers, layer, tpb, hkv, head_dim, axes, pages = ( + spec[k] for k in ("layers", "layer", "tokens_per_block", "hkv", "head_dim", "axes", "pages") + ) + pool = torch.full( + (pages, layers, tpb, hkv, head_dim), KEY_FILL, dtype=torch.bfloat16, device=DEV + ) + index_cache = pool[:, layer] + assert index_cache.stride() == (spec["page_stride"], hkv * head_dim, head_dim, 1) + position_cache = torch.full((pages, tpb, axes), POS_FILL, dtype=torch.int32, device=DEV) + extras["coalesced"] = (pool, layer) + return index_cache, position_cache + + +def _other_layers_untouched(pool: torch.Tensor, layer: int) -> None: + fill, bits = _key_fill_bits(), pool.view(torch.int16) + for start in range(0, pool.shape[0], 16384): + band = bits[start : start + 16384] + for other in range(pool.shape[1]): + if other != layer: + assert bool((band[:, other] == fill).all()), ( + f"layer {other}'s index plane moved in pages {start}..{start + band.shape[0]}" + ) + + +def _index_store_build(spec, seed: int, planes=None): + """The op's kwargs and extras for one cell, storing into `planes` when given.""" + gen = torch.Generator(device="cpu").manual_seed(seed) + tpb, hkv, head_dim, axes = (spec[k] for k in ("tokens_per_block", "hkv", "head_dim", "axes")) + slots, width, pages = spec["slots"], spec["width"], spec["pages"] + segments = spec["segments"] + requests = [r for r, (_, n) in enumerate(segments) for _ in range(n)] + positions = [p for start, n in segments for p in range(start, start + n)] + rows = len(positions) + + table = torch.randint(0, pages, (slots, width), generator=gen, dtype=torch.int32) + order = torch.randperm(pages, generator=gen).to(torch.int32) + used, allocated_cols = 0, [] + for r, (start, n) in enumerate(segments): + cols = (start + n - 1) // tpb + 1 + table[r] = -1 + table[r, :cols] = order[used : used + cols] + used += cols + allocated_cols.append(cols) + assert used <= pages and len(segments) <= slots, "pool or table too small for the batch" + if spec.get("straddle"): + # The last row stores into the pool's last page: rows land on both sides of 2^31. + last_start, last_rows = segments[-1] + column = (last_start + last_rows - 1) // tpb + live = table[: len(segments)] + holder = (live == pages - 1).nonzero() + if len(holder): + live[holder[0, 0], holder[0, 1]] = live[len(segments) - 1, column] + live[len(segments) - 1, column] = pages - 1 + if spec["hole"] is not None: + request, position = _hole(spec, table, allocated_cols) + requests.append(request) + positions.append(position) + rows += 1 + table = table.to(DEV) + + projection = torch.randn( + rows, spec["index_heads"] + hkv, head_dim, generator=gen, dtype=torch.float32 + ).to(device=DEV, dtype=spec["key_dtype"]) + token_k = projection[:, spec["index_heads"] :] + if spec["key_layout"] == "fused_row": + # The fused projection row in heads of 128; the raw index key is the second to last. + heads = spec.get("row_width", 14080) // head_dim + row = torch.randn(rows, heads, head_dim, generator=gen, dtype=torch.float32) + token_k = row.to(device=DEV, dtype=spec["key_dtype"])[:, heads - 2 : heads - 2 + hkv] + assert token_k.stride(0) == heads * head_dim + + request_indices = torch.tensor(requests, dtype=spec["request_dtype"], device=DEV) + logical_positions = torch.tensor(positions, dtype=spec["position_dtype"], device=DEV) + # Three distinct axes (an image span) in the engine's `[3, max_num_tokens]` buffer. + coords = logical_positions.to(torch.int32)[:, None] + 7 * torch.arange( + 3, dtype=torch.int32, device=DEV + ) + buffer = torch.full((3, ENGINE_MAX_NUM_TOKENS), -1, dtype=torch.int32, device=DEV) + buffer[:, :rows] = coords.t() + coords = buffer[:, :rows].t() + + extras = {} + position_cache = None + if planes is not None: + index_cache, position_cache = planes + elif spec["cache_layout"] == "shared_page": + index_cache, position_cache = _shared_page_planes(spec, extras) + elif spec["cache_layout"] == "coalesced": + index_cache, position_cache = _coalesced_planes(spec, extras) + else: + index_cache = torch.full((pages, tpb, hkv, head_dim), KEY_FILL, device=DEV) + index_cache = index_cache.to(torch.bfloat16) + if position_cache is None: + position_cache = torch.full((pages, tpb, axes), POS_FILL, dtype=torch.int32, device=DEV) + args = dict( + token_k=token_k, position_coordinates=coords, index_cache=index_cache, + position_cache=position_cache, request_indices=request_indices, + logical_positions=logical_positions, block_table=table, tokens_per_block=tpb, + ) # fmt: skip + return args, extras + + +def _check_index_store(cell, monkeypatch) -> None: + args, extras = _index_store_build(cell.spec, seed=_seed(cell.why)) + if cell.spec.get("straddle"): + assert (cell.spec["pages"] - 1) * args["index_cache"].stride(0) >= _OFFSET_EDGE + ref = qsa_paged_index_store.reference(**args) + out = qsa_paged_index_store(**args) + assert out[0] is args["index_cache"] and out[1] is args["position_cache"] + _index_store_compare(out, ref) + if "roles" in extras: + assert bool((extras["roles"].view(torch.int16) == _key_fill_bits()).all()), ( + "another index-K role of the shared page moved" + ) + if "coalesced" in extras: + _other_layers_untouched(*extras["coalesced"]) + + +def _kv_store_compare(out, ref) -> None: + """Bitwise over both role planes, which together are the whole pool: the stray-write check.""" + for name, got, want in zip(("k_cache", "v_cache"), out, ref): + assert got.dtype == want.dtype, (name, got.dtype, want.dtype) + assert got.shape == want.shape, (name, got.shape, want.shape) + torch.testing.assert_close( + _bits(got), _bits(want), rtol=0, atol=0, msg=lambda m, n=name: f"{n}: {m}" + ) + + +FILL = -2.0 + + +def _check(result): + err, *value = result + assert err == cuda_drv.CUresult.CUDA_SUCCESS, err + return value[0] if len(value) == 1 else value + + +class _ArrayInterface: + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = dict( + shape=(nbytes,), strides=None, typestr="|u1", data=(ptr, False), version=2 + ) + + +class _KvEnginePool: + """Layer `layer_slot`'s K and V views of the stock manager's QSA pool at `pages` pages. The + range is reserved whole; memory is mapped only under the K and V planes of `touched` pages + and page 0 (the tensor's base pointer).""" + + def __init__(self, spec, touched): + dtype = spec["cache_dtype"] + elem = torch.empty((), dtype=dtype).element_size() + hkv, tpb, dim = spec["hkv"], spec["tokens_per_block"], spec["head_dim"] + self.pages, self.touched = spec["pages"], sorted(set(touched)) + self.role = hkv * tpb * dim + self.stride = spec["role_pages"] * self.role + self.own = spec["layer_slot"] * 2 * self.role + + def planes(page): + """This layer's K and V planes of `page`, in bytes of the range.""" + lo = (page * self.stride + self.own) * elem + return lo, lo + 2 * self.role * elem + + self.written = [planes(p) for p in self.touched] + dev = torch.cuda.current_device() + prop = cuda_drv.CUmemAllocationProp() + prop.type = cuda_drv.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = cuda_drv.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + prop.location.id = dev + gran = _check( + cuda_drv.cuMemGetAllocationGranularity( + prop, + cuda_drv.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED, + ) + ) + self.size = -(-self.pages * self.stride * elem // gran) * gran + self.va = int(_check(cuda_drv.cuMemAddressReserve(self.size, gran, 0, 0))) + merged = [] + for lo, hi in sorted({planes(0), *self.written}): + lo, hi = lo // gran * gran, min(-(-hi // gran) * gran, self.size) + if merged and lo <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], hi) + else: + merged.append([lo, hi]) + access = cuda_drv.CUmemAccessDesc() + access.location.type = cuda_drv.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + access.location.id = dev + access.flags = cuda_drv.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + self.maps, self.spans = [], merged + for lo, hi in merged: + handle = _check(cuda_drv.cuMemCreate(hi - lo, prop, 0)) + ptr = cuda_drv.CUdeviceptr(self.va + lo) + _check(cuda_drv.cuMemMap(ptr, hi - lo, 0, handle, 0)) + self.maps.append((ptr, hi - lo, handle)) + _check(cuda_drv.cuMemSetAccess(ptr, hi - lo, [access], 1)) + + self.raw = torch.as_tensor(_ArrayInterface(self.va, self.size), device=DEV) + pool = self.raw.view(dtype).as_strided( + (self.pages, 2, hkv, tpb, dim), + (self.stride, self.role, tpb * dim, dim, 1), + self.own, + ) + self.k, self.v = pool[:, 0], pool[:, 1] + self.index = torch.tensor(self.touched, dtype=torch.long, device=DEV) + self.fill() + + def fill(self) -> None: + pattern = _bits(torch.full((2,), FILL, dtype=torch.bfloat16).to(self.k.dtype)) + for lo, hi in self.spans: + self.raw[lo:hi].view(pattern.dtype).fill_(int(pattern[0])) + torch.cuda.synchronize() + self.before = [self.raw[lo:hi].clone() for lo, hi in self.spans] + + def dense(self, table: torch.Tensor): + """`(k, v, table)`: copies of the touched pages and `table` renumbered into them.""" + remap = torch.full((self.pages,), -1, dtype=torch.int32, device=DEV) + remap[self.index] = torch.arange(len(self.touched), dtype=torch.int32, device=DEV) + renamed = torch.where(table >= 0, remap[table.clamp_min(0).long()], -1).to(torch.int32) + return self.k[self.index].clone(), self.v[self.index].clone(), renamed.contiguous() + + def gathered(self): + return self.k[self.index], self.v[self.index] + + def assert_rest_unchanged(self) -> None: + torch.cuda.synchronize() + for (lo, hi), before in zip(self.spans, self.before): + keep = torch.ones(hi - lo, dtype=torch.bool, device=DEV) + for p_lo, p_hi in self.written: + if p_hi > lo and p_lo < hi: + keep[max(p_lo, lo) - lo : min(p_hi, hi) - lo] = False + assert torch.equal(self.raw[lo:hi][keep], before[keep]), ( + f"bytes [{lo}, {hi}) of the pool changed outside the touched pages' planes" + ) + + def close(self) -> None: + torch.cuda.synchronize() + del self.k, self.v, self.raw + for ptr, size, handle in self.maps: + _check(cuda_drv.cuMemUnmap(ptr, size)) + _check(cuda_drv.cuMemRelease(handle)) + _check(cuda_drv.cuMemAddressFree(cuda_drv.CUdeviceptr(self.va), self.size)) + + +def _kv_store_build(spec, seed: int): + """`(op kwargs, pool)`: a contiguous pool, or a `_KvEnginePool` for a `pool="engine"` cell.""" + gen = torch.Generator(device="cpu").manual_seed(seed) + tpb, hkv, head_dim = spec["tokens_per_block"], spec["hkv"], spec["head_dim"] + slots, width, pages = spec["slots"], spec["width"], spec["pages"] + engine = spec.get("pool") == "engine" + segments = spec["segments"] + requests = [r for r, (_, n) in enumerate(segments) for _ in range(n)] + positions = [p for start, n in segments for p in range(start, start + n)] + + table = torch.randint(0, pages, (slots, width), generator=gen, dtype=torch.int32) + order = torch.randperm(pages, generator=gen).to(torch.int32) + if engine: + # The first row's page is the pool's last one, whose planes are its highest bytes. + first = segments[0][0] // tpb + last = int((order == pages - 1).nonzero()[0]) + order[[first, last]] = order[[last, first]] + used, allocated_cols = 0, [] + for r, (start, n) in enumerate(segments): + cols = (start + n - 1) // tpb + 1 + assert cols <= width, f"request {r} ends past the table's {width} columns" + table[r] = -1 + table[r, :cols] = order[used : used + cols] + used += cols + allocated_cols.append(cols) + assert used <= pages and len(segments) <= slots, "pool or table too small for the batch" + if spec["hole"] is not None: + request, position = _hole(spec, table, allocated_cols) + requests.append(request) + positions.append(position) + + rows = len(positions) + assert rows == spec.get("rows", rows), (rows, spec["rows"]) + heads = spec["heads_q"] + 2 * hkv + spec.get("row_pad_heads", 0) + qkv = torch.randn(rows, heads, head_dim, generator=gen, dtype=torch.float32) + qkv = qkv.to(device=DEV, dtype=spec["row_dtype"]) + assert qkv.stride(0) == spec.get("row_stride", qkv.stride(0)), qkv.stride(0) + k = qkv[:, spec["heads_q"] : spec["heads_q"] + hkv] + v = qkv[:, spec["heads_q"] + hkv : spec["heads_q"] + 2 * hkv] + touched = [int(table[r, p // tpb]) for r, p in zip(requests, positions)] + table = table.to(DEV) + + if engine: + pool = _KvEnginePool(spec, [t for t in touched if t >= 0]) + k_cache, v_cache = pool.k, pool.v + else: + pool = torch.full( + (pages, 2, hkv, tpb, head_dim), FILL, dtype=torch.bfloat16, device=DEV + ).to(spec["cache_dtype"]) + k_cache, v_cache = pool[:, 0], pool[:, 1] + args = dict( + k=k, v=v, k_cache=k_cache, v_cache=v_cache, + request_indices=torch.tensor(requests, dtype=spec["request_dtype"], device=DEV), + logical_positions=torch.tensor(positions, dtype=spec["position_dtype"], device=DEV), + block_table=table, tokens_per_block=tpb, + ) # fmt: skip + return args, pool + + +def _kv_engine_reference(args, pool: _KvEnginePool): + """The entry's reference over a dense copy of the touched pages.""" + dense_k, dense_v, renamed = pool.dense(args["block_table"]) + return qsa_paged_kv_store.reference( + **{**args, "k_cache": dense_k, "v_cache": dense_v, "block_table": renamed} + ) + + +def _check_kv_store(cell, monkeypatch) -> None: + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + if not isinstance(pool, _KvEnginePool): + before = pool.clone() + ref = qsa_paged_kv_store.reference(**args) + out = qsa_paged_kv_store(**args) + assert out[0] is args["k_cache"] and out[1] is args["v_cache"] + _kv_store_compare(out, ref) + wrote = not torch.equal(_bits(pool), _bits(before)) + assert wrote == bool(cell.spec["segments"]), "a cell with rows writes, an empty one not" + return + try: + top = max(pool.touched) + assert top == pool.pages - 1, (top, pool.pages) + assert top * pool.k.stride(0) >= 2**32, top * pool.k.stride(0) + before = tuple(t.clone() for t in pool.gathered()) + ref = _kv_engine_reference(args, pool) + out = qsa_paged_kv_store(**args) + assert out[0] is args["k_cache"] and out[1] is args["v_cache"] + got = pool.gathered() + _kv_store_compare(got, ref) + pool.assert_rest_unchanged() + assert any(not torch.equal(_bits(g), _bits(b)) for g, b in zip(got, before)), "no write" + finally: + pool.close() + + +def _gqa_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """4 ulp of each (row, head)'s magnitude and 2 ulp of relative RMS in q's dtype, the band of + every sparse GQA entry here. The kernels round softmax probabilities to the cache dtype + before P.V, so the error tracks the averaged values rather than the output, and a gated + output is rounded twice; a page-shifted table or a dropped scale or token falls outside it.""" + assert out.dtype == ref.dtype, (out.dtype, ref.dtype) + assert out.shape == ref.shape, (out.shape, ref.shape) + if out.numel() == 0: + return + assert_within_ulp(out, ref, element_ulp=4.0, rms_ulp=2.0) + + +#: NaN's bit pattern per pool dtype, filled through an integer view (not every fill takes e4m3). +_NAN_BITS = {torch.bfloat16: (torch.int16, 0x7FC0), torch.float8_e4m3fn: (torch.uint8, 0x7F)} + + +def _selected_tokens(positions, seed, compress_ratio, token_topk, shared=False, chunk=None): + """QSA's expansion layout: a random `min(available, block_topk)`-subset of the complete + visible groups expanded to their tokens, the incomplete trailing group, then `-1`, front-packed. + `shared`: every row ranks the groups by one score vector, as real selections overlap.""" + block_topk = token_topk // compress_ratio + final_topk = token_topk + compress_ratio - 1 + gen = torch.Generator(device=DEV).manual_seed(seed) + rows = positions.numel() + visible = positions.long() + 1 + available = visible // compress_ratio + taken = available.clamp(max=block_topk) + width = max(block_topk, int(available.max().item()) if rows else 0) + out = torch.empty(rows, final_topk, dtype=torch.int32, device=DEV) + column = torch.arange(final_topk, device=DEV)[None, :] + step = chunk or max(rows, 1) + for start in range(0, rows, step): + stop = min(start + step, rows) + avail, take, vis = available[start:stop], taken[start:stop], visible[start:stop] + if shared: + scores = torch.rand(1, width, generator=gen, device=DEV).expand(stop - start, -1) + gen.manual_seed(seed) + else: + scores = torch.rand(stop - start, width, generator=gen, device=DEV) + group_ids = torch.arange(width, device=DEV)[None, :] + scores = torch.where(group_ids < avail[:, None], scores, torch.inf) + chosen = scores.argsort(dim=1)[:, :block_topk] + group_of_column = (column // compress_ratio).clamp(max=block_topk - 1) + body = torch.gather(chosen, 1, group_of_column.expand(stop - start, -1)) + body = body * compress_ratio + column % compress_ratio + in_body = column < (take * compress_ratio)[:, None] + tail_column = column - (take * compress_ratio)[:, None] + tail = (avail * compress_ratio)[:, None] + tail_column + in_tail = ~in_body & (tail_column >= 0) & (tail < vis[:, None]) + out[start:stop] = torch.where(in_body, body, torch.where(in_tail, tail, -1)).to(torch.int32) + return out + + +def _straddling_pages(spec, needed: int, edge: int = _OFFSET_EDGE) -> int: + """Pool size whose last `needed` pages lie on both sides of `edge` elements of the K plane.""" + role_page = spec["kv_heads"] * spec["tokens_per_block"] * spec["head_dim"] + edge_page = math.ceil((edge - 2 * spec["layer"] * role_page) / (spec["slot_pages"] * role_page)) + return max(edge_page + needed // 2, needed) + + +def _paged_kv_state(spec, seed, dtype, pool_pages_of, slot_pages=2, layer=0, chunk=None, + scratch=False) -> SimpleNamespace: # fmt: skip + """The engine's paged K/V: a `[slots, pages]` table over seeded-permuted slots, `-1` past each + sequence; K and V layer `layer`'s role planes of a NaN `[pool_pages, slot_pages, kv, tpb, D]` + pool, or with `scratch` two contiguous pools (what `qsa_fp8_pages_to_bf16` writes).""" + gen = torch.Generator(device=DEV).manual_seed(seed) + tpb, kv, d = spec["tokens_per_block"], spec["kv_heads"], spec["head_dim"] + segments = list(spec["segments"]) + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in segments]) + req = torch.cat([torch.full((n,), i, device=DEV) for i, (_, n) in enumerate(segments)]) + slots = spec["slots"] + assert len(segments) <= slots + slot_of = torch.randperm(slots, generator=torch.Generator().manual_seed(seed))[: len(segments)] + mapped = [math.ceil((s + n) / tpb) for s, n in segments] + assert max(mapped) <= spec["pages"] + needed = sum(mapped) + 7 + pool_pages = pool_pages_of(needed) + assert pool_pages >= needed + first = pool_pages - needed + page_ids = first + torch.randperm(needed, generator=gen, device=DEV).to(torch.int32) + table = torch.full((slots, spec["pages"]), -1, dtype=torch.int32, device=DEV) + cursor = 0 + for slot, count in zip(slot_of.tolist(), mapped): + table[slot, :count] = page_ids[cursor : cursor + count] + cursor += count + if scratch: + k, v = (torch.randn((pool_pages, kv, tpb, d), generator=gen, device=DEV).to(dtype) + for _ in range(2)) # fmt: skip + else: + bits, nan = _NAN_BITS[dtype] + pool = torch.full((pool_pages, slot_pages, kv, tpb, d), nan, dtype=bits, device=DEV) + pool = pool.view(dtype) + for p0 in range(first, pool_pages, chunk or needed): + p1 = min(p0 + (chunk or needed), pool_pages) + fresh = torch.randn(p1 - p0, 2, kv, tpb, d, generator=gen, device=DEV) + pool[p0:p1, 2 * layer : 2 * layer + 2] = fresh.to(dtype) + k, v = pool[:, 2 * layer], pool[:, 2 * layer + 1] + rows = slot_of.to(DEV)[req] + return SimpleNamespace(gen=gen, positions=positions, rows=rows, table=table, k=k, v=v) + + +def _assert_straddles(args, edge: int = _OFFSET_EDGE) -> None: + """The mapped pages put both planes' element offsets on both sides of `edge`.""" + table = args["block_table"] + pages = table[(table >= 0) & (table < args["k_cache"].shape[0])].long() + for name in ("k_cache", "v_cache"): + plane = args[name] + ends = plane.storage_offset() + pages * plane.stride(0) + plane[0].numel() - 1 + assert bool((ends >= edge).any()), f"{name}: no mapped page lies past {edge}" + assert bool((ends < edge).any()), f"{name}: every mapped page lies past {edge}" + + +def _assert_prefill_selection(selected, positions, spec) -> None: + """The precondition the prefill kernels cannot check: front-packed, and exactly as many + valid entries as the per-row scan bound the positions give.""" + valid = selected >= 0 + columns = torch.arange(selected.shape[1], device=selected.device)[None, :] + assert bool((valid == (columns < valid.sum(1)[:, None])).all()) + cr, visible = spec["compress_ratio"], positions.long() + 1 + bound = (visible // cr).clamp(max=spec["token_topk"] // cr) * cr + visible % cr + assert torch.equal(valid.sum(1), bound) + + +def _refill(args, seed, selection) -> None: + """The engine's next step in the call's own buffers: new q, gate, selection and mapped K/V.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + for name in ("q", "output_gate"): + if args.get(name) is not None: + view = args[name] + view.copy_(torch.randn(view.shape, generator=gen, device=DEV).to(view.dtype)) + args["selected_tokens"].copy_(selection(args)) + pages = args["block_table"][args["block_table"] >= 0].long() + for name in ("k_cache", "v_cache"): + plane = args[name] + bits = _NAN_BITS[plane.dtype][0] + fresh = torch.randn((pages.numel(), *plane.shape[1:]), generator=gen, device=DEV) + plane.view(bits)[pages] = fresh.to(plane.dtype).view(bits) + + +def _captured(fn, refill): + """`fn()` captured in a CUDA graph, `refill()` run, the graph replayed: the captured output.""" + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fn() + refill() + graph.replay() + torch.cuda.synchronize() + return out + + +def _replay_in_a_graph(entry, args, refill, first) -> None: + """Captured once, inputs refilled in place, replayed: within the band of the new inputs' + reference and bitwise an eager call on them.""" + replayed = _captured(lambda: entry(**args), refill).clone() + assert not torch.equal(replayed, first), "the replay did not read the refilled inputs" + _gqa_compare(replayed, entry.reference(**args)) + assert torch.equal(replayed, entry(**args)), "the replay differs from an eager call" + + +class _LaunchRecorder: + """Records a kernel's grid and PDL flag, then delegates to the kernel.""" + + def __init__(self, inner, name, log): + self._inner, self._name, self._log = inner, name, log + + def __getitem__(self, grid): + entry = [self._name, tuple(grid), False] + self._log.append(entry) + launch = self._inner[grid] + + def run(*args, **kwargs): + entry[2] = bool(kwargs.get("launch_pdl", False)) + return launch(*args, **kwargs) + + return run + + def __call__(self, selected_tokens, *args, **kwargs): + # the row-tile token plan: one CTA a group (its 10th argument); the block plan: its 9th + grid = (args[8],) if self._name == "_group_plan" else (args[7],) + self._log.append([self._name, grid, False]) + return self._inner(selected_tokens, *args, **kwargs) + + def warmup(self, *args, **kwargs): + """A compile-only call (`jit_variants.compile_only`): no launch to record.""" + return self._inner.warmup(*args, **kwargs) + + +def _record_launches(monkeypatch, module, names) -> list: + """Which of `module`'s kernels the next calls launch, in order: [name, grid, PDL].""" + log: list = [] + for name in names: + monkeypatch.setattr(module, name, _LaunchRecorder(getattr(module, name), name, log)) + return log + + +_KERNELS = { + "_qsa_prefill_pair_stable_kernel": "prefill", + "_qsa_splitk_stable_kernel": "splitk", + "_qsa_merge_stable_kernel": "merge", +} + + +def _row_gate(q, spec): + """The output gate's columns of the fused row `q` is read off: after q, k and v.""" + rows, d = q.shape[0], spec["head_dim"] + row = q.as_strided((rows, q.stride(0)), (q.stride(0), 1)) + start = (spec["q_heads"] + 2 * spec["kv_heads"]) * d + return row[:, start : start + spec["q_heads"] * d] + + +def _sparse_gqa_selection(spec, seed): + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in spec["segments"]]) + return _selected_tokens(positions, seed, spec["compress_ratio"], spec["token_topk"]) + + +def _sparse_gqa_build(spec, seed): + """One cell's args: q off the packed or the fused projection row (`row_heads` heads).""" + st = _paged_kv_state( + spec, seed, spec["cache_dtype"], lambda needed: spec.get("pool_pages", needed), + slot_pages=spec.get("slot_pages", 2), layer=spec.get("layer", 0), + ) # fmt: skip + rows, d, q_heads = st.positions.numel(), spec["head_dim"], spec["q_heads"] + fused = spec["q_layout"] == "fused_row" + row_heads = spec.get("row_heads", 55 if fused else q_heads + 2 * spec["kv_heads"]) + row = torch.randn(rows, row_heads * d, generator=st.gen, device=DEV).to(spec["q_dtype"]) + q = row.view(rows, row_heads, d)[:, :q_heads] + args = dict( + q=q, k_cache=st.k, v_cache=st.v, block_table=st.table, + selected_tokens=_sparse_gqa_selection(spec, seed + 3), + request_indices=st.rows.to(spec["request_dtype"]), + tokens_per_block=spec["tokens_per_block"], softmax_scale=d**-0.5, + ) # fmt: skip + if spec["prefill_pair"]: + args.update(query_positions=st.positions, compress_ratio=spec["compress_ratio"]) + if spec.get("gate") == "fused_row": + assert fused, "the gate is read off the fused row" + if spec.get("gate_fused", True): + args.update(output_gate=_row_gate(q, spec)) + return args + + +def _check_sparse_gqa(cell, monkeypatch) -> None: + """The cell's branch (its kernels, grid and PDL, recorded), then the band; a launch with no + merge refuses the gate (the call site applies it after); a `capture` cell is replayed.""" + launches = _record_launches(monkeypatch, _stable, _KERNELS) + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sparse_gqa_build(spec, seed) + q, rows = args["q"], args["q"].shape[0] + assert rows == spec.get("rows", rows) + if "row_heads" in spec: + assert q.stride(0) == spec["row_heads"] * spec["head_dim"] + if "output_gate" in args: + gate = args["output_gate"] + assert gate.stride(0) == q.stride(0) + assert gate.untyped_storage().data_ptr() == q.untyped_storage().data_ptr() + if "rows_per_request" in spec: + assert {n for _, n in spec["segments"]} == {spec["rows_per_request"]} + if "offset_edge" in spec: + _assert_straddles(args, spec["offset_edge"]) + if spec["prefill_pair"]: + _assert_prefill_selection(args["selected_tokens"], args["query_positions"], spec) + out = qsa_paged_sparse_gqa(**args) + assert [_KERNELS[name] for name, _, _ in launches] == list(spec["launches"]) + assert launches[0][1][:2] == (rows, spec["kv_heads"]) + if "splits" in spec: + assert launches[0][1][2] == spec["splits"], launches + assert all(pdl == spec.get("pdl", False) for _, _, pdl in launches), launches + if "output_gate" in args: + gated = (q, args["k_cache"], args["selected_tokens"], args["output_gate"]) + assert qsa_paged_sparse_gqa.fuses_output_gate(*gated) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == q.dtype and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, qsa_paged_sparse_gqa.reference(**args)) + if spec.get("gate") == "fused_row" and not spec.get("gate_fused", True): + gate = _row_gate(q, spec) + assert not qsa_paged_sparse_gqa.fuses_output_gate( + q, args["k_cache"], args["selected_tokens"], gate + ) + with pytest.raises(ValueError, match="output-gate fusion requires"): + qsa_paged_sparse_gqa(**args, output_gate=gate) + if spec.get("capture"): + eager = [tuple(entry) for entry in launches] + del launches[:] + + def refill(): + assert [tuple(entry) for entry in launches] == eager, "the capture's launches" + _refill(args, seed + 1, lambda a: _sparse_gqa_selection(spec, seed + 4)) + + _replay_in_a_graph(qsa_paged_sparse_gqa, args, refill, out) + + +PRO_D, PRO_IH, PRO_ID, PRO_TPB, PRO_CR, PRO_ROT, PRO_WIDTH, PRO_SLOTS = ( + 256, 4, 128, 32, 4, 64, 8196, 8 +) # fmt: skip +#: (q heads, KV heads) per rank; fused row `[q | k | v | gate | index q 4 x 128, k | v slot]`. +PRO_GEOMS = {"tp4": (6, 1), "tp2": (12, 1), "dep4": (24, 2)} +PRO_ROLE_PAGES, PRO_LAYER_SLOT, PRO_INDEX_LAYERS = 24, 11, 12 +#: One bf16 step in 2^-8 units of the row scale; the relative RMS bound is empirical. +PRO_ELEMENT_ULP, PRO_RMS_ULP = 2.0, 0.05 + + +def _pro_geometry(geom): + hq, hk = PRO_GEOMS[geom] + qkv = (hq + 2 * hk) * PRO_D + iq_col = qkv + hq * PRO_D + return hq, hk, qkv, iq_col, iq_col + (PRO_IH + 2) * PRO_ID + + +def _pro_edge_rows(proj, hq, hk) -> None: + """Zero, -0.0 and outlier blocks in the k head past the rotary prefix and in the v head.""" + k0, v0 = hq * PRO_D, (hq + hk) * PRO_D + edges = ( + (0, slice(k0 + 128, k0 + 144), 0.0), + (1, slice(k0 + 160, k0 + 176), -0.0), + (2, slice(k0 + 200, k0 + 201), 900.0), + (0, slice(v0 + 16, v0 + 32), 0.0), + (1, slice(v0 + 48, v0 + 64), torch.tensor([-0.0] * 8 + [0.0] * 8)), + (3, slice(v0 + 40, v0 + 41), -900.0), + (2, slice(v0 + 64, v0 + 80), 1e-30), + ) + for r, cols, value in edges: + if r < proj.shape[0]: + proj[r, cols] = value + + +def _pro_build(spec, seed): + g = torch.Generator(device=DEV).manual_seed(seed) + hq, hk, qkv_width, iq_col, row_width = _pro_geometry(spec["geom"]) + per = spec.get("rows_per_request", 1) + layout = [(r, p + i) for r, p in enumerate(spec["pos"]) for i in range(per)] + pos = [p for _, p in layout] + rows = len(pos) + last = {} + for req, p in layout: + last[req] = max(last.get(req, 0), p) + slots = max(PRO_SLOTS, len(last) + 3) + engine = spec.get("engine", False) + if engine: + # Enough pages that the last ones start past 2^31 elements; rows go on those. + page_elems = PRO_ROLE_PAGES * hk * PRO_TPB * PRO_D + pages = _OFFSET_EDGE // page_elems + 2 + sum(p // PRO_TPB + 1 for p in last.values()) + perm = torch.arange(pages - 1, -1, -1, device=DEV, dtype=torch.int32) + else: + pages = sum(p // PRO_TPB + 1 for p in last.values()) + 64 + perm = torch.randperm(pages, generator=g, device=DEV).to(torch.int32) + block_table = torch.full((slots, PRO_WIDTH), -1, dtype=torch.int32, device=DEV) + used = 0 + for req, p in last.items(): + n = p // PRO_TPB + 1 + block_table[req, :n] = perm[used : used + n] + used += n + request = torch.tensor([req for req, _ in layout], dtype=torch.int32, device=DEV) + if spec.get("unmapped") is not None: + request[spec["unmapped"]] = slots + 5 + logical = torch.tensor(pos, dtype=torch.int64, device=DEV) + ids = torch.stack([logical, logical + 3, logical + 7]).to(torch.int32).contiguous() + buf = torch.empty(3, 1, rows + 5, dtype=torch.int32, device=DEV) + buf[:, 0, :rows] = ids + coords = torch.transpose(buf[:, 0, :rows], 0, 1) + table_rows = max(pos) + 8 + angles = torch.rand(table_rows, PRO_ROT // 2, generator=g, device=DEV) * 6.28 + cos_sin = torch.cat([torch.cos(angles), torch.sin(angles)], dim=1).contiguous() + proj = (torch.randn(rows, row_width, generator=g, device=DEV) * 2.0).to(torch.bfloat16) + if spec.get("edge"): + _pro_edge_rows(proj, hq, hk) + out = torch.randn(rows, qkv_width + 64, generator=g, device=DEV).to(torch.bfloat16) + index_scratch = torch.randn(rows, PRO_IH + 2, PRO_ID, generator=g, device=DEV) + index_scratch = index_scratch.to(torch.bfloat16) + weights = [ + (0.3 * torch.randn(n, generator=g, device=DEV)).to(torch.bfloat16) + for n in (PRO_D, PRO_D, PRO_ID, PRO_ID) + ] + kv_dtype = torch.float8_e4m3fn if spec.get("fp8") else torch.bfloat16 + if engine: + roles = torch.zeros(pages, PRO_ROLE_PAGES, hk, PRO_TPB, PRO_D, dtype=kv_dtype, device=DEV) + pool = roles[:, 2 * PRO_LAYER_SLOT : 2 * PRO_LAYER_SLOT + 2] + planes = torch.randn(pages, PRO_INDEX_LAYERS, PRO_TPB, 1, PRO_ID, generator=g, device=DEV) + index_cache = planes.to(torch.bfloat16)[:, PRO_LAYER_SLOT] + else: + pool = torch.randn(pages, 2, hk, PRO_TPB, PRO_D, generator=g, device=DEV).to(kv_dtype) + index_cache = torch.randn(pages, PRO_TPB, 1, PRO_ID, generator=g, device=DEV) + index_cache = index_cache.to(torch.bfloat16) + position_cache = torch.randint( + 0, table_rows, (pages, PRO_TPB, 3), generator=g, device=DEV, dtype=torch.int32 + ) + return dict( + proj=proj, out=out[:, :qkv_width], index_q=index_scratch[:, :PRO_IH], position_ids=ids, + coords=coords, cos_sin=cos_sin, q_weight=weights[0], k_weight=weights[1], + index_q_weight=weights[2], index_k_weight=weights[3], k_cache=pool[:, 0], + v_cache=pool[:, 1], index_cache=index_cache, position_cache=position_cache, + request_indices=request, logical_positions=logical, block_table=block_table, + tokens_per_block=PRO_TPB, compress_ratio=PRO_CR, num_heads_q=hq, num_heads_k=hk, head_dim=PRO_D, + index_heads=PRO_IH, index_head_dim=PRO_ID, index_q_col=iq_col, rotary_dim=PRO_ROT, eps=1e-6, + mrope_section1=11, mrope_section2=10, rows_per_request=per, + ) # fmt: skip + + +def _pro_clone(args): + """The arguments over fresh copies of every allocation they view, and the copies keyed by + the first argument viewing each.""" + out, copies, keys = dict(args), {}, {} + for name, t in args.items(): + if not isinstance(t, torch.Tensor): + continue + key = keys.setdefault(t.untyped_storage().data_ptr(), name) + if key not in copies: + storage = torch.empty(0, dtype=torch.uint8, device=t.device).set_(t.untyped_storage()) + copies[key] = storage.clone() + view = torch.empty(0, dtype=t.dtype, device=t.device) + view.set_(copies[key].untyped_storage(), t.storage_offset(), t.shape, t.stride()) + out[name] = view + return out, copies + + +def _pro_written_slots(args): + """(row, page, slot) of every row whose page lookup holds, from the host's view.""" + table = args["block_table"].cpu() + pages = args["k_cache"].shape[0] + found = [] + for t, (req, p) in enumerate( + zip(args["request_indices"].tolist(), args["logical_positions"].tolist()) + ): + column = p // PRO_TPB + if 0 <= req < table.shape[0] and column < table.shape[1]: + page = int(table[req, column]) + if 0 <= page < pages: + found.append((t, page, p % PRO_TPB)) + return found + + +def _pro_check(got, want) -> None: + """Band the rotated values (only the rms's reduction order differs), overwrite them in `got`, + then require every byte bitwise.""" + g, w = got[0], want[0] + hq, hk = g["num_heads_q"], g["num_heads_k"] + width = (hq + hk) * PRO_D + + def banded(a, b): + a, b = a.float().reshape(-1, a.shape[-1]), b.float().reshape(-1, b.shape[-1]) + assert_within_ulp(a.to(torch.bfloat16), b.to(torch.bfloat16), PRO_ELEMENT_ULP, PRO_RMS_ULP) + + banded(g["out"][:, :width], w["out"][:, :width]) + banded(g["index_q"], w["index_q"]) + slots = _pro_written_slots(g) + k_got = g["out"][:, hq * PRO_D : width].reshape(-1, hk, PRO_D) + for t, page, slot in slots: + own = k_got[t].to(g["k_cache"].dtype) + assert torch.equal(g["k_cache"][page, :, slot].view(torch.uint8), own.view(torch.uint8)), ( + f"row {t}: the K pool's row is not the packed k converted" + ) + g["k_cache"][page, :, slot] = w["k_cache"][page, :, slot] + ends = [(t, page, slot) for t, page, slot in slots if (slot + 1) % PRO_CR == 0] + if ends: + a = torch.stack([g["index_cache"][page, slot, 0] for _, page, slot in ends]) + b = torch.stack([w["index_cache"][page, slot, 0] for _, page, slot in ends]) + banded(a, b) + for _, page, slot in ends: + g["index_cache"][page, slot, 0] = w["index_cache"][page, slot, 0] + g["out"][:, :width] = w["out"][:, :width] + g["index_q"].copy_(w["index_q"]) + assert got[1].keys() == want[1].keys() + for key, copy in got[1].items(): + differ = (copy != want[1][key]).nonzero() + assert differ.numel() == 0, f"{key}: {differ.shape[0]} bytes differ" + + +def _check_decode_prologue(cell, monkeypatch) -> None: + args = _pro_build(cell.spec, seed=_seed(cell.why) % (2**31)) + want = _pro_clone(args) + _pro_op.reference(**want[0]) + got = _pro_clone(args) + _pro_op(**got[0]) + torch.cuda.synchronize() + _pro_check(got, want) + + +def _plan_operands(spec, g): + n, p = spec["requests"], spec["rows_per_request"] + kind = spec["kind"] + if kind == "equal": + windows = torch.full((n,), 25000, dtype=torch.int64) + else: + windows = torch.randint(500, 62000, (n,), generator=g) + if kind == "padded": + windows[-3:] = 1 + rows = n * p + visible = windows.repeat_interleave(p) + torch.arange(p).repeat(n) // 2 + return ( + visible.to(torch.int32).to(DEV), + torch.arange(n, dtype=torch.int32).repeat_interleave(p).to(DEV), + (visible * 4 + torch.randint(0, 4, (rows,), generator=g)).to(DEV), + (visible * 4 + 7).to(torch.int32).to(DEV), + torch.zeros(rows, dtype=torch.int32, device=DEV), + ) + + +def _check_index_split_plan(cell, monkeypatch) -> None: + """Bitwise the reference, eager and replayed on new windows; the ranks' summed windows + balanced and the padding requests spread over them.""" + sp = cell.spec + n, p, world, rank = sp["requests"], sp["rows_per_request"], sp["world"], sp["rank"] + g = torch.Generator().manual_seed(_seed(cell.why)) + + def check(got, vectors): + want = index_split_plan.reference(*vectors, n, p, world, rank) + for a, b in zip(got, want): + assert a.dtype == b.dtype and torch.equal(a, b) + perm = got.perm_rows.long() + share = n // world * p + win = vectors[0].long() + sums = [int(win[perm[r * share : (r + 1) * share]].sum()) for r in range(world)] + assert max(sums) - min(sums) <= int(win.max()) * p, sums + if sp["kind"] == "padded": + pads = set(range((n - 3) * p, n * p)) + owners = {int(torch.nonzero(perm == r_).item()) // share for r_ in pads} + assert len(owners) > 1, "the padding requests land on more than one rank" + + vectors = _plan_operands(sp, g) + check(index_split_plan(*vectors, n, p, world, rank), vectors) + s_vec, fresh = tuple(v.clone() for v in vectors), _plan_operands(sp, g) + out = _captured( + lambda: index_split_plan(*s_vec, n, p, world, rank), + lambda: [a.copy_(b) for a, b in zip(s_vec, fresh)], + ) + check(out, fresh) + + +def _check_draft_continue_select(cell, monkeypatch) -> None: + """Bitwise `_draft_continue_rows` and the row metadata's `index_select`s, eager and replayed.""" + seqs, p = cell.spec["seqs"], cell.spec["rows_per_request"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + rows = seqs * p + 5 + + def operands(): + accepted = torch.randint(1, p + 1, (seqs + 3,), device=DEV, generator=g).to(torch.int32) + vectors = ( + torch.randint(0, 1 << 20, (rows,), device=DEV, generator=g).to(torch.int32), + torch.randint(0, 1 << 20, (rows,), device=DEV, generator=g).to(torch.int32), + torch.randint(0, 1 << 40, (rows,), device=DEV, generator=g), + torch.randint(0, seqs, (rows,), device=DEV, generator=g).to(torch.int32), + ) + return accepted, vectors + + def check(got, accepted, vectors): + want = draft_continue_select.reference(accepted, seqs, p, vectors) + assert torch.equal(got[0], want[0]) + for a, b in zip(got[1], want[1]): + assert a.dtype == b.dtype and torch.equal(a, b) + core = types.SimpleNamespace(dp_size=1) + md = types.SimpleNamespace(qsa_mtp_num_accepted=accepted) + assert torch.equal(got[0], M._draft_continue_rows(core, md, seqs, p, False)) + + accepted, vectors = operands() + check(draft_continue_select(accepted, seqs, p, vectors), accepted, vectors) + s_acc, s_vec = accepted.clone(), tuple(v.clone() for v in vectors) + accepted, vectors = operands() + out = _captured( + lambda: draft_continue_select(s_acc, seqs, p, s_vec), + lambda: [a.copy_(b) for a, b in zip((s_acc, *s_vec), (accepted, *vectors))], + ) + check(out, accepted, vectors) + + +def _check_draft_rows_gather(cell, monkeypatch) -> None: + """Bitwise `index_select` of strided and one-column row operands, eager and replayed.""" + rows, widths = cell.spec["rows"], cell.spec["widths"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + src_rows = 3 * rows + 2 + index = torch.randperm(src_rows, device=DEV, generator=g)[:rows].to(torch.int64) + operands = [] + for k, w in enumerate(widths): + pad = 64 * (k % 2) # every other operand a column slice of a wider row + buf = torch.randn(src_rows, w + pad, device=DEV, generator=g).to(torch.bfloat16) + operands.append(buf[:, pad : pad + w]) + got = draft_rows_gather(index, tuple(operands)) + want = draft_rows_gather.reference(index, tuple(operands)) + for a, b, t in zip(got, want, operands): + assert a.is_contiguous() and torch.equal(a, b) + assert torch.equal(a, torch.index_select(t, 0, index)) + s_idx = index.clone() + index2 = torch.randperm(src_rows, device=DEV, generator=g)[:rows].to(torch.int64) + out = _captured(lambda: draft_rows_gather(s_idx, tuple(operands)), lambda: s_idx.copy_(index2)) + for a, t in zip(out, operands): + assert torch.equal(a, torch.index_select(t, 0, index2)) + + +def _sgc_build(spec, seed): + st = _paged_kv_state( + spec, seed, spec["kv"], lambda needed: _straddling_pages(spec, needed), + slot_pages=spec["slot_pages"], layer=spec["layer"], chunk=4096, + ) # fmt: skip + rows, d, hq, kv = st.positions.numel(), spec["head_dim"], spec["q_heads"], spec["kv_heads"] + qkv = torch.randn(rows, hq + 2 * kv, d, generator=st.gen, device=DEV).to(torch.bfloat16) + q = qkv[:, :hq] + cr, topk = spec["compress_ratio"], spec["token_topk"] + selected = _selected_tokens(st.positions, seed + 3, cr, topk) + if spec.get("holes"): + selected[:, 1::3] = -1 + offset = spec.get("row_offset", 0) + if offset: + # The generation rows of a mixed step: a row slice of a wider selection buffer. + wide = torch.full((offset + rows, selected.shape[1]), -1, dtype=torch.int32, device=DEV) + wide[offset:] = selected + selected = wide[offset:] + request_indices = st.rows.to(torch.int32) + if spec.get("foreign"): + request_indices[0] = -1 + args = dict( + q=q, k_cache=st.k, v_cache=st.v, block_table=st.table, selected_tokens=selected, + request_indices=request_indices, tokens_per_block=spec["tokens_per_block"], + softmax_scale=d**-0.5, + ) # fmt: skip + gate = spec.get("gate") + if gate == "row": + # The trunk's fused input projection row `[q | k | v | gate | index q, k]`. + row = torch.randn(rows, spec["row_heads"] * d, generator=st.gen, device=DEV).to(q.dtype) + start = (hq + 2 * kv) * d + args["output_gate"] = row[:, start : start + hq * d] + elif gate == "interleaved": + # The draft layer's q projection `[T, Hq, 2 D]`: q then the gate, per head. + qg = torch.randn(rows, hq, 2 * d, generator=st.gen, device=DEV).to(q.dtype) + args["output_gate"] = qg[:, :, d:] + elif gate == "contiguous": + # A gate gathered out of the projection row (a draft-extend step's continuing rows). + args["output_gate"] = torch.randn(rows, hq * d, generator=st.gen, device=DEV).to(q.dtype) + return args + + +def _rolled_selection(args): + """The same selection rolled by one column: other tokens, the layout kept.""" + sel = args["selected_tokens"] + return torch.where(sel >= 0, torch.roll(sel, 1, dims=1).clamp_min(0), sel) + + +def _check_sparse_gqa_splitk(cell, monkeypatch) -> None: + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sgc_build(spec, seed) + _assert_straddles(args) + q, k = args["q"], args["k_cache"] + rows = q.shape[0] + assert _sgc_op.serves(q, k, args["selected_tokens"]), "the target sends this call here" + if "splits" in spec: + width = args["selected_tokens"].shape[1] + elem = 1 if spec["kv"] == torch.float8_e4m3fn else 2 + assert mod.plan_splits(rows, spec["kv_heads"], width, elem)[0] == spec["splits"] + out = _sgc_op(**args) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == torch.bfloat16 and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _sgc_op.reference(**args)) + if spec.get("foreign"): + assert bool((out[0] == 0).all()), "a row of no request attends nothing" + if spec.get("capture"): + _replay_in_a_graph(_sgc_op, args, lambda: _refill(args, seed + 1, _rolled_selection), out) + + +def _sgg_selection(spec, seed): + """Rows of a request select from its block scores plus `overlap` x noise (None: independent).""" + cr, token_topk, spread = spec["compress_ratio"], spec["token_topk"], spec.get("overlap") + block_topk, final_topk = token_topk // cr, token_topk + cr - 1 + segments = list(spec["segments"]) + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in segments]) + requests = torch.cat([torch.full((n,), i, device=DEV) for i, (_, n) in enumerate(segments)]) + gen = torch.Generator(device=DEV).manual_seed(seed) + rows = positions.numel() + visible = positions.long() + 1 + available = visible // cr + taken = available.clamp(max=block_topk) + width = max(block_topk, int(available.max().item())) + if spread is None: + scores = torch.rand(rows, width, generator=gen, device=DEV) + else: + base = torch.rand(int(requests.max()) + 1, width, generator=gen, device=DEV) + scores = base[requests] + spread * torch.rand(rows, width, generator=gen, device=DEV) + group_ids = torch.arange(width, device=DEV)[None, :] + scores = torch.where(group_ids < available[:, None], scores, -torch.inf) + chosen = scores.topk(block_topk, dim=1).indices.sort(dim=1).values + column = torch.arange(final_topk, device=DEV)[None, :] + group_of_column = (column // cr).clamp(max=block_topk - 1).expand(rows, -1) + body = torch.gather(chosen, 1, group_of_column) * cr + column % cr + in_body = column < (taken * cr)[:, None] + tail_column = column - (taken * cr)[:, None] + tail = (available * cr)[:, None] + tail_column + in_tail = ~in_body & (tail_column >= 0) & (tail < visible[:, None]) + selected = torch.where(in_body, body, torch.where(in_tail, tail, torch.full_like(body, -1))) + return selected.to(torch.int32) + + +def _check_sparse_gqa_grouped(cell, monkeypatch) -> None: + """Within the band of the reference and of the per-row entry, and deterministic.""" + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sgc_build({k: v for k, v in spec.items() if k != "holes"}, seed) + args["selected_tokens"].copy_(_sgg_selection(spec, seed + 5)) + args["rows_per_request"] = spec["rows_per_request"] + _assert_straddles(args) + q, k = args["q"], args["k_cache"] + rows = q.shape[0] + assert _sgg_op.serves(q, k, args["selected_tokens"], args["rows_per_request"]) + out = _sgg_op(**args) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == torch.bfloat16 and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _sgg_op.reference(**args)) + _gqa_compare(out, _sgc_op(**{k_: v for k_, v in args.items() if k_ != "rows_per_request"})) + assert torch.equal(out, _sgg_op(**args)), "two calls on the same inputs differ" + if spec.get("foreign"): + assert bool((out[0] == 0).all()), "a row of no request attends nothing" + if spec.get("capture"): + refill = lambda: _refill(args, seed + 1, lambda a: _sgg_selection(spec, seed + 8)) # noqa: E731 + _replay_in_a_graph(_sgg_op, args, refill, out) + + +TKC_K, TKC_CR, TKC_TOKEN_TOPK = 512, 4, 2048 + + +def _tkc_build(spec, seed): + """One cell's arguments, with garbage past each row's visible blocks that must not be read.""" + g = torch.Generator(device=DEV).manual_seed(seed) + W = spec["width"] + if "requests" in spec: + # A verify step's 1 + draft rows: every row's KV length is the request's last row + 1. + pos = torch.cat( + [torch.arange(s, s + c, dtype=torch.int64, device=DEV) for s, c in spec["requests"]] + ) + seqlen = torch.cat( + [torch.full((c,), s + c, dtype=torch.int32, device=DEV) for s, c in spec["requests"]] + ) + else: + pos = torch.tensor(spec["pos"], dtype=torch.int64, device=DEV) + seqlen = (pos + 1).to(torch.int32) + rows = pos.numel() + assert rows == spec["rows"], (rows, spec["rows"]) + scores = torch.randn(rows, W, generator=g, device=DEV).abs() * 3.0 + if spec.get("relu_sparse"): + # Fewer positive scores than the budget, so the zero class straddles it. + scores = torch.relu(torch.randn(rows, W, generator=g, device=DEV) - 2.2) + scores[:, ::7] = -0.0 # negative zero ties with zero + if spec.get("relu_sum"): + # The indexer's form: four heads' ReLUs summed, times a scale (zero in 1 of 16). + scores = torch.zeros(rows, W, device=DEV) + for _ in range(4): + scores += torch.relu(torch.randn(rows, W, generator=g, device=DEV)) + scores *= 0.0884 + if spec.get("equal"): + # Every score tied: the budget is the highest-index blocks. + scores = torch.full((rows, W), 0.5, device=DEV) + if spec.get("lognormal"): + # bf16-like resolution, so equal scores tie; a zero in every other row. + scores = torch.exp(1.74 + 0.2 * torch.randn(rows, W, generator=g, device=DEV)) + scores = scores.to(torch.bfloat16).float() + scores[::2, 5] = 0.0 + ends = ((pos + 1) // TKC_CR).to(torch.int32) + assert int(ends.max()) <= W, "a window wider than the score table" + cols = torch.arange(W, device=DEV) + scores = torch.where(cols[None, :] >= ends[:, None], 1.0e30, scores) + scores = torch.where(cols[None, :] == ends[:, None] + 3, float("nan"), scores) + topk = torch.full((rows, TKC_K), -7, dtype=torch.int32, device=DEV) + return [scores, ends, pos, seqlen, topk, TKC_CR, TKC_TOKEN_TOPK] + + +def _check_topk_expand_cluster(cell, monkeypatch) -> None: + args = _tkc_build(cell.spec, seed=_seed(cell.why) % 2**31) + assert supports(args[0]), "a cell outside the target's routing to this entry" + want_topk, want_sel = _tkc_op.reference(*args) + sel = _tkc_op(*args) + assert sel.dtype == torch.int32 and sel.shape == want_sel.shape + assert torch.equal(args[4], want_topk), "block list" + assert torch.equal(sel, want_sel), "token selection" + + +# Scratch and table are pre-filled with sentinels so a missing or a stray write shows. + +F2B_SCRATCH_FILL = 0x7FC1 # a bf16 NaN +F2B_TABLE_FILL = -7 + + +def _f2b_build(spec, seed): + gen = torch.Generator(device=DEV).manual_seed(seed) + tpb, kv, d = spec["tokens_per_block"], spec["kv_heads"], spec["head_dim"] + counts = [math.ceil(n / tpb) for n in spec["kv_lens"]] + needed = sum(counts) + 7 + pool_pages = (_straddling_pages(spec, needed) if spec["straddle"] else needed) + 5 + role_pages, layer = spec["slot_pages"], spec["layer"] + # Random e4m3 codes (NaN codes 0x7F / 0xFF excluded), in the layer's two planes only. + data = torch.zeros((pool_pages, role_pages, kv, tpb, d), dtype=torch.uint8, device=DEV) + first = pool_pages - needed + for p0 in range(first, pool_pages, 1024): + p1 = min(p0 + 1024, pool_pages) + codes = torch.randint(0, 254, (p1 - p0, 2, kv, tpb, d), generator=gen, device=DEV) + codes = torch.where(codes >= 0x7F, codes + 1, codes) + data[p0:p1, 2 * layer : 2 * layer + 2] = codes.to(torch.uint8) + pool = data.view(torch.float8_e4m3fn) + width, slots_total = spec["width"], spec["slots"] + table = torch.full((slots_total, width), -1, dtype=torch.int32, device=DEV) + slots = torch.randperm(slots_total, generator=torch.Generator().manual_seed(seed)) + slots = slots[: len(counts)] + page_ids = first + torch.randperm(needed, generator=gen, device=DEV).to(torch.int32) + cursor = 0 + for slot, count in zip(slots.tolist(), counts): + table[slot, :count] = page_ids[cursor : cursor + count] + cursor += count + if spec["holes"]: + head = slots[0].item() + table[head, 1] = -1 # unmapped inside the sequence + table[head, 2] = pool_pages + 3 # past the pool + counts_t = torch.tensor(counts, dtype=torch.int64, device=DEV) + out_shape = (int(counts_t.sum()), kv, tpb, d) + + def scratch(): + fill = torch.full(out_shape, F2B_SCRATCH_FILL, dtype=torch.int16, device=DEV) + return fill.view(torch.bfloat16) + + return dict( + k_cache=pool[:, 2 * layer], v_cache=pool[:, 2 * layer + 1], block_table=table, + slots=slots.to(DEV, torch.int32), counts=counts_t, + offsets=torch.cumsum(counts_t, 0) - counts_t, table=torch.full_like(table, F2B_TABLE_FILL), + k_out=scratch(), v_out=scratch(), + ) # fmt: skip + + +def _check_fp8_pages_to_bf16(cell, monkeypatch) -> None: + args = _f2b_build(cell.spec, _seed(cell.why)) + if cell.spec["straddle"]: + _assert_straddles(args) + ref = _f2b_op.reference(**{k: v.clone() if torch.is_tensor(v) else v for k, v in args.items()}) + out = _f2b_op(**args) + for name, got, want in zip(("k_out", "v_out", "table"), out, ref): + assert got.shape == want.shape and torch.equal(_bits(got), _bits(want)), name + # Every listed page was written, and only the listed requests' table entries. + assert not bool(out[0].isnan().any()) and not bool(out[1].isnan().any()) + listed_rows = torch.zeros_like(args["block_table"], dtype=torch.bool) + for slot, count in zip(args["slots"].tolist(), args["counts"].tolist()): + listed_rows[slot, :count] = True + assert bool((out[2][~listed_rows] == F2B_TABLE_FILL).all()) + + +#: The entry's launches with the token plan and with the block plan (where it serves). +_RTL_FUSED = ("_group_plan", "_rowtile_lone_kernel", "_rowtile_attention_cute") +_RTL_BLOCK = ("_block_plan", "_rowtile_lone_kernel", "_rowtile_attention_cute") + + +def _rtl_build(spec, seed): + trunk = spec["cache_layout"] == "trunk" + assert trunk or spec["cache_layout"] == "scratch" + assert spec["cache_dtype"] == torch.bfloat16 and spec["q_layout"] == "qkv_slice" + st = _paged_kv_state( + spec, seed, spec["cache_dtype"], + lambda needed: _straddling_pages(spec, needed, spec["offset_edge"]) if trunk else needed, + slot_pages=spec.get("slot_pages", 2), layer=spec.get("layer", 0), chunk=1024, + scratch=not trunk, + ) # fmt: skip + rows, kv, d, q_heads = st.positions.numel(), spec["kv_heads"], spec["head_dim"], spec["q_heads"] + qkv = torch.randn(rows, q_heads + 2 * kv, d, generator=st.gen, device=DEV).to(spec["q_dtype"]) + selected = _selected_tokens( + st.positions, seed + 3, spec["compress_ratio"], spec["token_topk"], + shared=spec.get("shared_scores", False), chunk=1024, + ) # fmt: skip + return dict( + q=qkv[:, :q_heads], k_cache=st.k, v_cache=st.v, block_table=st.table, + selected_tokens=selected, request_indices=st.rows.to(spec["request_dtype"]), + tokens_per_block=spec["tokens_per_block"], softmax_scale=d**-0.5, + query_positions=st.positions.to(spec["request_dtype"]), + compress_ratio=spec["compress_ratio"], + ) # fmt: skip + + +def _check_sparse_gqa_rowtile(cell, monkeypatch) -> None: + """The plan, lone-row and row-tile passes with their grids, then the band of the reference.""" + launches = _record_launches(monkeypatch, _module, dict.fromkeys(_RTL_FUSED + _RTL_BLOCK)) + spec = cell.spec + args = _rtl_build(spec, seed=sum(map(ord, cell.why))) + if spec["cache_layout"] == "trunk": + _assert_straddles(args, spec["offset_edge"]) + rows = args["q"].shape[0] + _assert_prefill_selection(args["selected_tokens"], args["query_positions"], spec) + out = _rtl_op(**args) + group_rows = rows_per_group(spec["q_heads"] // spec["kv_heads"]) + assert group_rows == {12: 10, 6: 20}[spec["q_heads"] // spec["kv_heads"]] + groups = math.ceil(rows / group_rows) + blocks = _module.block_plan_serves(spec["compress_ratio"], spec["tokens_per_block"], group_rows) + assert [name for name, _, _ in launches] == list(_RTL_BLOCK if blocks else _RTL_FUSED) + assert launches[0][1] == (groups,) + assert launches[1][1] == (min(rows, 256), spec["kv_heads"]) + # The row-tile pass is persistent: one CTA per SM, never more than (group, KV head) items. + sms = torch.cuda.get_device_properties(0).multi_processor_count + assert launches[2][1] == (min(groups * spec["kv_heads"], sms),) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == args["q"].dtype and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _rtl_op.reference(**args)) + + +def _check_spec_step_metadata(cell, monkeypatch) -> None: + """Bitwise the engine's composed launches over a column slice of a wider position buffer, + eager and replayed on new offsets.""" + n, p = cell.spec["n"], cell.spec["p"] + rows = n * p + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why) % 2**31) + + def operands(): + buf = torch.randint(0, 200000, (3, 1, rows + 9), generator=g, device=DEV, dtype=torch.int32) + pos_off = torch.randint(0, 4, (rows + 5,), generator=g, device=DEV, dtype=torch.int32) + kv = torch.randint(90000, 100000, (n,), generator=g, device=DEV, dtype=torch.int32) + kv_off = torch.randint(-3, 1, (n + 2,), generator=g, device=DEV, dtype=torch.int32) + seq = torch.full((n + 1,), p, device=DEV, dtype=torch.int32) + mapping = tuple( + torch.full((rows + 4,), -9, device=DEV, dtype=dt) + for dt in (torch.int32, torch.int64, torch.int32, torch.int32) + ) + return buf, buf[:, :, :rows], pos_off, kv, kv_off, seq, mapping + + def copy(ops): + c = tuple(x.clone() if torch.is_tensor(x) else tuple(y.clone() for y in x) for x in ops) + return (c[0], c[0][:, :, :rows]) + c[2:] + + def composed(buf, pos, pos_off, kv, kv_off, seq, mapping): + pos += pos_off[:rows] + kv += kv_off[:n] + triton_qsa_generation_token_mapping( + kv_lens=kv, seq_lens=seq[:n], request_indices=mapping[0], logical_positions=mapping[1], + sequence_lengths=mapping[2], visible_blocks=mapping[3], num_rows=rows, compress_ratio=4, + ) # fmt: skip + + a = operands() + b, c = copy(a), copy(a) + pos_out = torch.full((3, rows), -9, device=DEV, dtype=torch.int32) + qsa_spec_step_metadata(a[1], a[2], a[3], a[4], a[5][:n], a[6], 4, pos_out=pos_out) + composed(*b) + assert torch.equal(pos_out, b[1].reshape(3, rows)), "the contiguous positions" + qsa_spec_step_metadata.reference(c[1], c[2], c[3], c[4], c[5][:n], c[6], 4) + for got, want, ref in zip((a[0], a[3]) + a[6], (b[0], b[3]) + b[6], (c[0], c[3]) + c[6]): + assert torch.equal(got, want) and torch.equal(got, ref) + if not cell.spec["capture"]: + return + s, fresh = operands(), operands() + want = copy(fresh) + _captured( + lambda: qsa_spec_step_metadata(s[1], s[2], s[3], s[4], s[5][:n], s[6], 4), + lambda: [dst.copy_(fresh[i]) for i, dst in ((0, s[0]), (2, s[2]), (3, s[3]), (4, s[4]))], + ) + composed(*want) + assert torch.equal(s[0], want[0]) and torch.equal(s[3], want[3]) + for got, exp in zip(s[6], want[6]): + assert torch.equal(got[:rows], exp[:rows]) + + +_ENTRIES = ( + (qsa_paged_index_store, qsa_paged_index_store.CELLS, _check_index_store), + (qsa_paged_kv_store, qsa_paged_kv_store.CELLS, _check_kv_store), + (qsa_paged_sparse_gqa, qsa_paged_sparse_gqa.CELLS, _check_sparse_gqa), + (_pro_op, _pro_op.CELLS, _check_decode_prologue), + (index_split_plan, index_split_plan.CELLS, _check_index_split_plan), + (draft_continue_select, draft_continue_select.CELLS, _check_draft_continue_select), + (draft_rows_gather, draft_rows_gather.CELLS, _check_draft_rows_gather), + (_sgc_op, _sgc_op.CELLS, _check_sparse_gqa_splitk), + (_sgg_op, _sgg_op.CELLS, _check_sparse_gqa_grouped), + (_tkc_op, _tkc_op.CELLS, _check_topk_expand_cluster), + (_f2b_op, _f2b_op.CELLS, _check_fp8_pages_to_bf16), + (_rtl_op, _rtl_op.CELLS, _check_sparse_gqa_rowtile), + (qsa_spec_step_metadata, qsa_spec_step_metadata.CELLS, _check_spec_step_metadata), +) +CASES = [ + pytest.param( + entry, cell, check, + id=f"{type(entry).__name__}: {cell.why[:44]}", + marks=[pytest.mark.large] if cell.spec.get("large") else [], + ) + for entry, cells, check in _ENTRIES + for cell in cells +] # fmt: skip + + +@pytest.mark.parametrize("entry, cell, check", CASES) +def test_certified_cells(entry, cell, check, certifying, monkeypatch) -> None: # noqa: F811 + """Every (entry, cell) pair against the entry's own `reference`, with `certifying` armed.""" + certifying(entry) + check(cell, monkeypatch) + + +# `DecodeTarget._attention_block` at the checkpoint's shapes for one rank of a layout. + +PRL_DEV, PRL_BF, PRL_F8 = "cuda", torch.bfloat16, torch.float8_e4m3fn +PRL_D, PRL_ROT, PRL_IH, PRL_ID, PRL_TPB, PRL_CR, PRL_HIDDEN, PRL_LAYERS, PRL_LAYER = ( + 256, 64, 4, 128, 32, 4, 2560, 12, 5 +) # fmt: skip +PRL_CFG = dict(theta=1.0e7, eps=1e-6, mrope_section1=11, mrope_section2=10, max_positions=262144) +#: (q heads, KV heads, context lengths after the step, rows a request, pool dtype). +PRL_LAYOUTS = { + "tep4_verify_bf16": (6, 1, (7003,), 4, PRL_BF), + "tep2_decode_bf16": (12, 1, (4096, 5003, 6150), 1, PRL_BF), + "dep4_verify_e4m3": (24, 2, (2500, 4100, 6000, 8000), 4, PRL_F8), +} +#: A tep2 rank, kv_fp8, MTP2 verify (3 rows a request) at 90 requests (270 rows). +IDX_LAYOUT = (12, 1, tuple(2400 + 61 * r for r in range(90)), 3, PRL_F8) + + +def _prl_core(hq, hk): + c = types.SimpleNamespace( + **PRL_CFG, head_dim=PRL_D, rotary_dim=PRL_ROT, heads_q=hq, heads_kv=hk + ) + c.index_heads, c.index_kv_heads, c.index_head_dim = PRL_IH, 1, PRL_ID + c.token_topk, c.compress_ratio = 2048, PRL_CR + c.block_topk = c.token_topk // PRL_CR + c.index_width = (PRL_IH + 1) * PRL_ID + c.index_scratch_width = c.index_width + PRL_ID + c.softmax_scale = PRL_D**-0.5 + c.q_width, c.kv_width = hq * PRL_D, hk * PRL_D + c.qkv_width = c.q_width + 2 * c.kv_width + c.attn_in_width = c.qkv_width + c.q_width + c.index_scratch_width + c.attn_in_tail_heads = c.attn_in_width // PRL_D - hq - hk + c.index_col = c.qkv_width + c.q_width + c._cos_sin = M.Qwen4ExpUnifiedModelingV2._index_rope_table(c, c.theta, PRL_DEV) + c.tp_sum = lambda x: x + return c + + +def _prl_build(hq, hk, ctx, p, kv_dtype, seed): + g = torch.Generator(device=PRL_DEV).manual_seed(seed) + core = _prl_core(hq, hk) + need = [-(-n // PRL_TPB) for n in ctx] + pages = sum(need) + 16 + perm = torch.randperm(pages, generator=g, device=PRL_DEV).to(torch.int32) + table = torch.full((max(8, len(ctx)), 8196), -1, dtype=torch.int32, device=PRL_DEV) + used = 0 + for r, n in enumerate(need): + table[r, :n] = perm[used : used + n] + used += n + roles = torch.randn(pages, 2 * PRL_LAYERS, hk, PRL_TPB, PRL_D, generator=g, device=PRL_DEV) + roles = roles.to(kv_dtype) + # Index keys 2..3x larger at 512 groups a request win the block top-k by a 2x margin, so + # paths a sub-ulp apart in the query select the same tokens. + scale = 0.25 + 0.75 * torch.rand(pages, PRL_TPB, generator=g, device=PRL_DEV) + for r, n in enumerate(ctx): + hot = torch.randperm((n - p) // PRL_CR - 1, generator=g, device=PRL_DEV)[:512] + token = hot * PRL_CR + PRL_CR - 1 + scale[table[r, token // PRL_TPB].long(), token % PRL_TPB] = 2.0 + torch.rand( + hot.shape[0], generator=g, device=PRL_DEV + ) + planes = ( + scale[:, None, :, None, None] + * torch.randn(PRL_LAYERS, 1, 1, PRL_ID, generator=g, device=PRL_DEV)[None] + ).to(PRL_BF) + position = torch.randint( + 0, 8192, (pages, PRL_TPB, 3), generator=g, device=PRL_DEV, dtype=torch.int32 + ) + core._bound = types.SimpleNamespace( + pools=types.SimpleNamespace( + kv={PRL_LAYER: P.KvPlanes(roles[:, 2 * PRL_LAYER], roles[:, 2 * PRL_LAYER + 1])}, + index_cache={PRL_LAYER: planes[:, PRL_LAYER]}, + position_cache=position, + ) + ) + n = len(ctx) + req = torch.arange(n, device=PRL_DEV, dtype=torch.int32).repeat_interleave(p) + lens = torch.tensor(ctx, device=PRL_DEV, dtype=torch.int64).repeat_interleave(p) + logical = lens - p + torch.arange(p, device=PRL_DEV, dtype=torch.int64).repeat(n) + ids = logical.to(torch.int32).expand(3, n * p).contiguous() + qsa = ( + req, logical, lens.to(torch.int32), ((logical + 1) // PRL_CR).to(torch.int32), + torch.empty(n * p, core.block_topk, dtype=torch.int32, device=PRL_DEV), + torch.zeros(n * p, dtype=torch.int32, device=PRL_DEV), table, PRL_TPB, ids.clone(), ids[0], + torch.transpose(ids, 0, 1), True, + ) # fmt: skip + w = lambda *shape: torch.randn(*shape, generator=g, device=PRL_DEV) # noqa: E731 + L = types.SimpleNamespace( + attn_in=(w(core.attn_in_width, PRL_HIDDEN) * 0.02).to(PRL_BF).t(), + o=(w(PRL_HIDDEN, core.q_width) * 0.02).to(PRL_BF).t(), + q_norm=(w(PRL_D) * 0.3).to(PRL_BF), k_norm=(w(PRL_D) * 0.3).to(PRL_BF), + idx_qn=(w(PRL_ID) * 0.3).to(PRL_BF), idx_kn=(w(PRL_ID) * 0.3).to(PRL_BF), + ) # fmt: skip + x = w(n * p, PRL_HIDDEN).to(PRL_BF) + return core, L, x, qsa, (roles, planes, position), p + + +def test_decode_index_selection_split_over_tp_is_the_replicated_selection(monkeypatch) -> None: + """The decode index selection split over two attention-TP ranks, gathers emulated in one + process: split by position or by `_index_split_plan`, the gathered selection and the layer + over it are bitwise the replicated ones.""" + core, L, x, qsa, pools, p = _prl_build(*IDX_LAYOUT, seed=37) + rows = x.shape[0] + monkeypatch.setattr(M, "_INDEX_SPLIT_MIN_REQUESTS", 32) + # bitwise halves need one scoring kernel for 45 and 90 groups alike (the family's) + monkeypatch.setattr(M, "_INDEX_SCORES_CUTE_MIN_GROUPS", 1 << 30) + q8 = qsa[:6] + (qsa[6], qsa[7]) + cache = core._bound.pools.index_cache[PRL_LAYER] + saved = [t.clone() for t in pools] + assert M._index_split_rows(None, rows, p) is None + assert M._index_split_rows(([0, 1], None, 1), rows, p) == (135, 270) + assert M._index_split_rows(([0, 1], None, 1), 31 * p, p) is None # below the threshold + + def restore(): + for t, sv in zip(pools, saved): + t.copy_(sv) + + def layer(target, gather): + qkv = torch.empty(rows, core.qkv_width, dtype=PRL_BF, device=PRL_DEV) + scratch = torch.empty(rows, core.index_scratch_width, dtype=PRL_BF, device=PRL_DEV) + with monkeypatch.context() as m: + m.setattr(M, "allgather", gather) + out = M.DecodeTarget._attention_block( + target, x, PRL_LAYER, L, qkv, scratch, None, qsa, + {"predicted_tokens_per_seq": p}, False, + ) # fmt: skip + torch.cuda.synchronize() + return out.clone(), scratch + + def select(rank, q, vectors, n, rpr, gather=lambda part, sizes, g_, pg: part, **kw): + with monkeypatch.context() as m: + m.setattr(M, "allgather", gather) + split = None if rank is None else ([0, 1], None, rank) + return M._decode_index_selection(core, split, q, vectors, n, rpr, PRL_DEV, cache, **kw) + + # The selections are taken over the planes the layer left (its rows' index keys stored). + full_out, scratch = layer(types.SimpleNamespace(core=core, _index_split=None), None) + q = M._index_query(core, scratch) + full = select(None, q, q8, rows, p) + halves = [select(rank, q, q8, rows, p) for rank in (0, 1)] + assert [h.shape[0] for h in halves] == [135, 135] + assert torch.equal(torch.cat(halves), full), "the halves are bitwise the replicated selection" + # A draft step's shape: one row a request (each request's last verify row here). + last = torch.arange(p - 1, rows, p, device=PRL_DEV) + q1, d8 = q.index_select(0, last), tuple(t.index_select(0, last) for t in q8[:6]) + q8[6:] + halves1 = [select(rank, q1, d8, last.numel(), 1) for rank in (0, 1)] + assert [h.shape[0] for h in halves1] == [45, 45] + assert torch.equal(torch.cat(halves1), select(None, q1, d8, last.numel(), 1)) + + lam = object() # `_index_split_plan` needs a Lamport gather; the emulation below is NCCL's + plans = [M._index_split_plan(([0, 1], None, r, lam), q8, rows, p) for r in (0, 1)] + assert all(pl is not None for pl in plans) + assert torch.equal(plans[0].perm_rows, plans[1].perm_rows) + perm = plans[0].perm_rows.long() + assert torch.equal(torch.sort(perm).values, torch.arange(rows, device=PRL_DEV)) + assert torch.equal(plans[0].iq_rows.long()[perm], torch.arange(rows, device=PRL_DEV)) + windows = qsa[3].long() + by_plan = [int(windows[perm[r * 135 : (r + 1) * 135]].sum()) for r in (0, 1)] + by_position = [int(windows[r * 135 : (r + 1) * 135].sum()) for r in (0, 1)] + assert max(by_position) / min(by_position) > 1.1 + assert max(by_plan) / min(by_plan) < 1.01, by_plan + parts = {} + for permuted in (False, True): + qs = q.index_select(0, perm) if permuted else q + for rank in (0, 1): + kw = dict(plan=plans[rank], q_permuted=permuted) + keep = lambda part, sizes, g_, pg, key=(permuted, rank): parts.setdefault( # noqa: E731 + key, part.clone() + ).repeat(2, 1) + select(rank, qs, q8, rows, p, gather=keep, **kw) + gathered = torch.cat([parts[(permuted, 0)], parts[(permuted, 1)]]) + assert torch.equal(gathered, full.index_select(0, perm)) + for rank in (0, 1): + kw = dict(plan=plans[rank], q_permuted=permuted) + got = select(rank, qs, q8, rows, p, gather=lambda *a, g=gathered: g, **kw) + assert torch.equal(got, full), (permuted, rank) + restore() + + for rank in (0, 1): + # By position: this rank's half handed over, the gathered selection back. + handed = [] + + def nccl(part, sizes, group, process_group): + assert sizes is None and group == [0, 1] + handed.append(part.clone()) + return torch.cat(halves) + + out, _ = layer(types.SimpleNamespace(core=core, _index_split=([0, 1], None, rank)), nccl) + restore() + assert len(handed) == 1 and torch.equal(handed[0], halves[rank]) + assert torch.equal(out, full_out), "the layer over the gathered selection" + # The plan: the prologue stores the index queries in the permuted order. + target = types.SimpleNamespace( + core=core, _index_split=([0, 1], None, rank), _index_plan=plans[rank] + ) + out, scr = layer(target, lambda part, sizes, g_, pg: gathered) + restore() + assert torch.equal(M._index_query(core, scr), q.index_select(0, perm)) + assert torch.equal(out, full_out), rank + + +def _replay_index_store(prefix, monkeypatch) -> None: + """A store captured at the cell's batch, replayed after the next step's metadata is written + into the captured buffers, stores that step bitwise (the pool is sized to the batch).""" + cell = next(c.spec for c in qsa_paged_index_store.CELLS if c.why.startswith(prefix)) + spec = dict(cell, pages=24576, straddle=False) + args, extras = _index_store_build(spec, seed=101) + index_cache, position_cache = args["index_cache"], args["position_cache"] + qsa_paged_index_store(**args) # compiles the launch before the capture + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_paged_index_store(**args) + shifted = dict(spec, segments=tuple((max(0, s - 3), n) for s, n in spec["segments"])) + following, _ = _index_store_build(shifted, seed=202, planes=(index_cache, position_cache)) + before = qsa_paged_index_store.reference(**args) + names = ("token_k", "position_coordinates", "request_indices", "logical_positions") + for name in (*names, "block_table"): + args[name].copy_(following[name]) + index_cache.fill_(KEY_FILL) + position_cache.fill_(POS_FILL) + ref = qsa_paged_index_store.reference(**args) + assert not torch.equal(_bits(ref[0]), _bits(before[0])), "the next step stores nothing new" + graph.replay() + torch.cuda.synchronize() + _index_store_compare((index_cache, position_cache), ref) + _other_layers_untouched(*extras["coalesced"]) + + +def _replay_kv_store(prefix, monkeypatch) -> None: + """A step over the engine's pool captured with every position at -1 (nothing read or + written), replayed after the real positions are written into the same buffer.""" + cell = next(c for c in qsa_paged_kv_store.CELLS if c.why.startswith(prefix)) + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + try: + positions = args["logical_positions"] + saved = positions.clone() + positions.fill_(-1) + before = tuple(t.clone() for t in pool.gathered()) + for _ in range(3): # compile the Triton kernel before capture + qsa_paged_kv_store(**args) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_paged_kv_store(**args) + torch.cuda.synchronize() + _kv_store_compare(pool.gathered(), before) # a row at position -1 writes nothing + positions.copy_(saved) + ref = _kv_engine_reference(args, pool) + graph.replay() + torch.cuda.synchronize() + _kv_store_compare(pool.gathered(), ref) + pool.assert_rest_unchanged() + finally: + pool.close() + + +@pytest.mark.parametrize( + "replay, prefix", + [ + pytest.param(_replay_index_store, "tp2 decode: 128 requests", id="index tp2-decode"), + pytest.param(_replay_index_store, "dep4 MTP3 verify", id="index dep4-mtp3"), + pytest.param(_replay_kv_store, "tp2 decode, 128 rows (max_batch_size)", id="kv tp2-decode"), + pytest.param(_replay_kv_store, "dep4 MTP3 verify, 128 requests x 4", id="kv dep4-mtp3"), + ], +) +def test_captured_stores_replay_rewritten_metadata(replay, prefix, monkeypatch) -> None: + replay(prefix, monkeypatch) + + +def _index_pool(n_requests, need, seed, pool_pages, slots): + """A trunk-like index pool's layer-11 plane and a `[slots, 8196]` table of random pages.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + pages = max(pool_pages, sum(need) + 8) + pool = torch.randn(pages, 12, 32, 1, 128, generator=gen, device=DEV).to(torch.bfloat16) + perm = torch.randperm(pages, generator=gen, device=DEV).to(torch.int32) + table = torch.full((slots, 8196), -1, dtype=torch.int32, device=DEV) + used = 0 + slot_of = torch.randperm(slots, generator=torch.Generator().manual_seed(seed))[:n_requests] + for i, k in enumerate(need): + table[int(slot_of[i]), :k] = perm[used : used + k] + used += k + return gen, pool[:, 11], table, slot_of + + +def _family_index_scores_groups(contexts, monkeypatch) -> None: + """The CuTe request-group index scores: within fp32 summation-order noise of the family's + kernel, the same top-512 except at near-ties, deterministic, and replayable.""" + rpr = 3 + gen, cache, table, slot_of = _index_pool( + len(contexts), [-(-c // 32) for c in contexts], len(contexts) * 7 + 1, 6000, 128 + ) + req = slot_of.to(DEV).to(torch.int32).repeat_interleave(rpr) + lens = torch.tensor(contexts, device=DEV, dtype=torch.int64).repeat_interleave(rpr) + pos = lens - rpr + torch.arange(rpr, device=DEV, dtype=torch.int64).repeat(len(contexts)) + q = torch.randn(len(contexts) * rpr, 6, 128, generator=gen, device=DEV) + q = q.to(torch.bfloat16)[:, :4] + assert _igc.serves(q, cache, rpr) + fam = M.qsa_paged_index_scores( + q, cache, table, pos, req, 32, 4, only_visible_blocks=True, context_rows=False, + rows_per_request=rpr, + ) # fmt: skip + out = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + vis = (pos + 1) // 4 + inside = torch.arange(out.shape[1], device=DEV)[None, :] < vis[:, None] + torch.testing.assert_close(out[inside], fam[inside], rtol=1e-5, atol=1e-6) + neg = torch.full_like(out, -1.0) + k = min(512, int(vis.min().item())) if int(vis.min().item()) > 0 else 0 + if k: + a = torch.where(inside, out, neg).topk(k, dim=1) + b = torch.where(inside, fam, neg).topk(k, dim=1) + differ = (a.indices.sort(dim=1).values != b.indices.sort(dim=1).values).any(dim=1) + # a row may differ only where its k-th and (k+1)-th scores are within the noise + next_value = torch.where(inside, fam, neg).topk(k + 1, dim=1).values[:, -1] + gap = (b.values[:, -1] - next_value).abs() + assert bool((~differ | (gap <= 1e-5 * b.values[:, -1].abs())).all()) + out2 = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + assert torch.equal(out2[inside], out[inside]), "two calls on the same inputs differ" + refill = lambda: q.copy_(torch.randn(q.shape, device=DEV).to(q.dtype)) # noqa: E731 + captured = _captured( + lambda: _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr), refill + ) + eager = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + assert torch.equal(captured[inside], eager[inside]), "the replay differs from an eager call" + + +def _family_index_scores_prefill(requests, monkeypatch) -> None: + """The tcgen05 context-row index scores, `requests` = ((prefix, rows), ...): bitwise the + family's kernel over every row's visible columns, and replayable.""" + need = [-(-(p + r) // 32) for p, r in requests] + gen, cache, table, slot_of = _index_pool( + len(requests), need, sum(r for _, r in requests), 12000, 64 + ) + req = torch.cat( + [torch.full((r,), int(slot_of[i]), dtype=torch.int32) for i, (_, r) in enumerate(requests)] + ).to(DEV) + pos = torch.cat([torch.arange(p, p + r, dtype=torch.int32) for p, r in requests]).to(DEV) + q = torch.randn(pos.numel(), 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + assert _isc.supports(q, cache, 32, 4) + fam = M.qsa_paged_index_scores( + q, cache, table, pos, req, 32, 4, only_visible_blocks=True, context_rows=True + ) + out = _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4) + vis = ((pos + 1) // 4).long() + inside = torch.arange(out.shape[1], device=DEV)[None, :] < vis[:, None] + assert torch.equal(out[inside], fam[inside]), "visible scores differ from the family's" + refill = lambda: q.copy_(torch.randn(q.shape, device=DEV).to(q.dtype)) # noqa: E731 + captured = _captured( + lambda: _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4), refill + ) + eager = _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4) + assert torch.equal(captured[inside], eager[inside]), "the replay differs from an eager call" + + +def _family_topk_rows(case, monkeypatch) -> None: + """The prefill row top-k: bitwise the family's `qsa_topk_expand_decode` on a wide table.""" + name, spec = case + args = _tkc_build(dict(spec, width=65568), seed=_seed(name) % 2**31) + assert _tkr_supports(args[0], args[4]), "a table outside the row kernel's range" + family_args = [a.clone() if torch.is_tensor(a) else a for a in args] + sel = _tkr_op(*args) + family_sel = family_op(*family_args) + assert torch.equal(args[4], family_args[4]), "block list" + assert torch.equal(sel, family_sel), "token selection" + + +_FAMILY_CASES = [ + *( + pytest.param(_family_index_scores_groups, contexts, id=f"index scores groups {name}") + for name, contexts in ( + ("32x50k", tuple([50000] * 32)), + ("64x20k", tuple([20000] * 64)), + ("32_spread", tuple(4400 + 6007 * i for i in range(32))), + ("edges", (3, 4, 5, 515, 516, 517, 2051, 66000, 131071, 262143, 9000, 31)), + ("8x100k", tuple([100000] * 8)), + ("20_spread", tuple(70000 + 1500 * i for i in range(20))), + ) + ), + *( + pytest.param(_family_index_scores_prefill, requests, id=f"index scores prefill {name}") + for name, requests in ( + ("88k_one", ((88064, 1024),)), + ("three_requests", ((88064, 700), (40000, 333), (3, 300))), + ("windows_from_0", ((0, 1500),)), + ("131k_250k", ((131072, 320), (250000, 64))), + ) + ), + *( + pytest.param(_family_topk_rows, (name, spec), id=f"topk rows {name}") + for name, spec in ( + ("chunk_88k_relu_sum", dict(rows=1024, requests=((88064, 1024),), relu_sum=True)), + ("chunk_131k_relu_sum", dict(rows=512, requests=((131072, 512),), relu_sum=True)), + ("requests_12_relu_sum", dict(rows=768, requests=tuple((88064 + 97 * j, 64) + for j in range(12)), relu_sum=True)), + ("chunk_250k_relu_sum", dict(rows=256, requests=((250000, 256),), relu_sum=True)), + ("zero_class_straddles", dict(rows=256, requests=((70000, 256),), relu_sparse=True)), + ("every_score_tied", dict(rows=256, requests=((88064, 256),), equal=True)), + ("lognormal_bf16_ties", dict(rows=512, requests=((60000, 512),), lognormal=True)), + ("windows_0_to_past_k", dict(rows=2600, requests=((0, 2600),), relu_sum=True)), + ) + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, case", _FAMILY_CASES) +def test_kernels_without_cells_match_the_family(check, case, monkeypatch) -> None: + check(case, monkeypatch) + + +def _prl_site(monkeypatch, built, prologue, site): + """One `trunk` (`DecodeTarget._attention_block`) or `draft` (`_mtp_attention_planes`) run + over the built pools, restored after, with the fused prologue on or off.""" + core, L, x, qsa, pools, p = built + saved = [t.clone() for t in pools] + calls, entry = [], M.qwen4_exp_attn_decode_prologue + + def counted(*args, **kwargs): + calls.append(kwargs["rows_per_request"]) + return entry(*args, **kwargs) + + rows = x.shape[0] + with monkeypatch.context() as m: + m.setattr(M, "qwen4_exp_attn_decode_prologue", counted) + if not prologue: + m.setattr(M, "_decode_prologue", lambda kv, sparse: False) + if site == "trunk": + qkv = torch.empty(rows, core.qkv_width, dtype=PRL_BF, device=PRL_DEV) + scratch = torch.empty(rows, core.index_scratch_width, dtype=PRL_BF, device=PRL_DEV) + out = M.DecodeTarget._attention_block( + types.SimpleNamespace(core=core), x, PRL_LAYER, L, qkv, scratch, None, qsa, + {"predicted_tokens_per_seq": p}, False, + ) # fmt: skip + else: + roles, planes, position = pools + draft = types.SimpleNamespace( + kv=P.KvPlanes(roles[:, 2 * PRL_LAYER], roles[:, 2 * PRL_LAYER + 1]), + index_cache=planes[:, PRL_LAYER], + position_cache=position, + ) + vectors = (qsa[0], qsa[1], qsa[6], qsa[7], qsa[8], qsa[9], qsa[10]) + qkv, gate, iq = M._mtp_attention_planes(core, L, x, draft, vectors, p, True, True) + torch.cuda.synchronize() + if site == "trunk": + got = dict(out=out, qkv=qkv, iq=M._index_query(core, scratch).clone()) + else: + got = dict(qkv=qkv.clone(), gate=gate.clone(), iq=iq.clone()) + got.update(calls=calls, planes=[t.clone() for t in pools]) + for t, s in zip(pools, saved): + t.copy_(s) + return got + + +def _site_prologue(case, monkeypatch) -> None: + """The one-launch prologue against the composition it replaces: v, index keys, coordinates + and the draft's gate bitwise; rotated values within rounding, since the entry reads the fp32 + table and `fused_qk_norm_rope` takes `__sincosf`.""" + site, layout = case + hq, hk, ctx, p, kv_dtype = PRL_LAYOUTS[layout] + built = _prl_build(hq, hk, ctx, p, kv_dtype, seed=len(layout) + (7 if site == "draft" else 0)) + if site == "draft": + core = built[0] + core.mtp_theta = 5.0e6 + core._mtp_cos_sin = M.Qwen4ExpUnifiedModelingV2._index_rope_table(core, 5.0e6, PRL_DEV) + fused = _prl_site(monkeypatch, built, True, site) + composed = _prl_site(monkeypatch, built, False, site) + assert fused["calls"] == [p] and composed["calls"] == [] + width = (hq + hk) * PRL_D + assert_within_ulp(fused["qkv"][:, :width], composed["qkv"][:, :width], 2.0, 0.25) + assert torch.equal(fused["qkv"][:, width:], composed["qkv"][:, width:]), "v" + if site == "draft": + assert torch.equal(fused["gate"], composed["gate"]), "the gate columns" + assert_within_ulp(fused["iq"], composed["iq"], 2.0, 0.25) + (roles_f, index_f, pos_f), (roles_c, index_c, pos_c) = fused["planes"], composed["planes"] + assert torch.equal(index_f, index_c), "raw and compressed index keys" + assert torch.equal(pos_f, pos_c), "coordinates" + v_f, v_c = roles_f[:, 2 * PRL_LAYER + 1], roles_c[:, 2 * PRL_LAYER + 1] + assert torch.equal(v_f.view(torch.uint8), v_c.view(torch.uint8)), "the V pool" + others = [s for s in range(2 * PRL_LAYERS) if s != 2 * PRL_LAYER] + assert torch.equal(roles_f[:, others].view(torch.uint8), roles_c[:, others].view(torch.uint8)) + k_f, k_c = roles_f[:, 2 * PRL_LAYER], roles_c[:, 2 * PRL_LAYER] + if kv_dtype == PRL_BF: + assert_within_ulp(k_f, k_c, 2.0, 0.25) + else: + a, b = k_f.view(torch.uint8).int(), k_c.view(torch.uint8).int() + moved = a != b + assert moved.float().mean().item() < 1e-3 + if site == "trunk": + # One bf16 step of k moves an e4m3 rounding by at most one code: the codes' + # magnitudes are monotonic, a sign may differ only next to zero. + mag = ((a & 0x7F) - (b & 0x7F)).abs() + near_zero = ((a & 0x7F) <= 1) & ((b & 0x7F) <= 1) + assert ((mag <= 1) & (((a ^ b) & 0x80 == 0) | near_zero))[moved].all() + if site == "trunk": + # The layer output follows within the selection both paths make from those keys. + o_f, o_c = fused["out"].float(), composed["out"].float() + assert ((o_f - o_c).norm() / o_c.norm()).item() < 1e-2 + + +def _site_rowtile_q_in_place(segments, monkeypatch) -> None: + """q in place on the attention projection row gives the compact row's bits, and the + lone-row kernel (q's row stride a runtime argument) compiles once for both strides.""" + spec = {**_rtl_op.CELLS[0].spec, "segments": tuple(segments), "slots": 16} + assert (spec["q_heads"], spec["kv_heads"], spec["head_dim"]) == (6, 1, 256) + args = _rtl_build(spec, seed=7900 + len(segments)) + q = args["q"] + rows, heads, d = q.shape + # A projection row: q | k | v | gate | index planes, wider than the compact row. + width = (2 * heads + 5) * d + proj = torch.randn(rows, width, device=DEV).to(q.dtype) + proj[:, : heads * d].copy_(q.reshape(rows, heads * d)) + in_place = torch.reshape(proj, [rows, width // d, d])[:, :heads] + assert in_place.stride(0) != q.stride(0) and torch.equal(in_place, q) + dev = torch.cuda.current_device() + compact = _rtl_op(**args) + variants = len(_module._rowtile_lone_kernel.device_caches[dev][0]) + got = _rtl_op(**{**args, "q": in_place}) + assert len(_module._rowtile_lone_kernel.device_caches[dev][0]) == variants, "recompiled" + assert torch.equal(_bits(got), _bits(compact)), "q in place vs compact" + _gqa_compare(got, _rtl_op.reference(**args)) + + +def _site_rowtile_scratch_groups(_, monkeypatch) -> None: + """`_rowtile_scratch_groups`: one launch (None) while the pages fit the scratch, else the + fewest launches of consecutive requests that fit, each request in exactly one.""" + tpb = 32 + page_bytes = 32 * 256 * 4 # one fp8 K/V head of 256 at 32 tokens a page, widened to bf16 + cap_pages = M._FP8_ROWTILE_MAX_SCRATCH // page_bytes + + def md(q_lens, kv_lens): + return types.SimpleNamespace( + num_contexts=len(q_lens), + tokens_per_block=tpb, + seq_lens=torch.tensor(q_lens + [1, 1], dtype=torch.int32), + kv_lens_runtime=torch.tensor(kv_lens + [9, 9], dtype=torch.int32), + ) + + assert M._rowtile_scratch_groups(md([1024] * 4, [68608] * 4), page_bytes) is None + # Sixteen 1024-token chunks at a 67k prefix: 2144 pages each, 34304 in all, past the cap. + q_lens, kv_lens = [1024] * 16, [67584 + 1024] * 16 + groups = M._rowtile_scratch_groups(md(q_lens, kv_lens), page_bytes) + pages = [-(-k // tpb) for k in kv_lens] + assert sum(pages) > cap_pages and groups is not None and len(groups) > 1 + starts = [0] + for q in q_lens: + starts.append(starts[-1] + q) + first = 0 + for a, b, n in groups: + assert n <= cap_pages + last = starts.index(b) + assert a == starts[first] and n == sum(pages[first:last]) + first = last + assert first == len(q_lens) and groups[-1][1] == sum(q_lens) + assert len(groups) == -(-sum(pages) // cap_pages) == 2 + assert [b - a for a, b, _ in groups] == [8192, 8192] + # Uneven requests: still launches that fit, every page in one. + q_lens = [512, 4096, 256, 2048, 1024, 6000, 300, 2000] + kv_lens = [200000, 150000, 180000, 90000, 210000, 120000, 160000, 70000] + groups = M._rowtile_scratch_groups(md(q_lens, kv_lens), page_bytes) + assert groups[0][0] == 0 and groups[-1][1] == sum(q_lens) + assert all(n <= cap_pages for _, _, n in groups) + assert sum(n for _, _, n in groups) == sum(-(-k // tpb) for k in kv_lens) + assert all(prev[1] == nxt[0] for prev, nxt in zip(groups, groups[1:])) + + +_SITE_CASES = [ + *( + pytest.param(_site_prologue, (site, layout), id=f"{site} prologue {layout}") + for site in ("trunk", "draft") + for layout in PRL_LAYOUTS + ), + pytest.param(_site_rowtile_q_in_place, [(5000, 2048)], id="rowtile q in place, one request"), + pytest.param( + _site_rowtile_q_in_place, [(3000, 1040), (7000, 1043)], id="rowtile q in place, lone group" + ), + pytest.param(_site_rowtile_scratch_groups, None, id="rowtile fp8 scratch groups"), +] + + +@pytest.mark.parametrize("check, case", _SITE_CASES) +def test_call_sites(check, case, monkeypatch) -> None: + """What the target's call sites hand the entries beyond a cell.""" + check(case, monkeypatch) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py new file mode 100644 index 000000000000..71baf8bba059 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's hyper-connection entries and their sites' paths.""" + +import zlib +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_rstd import ( # noqa: E501 + qwen4_exp_hc_inject_rstd as rstd_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( # noqa: E501 + qwen4_exp_hc_decode_rows_first as first_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( # noqa: E501 + qwen4_exp_hc_decode_rows_mix as mix_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_tc_first__decode import ( # noqa: E501 + qwen4_exp_hc_decode_tc_first as op, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + assert_within_ulp, + certifying, + validating, +) + +assert torch.cuda.is_available(), "the hyper-connection entries require a CUDA device" + +DEV = "cuda" +HC, HIDDEN, RANK, FIRST_ROWS = 4, 2560, 320, 384 +WIDTH = HC * HIDDEN +EPS = 1e-6 +CORE = SimpleNamespace(hc_count=HC, hc_lowrank=RANK, hc_first_cols=352, eps=EPS) + + +def _cell_seed(cell) -> int: + return zlib.crc32(cell.why.encode()) % (2**31) + + +def _ratio(out, ref) -> float: + """Worst error in bands of one ulp plus `eps / 16` of the max (SiLU cancels near zero).""" + eps = torch.finfo(ref.dtype).eps + r = ref.double() + return ((out.double() - r).abs() / (eps * r.abs() + eps / 16 * r.abs().max())).max().item() + + +def _site(rows, inject, seed): + """One site's normed rows and weights at the checkpoint's statistics.""" + g = torch.Generator(device=DEV).manual_seed(seed) + parts = [torch.randn(RANK, WIDTH, generator=g, device=DEV) * 0.0296] + if inject: + parts.append(torch.randn(HC, WIDTH, generator=g, device=DEV) * 0.0107) + w = torch.cat(parts) + first = torch.cat([w, torch.zeros(FIRST_ROWS - w.shape[0], WIDTH, device=DEV)]) + up = torch.randn(WIDTH, RANK, generator=g, device=DEV) * 0.05 + gamma = torch.randn(WIDTH, generator=g, device=DEV) * 0.477 + x = torch.randn(rows, HC, HIDDEN, generator=g, device=DEV) + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) + normed = (x.flatten(-2) * (1.0 + gamma)).to(torch.bfloat16) + return normed, first.to(torch.bfloat16).t(), up.to(torch.bfloat16).t() + + +def _check_tc_first(cell): + rows, inject = cell.spec["rows"], cell.spec.get("inject", True) + normed, first, _ = _site(rows, inject, _cell_seed(cell)) + args = (normed, first, RANK, HC, inject) + low, inj = op(*args) + assert low.shape == (rows, RANK) + for got, want in zip((low, inj), op.reference(*args)): + assert (got is None) == (want is None) + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape and got.is_contiguous() + assert _ratio(got, want) <= 1.0, f"{_ratio(got, want):.4g}x the band" + + +def _mixer(monkeypatch, cap, normed, first, up, inject): + calls = [] + entry = M.qwen4_exp_hc_decode_tc_first + + def counted(*args, **kwargs): + calls.append(args[0].shape[0]) + return entry(*args, **kwargs) + + with monkeypatch.context() as m: + m.setattr(M, "_HC_TC_FIRST_MAX_ROWS", cap) + m.setattr(M, "qwen4_exp_hc_decode_tc_first", counted) + mixed, inj = M._hc_mixer(CORE, normed, first, up, inject) + return mixed, inj, calls + + +def _mixer_first_halves(rows, inject, monkeypatch): + """`_hc_mixer`'s entry against its cuBLAS path: bf16 roundings of fp32 sums in other orders.""" + normed, first, up = _site(rows, inject, seed=100 + rows) + tc = _mixer(monkeypatch, 640, normed, first, up, inject) + cb = _mixer(monkeypatch, 0, normed, first, up, inject) + assert tc[2] == [rows] and cb[2] == [] + assert_within_ulp(tc[0], cb[0], element_ulp=2.0, rms_ulp=0.25) + if inject: + assert_within_ulp(tc[1], cb[1], element_ulp=2.0, rms_ulp=1.0) + else: + assert tc[1] is None and cb[1] is None + + +def _operands(rows, seed): + """One site in the forward's layouts; `gate` is column 1280 of a [rows, 1288] gate|up row.""" + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + + def rnd(*shape, scale=1.0): + return (torch.randn(*shape, generator=g, device=DEV) * scale).to(bf) + + w = torch.cat( + [ + torch.randn(RANK, WIDTH, generator=g, device=DEV) * 0.0296, + torch.randn(HC, WIDTH, generator=g, device=DEV) * 0.0107, + torch.zeros(FIRST_ROWS - RANK - HC, WIDTH, device=DEV), + ] + ) + return dict( + first=w.to(bf).t(), + up=rnd(WIDTH, RANK, scale=0.05).t(), + norm=rnd(WIDTH, scale=0.477), + hc=rnd(rows, WIDTH, scale=2.0), + block=rnd(rows, HIDDEN, scale=0.5), + shared=rnd(rows, HIDDEN, scale=0.5), + gate=rnd(rows, 1288)[:, 1280:1281], + inject=(2.0 * torch.sigmoid(torch.randn(rows, HC, generator=g, device=DEV))).to(bf), + ) + + +def _site_inputs(o, kind): + blk, inj = (None, None) if kind == "norm" else (o["block"], o["inject"]) + sh, gate = (o["shared"], o["gate"]) if kind == "combine" else (None, None) + return blk, inj, sh, gate + + +def _first_args(o, site, cols): + blk, inj, sh, gate = _site_inputs(o, site) + return (o["hc"], blk, inj, o["norm"], o["first"], HC, cols, EPS, sh, gate) + + +def _check_rows_first(cell): + rows, site, cols = cell.spec["rows"], cell.spec["site"], cell.spec["cols"] + args = _first_args(_operands(rows, _cell_seed(cell)), site, cols) + hc_out, low, inj, rstd = first_op(*args) + ref = first_op.reference(*args) + assert low.shape == (rows, RANK) and rstd.shape == (rows, HC) and rstd.dtype == torch.float32 + assert (inj is None) == (cols == RANK) + if site == "norm": + assert hc_out is args[0], "without a pending block the stream is the input" + assert _ratio(hc_out, ref[0]) <= 1.0, "hc_out" + assert _ratio(low, ref[1]) <= 1.0, f"low {_ratio(low, ref[1]):.3g}x the band" + if inj is not None: + assert _ratio(inj, ref[2]) <= 1.0, f"inject {_ratio(inj, ref[2]):.3g}x the band" + assert ((rstd - ref[3]).abs() / ref[3]).max() <= 1e-6, "rstd in fp32" + # The residual every later site reads: bitwise what `qwen4_exp_hc_inject_norm` writes. + want, _ = qwen4_exp_hc_inject_norm(*args[:4], EPS, HC, *args[8:]) + assert torch.equal(hc_out, want) + + +def _check_rows_mix(cell): + rows = cell.spec["rows"] + o = _operands(rows, _cell_seed(cell)) + with validating(first_op): + hc_out, low, _, rstd = first_op(*_first_args(o, "combine", RANK + HC)) + args = (low, rstd, o["up"], hc_out, o["norm"], HC) + mixed = mix_op(*args) + assert mixed.shape == (rows, HIDDEN) and mixed.is_contiguous() + assert _ratio(mixed, mix_op.reference(*args)) <= 1.0 + + +def _site_run(monkeypatch, lo, o, kind): + """`_hc_site` with the two-launch range starting at `lo`; counts its first launches.""" + calls = [] + entry = M.qwen4_exp_hc_decode_rows_first + + def counted(*args, **kwargs): + calls.append(args[0].shape[0]) + return entry(*args, **kwargs) + + blk, inj, sh, gate = _site_inputs(o, kind) + extra = (sh, gate) if kind == "combine" else () + with monkeypatch.context() as m: + m.setattr(M, "_HC_ROWS_TC_MIN_ROWS", lo) + m.setattr(M, "qwen4_exp_hc_decode_rows_first", counted) + hc, mixed, inject = M._hc_site( + CORE, o["hc"], blk, inj, o["norm"], o["first"], o["up"], kind != "norm", *extra + ) + return hc, mixed, inject, calls + + +def _rows_tc_sites(rows, kind, monkeypatch): + """`_hc_site`'s two launches against `qwen4_exp_hc_inject_norm` + `_hc_mixer`: the residual + bitwise, `mixed` within rounding since the grouped norm reads other bf16 operands.""" + o = _operands(rows, 100 + rows) + two = _site_run(monkeypatch, 1, o, kind) + old = _site_run(monkeypatch, 99, o, kind) + assert two[3] == [rows] and old[3] == [] + assert torch.equal(two[0], old[0]) + assert_within_ulp(two[1], old[1], element_ulp=3.0, rms_ulp=1.0) + if kind == "norm": + assert two[2] is None and old[2] is None + else: + a, b = two[2].double(), old[2].double() + step = torch.finfo(torch.bfloat16).eps * torch.maximum(a.abs(), b.abs()) + assert ((a - b).abs() <= step).all(), (a - b).abs().max().item() + + +def _mixer_rstd(rows, inject): + """The prefill rstd site (normed stream formed on chip in fp32) against `_hc_mixer` on the + stored bf16 stream; the planted |1 + gamma| = 7.6 column sets the 4-ulp element bound.""" + o = _operands(rows, 900 + rows) + o["norm"][0], o["norm"][WIDTH // 2] = 6.625, -6.625 # this checkpoint's largest |gamma| + hc, rstd = rstd_op(o["hc"], o["block"], o["inject"], EPS, HC) + hc2, normed = qwen4_exp_hc_inject_norm(o["hc"], o["block"], o["inject"], o["norm"], EPS, HC) + assert torch.equal(hc, hc2) + w1 = {o["norm"].data_ptr(): (1.0 + o["norm"].float()).contiguous()} + core = SimpleNamespace(**vars(CORE), _hc_norm_w1=w1) + a = M._hc_mixer_rstd(core, hc, rstd, o["norm"], o["first"], o["up"], inject) + b = M._hc_mixer(core, normed, o["first"], o["up"], inject) + assert_within_ulp(a[0], b[0], element_ulp=4.0, rms_ulp=1.0) + if inject: + assert_within_ulp(a[1], b[1], element_ulp=2.0, rms_ulp=1.0) + else: + assert a[1] is None and b[1] is None + + +def _tp_partial(): + """`_tp_partial` defers the sum only where the next site's fused all-reduce can take it; + `_hc_site` sums a partial it cannot fuse with `tp_sum` first, bitwise the summed call.""" + calls = [] + + def tp_sum(x): + calls.append(x.shape[0]) + return (x.float() * 0.75).to(x.dtype) + + lo = M._HC_AR_INJECT_MIN_ROWS + x_lo = torch.zeros(lo, HIDDEN, dtype=torch.bfloat16, device=DEV) + x16 = torch.zeros(16, HIDDEN, dtype=torch.bfloat16, device=DEV) + off = SimpleNamespace(tp_sum=tp_sum) + on = SimpleNamespace(hc_ar_inject=object(), tp_sum=tp_sum) + assert not isinstance(M._tp_partial(off, x_lo), M._TpPartial) + assert isinstance(M._tp_partial(on, x_lo), M._TpPartial) + assert not isinstance(M._tp_partial(on, x16), M._TpPartial) + assert calls == [lo, 16] + o = _operands(lo, 77) + core = SimpleNamespace(**vars(CORE), tp_sum=tp_sum) + for sh in ((o["shared"], o["gate"]), ()): + rest = (o["norm"], o["first"], o["up"], True, *sh) + a = M._hc_site(core, o["hc"], M._TpPartial(o["block"]), o["inject"], *rest) + b = M._hc_site(core, o["hc"], tp_sum(o["block"]), o["inject"], *rest) + for x, y in zip(a, b): + assert torch.equal(x, y) + + +def _inject_norm_launch(rows, kind): + """`_hc_inject_norm_launch` against the family op: residual bitwise, normed stream within one + rounding at the site's warps; all bitwise at eight, eager and replayed behind a producer.""" + + def inputs(seed): + o = _operands(rows, seed) + blk, inj, sh, gate = _site_inputs(o, kind) + return o, (o["hc"], blk, inj, o["norm"], EPS, HC, sh, gate) + + _, args = inputs(500 + rows) + hc8, normed8 = qwen4_exp_hc_inject_norm(*args) + for warps in (M._HC_INJECT_WARPS, 2): + hc_w, normed_w = M._hc_inject_norm_launch(*args, warps) + assert torch.equal(hc_w, hc8) + assert_within_ulp(normed_w, normed8, element_ulp=1.0, rms_ulp=0.25) + o, args = inputs(700 + rows) + hc8, normed8 = qwen4_exp_hc_inject_norm(*args) + hc_l, normed_l = M._hc_inject_norm_launch(*args, 8) + assert torch.equal(hc_l, hc8) and torch.equal(normed_l, normed8) + src = o["hc"].clone() + hc_in = src.clone() + M._hc_inject_norm_launch(hc_in, *args[1:], 8) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + hc_in.copy_(src * 1.5) + hc_g, normed_g = M._hc_inject_norm_launch(hc_in, *args[1:], 8) + graph.replay() + torch.cuda.synchronize() + hc_e, normed_e = M._hc_inject_norm_launch(src * 1.5, *args[1:], 8) + torch.cuda.synchronize() + assert torch.equal(hc_g, hc_e) and torch.equal(normed_g, normed_e) + + +_CELL_CHECKS = ((op, _check_tc_first), (first_op, _check_rows_first), (mix_op, _check_rows_mix)) +CASES = [(entry, check, cell) for entry, check in _CELL_CHECKS for cell in entry.CELLS] + + +@pytest.mark.parametrize( + "entry, check, cell", CASES, ids=[f"{type(e).__name__}-{c.why[:40]}" for e, _, c in CASES] +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + certifying(entry) + check(cell) + + +_KINDS = ("combine", "inject", "norm") +#: `(path, rows, variant)`: `inject` for the mixers, the site kind for the decode sites. +# fmt: off +_SITE_CASES = ( + [("mixer_first_halves", r, i) for r in (4, 64, 288, 576) for i in (True, False)] + + [("rows_tc_sites", r, k) for r in (1, 8, 16) for k in _KINDS] + + [("mixer_rstd", r, i) for r in (3072, 8192) for i in (True, False)] + + [("tp_partial", 0, None)] + + [("inject_norm_launch", r, k) for r in (17, 192, 288) for k in _KINDS] +) +# fmt: on + + +@pytest.mark.parametrize("path, rows, variant", _SITE_CASES, ids=lambda v: str(v)) +def test_hc_site_paths(path, rows, variant, monkeypatch) -> None: + """Site paths no certified cell reaches, each against its alternative.""" + if path == "mixer_first_halves": + _mixer_first_halves(rows, variant, monkeypatch) + elif path == "rows_tc_sites": + _rows_tc_sites(rows, variant, monkeypatch) + elif path == "mixer_rstd": + _mixer_rstd(rows, variant) + elif path == "tp_partial": + _tp_partial() + else: + _inject_norm_launch(rows, variant) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py new file mode 100644 index 000000000000..18ef562a97c1 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py @@ -0,0 +1,360 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""In-forward Mamba snapshots in the FP8 unified target's prefill: one step over [.., L) leaves +the state at P = L - tail in the live slot and stages the state at L, checked against the main +chunk [.., P) then the tail [P, L) alone in the next step, at a tep2 rank's GDN geometry.""" + +from __future__ import annotations + +import types + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ( # noqa: E501 + CKPT_MULTIPLIERS, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( # noqa: E501 + ngram_token_history, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_gate_conv_inject_norm import ( # noqa: E501 + CKPT_CLAMP_MIN, + CKPT_DILATION, + CKPT_HC_COUNT, + CKPT_HIDDEN, + ple_gate_conv_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _m, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm import ( # noqa: E501 + flashinfer_chunk_gated_delta_rule as fi, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_inforward_tail import ( # noqa: E501 + gdn_inforward_tail_reference, +) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata +from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import InforwardSnapshotEntry + +assert torch.cuda.is_available(), "the in-forward snapshot test requires a CUDA device" + +DEV = "cuda" +H, HV, DK, DV = fi.RANK2_GEOMETRY +CONV_DIM, D_CONV, SLOTS, ROWS, EOS = 2 * H * DK + HV * DV, 4, 24, 8, 2 # ROWS: staging rows +NGRAM, PLE_STATE, STREAM = len(CKPT_MULTIPLIERS), 3 * CKPT_DILATION, CKPT_HC_COUNT * CKPT_HIDDEN + +# (request id, live slot, chunk length, carries a state, tail or None). One request: CP pipeline. +BATCHES = { + "1req": [(31, 4, 3000, True, 2)], + "1fresh": [(32, 9, 16384, False, 2)], + "3req": [(11, 5, 300, True, None), (12, 2, 1000, True, 2), (13, 7, 150, False, 2)], + "8req": [(21, 3, 96, True, 2), (22, 9, 700, True, None), (23, 1, 67, True, 2), + (24, 17, 2048, False, None), (25, 12, 130, True, 2), (26, 0, 515, True, 2), + (27, 20, 64, True, None), (28, 6, 1500, False, 2)], +} # fmt: skip + + +def _batch_split(batch): + plan, row = {}, 0 + for rid, _, length, _, tail in batch: + if tail is not None: + plan[rid] = InforwardSnapshotEntry(row, length - tail, length - tail, tail) + row += 1 + slots = {rid: slot for rid, slot, *_ in batch} + manager = types.SimpleNamespace( + conv_state_shape=[CONV_DIM, D_CONV - 1], + inforward_snapshot_rows=ROWS, + inforward_snapshot_max_tail=2, + inforward_snapshot_plan=plan, + state_index=lambda rid: slots[rid], + ) + attn = types.SimpleNamespace( + request_ids=[rid for rid, *_ in batch], + seq_lens=torch.tensor([length for _, _, length, _, _ in batch], dtype=torch.int32), + ) + metadata = Mamba2Metadata(max_batch_size=32, chunk_size=128) + metadata._prepare_inforward_split(attn, manager, plan, len(batch), len(batch)) + return metadata + + +class _Rows: + def __init__(self, batch): + self.main, self.tail, self.main_cu, self.tail_cu = [], [], [0], [0] + self.tail_slots, self.starts = [], [] + start = 0 + for _, slot, length, _, tail in batch: + before = length - (tail or 0) + self.starts.append(start) + self.main += range(start, start + before) + self.main_cu.append(self.main_cu[-1] + before) + if tail: + self.tail += range(start + before, start + length) + self.tail_cu.append(self.tail_cu[-1] + tail) + self.tail_slots.append(slot) + start += length + self.total = start + self.slots = [slot for _, slot, *_ in batch] + self.carry = [c for *_, c, _ in batch] + self.split = [(slot, start, length - tail, start + length) for (_, slot, length, _, tail), start + in zip(batch, self.starts) if tail] # fmt: skip + + def t(self, values, dtype=torch.long): + return torch.tensor(values, dtype=dtype, device=DEV) + + +def _fake_target(metadata, batch, pools, stage, input_ids=None): + core = types.SimpleNamespace(_bound=types.SimpleNamespace(pools=pools, stage=stage)) + target = types.SimpleNamespace(core=core, _step=None) + hi = torch.tensor([c for *_, c, _ in batch], dtype=torch.bool, device=DEV) + total = sum(length for _, _, length, _, _ in batch) + split = _m.PrefillTarget._split_of(target, metadata, len(batch), 0, hi, total) + target._step = types.SimpleNamespace(split=split, input_ids=input_ids) + return target + + +def _strided_pool(gen, *shape, dtype=torch.bfloat16): + """`[SLOTS, *shape]` whose slot stride is two states: the coalesced page's layout.""" + page = (torch.randn(SLOTS, 2, *shape, generator=gen, device=DEV) * 0.1).to(dtype) + return page[:, 1] + + +def _stage(gdn): + bf = torch.bfloat16 + ssm = {0: torch.full((ROWS, HV, DV, DK), 7.0, dtype=bf, device=DEV)} if gdn else {} + conv = {0: torch.full((ROWS, CONV_DIM, D_CONV - 1), 7.0, dtype=bf, device=DEV)} if gdn else {} + return _m._InforwardStage( + ssm=ssm, + conv=conv, + ple_conv=torch.full((ROWS, STREAM, PLE_STATE), 7.0, dtype=bf, device=DEV), + ple_ngram=torch.full((ROWS, NGRAM - 1), -1, dtype=torch.long, device=DEV), + ) + + +def _gdn_inputs(total, gen): + q = torch.randn(1, total, H, DK, generator=gen, device=DEV) + k = torch.randn(1, total, H, DK, generator=gen, device=DEV) + q = (q / q.norm(dim=-1, keepdim=True)).bfloat16() + k = (k / k.norm(dim=-1, keepdim=True)).bfloat16() + v = torch.randn(1, total, HV, DV, generator=gen, device=DEV).bfloat16() + g = torch.exp(-torch.rand(1, total, HV, generator=gen, device=DEV) * 0.2) + beta = torch.rand(1, total, HV, generator=gen, device=DEV) + return q, k, v, g, beta + + +def _fi_step(qkvgb, sel, pool, slots, cu, carry): + out, _ = fi.flashinfer_chunk_gated_delta_rule( + *(x.index_select(1, sel) for x in qkvgb), scale=None, initial_state=pool, + initial_state_indices=slots, inplace_indexed_state_update=True, output_final_state=False, + cu_seqlens=cu, head_first=False, use_qk_l2norm_in_kernel=False, output=None, + state_workspace=None, has_initial_state=carry, linear_gate=True, + ) # fmt: skip + return out + + +def _ple_inputs(gen, n): + def rand(*shape, scale=1.0): + return (torch.randn(*shape, generator=gen, device=DEV) * scale).bfloat16() + + hc, key, value = rand(n, STREAM, scale=2.0), rand(n, STREAM), rand(n, CKPT_HIDDEN, scale=0.5) + norms = [rand(STREAM, scale=0.5) for _ in range(4)] + return hc, key, value, norms, rand(STREAM, 1, 4, scale=0.3) + + +def _ple(inputs, sel, pool, slots, cu, carry, max_len, **kw): + """`ple_gate_conv_inject_norm` over rows `sel` (None: every row) at the checkpoint's shapes.""" + hc, key, value, norms, conv_w = inputs + if sel is not None: + hc, key, value = (t.index_select(0, sel) for t in (hc, key, value)) + return ple_gate_conv_inject_norm( + hc, key, value, norms[0], norms[1], norms[2], conv_w, pool, slots, cu, norms[3], 1e-6, + CKPT_HC_COUNT, CKPT_CLAMP_MIN, CKPT_DILATION, -1, max_len, carry, **kw, + ) # fmt: skip + + +def _gdn_split(batch, metadata, rows): + """`_gdn_split`: the state at P, main rows and conv windows bitwise the main step's; the tails' + outputs and staged states within rounding of the tail step and the float64 reference.""" + gen = torch.Generator(device=DEV).manual_seed(7 + len(batch)) + qkvgb = _gdn_inputs(rows.total, gen) + ssm = _strided_pool(gen, HV, DV, DK) + conv = _strided_pool(gen, CONV_DIM, D_CONV - 1) + proj = torch.randn(rows.total, CONV_DIM + 96, generator=gen, device=DEV).bfloat16() + mixed_qkv = proj[:, :CONV_DIM] + main, tail = rows.t(rows.main), rows.t(rows.tail) + tail_slots = rows.t(rows.tail_slots) + tail_carry = torch.ones(len(rows.tail_slots), dtype=torch.bool, device=DEV) + + ref_ssm = ssm.clone() + slots32, carry = rows.t(rows.slots, torch.int32), rows.t(rows.carry, torch.bool) + out_main = _fi_step(qkvgb, main, ref_ssm, slots32, rows.t(rows.main_cu), carry) + state_p = ref_ssm.clone() + tail32 = rows.t(rows.tail_slots, torch.int32) + out_tail = _fi_step(qkvgb, tail, ref_ssm, tail32, rows.t(rows.tail_cu), tail_carry) + state_l = ref_ssm.index_select(0, tail_slots) + + for slot, start, _, end in rows.split: + conv[slot] = mixed_qkv[end - (D_CONV - 1) : end].t() + conv_before = conv.clone() + stage = _stage(gdn=True) + pools = types.SimpleNamespace(ssm={0: ssm}, conv={0: conv}) + target = _fake_target(metadata, batch, pools, stage) + out = torch.full((1, rows.total, HV, DV), -7.0, dtype=torch.bfloat16, device=DEV) + _m.PrefillTarget._gdn_split(target, *qkvgb, mixed_qkv, 0, out) + torch.cuda.synchronize() + + assert torch.equal(ssm, state_p), "live slots differ from the main step's states" + assert torch.equal(out[0].index_select(0, main), out_main[0]), "main rows' outputs differ" + nsplit = len(rows.split) + staged = stage.ssm[0][:nsplit] + got_tail = out[0].index_select(0, tail) + torch.testing.assert_close(staged.float(), state_l.float(), rtol=2e-2, atol=2e-2) + torch.testing.assert_close(got_tail.float(), out_tail[0].float(), rtol=2e-2, atol=2e-2) + ref_stage, ref_out = torch.zeros_like(stage.ssm[0]), torch.zeros_like(out) + sp = target._step.split + gdn_inforward_tail_reference( + *qkvgb, sp.tail_tok, sp.cu_tail, state_p, sp.tail_slots, ref_stage, sp.stage_rows, ref_out + ) + torch.testing.assert_close(staged.float(), ref_stage[:nsplit].float(), rtol=1e-2, atol=1e-2) + torch.testing.assert_close( + got_tail.float(), ref_out[0].index_select(0, tail).float(), rtol=1e-2, atol=1e-2 + ) + for row, (slot, start, before, end) in enumerate(rows.split): + p = start + before + assert torch.equal(conv[slot], mixed_qkv[p - (D_CONV - 1) : p].t()) + assert torch.equal(stage.conv[0][row], conv_before[slot]) + others = [s for s in range(SLOTS) if s not in rows.tail_slots] + assert torch.equal(conv[others], conv_before[others]) + + +def _ple_split(batch, metadata, rows): + """`_split_ple_states`: PLE conv slots and n-gram histories bitwise the two-step schedule's.""" + gen = torch.Generator(device=DEV).manual_seed(100 + len(batch)) + n = rows.total + inputs = _ple_inputs(gen, n) + conv_pool = torch.randn(SLOTS, STREAM, PLE_STATE, generator=gen, device=DEV).bfloat16() + ids = torch.randint(3, 50000, (n,), generator=gen, device=DEV, dtype=torch.int32) + ngram_pool = torch.randint(3, 50000, (SLOTS, NGRAM - 1), generator=gen, device=DEV) + carry = rows.t(rows.carry, torch.bool) + slots32 = rows.t(rows.slots, torch.int32) + + def conv_step(pool, sel, cu, slots, carries, conv_input=None): + max_len = int((cu[1:] - cu[:-1]).max()) + _ple(inputs, sel, pool, slots, cu, carries, max_len, conv_input=conv_input) + + main, tail = rows.t(rows.main), rows.t(rows.tail) + tail_slots32 = rows.t(rows.tail_slots, torch.int32) + tail_carry = torch.ones(len(rows.tail_slots), dtype=torch.bool, device=DEV) + ref_conv = conv_pool.clone() + conv_step(ref_conv, main, rows.t(rows.main_cu), slots32, carry) + conv_p = ref_conv.clone() + conv_step(ref_conv, tail, rows.t(rows.tail_cu), tail_slots32, tail_carry) + _, ngram_p = ngram_token_history.reference( + ids.index_select(0, main), ngram_pool, slots32, carry, rows.t(rows.main_cu), EOS, NGRAM + ) + _, ngram_l = ngram_token_history.reference( + ids.index_select(0, tail), ngram_p, tail_slots32, tail_carry, rows.t(rows.tail_cu), EOS, + NGRAM, + ) # fmt: skip + + whole_cu = rows.t([0] + [s + length for s, (_, _, length, _, _) in zip(rows.starts, batch)]) + conv_input = torch.empty(n, STREAM, dtype=torch.bfloat16, device=DEV) + conv_step(conv_pool, None, whole_cu, slots32, carry, conv_input=conv_input) + _, ngram_whole = ngram_token_history.reference( + ids, ngram_pool, slots32, carry, whole_cu, EOS, NGRAM + ) + ngram_pool = ngram_whole.clone() + stage = _stage(gdn=False) + target = _fake_target(metadata, batch, types.SimpleNamespace(), stage, input_ids=ids) + _m.PrefillTarget._split_ple_states(target, conv_pool, ngram_pool, conv_input) + torch.cuda.synchronize() + + k = len(rows.split) + assert torch.equal(conv_pool, conv_p), "PLE conv slots differ from the main step's" + assert torch.equal(stage.ple_conv[:k], ref_conv.index_select(0, rows.t(rows.tail_slots))) + assert torch.equal(ngram_pool, ngram_p), "n-gram histories differ from the main step's" + assert torch.equal(stage.ple_ngram[:k], ngram_l.index_select(0, rows.t(rows.tail_slots))) + + +@pytest.mark.parametrize("state", ["gdn", "ple"]) +@pytest.mark.parametrize("name", list(BATCHES)) +def test_split_states_are_the_two_iteration_schedule(name, state) -> None: + batch = BATCHES[name] + (_gdn_split if state == "gdn" else _ple_split)(batch, _batch_split(batch), _Rows(batch)) + + +def _padded_gdn_call(rows, padded, seqs, zero_pad) -> None: + """`_gdn_core_graph`'s CP and carried calls over a bucket's `padded` rows are bitwise the + unpadded call; rows past the batch are zero or, without `zero_pad`, values never read.""" + gen = torch.Generator(device=DEV).manual_seed(rows + padded) + q, k, v, g, beta = _gdn_inputs(padded, gen) + lens = [rows // seqs] * seqs + lens[-1] += rows - sum(lens) + cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.long, device=DEV) + slots = torch.tensor([3 + 5 * i for i in range(seqs)], dtype=torch.int32, device=DEV) + carry = torch.tensor([i % 2 == 0 for i in range(seqs)], dtype=torch.bool, device=DEV) + dev = torch.device(DEV) + cp = fi._reaches_cp(seqs, HV, dev) + keep = (torch.arange(padded, device=DEV) < rows).view(1, padded) + padded_in = [ + torch.where(keep.view(1, padded, *([1] * (t.dim() - 2))), t, 0) if zero_pad else t + for t in (q, k, v, g, beta) + ] + chunk = fi.cp_chunk_len_for(rows, max(H, HV), dev) if cp else None + fi_kwargs = {"_cp_chunk_len": chunk, "max_seqlen": padded} if cp else None + for path in [fi._chunk_carried] + ([fi._chunk_cp_inplace] if cp else []): + results = [] + for unpadded in (True, False): + pool = _strided_pool(torch.Generator(device=DEV).manual_seed(5), HV, DV, DK) + out = torch.full((1, padded, HV, DV), -7.0, dtype=torch.bfloat16, device=DEV) + if unpadded: + path( + *(t[:, :rows] for t in (q, k, v, g, beta)), None, pool, slots, carry, cu, + out[:, :rows], True, + ) # fmt: skip + else: + path(*padded_in, None, pool, slots, carry, cu, out, True, fi_kwargs=fi_kwargs) + results.append((out[:, :rows].clone(), pool.clone())) + torch.cuda.synchronize() + assert torch.equal(results[0][0], results[1][0]), f"{path.__name__}: outputs" + assert torch.equal(results[0][1], results[1][1]), f"{path.__name__}: states" + + +def _ple_publish_end() -> None: + """`publish_end=k` (`DecodeTarget._ple_from_pool`): outputs bitwise the full launch's; conv + pool rows of sequences below `k` published, the later ones left exactly as they were.""" + gen = torch.Generator(device=DEV).manual_seed(7) + # A 5-row context-shaped prefix, then three 3-row verify requests. + inputs = _ple_inputs(gen, 14) + pool0 = torch.randn(SLOTS, STREAM, PLE_STATE, generator=gen, device=DEV).bfloat16() + slots = torch.tensor([2, 5, 7, 9], dtype=torch.int32, device=DEV) + cu = torch.tensor([0, 5, 8, 11, 14], dtype=torch.int32, device=DEV) + carry = torch.ones(4, dtype=torch.bool, device=DEV) + outs, pools = [], [pool0.clone(), pool0.clone()] + for publish_end, pool in zip((None, 1), pools): + outs.append(_ple(inputs, None, pool, slots, cu, carry, 5, publish_end=publish_end)) + torch.cuda.synchronize() + assert torch.equal(outs[0][0], outs[1][0]) and torch.equal(outs[0][1], outs[1][1]), "outputs" + assert torch.equal(pools[1][2], pools[0][2]), "the sequence below publish_end publishes" + for slot in (5, 7, 9): + assert torch.equal(pools[1][slot], pool0[slot]), f"slot {slot} must stay unpublished" + assert not torch.equal(pools[0][slot], pool0[slot]), "the full launch publishes it" + others = [s for s in range(SLOTS) if s not in (2, 5, 7, 9)] + assert torch.equal(pools[1][others], pool0[others]) + + +# (eager rows, padded bucket rows, sequences): every FlashInfer CP chunk-length class below 4096. +PADDED_CALLS = [ + (126, 128, 1), (300, 512, 1), (1000, 1024, 1), (1300, 2048, 1), (2046, 2048, 1), + (3070, 4096, 1), (900, 1024, 2), (3000, 4096, 3), +] # fmt: skip +_VARIANTS = [ + (f"padded-{r}-{p}-{s}-{'zero' if z else 'stale'}", _padded_gdn_call, (r, p, s, z)) + for r, p, s in PADDED_CALLS + for z in (True, False) +] + [("ple_publish_end", _ple_publish_end, ())] + + +@pytest.mark.parametrize("helper,args", [v[1:] for v in _VARIANTS], ids=[v[0] for v in _VARIANTS]) +def test_launch_variants_are_bitwise_the_plain_launch(helper, args) -> None: + """A padded CUDA-graph GDN call and a partly publishing PLE launch against the plain ones.""" + helper(*args) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py new file mode 100644 index 000000000000..5c84c8777a1e --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py @@ -0,0 +1,1000 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's MoE and dense entries and the paths no cell reaches.""" + +import zlib +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_post_inject import ( # noqa: E501 + qwen4_exp_post_inject, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( # noqa: E501 + renorm_moe_routing_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.fp8_quantize_1x128 import ( # noqa: E501 + fp8_quantize_1x128, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.embedding.embed_rows__decode import ( # noqa: E501 + embed_rows, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + low_m_bf16_mm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + plan as low_m_plan, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + fp8_block_scale_moe_runner as _runner_mod, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + moe_prefill_grouped__prefill as _grouped, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + moe_tpb_decode__decode as _tpb, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.fp8_block_scale_moe_runner import ( # noqa: E501 + BLOCK, + WIDE_DECODE_MIN_ROWS, + WIDE_DECODE_SMALL_TACTICS, + WIDE_DECODE_TACTIC, + fp8_block_scale_moe_front, + fp8_block_scale_moe_runner, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.moe_front_router_quant__decode import ( # noqa: E501 + moe_front_router_quant, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.mtp_moe_combine_inject__decode import ( # noqa: E501 + mtp_moe_combine_inject, +) +from tensorrt_llm._torch.moe.fused_shared_expert import fused_sigmoid_gate_mul_add + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + ULP, + assert_within_ulp, + certifying, +) + +assert torch.cuda.is_available(), "the MoE and dense entries require a CUDA device" + +DEV = "cuda" +_BF16_ULP = 2.0**-8 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) + + +# ------------------------------------------------------- fp8_block_scale_moe_runner --- + + +def _moe_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """Gate in bf16 ulp of each row's max: 16 element-wise, 0.75 RMS. Kernel and reference differ + only by accumulation order and FC1 sums at e4m3 rounding boundaries (measured 6.65 / 0.306).""" + assert out.dtype == ref.dtype == torch.bfloat16, (out.dtype, ref.dtype) + assert out.shape == ref.shape, (out.shape, ref.shape) + o, r = out.float(), ref.float() + rowmax = r.abs().amax(dim=-1, keepdim=True).clamp(min=1e-6) + d = (o - r) / (rowmax * 2.0**-8) + elt, rms = d.abs().max().item(), d.pow(2).mean().sqrt().item() + assert elt <= 16.0, f"{elt:.2f} ulp element-wise > 16" + assert rms <= 0.75, f"{rms:.3f} ulp RMS > 0.75" + + +CKPT = dict(hidden=2560, inter=640, num_experts=512, top_k=10) + +#: The arguments a step writes: what a captured graph reads afresh on each replay. +STEP_INPUTS = ("hidden_states", "hidden_states_scale", "topk_weights", "topk_ids") + +_WEIGHTS: dict = {} + + +def _weights(hidden: int, inter: int, experts: int, seed: int): + """`(w1, w1s, w2, w2s)` drawn directly in e4m3 / fp32, so every value is exact. Cached: the + checkpoint stack is 2.5 GB. Drawn 32 experts at a time to cap the fp32 transient.""" + key = (hidden, inter, experts, seed) + if key not in _WEIGHTS: + g = torch.Generator(device=DEV).manual_seed(seed) + w1 = torch.empty(experts, 2 * inter, hidden, dtype=torch.float8_e4m3fn, device=DEV) + w2 = torch.empty(experts, hidden, inter, dtype=torch.float8_e4m3fn, device=DEV) + for lo in range(0, experts, 32): + hi = min(lo + 32, experts) + w1[lo:hi] = torch.randn(hi - lo, 2 * inter, hidden, device=DEV, generator=g) + w2[lo:hi] = torch.randn(hi - lo, hidden, inter, device=DEV, generator=g) + w1s = 0.01 + 0.02 * torch.rand( + experts, 2 * inter // BLOCK, hidden // BLOCK, device=DEV, generator=g + ) + w2s = 0.01 + 0.02 * torch.rand( + experts, hidden // BLOCK, inter // BLOCK, device=DEV, generator=g + ) + _WEIGHTS[key] = (w1, w1s, w2, w2s) + return _WEIGHTS[key] + + +def _stack_window(offset: int, width: int): + """`(w1, w1s, w2, w2s)` of experts `[offset, offset + width)` of the checkpoint-shape stack.""" + win = slice(offset, offset + width) + return tuple(t[win] for t in _weights(2560, 640, 512, seed=2560 + 640 + 512)) + + +def _kwargs(hs, hs_scale, weights, offset, local, num_experts=512, top_k=10, inter=640, **routing): + w1, w1s, w2, w2s = weights + # fmt: off + base = dict( + routing_logits=None, routing_bias=None, hidden_states=hs, hidden_states_scale=hs_scale, + gemm1_weights=w1, gemm1_weights_scale=w1s, gemm2_weights=w2, gemm2_weights_scale=w2s, + num_experts=num_experts, top_k=top_k, num_fused_shared_experts=None, n_group=None, + topk_group=None, intermediate_size=inter, local_expert_offset=offset, + local_num_experts=local, routed_scaling_factor=None, routing_method_type=1, + topk_weights=None, topk_ids=None, gemm1_clamp_limit=None, + ) + # fmt: on + return dict(base, **routing) + + +def _runner_kwargs(xq, xsf, offset, width, **routing) -> dict: + return _kwargs(xq, xsf, _stack_window(offset, width), offset, width, **routing) + + +def _args( + tokens, hidden, inter, num_experts, top_k, local_expert_offset=0, local_num_experts=None, seed=0 +): + """Pre-routed op arguments over a window of one stack; one seed's windows split one layer.""" + local = num_experts if local_num_experts is None else local_num_experts + window = slice(local_expert_offset, local_expert_offset + local) + stack = _weights(hidden, inter, num_experts, seed=hidden + inter + num_experts) + g = torch.Generator(device=DEV).manual_seed(seed) + hs = (2.0 * torch.randn(tokens, hidden, device=DEV, generator=g)).to(torch.float8_e4m3fn) + # [H/128, T]: the transposed layout fp8_quantize_1x128 returns here. + hs_scale = 0.5 + torch.rand(hidden // BLOCK, tokens, device=DEV, generator=g) + logits = torch.randn(tokens, num_experts, device=DEV, generator=g) + weights, ids = torch.topk(torch.softmax(logits, dim=-1), top_k, dim=-1) + return _kwargs( + hs, + hs_scale, + tuple(t[window] for t in stack), + local_expert_offset, + local, + num_experts, + top_k, + inter, + topk_weights=weights.to(torch.bfloat16).contiguous(), + topk_ids=ids.to(torch.int32).contiguous(), + ) + + +def _window_args(cell, salt: int = 0) -> tuple[dict, torch.Tensor]: + """The cell's arguments and in-window slots, redrawn while the routing misses the window.""" + geometry = {k: v for k, v in cell.spec.items() if k != "capture"} + for attempt in range(8): + args = _args(**geometry, seed=_seed(cell.why) + salt + attempt) + hit = _in_window(args) + if hit.any(): + return args, hit + raise AssertionError(f"no slot routes into the window: {cell.why!r} would certify nothing") + + +def _in_window(args) -> torch.Tensor: + """`[T, top_k]`: which slots the call's window answers for.""" + local = args["topk_ids"].long() - args["local_expert_offset"] + return (local >= 0) & (local < args["local_num_experts"]) + + +def _moe_check(out: torch.Tensor, args: dict, hit: torch.Tensor) -> None: + assert out.shape == args["hidden_states"].shape and out.dtype == torch.bfloat16 + miss = ~hit.any(-1) + assert torch.equal(out[miss], torch.zeros_like(out[miss])) + _moe_compare(out, fp8_block_scale_moe_runner.reference(**args)) + + +def _warm_on_side_stream(fn) -> None: + """One eager call on a side stream before a capture, as `torch.cuda.graph` asks.""" + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + fn() + torch.cuda.current_stream().wait_stream(side) + + +def _check_moe_runner(cell): + args, hit = _window_args(cell) + out = fp8_block_scale_moe_runner(**args) + _moe_check(out, args, hit) + if not cell.spec.get("capture"): + return + static = {k: (v.clone() if k in STEP_INPUTS else v) for k, v in args.items()} + _warm_on_side_stream(lambda: fp8_block_scale_moe_runner(**static)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + replayed = fp8_block_scale_moe_runner(**static) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(replayed, out), "the replay differs from the eager call it captured" + second, hit2 = _window_args(cell, salt=1 << 20) + assert not torch.equal(second["topk_ids"], args["topk_ids"]), "the second step reroutes" + for name in STEP_INPUTS: + static[name].copy_(second[name]) + graph.replay() + torch.cuda.synchronize() + step2 = replayed.clone() + assert torch.equal(step2, fp8_block_scale_moe_runner(**second)), "second replay vs eager" + _moe_check(step2, second, hit2) + + +def _windows_sum(width, tokens): + """A member's windows summed in fp32 pass the 512-expert gate; without any one they fail.""" + seed = 900 + tokens + width + offsets = range(0, CKPT["num_experts"], width) + whole = _args(tokens, **CKPT, seed=seed) + parts = [ + fp8_block_scale_moe_runner( + **_args(tokens, **CKPT, local_expert_offset=o, local_num_experts=width, seed=seed) + ) + for o in offsets + ] + ref = fp8_block_scale_moe_runner.reference(**whole) + _moe_compare(torch.stack([p.float() for p in parts]).sum(0).to(torch.bfloat16), ref) + ids = whole["topk_ids"].long() + for i, o in enumerate(offsets): + if not ((ids >= o) & (ids < o + width)).any(): + continue + short = torch.stack([p.float() for j, p in enumerate(parts) if j != i]).sum(0) + with pytest.raises(AssertionError): + _moe_compare(short.to(torch.bfloat16), ref) + + +def _pad_window(offset: int, width: int): + return M._weights.ExpertWindow(*_stack_window(offset, width), offset=offset, count=width) + + +def _padded_rows(bucket: int, live: int, seed: int) -> torch.Tensor: + """`[bucket, 2560]` bf16: `live` random rows, then the zero rows of a bucket's padding.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.zeros(bucket, 2560, dtype=torch.bfloat16, device=DEV) + x[:live] = torch.randn(live, 2560, device=DEV, generator=g).to(torch.bfloat16) + return x + + +def _padding_rows(offset, width): + """A prefill graph's padding rows (`PrefillTarget._pad_mask`) route to expert -1 and come out + zero, real rows bitwise unchanged, eager and replayed; unmasked they tie onto the lowest ids.""" + core = SimpleNamespace(topk=10, num_experts=512, moe_inter=640) + g = torch.Generator(device=DEV).manual_seed(77) + router = (0.02 * torch.randn(512, 2560, device=DEV, generator=g)).to(torch.bfloat16).t() + win = _pad_window(offset, width) + bucket, live = 2048, 1500 + x = _padded_rows(bucket, live, seed=5) + pad_ids, _ = renorm_moe_routing_op( + cublas_mm(x[live:], router), topk=10, output_dtype=torch.bfloat16 + ) + assert int(pad_ids.max()) < 10, "padding rows no longer tie onto the lowest expert ids" + + plain = M._routed_experts(core, x, router, win) + mask = torch.unsqueeze(torch.arange(bucket, device=DEV) >= live, 1) + masked = M._routed_experts(core, x, router, win, pad_mask=mask) + assert torch.equal(masked[:live], plain[:live]), "the mask changed a real row" + assert not masked[live:].any() and not plain[live:].any(), "a padding row is not zero" + + # As the prefill graph captures it: the mask from a device row count rewritten each step. + live_dev = torch.full([1], live, dtype=torch.int32, device=DEV) + xs = x.clone() + + def body(): + rows = torch.arange(bucket, dtype=torch.int32, device=DEV) + return M._routed_experts( + core, xs, router, win, pad_mask=torch.unsqueeze(rows >= live_dev, 1) + ) + + _warm_on_side_stream(body) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = body() + for n, seed in ((live, 5), (700, 6)): + xs.copy_(_padded_rows(bucket, n, seed)) + live_dev.fill_(n) + graph.replay() + torch.cuda.synchronize() + eager = M._routed_experts(core, xs, router, win) + assert torch.equal(out[:n], eager[:n]), f"live {n}: the replay changed a real row" + assert not out[n:].any(), f"live {n}: a padding row is not zero" + + +@pytest.mark.parametrize( + "kind, width, offset", + [("sum", 128, 0), ("sum", 256, 0), ("pad", 512, 0), ("pad", 256, 0), ("pad", 256, 256)], + ids=lambda v: str(v), +) +def test_moe_windows(kind, width, offset): + """A member's MoE windows sum to the stack; a prefill graph's padding rows route to none.""" + if kind == "sum": + _windows_sum(width, tokens=width) + else: + _padding_rows(offset, width) + + +# ---------------------------------------------------------------- low_m_bf16_mm__decode --- + + +def _low_m_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """Accumulation order only, as `cublas_mm`: 4 bf16 ulp element-wise, 2 ulp relative RMS.""" + assert_within_ulp(out, ref, element_ulp=4.0, rms_ulp=2.0, ulp=ULP[torch.bfloat16]) + + +def _low_m_build(m, n, k, seed): + """`mat_b` is a `[N, K]` weight's `.t()`, as a target stores its projections.""" + g = torch.Generator(device="cuda").manual_seed(seed) + mat_a = torch.randn(m, k, generator=g, device="cuda").to(torch.bfloat16) + weight = (torch.randn(n, k, generator=g, device="cuda") * k**-0.5).to(torch.bfloat16) + return mat_a, weight.t() + + +def _check_low_m(cell): + s = cell.spec + assert low_m_plan(s["m"], s["n"], s["k"]).kind == s["kind"], low_m_plan(s["m"], s["n"], s["k"]) + mat_a, mat_b = _low_m_build(s["m"], s["n"], s["k"], seed=_seed(cell.why) % 2**31) + out = low_m_bf16_mm(mat_a, mat_b, s["pdl"]) + assert out.shape == (s["m"], s["n"]) and out.dtype == torch.bfloat16 and out.is_contiguous() + _low_m_compare(out, low_m_bf16_mm.reference(mat_a, mat_b, s["pdl"])) + + +def test_the_low_m_dependent_chain_under_graph_replay() -> None: + """Decode replays these in CUDA graphs with PDL, each link reading the one before it, so a + kernel that read its activation before `griddepcontrol.wait` would see the previous replay's + values. One chain per branch at the target's rows: warp at M=4, direct at M=2.""" + chains = [] + for m, branches in ((4, ("warp", "warp", "warp")), (2, ("direct", "direct", "warp"))): + o = _low_m_build(m, 2560, 1536, seed=m)[1] + gdn_in = _low_m_build(m, 4128, 2560, seed=10 + m)[1] + attn_in = _low_m_build(m, 4352, 2560, seed=20 + m)[1] + got = tuple(low_m_plan(m, w.shape[1], w.shape[0]).kind for w in (o, gdn_in, attn_in)) + assert got == branches, got + x = torch.zeros(m, 1536, dtype=torch.bfloat16, device="cuda") + chains.append((x, o, gdn_in, attn_in)) + + def run(): + outs = [] + for x, o, gdn_in, attn_in in chains: + h = low_m_bf16_mm(x, o, True) + outs.append((h, low_m_bf16_mm(h, gdn_in, True), low_m_bf16_mm(h, attn_in, True))) + return outs + + _warm_on_side_stream(lambda: (run(), run())) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = run() + for trial in range(3): + g = torch.Generator(device="cuda").manual_seed(100 + trial) + for x, *_ in chains: + x.copy_(torch.randn(x.shape, generator=g, device="cuda").to(x.dtype)) + graph.replay() + torch.cuda.synchronize() + for (x, o, gdn_in, attn_in), (h, g_out, a_out) in zip(chains, outs): + _low_m_compare(h, low_m_bf16_mm.reference(x, o)) + _low_m_compare(g_out, low_m_bf16_mm.reference(h, gdn_in)) + _low_m_compare(a_out, low_m_bf16_mm.reference(h, attn_in)) + + +# ------------------------------------------------------- moe_front_router_quant__decode --- + + +def _front_operands(rows: int, seed: int) -> tuple[torch.Tensor, torch.Tensor]: + """bf16 MoE rows whose 1x128 blocks span many octaves (one zero block), and a router.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(rows, 2560, device=DEV, generator=g) + octaves = torch.randint(-24, 12, (rows, 20, 1), device=DEV, generator=g).float() + x = (x.reshape(rows, 20, 128) * torch.exp2(octaves)).reshape(rows, 2560) + x[0, :128] = 0.0 + router = 0.02 * torch.randn(512, 2560, device=DEV, generator=g) + return x.to(torch.bfloat16).contiguous(), router.to(torch.bfloat16).contiguous() + + +def _front_check(x, router, logits, xq, xsf, label: str) -> None: + """Logits within one bf16 ulp of each row's largest |logit| of the exact product; codes and + scales bitwise `fp8_quantize_1x128`'s, the scales within 4 fp32 ulp of the exact quotient.""" + _, _, ref_scales = moe_front_router_quant.reference(x, router) + exact = x.double() @ router.double().t() + rowmax = exact.abs().amax(dim=-1, keepdim=True) + err = ((logits.double() - exact) / rowmax).abs().max().item() + assert err <= _BF16_ULP, f"{label}: logits {err / _BF16_ULP:.3f} bf16 ulp off" + want_q, want_s = fp8_quantize_1x128(x, use_ue8m0=False) + assert torch.equal(xq.view(torch.uint8), want_q.view(torch.uint8)), f"{label}: e4m3 codes" + assert torch.equal(xsf, want_s), f"{label}: activation scales" + ratio = (xsf.double() / ref_scales.double() - 1.0).abs().max().item() + assert ratio <= 4 * 2.0**-23, f"{label}: scales {ratio:.3g} from the exact quotient" + + +def _check_front(cell): + rows = cell.spec["rows"] + x, router = _front_operands(rows, _seed(cell.why)) + moe_front_router_quant.is_valid(x, router) + logits, xq, xsf = moe_front_router_quant(x, router) + _front_check(x, router, logits, xq, xsf, cell.why[:44]) + if not cell.spec["capture"]: + return + static = x.clone() + _warm_on_side_stream(lambda: moe_front_router_quant(static, router)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = moe_front_router_quant(static, router) + graph.replay() + torch.cuda.synchronize() + for got, want in zip(outs, (logits, xq, xsf)): + assert torch.equal(got.view(torch.uint8), want.view(torch.uint8)) + x2, _ = _front_operands(rows, _seed(cell.why) + 1) + static.copy_(x2) + graph.replay() + torch.cuda.synchronize() + for got, want in zip(outs, moe_front_router_quant(x2, router)): + assert torch.equal(got.view(torch.uint8), want.view(torch.uint8)), "second replay" + _front_check(x2, router, *[o.clone() for o in outs], f"{cell.why[:44]} (second replay)") + + +def _coarse(logits: torch.Tensor) -> torch.Tensor: + """Three values a row: exact ties decide most of the ranking.""" + return (logits.float().sign() * (logits.float().abs() > 0.5)).to(torch.bfloat16) + + +def _runner_routes_as_renorm(rows, width, offset, ties): + """`routing_logits=` is bitwise the runner pre-routed by `renorm_moe_routing_op`.""" + x, router = _front_operands(rows, 7 * rows + width + offset) + logits, xq, xsf = moe_front_router_quant(x, router) + if ties: + logits = _coarse(logits) + ids, weights = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + pre_routed = fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, topk_weights=weights, topk_ids=ids) + ) + routed = fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, routing_logits=logits) + ) + torch.cuda.synchronize() + assert torch.equal(routed, pre_routed), ( + f"{(routed != pre_routed).sum().item()} of {routed.numel()} elements differ" + ) + + +def _front_is_the_logits_routed_runner(rows, width, offset, ties): + """`fp8_block_scale_moe_front` and its folded quantization are bitwise the logits-routed + call and `fp8_quantize_1x128`; a capture replays on new rows (counters reset each launch).""" + + def operands(seed): + x, router = _front_operands(rows, seed) + return x, cublas_mm(x, router.t()) + + def want(x, logits): + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + return fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, routing_logits=logits) + ) + + x, logits = operands(13 * rows + width + offset) + if ties: + logits = _coarse(logits) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + weights = _stack_window(offset, width) + geometry = (512, 10, 640, offset, width, 1) + call = fp8_block_scale_moe_front(logits, xq, xsf, *weights, *geometry) + assert call is not None + routed = call(None) + fq, fsf = torch.empty_like(xq), torch.empty_like(xsf) + folded = fp8_block_scale_moe_front(logits, fq, fsf, *weights, *geometry)(x) + ref = want(x, logits) + torch.cuda.synchronize() + assert torch.equal(routed, ref), f"{(routed != ref).sum().item()} of {ref.numel()} differ" + assert torch.equal(folded, ref), f"{(folded != ref).sum().item()} of {ref.numel()} differ" + assert torch.equal(fq.view(torch.uint8), xq.view(torch.uint8)) and torch.equal(fsf, xsf) + + static_x, static_logits = x.clone(), logits.clone() + gq, gsf = torch.empty_like(xq), torch.empty_like(xsf) + fp8_block_scale_moe_front(static_logits, gq, gsf, *weights, *geometry)(static_x) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fp8_block_scale_moe_front(static_logits, gq, gsf, *weights, *geometry)(static_x) + for seed in (1, 2): + x2, logits2 = operands(1000 * seed + rows) + static_x.copy_(x2) + static_logits.copy_(logits2) + graph.replay() + ref2 = want(x2, logits2) + torch.cuda.synchronize() + assert torch.equal(out, ref2), f"replay {seed}: {(out != ref2).sum().item()} differ" + + +def _routed_experts_past_the_front_end(rows, offset, ties, monkeypatch): + """Past `_MOE_FRONT_MAX_ROWS` decode routes in the runner front, bitwise the pre-routed call.""" + x, router = _front_operands(rows, 13 * rows + offset) + if ties: + # A router whose logits take three values a row: ties everywhere. + router = (router.float().sign() * (router.float().abs() > 0.03) * 0.001).to(torch.bfloat16) + window = _pad_window(offset, 256) + core = SimpleNamespace(num_experts=512, topk=10, moe_inter=640, dp_size=1, moe_max_rows=16384) + calls = [] + real = M.renorm_moe_routing_op + + def spy(*args, **kwargs): + calls.append(args[0].shape[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(M, "renorm_moe_routing_op", spy) + new = M._routed_experts(core, x, router.t(), window, mm=M._decode_projection) + assert calls == [], "the logits-routed call launched the renorm routing" + ids, w = real(M._decode_projection(x, router.t()), topk=10, output_dtype=x.dtype) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + old = M._window_runner(core, window, xq, xsf, ids, w) + torch.cuda.synchronize() + assert torch.equal(new, old), f"{(new != old).sum().item()} of {new.numel()} elements differ" + + +def _wide_tactic_under_skewed_routing(rows, offset): + """A logits-routed 256-expert window takes its pinned decode tactic for the row count, which + under skewed routing like a real decode batch's is bitwise the fallback and in the gate.""" + x, router = _front_operands(rows, 17 * rows + offset) + logits = (x.float() @ router.float().t()).to(torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + hot = offset + 7 + logits[:, hot] = logits.abs().amax() + 4.0 + logits[: rows // 3, offset + 100] = logits.abs().amax() + 2.0 + inputs = [logits, None, xq, xsf, *_stack_window(offset, 256), None, None] + geometry = (512, 10, 640, offset, 256, 1, 0, None, 8192, False) + _runner_mod._DISPATCH.clear() + runner, tactic = _runner_mod._dispatch(geometry, inputs) + _runner_mod._DISPATCH.clear() + fallback = [int(v) for v in runner.get_fallback_tactic(2560, rows)] + if rows >= WIDE_DECODE_MIN_ROWS: + want = WIDE_DECODE_TACTIC + else: + want = [t for start, t in WIDE_DECODE_SMALL_TACTICS if rows >= start][-1] + assert list(tactic) == list(want), (tactic, want) + out = runner.forward(inputs, tactic=list(tactic)) + ref = runner.forward(inputs, tactic=fallback) + ids, weights = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + assert bool((ids == hot).any(dim=-1).all()), "the hot expert is not in every row's top-10" + exact = fp8_block_scale_moe_runner.reference( + **_runner_kwargs(xq, xsf, offset, 256, topk_weights=weights, topk_ids=ids) + ) + torch.cuda.synchronize() + assert torch.equal(out, ref), "the chosen tactic changes the output" + _moe_compare(out, exact) + + +_FRONT_WINDOWS = ((128, 384), (256, 256), (512, 0)) +#: `(path, rows, window width, window offset, coarse logits)`. +# fmt: off +_FRONT_CASES = ( + [("runner", r, w, o, t) for r in (1, 16) for w, o in _FRONT_WINDOWS for t in (False, True)] + + [("front", r, w, o, t) for r in (1, 129, 576) for w, o in _FRONT_WINDOWS for t in (False, True)] + + [("routed_experts", r, 256, o, t) for r in (96, 576) for o in (0, 256) for t in (False, True)] + + [("wide_tactic", r, 256, o, False) for r in (3, 96, 167, 168, 576) for o in (0, 256)] +) +# fmt: on + + +@pytest.mark.parametrize("path, rows, width, offset, ties", _FRONT_CASES, ids=lambda v: str(v)) +def test_decode_moe_front_routes_as_renorm(path, rows, width, offset, ties, monkeypatch): + """The decode MoE front end routes as `renorm_moe_routing_op` does, along each `path`.""" + if path == "runner": + _runner_routes_as_renorm(rows, width, offset, ties) + elif path == "front": + _front_is_the_logits_routed_runner(rows, width, offset, ties) + elif path == "routed_experts": + _routed_experts_past_the_front_end(rows, offset, ties, monkeypatch) + else: + _wide_tactic_under_skewed_routing(rows, offset) + + +# ------------------------------------------------ contiguous grouped prefill experts --- + + +def _logits_args(tokens: int, offset: int, local: int, seed: int): + """bf16 router logits and the runner's arguments pre-routed from them over a stack window.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(tokens, 2560, device=DEV, generator=g).to(torch.bfloat16) + logits = torch.randn(tokens, 512, device=DEV, generator=g).to(torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + ids, w = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + routing = dict(topk_weights=w.contiguous(), topk_ids=ids.contiguous()) + return logits, _runner_kwargs(xq, xsf, offset, local, **routing) + + +def _reroute(args: dict, logits: torch.Tensor) -> None: + ids, w = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + args["topk_ids"], args["topk_weights"] = ids.contiguous(), w.contiguous() + + +def _grouped_call(args, **kw): + names = ("hidden_states", "hidden_states_scale", "topk_ids", "topk_weights", "gemm1_weights") + names += ("gemm1_weights_scale", "gemm2_weights", "gemm2_weights_scale", "local_expert_offset") + return _grouped.moe_prefill_grouped(*(args[n] for n in names), **kw) + + +def _grouped_matches_the_reference(tokens, offset, local): + """The contiguous grouped path is in the gate, zero outside the window, deterministic despite + atomic slot ranks, and bitwise the same with either FC1 epilogue.""" + _, args = _logits_args(tokens, offset, local, seed=_seed(f"grouped{tokens}-{offset}")) + out = _grouped_call(args) + _moe_check(out, args, _in_window(args)) + assert torch.equal(out, _grouped_call(args)) + epi = _grouped.EPI_QUANT + try: + _grouped.EPI_QUANT = not epi + assert torch.equal(out, _grouped_call(args)) + finally: + _grouped.EPI_QUANT = epi + + +def _grouped_skewed_and_empty(): + """All rows on ten experts (warmup dummy tokens) pass the gate; an empty window gives zeros.""" + logits, args = _logits_args(2048, 0, 256, seed=_seed("grouped-skew")) + hot = torch.tensor([3, 4, 40, 41, 42, 100, 101, 180, 250, 255], device=DEV) + logits = logits.clone() + logits[:, hot] += 30.0 + _reroute(args, logits) + out = _grouped_call(args) + _moe_check(out, args, _in_window(args)) + assert torch.equal(out, _grouped_call(args)) + args["local_expert_offset"] = 256 + assert torch.equal(_grouped_call(args), torch.zeros_like(out)) + + +def _grouped_fin_first(first): + """`fin_first` rows are final when `after_first` gets the output, and the whole output is + bitwise the one-launch finalize's.""" + logits, args = _logits_args(3000, 128, 256, seed=_seed("grouped-finfirst")) + logits = logits.clone() + logits[:, 140:150] += 4.0 + _reroute(args, logits) + ref = _grouped_call(args) + r0, r1 = (0, 1408) if first == "head" else (1500, ref.shape[0]) + seen = [] + + def after_first(out): + torch.cuda.synchronize() + seen.append(out[r0:r1].clone()) + + got = _grouped_call(args, fin_first=(r0, r1), after_first=after_first) + torch.cuda.synchronize() + assert len(seen) == 1 and torch.equal(seen[0], ref[r0:r1]) + assert torch.equal(got, ref) + + +def _grouped_graph_replay(): + """Replays over two steps' inputs are bitwise their eager calls (tiles rebuilt on device).""" + _, args = _logits_args(2048, 256, 256, seed=_seed("groupedcap")) + _, args2 = _logits_args(2048, 256, 256, seed=_seed("groupedcap2")) + eager, eager2 = _grouped_call(args), _grouped_call(args2) + static = {k: (v.clone() if k in STEP_INPUTS else v) for k, v in args.items()} + _warm_on_side_stream(lambda: _grouped_call(static)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = _grouped_call(static) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager) + for k in STEP_INPUTS: + static[k].copy_(args2[k]) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager2) + + +@pytest.mark.parametrize( + "case", + ["2048-0-256", "4096-256-256", "3000-128-128", "skewed", "fin_first-head", "fin_first-tail", + "graph"], +) # fmt: skip +def test_prefill_grouped(case): + """The contiguous grouped prefill experts against the runner's reference, case by case.""" + if case == "skewed": + _grouped_skewed_and_empty() + elif case.startswith("fin_first-"): + _grouped_fin_first(case.split("-")[1]) + elif case == "graph": + _grouped_graph_replay() + else: + _grouped_matches_the_reference(*map(int, case.split("-"))) + + +# ------------------------------------------------------------------ embed_rows__decode --- + + +def _check_embed_rows(cell): + """Bitwise the reference over int32 and int64 ids (-1 clamped); a captured launch replays.""" + rows, copies, clamp = cell.spec["rows"], cell.spec["copies"], cell.spec["clamp"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + table = torch.randn(1000, 2560, device=DEV, generator=g).to(torch.bfloat16) + for dtype in (torch.int32, torch.int64): + ids = torch.randint(0, 1000, (rows,), device=DEV, generator=g).to(dtype) + if clamp: + ids[-1] = -1 + got = embed_rows(ids, table, copies, clamp) + assert torch.equal( + got.view(torch.int16), embed_rows.reference(ids, table, copies, clamp).view(torch.int16) + ) + static = ids.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = embed_rows(static, table, copies, clamp) + static.copy_(torch.randint(0, 1000, (rows,), device=DEV, generator=g).to(static.dtype)) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, embed_rows.reference(static, table, copies, clamp)) + + +# ------------------------------------------------------- mtp_moe_combine_inject__decode --- + + +def _check_mtp_combine(cell): + """Bitwise the unfused ops, within bf16 rounding of the fp32 reference; a capture replays.""" + rows, hc, hidden = cell.spec["rows"], cell.spec["hc_count"], cell.spec["hidden"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + + def operands(): + buf = torch.randn(rows, 2 * hc * hidden, device=DEV, generator=g).to(torch.bfloat16) + x = buf[:, : hc * hidden] + routed = (0.5 * torch.randn(rows, hidden, device=DEV, generator=g)).to(torch.bfloat16) + shared = (0.5 * torch.randn(rows, hidden, device=DEV, generator=g)).to(torch.bfloat16) + gug = (2.0 * torch.randn(rows, 1288, device=DEV, generator=g)).to(torch.bfloat16) + gate = gug[:, 1280:1281] if cell.spec["gate"] == "strided" else gug[:, 1280].contiguous() + inject = (2.0 * torch.sigmoid(torch.randn(rows, hc, device=DEV, generator=g))).to( + torch.bfloat16 + ) + return x, routed, gate, shared, inject + + def composed(x, routed, gate, shared, inject): + return qwen4_exp_post_inject( + x, fused_sigmoid_gate_mul_add(routed.clone(), gate, shared), inject + ) + + ops = operands() + got = mtp_moe_combine_inject(*ops) + assert torch.equal(got.view(torch.int16), composed(*ops).view(torch.int16)) + torch.testing.assert_close( + got.float(), mtp_moe_combine_inject.reference(*ops).float(), atol=0.07, rtol=0.02 + ) + static = [t.clone() for t in ops] + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = mtp_moe_combine_inject(*static) + new = operands() + for s, n in zip(static, new): + s.copy_(n) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, composed(*new)) + + +# ------------------------------------------------------------ moe_tpb_decode__decode --- + + +def _tpb_blocks(rank: int): + return _tpb.block_major(*_weights(2560, 640, 512, seed=2560 + 640 + 512), rank) + + +def _tpb_call(fn, logits, args, blocks, **kw): + return fn(logits, args["hidden_states"], args["hidden_states_scale"], blocks, **kw) + + +def _tpb_shared(rows: int, seed: int): + """A MoE site's bf16 rows and a bf16 shared expert: gate / up rows `[640, 2560]`, the + gate-logit row `[2560]` and the down projection `[2560, 640]`.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(rows, 2560, device=DEV, generator=g) + x = (x / x.abs().amax(-1, keepdim=True)).to(torch.bfloat16).contiguous() + gate = (0.02 * torch.randn(640, 2560, device=DEV, generator=g)).to(torch.bfloat16) + up = (0.02 * torch.randn(640, 2560, device=DEV, generator=g)).to(torch.bfloat16) + gl = (0.02 * torch.randn(2560, device=DEV, generator=g)).to(torch.bfloat16).contiguous() + dn = (0.02 * torch.randn(2560, 640, device=DEV, generator=g)).to(torch.bfloat16) + return x, gate, up, gl, dn + + +def _unfused_shared(x, gate, up, gl, dn): + """`(sigmoid(gate logit), shared)` of the unfused bf16 path, in fp32.""" + with true_fp32_matmul(): + gugate = (x.float() @ torch.cat([gate, up]).float().t()).to(torch.bfloat16).float() + act = ( + (torch.nn.functional.silu(gugate[:, :640]) * gugate[:, 640:]).to(torch.bfloat16).float() + ) + shared = (act @ dn.float().t()).to(torch.bfloat16).float() + logit = (x.float() @ gl.float()).to(torch.bfloat16).float().unsqueeze(1) + return torch.sigmoid(logit), shared + + +def _check_tpb(cell): + """Block-major decode experts in the gate of `reference`, deterministic, and replayable.""" + rows, rank = cell.spec["rows"], cell.spec["rank"] + logits, args = _logits_args(rows, 0, 512, seed=_seed(cell.why)) + blocks = _tpb_blocks(rank) + out = _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks) + _moe_compare(out, _tpb_call(_tpb.moe_tpb_decode.reference, logits, args, blocks)) + assert torch.equal(out, _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks)), "a re-run" + if not cell.spec["capture"]: + return + logits2, args2 = _logits_args(rows, 0, 512, seed=_seed(cell.why) + 1) + eager2 = _tpb_call(_tpb.moe_tpb_decode, logits2, args2, blocks) + slog = logits.clone() + sx = args["hidden_states"].clone() + ssf = args["hidden_states_scale"].clone() + _warm_on_side_stream(lambda: _tpb.moe_tpb_decode(slog, sx, ssf, blocks)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = _tpb.moe_tpb_decode(slog, sx, ssf, blocks) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, out) + slog.copy_(logits2) + sx.copy_(args2["hidden_states"]) + ssf.copy_(args2["hidden_states_scale"]) + graph.replay() + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, eager2) + + +def _tpb_ranks_sum(rows): + """The four round-robin ranks' partials summed in fp32 sit in the MoE gate of the 512-expert + runner reference and of the trtllm-gen runner's four 128-expert windows summed alike.""" + logits, args = _logits_args(rows, 0, 512, seed=_seed(f"tpb-sum-{rows}")) + parts = [_tpb_call(_tpb.moe_tpb_decode, logits, args, _tpb_blocks(r)) for r in range(4)] + total = sum(p.float() for p in parts).to(torch.bfloat16) + with true_fp32_matmul(): + ref = fp8_block_scale_moe_runner.reference(**args) + _moe_compare(total, ref) + xq, xsf = args["hidden_states"], args["hidden_states_scale"] + routing = dict(topk_weights=args["topk_weights"], topk_ids=args["topk_ids"]) + windows = [ + fp8_block_scale_moe_runner(**_runner_kwargs(xq, xsf, 128 * r, 128, **routing)) + for r in range(4) + ] + _moe_compare(total, sum(p.float() for p in windows).to(torch.bfloat16)) + + +def _tpb_shared_slice(rows): + """With the shared fold each rank's partial sits in its reference's gate, and the four summed + sit in the layer's: the routed reference + sigmoid(gate logit) x the unfused shared expert.""" + logits, args = _logits_args(rows, 0, 512, seed=_seed(f"tpb-shared-{rows}")) + x, gate, up, gl, dn = _tpb_shared(rows, _seed(f"tpb-shared-w-{rows}")) + total = torch.zeros(rows, 2560, device=DEV) + for r in range(4): + gu_r, dn_r = _tpb.shared_blocks(gate, up, dn, r) + sh = _tpb.SharedBlocks(x, gu_r, gl, dn_r) + blocks = _tpb_blocks(r) + out = _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks, shared=sh) + with true_fp32_matmul(): + ref = _tpb_call(_tpb.moe_tpb_decode.reference, logits, args, blocks, shared=sh) + _moe_compare(out, ref) + assert torch.equal(out, _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks, shared=sh)) + total += out.float() + with true_fp32_matmul(): + routed = fp8_block_scale_moe_runner.reference(**args).float() + scale, shared = _unfused_shared(x, gate, up, gl, dn) + _moe_compare(total.to(torch.bfloat16), (routed + scale * shared).to(torch.bfloat16)) + + +def _tpb_blocks_dealt(): + """The four ranks' 640 blocks are every (expert, block) of the 512 x 5 exactly once, each rank + holds a block of every expert, and `weights.tpb_blocks` agrees with the op's `block_ids`.""" + seen = set() + for r in range(4): + d = SimpleNamespace(moe_tpb=True, tp_rank=r, num_experts=512, moe_inter=640) + blocks = M._weights.tpb_blocks(d) + e, b = _tpb.block_ids(r) + assert blocks == list(zip(e, b)) + assert len(blocks) == 640 and {x for x, _ in blocks} == set(range(512)) + assert all((x + y) % 4 == r for x, y in blocks) + seen |= set(blocks) + assert seen == {(x, y) for x in range(512) for y in range(5)} + + +def _tpb_window(rank: int, shared) -> "M._weights.ExpertWindow": + """Tep4 rank `rank`'s 128-expert window with its block-major copy and shared-expert slice.""" + bl = _tpb_blocks(rank) + gate, up, gl, dn = shared + sh_gu, sh_dn = _tpb.shared_blocks(gate, up, dn, rank) + bw = M._weights.BlockWindow(bl.w1, bl.s1, bl.w2, bl.s2, rank, sh_gu, gl, sh_dn) + return _pad_window(128 * rank, 128)._replace(blocks=bw) + + +def _moe_block_folds_shared(rows, monkeypatch): + """A deferred tep4 decode step of `_MOE_TPB_SHARED_MIN_ROWS` (5) to 8 rows folds each rank's + shared-expert slice into its block launch, the four partials summing into the gate of the + unfused `routed + sigmoid(gate) * shared`. A 3-row step keeps the side-stream shared expert.""" + x, router = _front_operands(rows, 57 + rows) + x = (x.float() / x.float().abs().amax(dim=-1, keepdim=True)).to(torch.bfloat16).contiguous() + _, gate, up, gl, dn = _tpb_shared(rows, 91 + rows) + gug = torch.zeros(1288, 2560, dtype=torch.bfloat16, device=DEV) + gug[:640], gug[640:1280], gug[1280] = gate, up, gl + folded_calls = [] + real = M.moe_tpb_decode + + def spy(*args, **kwargs): + folded_calls.append( + len(args) > 4 and args[4] is not None or kwargs.get("shared") is not None + ) + return real(*args, **kwargs) + + monkeypatch.setattr(M, "moe_tpb_decode", spy) + + def block(rank): + core = SimpleNamespace( + num_experts=512, topk=10, moe_inter=640, shared_inter=640, dp_size=1, + tp_sum=lambda t: t, _side_stream=torch.cuda.Stream(), + _side_fork=torch.cuda.Event(), _side_join=torch.cuda.Event(), + ) # fmt: skip + L = SimpleNamespace( + router=router.t(), + experts=_tpb_window(rank, (gate, up, gl, dn)), + sh_gug=gug.t(), + sh_dn=dn.t(), + ) + return M._moe_block(core, x, L, None, mm=M._decode_projection, defer=True) + + out = [block(r) for r in range(4)] + torch.cuda.synchronize() + fold = rows >= M._MOE_TPB_SHARED_MIN_ROWS + assert folded_calls == [fold] * 4, folded_calls + # The folded partials already carry sigmoid(gate) * shared; the unfolded block returns it apart. + got = sum(p.float() for p, _, _ in out) + if fold: + assert all(g is None and s is None for _, g, s in out), "a folded block returned a shared" + else: + _, g0, s0 = out[0] + got = got + torch.sigmoid(g0.float()) * s0.float() + scale, shared = _unfused_shared(x, gate, up, gl, dn) + logits = moe_front_router_quant(x, router)[0] + ids, wts = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + args = _runner_kwargs(xq, xsf, 0, 512, topk_ids=ids, topk_weights=wts) + with true_fp32_matmul(): + ref_routed = fp8_block_scale_moe_runner.reference(**args).float() + _moe_compare(got.to(torch.bfloat16), (ref_routed + scale * shared).to(torch.bfloat16)) + + +_TPB_CASES = ( + [("ranks_sum", rows) for rows in (1, 3, 6, 8)] + + [("shared_slice", rows) for rows in (3, 6, 8)] + + [("moe_block", rows) for rows in (3, 5, 8)] + + [("blocks_dealt", 0)] +) + + +@pytest.mark.parametrize("case, rows", _TPB_CASES, ids=lambda v: str(v)) +def test_tpb_decode_moe(case, rows, monkeypatch): + """The TPB decode experts: rank sums, the shared-expert fold, and the block deal.""" + if case == "ranks_sum": + _tpb_ranks_sum(rows) + elif case == "shared_slice": + _tpb_shared_slice(rows) + elif case == "moe_block": + _moe_block_folds_shared(rows, monkeypatch) + else: + _tpb_blocks_dealt() + + +# ------------------------------------------------------------------ certified cells --- + +#: Each entry and its checker, which builds a cell's operands and compares entry and `reference`. +_CELL_CHECKS = ( + (fp8_block_scale_moe_runner, _check_moe_runner), + (low_m_bf16_mm, _check_low_m), + (moe_front_router_quant, _check_front), + (embed_rows, _check_embed_rows), + (mtp_moe_combine_inject, _check_mtp_combine), + (_tpb.moe_tpb_decode, _check_tpb), +) +CASES = [(entry, check, cell) for entry, check in _CELL_CHECKS for cell in entry.CELLS] + + +@pytest.mark.parametrize( + "entry, check, cell", CASES, ids=[f"{type(e).__name__}-{c.why[:40]}" for e, _, c in CASES] +) +def test_certified_cells(entry, check, cell, certifying): # noqa: F811 — fixture by name + certifying(entry) + check(cell) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py new file mode 100644 index 000000000000..daf8efa583b3 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -0,0 +1,1654 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Qwen3.8-Flash-Next-FP8 unified target: routing and deployment checks, each rank's share of the +core (on the meta device) and the per-rank KV / recurrent pool layout.""" + +import contextlib +from dataclasses import replace +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import routing +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + pool as P, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as W, +) +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight +from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache +from tensorrt_llm.llmapi import QSASparseAttentionConfig +from tensorrt_llm.llmapi.llm_args import ( + KvCacheConfig, + MTPDecodingConfig, + NGramDecodingConfig, + TorchLlmArgs, +) +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + +# ------------------------------------------------------------------ the members --- + +NUM_LAYERS = 48 + +#: The seven members; tp2 and tp4 leave the experts to Mapping's default split (the TP size). +MEMBERS = { + "tp1": dict(world_size=1, tp_size=1), + "tp2": dict(world_size=2, tp_size=2), + "tep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2), + "dep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2, enable_attention_dp=True), + "tp4": dict(world_size=4, tp_size=4), + "tep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4), + "dep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4, enable_attention_dp=True), +} + +_RANKS = [(m, r) for m, layout in MEMBERS.items() for r in range(layout["world_size"])] +#: The last rank of one member per attention layout: one rank, TP2, TP4, attention DP. +LAYOUT_RANKS = [("tp1", 0), ("tp2", 1), ("tp4", 3), ("dep4", 3)] + + +def _mapping(member: str, rank: int) -> Mapping: + return Mapping(rank=rank, gpus_per_node=4, pp_size=1, **MEMBERS[member]) + + +def _world(member: str) -> int: + return MEMBERS[member]["world_size"] + + +_needs_gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") + + +# --------------------------------------------------------------------- routing --- + +_DUMMY_MODEL = "/tmp/dummy_model" +_ARCH = "Qwen4ExpForConditionalGeneration" +_TARGET = ( + "models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.modeling." + "ModelingV2Qwen38FlashNextFp8Sm103Unified" +) +_NVFP4_DEFAULT_TARGET = ( + "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." + "ModelingV2Qwen38FlashNextNvfp4Sm103Default" +) + +# The quantization the engine resolves for the FP8 (128x128 blocks) and the NVFP4 exports. +_FP8 = QuantAlgo.FP8_BLOCK_SCALES +_NVFP4 = QuantAlgo.MIXED_PRECISION + + +def _qualified(target) -> str: + return f"{target.__module__.split('.modeling_v2.')[1]}.{target.__name__}" + + +@pytest.fixture +def on_sm103(monkeypatch): + """Route as if on a GB300, with the GDN cached replay at its default (on).""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + + +def _llm_args(mode="auto", **kwargs) -> TorchLlmArgs: + return TorchLlmArgs(model=_DUMMY_MODEL, modeling_v2=mode, **kwargs) + + +def _qwen4_exp_config(text_overrides=None, composite_overrides=None, **vision_overrides): + """Qwen3.8-Flash-Next's normalized composite config (the FP8 and NVFP4 exports alike).""" + text = dict( + num_hidden_layers=48, + hidden_size=2560, + num_experts=512, + num_attention_heads=24, + linear_num_value_heads=48, + vocab_size=248320, + ) + text.update(text_overrides or {}) + vision = dict( + depth=27, + hidden_size=1152, + num_heads=16, + spatial_merge_size=2, + out_hidden_size=2560, + num_position_embeddings=2304, + deepstack_visual_indexes=[], + hidden_act="gelu_pytorch_tanh", + ) + vision.update(vision_overrides) + return PretrainedConfig( + architectures=[_ARCH], + text_config=PretrainedConfig(**text), + vision_config=PretrainedConfig(**vision), + **(composite_overrides or {}), + ) + + +def _routed_config( + member=None, spec_config=None, pretrained_config=None, quant_algo=_FP8, **mapping_kwargs +) -> ModelConfig: + if member is not None: + mapping_kwargs = dict(MEMBERS[member], **mapping_kwargs) + return ModelConfig( + pretrained_config=pretrained_config or _qwen4_exp_config(), + mapping=Mapping(**mapping_kwargs), + spec_config=spec_config, + quant_config=QuantConfig(quant_algo=quant_algo, kv_cache_quant_algo=None), + ) + + +def _admitted(config, **llm_args_kwargs) -> None: + for mode in ("auto", "require"): + target = modeling_v2_resolve(config, _llm_args(mode, **llm_args_kwargs)) + assert target is not None and _qualified(target) == _TARGET, mode + + +def _mtp_with(**attributes): + """Sets post-load MTP fields directly: the constructor would remap or recompute them.""" + config = MTPDecodingConfig(max_draft_len=3) + for name, value in attributes.items(): + setattr(config, name, value) + return config + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, parallel", + [ + ("tp1", dict(tensor_parallel_size=1, moe_expert_parallel_size=1)), + ("tp2", dict(tensor_parallel_size=2, moe_expert_parallel_size=1)), + ("tep2", dict(tensor_parallel_size=2, moe_expert_parallel_size=2)), + ( + "dep2", + dict(tensor_parallel_size=2, moe_expert_parallel_size=2, enable_attention_dp=True), + ), + ("tp4", dict(tensor_parallel_size=4)), + ("tep4", dict(tensor_parallel_size=4, moe_expert_parallel_size=4)), + ( + "dep4", + dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=True), + ), + ], +) +def test_each_member_is_admitted_as_the_llm_api_spells_it(on_sm103, member, parallel, monkeypatch): + """Each member's LLM API spelling builds its mapping and is admitted. The node size is pinned: + inferred from one visible GPU, a multi-rank world would span nodes, which `layout` declines.""" + monkeypatch.setattr(torch.cuda, "device_count", lambda: 1) + mapping = _llm_args(gpus_per_node=4, **parallel).parallel_config.to_mapping() + assert routing.layout(mapping, routing._FP8_LAYOUTS) == member + _admitted(_routed_config(member)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, spec_config, llm_args_kwargs", + [ + ("tp1", MTPDecodingConfig(max_draft_len=1), {}), + ("tep2", MTPDecodingConfig(max_draft_len=7), dict(prefill_cuda_graph_backend="breakable")), + ( + "tep4", + MTPDecodingConfig(max_draft_len=3), + dict(kv_cache_config=KvCacheConfig(dtype="fp8")), + ), + ( + "dep4", + MTPDecodingConfig(max_draft_len=3), + dict(kv_cache_config=KvCacheConfig(dtype="nvfp4")), + ), + ("tp4", None, dict(kv_cache_config=KvCacheConfig(mamba_ssm_cache_dtype="bfloat16"))), + ], + ids=[ + "tp1-mtp1", + "tep2-mtp7-breakable", + "tep4-mtp3-fp8-kv", + "dep4-mtp3-nvfp4-kv", + "tp4-bf16-state", + ], +) +def test_the_engine_choices_the_target_runs_are_admitted( + on_sm103, member, spec_config, llm_args_kwargs +): + _admitted(_routed_config(member, spec_config), **llm_args_kwargs) + + +_MTP3 = MTPDecodingConfig(max_draft_len=3) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "config_kwargs, llm_args_kwargs, match", + [ + (dict(member="dep2", spec_config=_MTP3), {}, r"attention DP \(dep2\)"), + (dict(member="dep4"), dict(prefill_cuda_graph_backend="breakable"), "breakable"), + (dict(member="tep4"), dict(prefill_cuda_graph_backend="piecewise"), "piecewise"), + ( + dict(member="tep4", spec_config=_MTP3), + dict(kv_cache_config=KvCacheConfig(mamba_ssm_cache_dtype="float32")), + "recurrent-state cache", + ), + (dict(member="tep4"), dict(kv_cache_config=KvCacheConfig(dtype="fp8_ds_mla")), "KV cache"), + ( + dict( + member="tep4", + spec_config=NGramDecodingConfig(max_draft_len=3, max_matching_ngram_size=2), + ), + {}, + "one-model MTP-Eagle", + ), + ( + dict(member="tep4", spec_config=MTPDecodingConfig(max_draft_len=8)), + {}, + "max_draft_len=8", + ), + ( + dict(member="tep4", spec_config=_mtp_with(max_total_draft_tokens=5)), + {}, + "linear chain", + ), + ( + dict(member="tep4", spec_config=MTPDecodingConfig(max_draft_len=3, max_concurrency=8)), + {}, + "draft length", + ), + (dict(world_size=2, tp_size=1, pp_size=2), {}, "tp2 or tep2"), + (dict(MEMBERS["tep4"], gpus_per_node=2), {}, "tp2 or tep2"), + (dict(MEMBERS["dep4"], enable_lm_head_tp_in_adp=True), {}, "tp2 or tep2"), + (dict(world_size=8, tp_size=8, moe_tp_size=1, moe_ep_size=8), {}, "tp2 or tep2"), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(num_hidden_layers=60))), + {}, + "text shape", + ), + ( + dict( + member="tep4", + pretrained_config=_qwen4_exp_config(composite_overrides=dict(dtype="float16")), + ), + {}, + "torch.float16", + ), + ], + ids=[ + "dep2-mtp", + "dep4-breakable", + "piecewise", + "fp32-state", + "kv-algorithm", + "ngram", + "draft8", + "max-total-draft", + "runtime-draft-length", + "pp2", + "two-nodes", + "dep4-lm-head-tp", + "tep8", + "text-shape", + "fp16-model", + ], +) +def test_what_the_target_does_not_run_declines(on_sm103, config_kwargs, llm_args_kwargs, match): + """``auto`` falls back to the built-in model and ``require`` says why, once per reason.""" + config = _routed_config(**config_kwargs) + assert modeling_v2_resolve(config, _llm_args("auto", **llm_args_kwargs)) is None + with pytest.raises(ValueError, match=match): + modeling_v2_resolve(config, _llm_args("require", **llm_args_kwargs)) + + +@pytest.mark.cpu_only +def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypatch): + """The engine choices read off the `ModelConfig` or the process rather than `TorchLlmArgs`.""" + config = replace(_routed_config("tep4"), moe_backend_requested="CUTLASS") + with pytest.raises(ValueError, match="moe_config.backend"): + modeling_v2_resolve(config, _llm_args("require")) + monkeypatch.setenv("TRTLLM_USE_GDN_REPLAY", "0") + with pytest.raises(ValueError, match="TRTLLM_USE_GDN_REPLAY=0"): + modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 0)) + with pytest.raises(ValueError, match=r"compute capability \(10, 0\)"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) + + +@pytest.mark.cpu_only +def test_the_quantization_picks_the_checkpoint(on_sm103): + """MIXED_PRECISION routes to the NVFP4 target (its own checks mocked), FP8_BLOCK_SCALES here.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + modeling as nvfp4_modeling, + ) + + with mock.patch.object( + nvfp4_modeling.ModelingV2Qwen38FlashNextNvfp4Sm103Default, "check_deployment" + ): + target = modeling_v2_resolve( + _routed_config("tep4", quant_algo=_NVFP4), _llm_args("require") + ) + assert _qualified(target) == _NVFP4_DEFAULT_TARGET + _admitted(_routed_config("tep4")) + + +# ------------------------------------------------------------------------ the core --- + +# Qwen3.8-Flash-Next's text config (the FP8 export's `text_config`). +# fmt: off +_TEXT = dict( + architectures=["Qwen4ExpForCausalLM"], attention_bias=False, attention_dropout=0.0, + bos_token_id=248044, dtype="bfloat16", eos_token_id=248044, full_attention_interval=4, + hc_count=4, hc_lowrank=320, head_dim=256, heads_per_ngram=8, hidden_act="silu", + hidden_size=2560, indexer_budget=2048, indexer_compress_ratio=4, indexer_head_dim=128, + indexer_kv_heads=1, indexer_n_heads=4, + layer_types=["full_attention" if i % 4 == 3 else "linear_attention" for i in range(NUM_LAYERS)], + linear_conv_kernel_dim=4, linear_key_head_dim=128, linear_num_key_heads=16, + linear_num_value_heads=48, linear_value_head_dim=128, make_ngram_vocab_size_divisible_by=128, + mamba_ssm_dtype="float32", max_position_embeddings=262144, moe_intermediate_size=640, + mtp=dict(hybrid=True, layer_types=["full_attention"], mtp_use_hidden_state_from_layer=None, + num_hidden_layers=1, rope_theta=10000000), + mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=False, ngram_size=3, + ngram_vocab_size_base=20000000, num_attention_heads=24, num_experts=512, + num_experts_per_tok=10, num_hidden_layers=NUM_LAYERS, num_key_value_heads=2, + output_gate_type="sigmoid", partial_rotary_factor=0.25, ple_conv_kernel_size=4, + ple_embed_dim=2560, ple_layer_ids=[2], rms_norm_eps=1e-06, + rope_parameters=dict(mrope_interleaved=True, mrope_section=[11, 11, 10], + partial_rotary_factor=0.25, rope_theta=10000000, rope_type="default"), + shared_expert_intermediate_size=640, split_ngram_parts=128, tie_word_embeddings=False, + vocab_size=248320, +) +_VISION = dict( + deepstack_visual_indexes=[], depth=27, dtype="bfloat16", hidden_act="gelu_pytorch_tanh", + hidden_size=1152, in_channels=3, intermediate_size=4304, num_heads=16, + num_position_embeddings=2304, out_hidden_size=2560, patch_size=16, spatial_merge_size=2, + temporal_patch_size=2, +) +# fmt: on + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +# Per member: (q heads, KV heads, attn row, rope "v" heads, GDN k / v heads, in_proj split, o K). +_GEOMETRY = { + "tp1": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), + "tp2": (12, 1, 7424, 16, 8, 24, (5120, 3072, 24, 24), 3072), + "tep2": (12, 1, 7424, 16, 8, 24, (5120, 3072, 24, 24), 3072), + "dep2": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), + "tp4": (6, 1, 4352, 10, 4, 12, (2560, 1536, 12, 12), 1536), + "tep4": (6, 1, 4352, 10, 4, 12, (2560, 1536, 12, 12), 1536), + "dep4": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), +} + + +def _mtp(max_draft_len: int = 3) -> MTPDecodingConfig: + return MTPDecodingConfig(max_draft_len=max_draft_len) + + +def _core(member: str, rank: int, spec_config=None): + model_config = ModelConfig( + pretrained_config=Qwen4ExpTextConfig(**_TEXT), + mapping=_mapping(member, rank), + quant_config=QuantConfig(quant_algo=QuantAlgo.FP8_BLOCK_SCALES), + moe_backend="TRTLLM", + moe_backend_requested="TRTLLM", + sparse_attention_config=QSASparseAttentionConfig(), + spec_config=spec_config, + ) + composite = SimpleNamespace( + vision_config=Qwen4ExpVisionConfig(**_VISION), image_token_id=248056, video_token_id=248057 + ) + # The core asserts the device it runs on; the weights it declares stay on meta. + with ( + mock.patch.object(torch.cuda, "get_device_capability", return_value=M._SM), + torch.device("meta"), + ): + return M.Qwen4ExpUnifiedModelingV2(model_config, composite) + + +#: Every rank; MTP with 7 drafts at a member's last rank, 1 elsewhere, and none at dep2. +_CORE_CASES = [(m, r, None if m == "dep2" else 7 if r == _world(m) - 1 else 1) for m, r in _RANKS] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, rank, max_draft_len", + _CORE_CASES, + ids=[f"{m}-rank{r}-mtp{d}" for m, r, d in _CORE_CASES], +) +def test_each_rank_builds_its_share_of_the_core(member, rank, max_draft_len) -> None: + """This rank's heads (rank `r` of two holds KV head `r`, of four `r // 2`) and one + whole-expert window at the full intermediate, whatever the MoE TP/EP split; with MTP the draft + layer 48 is cut like a trunk layer and a verify step carries `max_draft_len + 1` rows.""" + hq, hkv, row, tail, hk, hv, split, out_k = _GEOMETRY[member] + world, adp = _world(member), MEMBERS[member].get("enable_attention_dp", False) + attn_tp = 1 if adp else world + core = _core(member, rank) + assert (core.heads_q, core.heads_kv) == (hq, hkv) + assert core.kv_head_start == (rank * 2 // attn_tp if attn_tp > 1 else 0) + assert (core.attn_in_width, core.attn_in_tail_heads) == (row, tail) + assert core.index_col == core.qkv_width + core.q_width == 2 * (hq + hkv) * 256 + assert row == 2 * hq * 256 + 2 * hkv * 256 + 640 + 128 + assert tuple(core.w["l3_attn_in_w"].shape) == (row, HIDDEN) + assert tuple(core.w["l3_o_w"].shape) == (HIDDEN, out_k) + assert (core.num_k_heads, core.num_v_heads) == (hk, hv) + assert tuple(core.in_proj_split) == split + # Padded with zero rows to a multiple of 16, the decode GEMM's whole-tile width. + assert core.in_proj_rows == -(-sum(split) // 16) * 16 + assert tuple(core.w["l0_in_proj_w"].shape) == (core.in_proj_rows, HIDDEN) + assert tuple(core.w["l0_out_w"].shape) == (HIDDEN, out_k) + assert (core.attn_tp_size, core.dp_size) == (attn_tp, world if adp else 1) + + count = NUM_EXPERTS // world + assert (core.local_experts, core.expert_offset, core.moe_inter) == (count, count * rank, 640) + window = W.expert_window(core.w, core.dims, 0) + assert (window.offset, window.count) == (count * rank, count) + assert tuple(window.fc1_w.shape) == (count, 2 * 640, HIDDEN) + assert window.fc1_w.dtype == window.fc2_w.dtype == torch.float8_e4m3fn + assert tuple(window.fc1_s.shape) == (count, 2 * 640 // 128, HIDDEN // 128) + assert window.fc1_s.dtype == torch.float32 + assert tuple(window.fc2_w.shape) == (count, HIDDEN, 640) + assert tuple(window.fc2_s.shape) == (count, HIDDEN // 128, 640 // 128) + assert tuple(core.w["l0_router"].shape) == (NUM_EXPERTS, HIDDEN) + assert not core.mtp_enabled and core.spec_rollback is None + assert not any("_mtp_" in k for k in core.w) + if max_draft_len is None: + return + + mtp = _core(member, rank, _mtp(max_draft_len)) + assert mtp.mtp_enabled and mtp.mtp_layer_idx == NUM_LAYERS and mtp.spec_rollback is not None + assert max_draft_len + 1 <= M._MAX_VERIFY_WIDTH + assert mtp.mtp_theta == mtp.theta == 1e7 + draft = W.expert_window(mtp.w, mtp.dims, NUM_LAYERS) + assert (draft.offset, draft.count) == (window.offset, window.count) + assert draft.fc1_w.shape == window.fc1_w.shape and draft.fc2_s.shape == window.fc2_s.shape + assert draft.fc1_w.dtype == draft.fc2_w.dtype == torch.float8_e4m3fn + p = f"l{NUM_LAYERS}_mtp_" + assert tuple(mtp.w[f"{p}attn_in_w"].shape) == (row, HIDDEN) + assert tuple(mtp.w[f"{p}o_w"].shape) == (HIDDEN, out_k) + assert tuple(mtp.w[f"{p}router"].shape) == (NUM_EXPERTS, HIDDEN) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "rows, dp_size, step, grouped_min, calls", + [ + (1, 1, 8192, 0, [("runner", 1)]), + (20000, 1, 8192, 0, [("runner", 8192), ("runner", 8192), ("runner", 3616)]), + (4095, 1, 8192, 4096, [("runner", 4095)]), + (16384, 1, 16384, 4096, [("grouped", 16384)]), + # An attention-DP window: up to four 8192-row chunks a runner call, never grouped. + (40000, 4, 8192, 4096, [("runner", 32768), ("runner", 7232)]), + ], + ids=["one-row", "tp-chunks", "tp-below-grouped", "tp-context-grouped", "adp-window"], +) +def test_a_moe_layer_enters_the_fp8_runner_over_its_window( + rows, dp_size, step, grouped_min, calls, monkeypatch +) -> None: + """`_routed_experts` enters the FP8 runner pre-routed over this rank's window in row-ordered + chunks of `core.moe_max_rows` rows, or `moe_prefill_grouped` for a large TP context chunk.""" + core = SimpleNamespace( + topk=TOPK, num_experts=NUM_EXPERTS, moe_inter=640, dp_size=dp_size, moe_max_rows=step + ) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384, 128) + x = torch.arange(rows, dtype=torch.float32)[:, None].expand(rows, 4).contiguous() + seen = [] + + def router(chunk, mat_b): + assert mat_b == "router" + return chunk[:, :1] + + def routing(logits, topk, output_dtype): + assert topk == TOPK + return logits.int().expand(-1, topk).contiguous(), logits.expand(-1, topk).contiguous() + + def quantize(chunk, use_ue8m0): + assert use_ue8m0 is False + return chunk + 0.5, chunk[:, :2].t().contiguous() + + def runner(**kw): + seen.append(("runner", kw["hidden_states"], kw["hidden_states_scale"], kw)) + assert kw["routing_logits"] is None and kw["routing_bias"] is None + assert (kw["gemm1_weights"], kw["gemm1_weights_scale"]) == ("fc1_w", "fc1_s") + assert (kw["gemm2_weights"], kw["gemm2_weights_scale"]) == ("fc2_w", "fc2_s") + assert (kw["num_experts"], kw["top_k"], kw["intermediate_size"]) == (512, 10, 640) + assert (kw["local_expert_offset"], kw["local_num_experts"]) == (384, 128) + assert kw["routing_method_type"] == 1 and kw["act_type"] == 0 + return kw["hidden_states"] * 2 + + def grouped(xq, xsf, ids, w, *weights): + assert weights == ("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384) + seen.append(("grouped", xq, xsf, dict(topk_ids=ids, topk_weights=w))) + return xq * 2 + + monkeypatch.setattr(M, "cublas_mm", router) + monkeypatch.setattr(M, "renorm_moe_routing_op", routing) + monkeypatch.setattr(M, "fp8_quantize_1x128", quantize) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", runner) + monkeypatch.setattr(M, "moe_prefill_grouped", grouped) + monkeypatch.setattr(M, "prefill_grouped_ready", lambda local: local == 128) + monkeypatch.setattr(M, "_MOE_PREFILL_GROUPED_MIN_ROWS", grouped_min) + out = M._routed_experts(core, x, "router", experts) + assert [(kind, xq.shape[0]) for kind, xq, *_ in seen] == calls + torch.testing.assert_close(out, (x + 0.5) * 2, rtol=0, atol=0) + lo = 0 + for _, xq, xsf, kw in seen: + n = xq.shape[0] + torch.testing.assert_close(xsf, x[lo : lo + n, :2].t(), rtol=0, atol=0) + torch.testing.assert_close(kw["topk_ids"][:, 0], x[lo : lo + n, 0].int(), rtol=0, atol=0) + torch.testing.assert_close(kw["topk_weights"][:, 0], x[lo : lo + n, 0], rtol=0, atol=0) + lo += n + + +def _pad(x: torch.Tensor, rows: int) -> torch.Tensor: + return torch.cat([x, x.new_zeros(rows - x.shape[0], x.shape[1])], 0) + + +def _expert_out(x, ids, wts, start, count): + """A stand-in MoE over experts `[start, start + count)`: whole-expert windows sum to all 512.""" + keep = ((ids >= start) & (ids < start + count)).to(x.dtype) + return (wts * keep * (ids + 1).to(x.dtype)).sum(-1, keepdim=True) * x + + +#: The core's side stream on the host, where there is nothing to order. +_HOST_STREAM = SimpleNamespace(wait_stream=lambda other: None) + + +class _HostMoe: + """A MoE layer's ops in float64 over a fixed router: top-10 softmax routing, `_expert_out` + over the window, chunks of `chunk_rows`, two chunks a runner call on an attention-DP window.""" + + def __init__(self, monkeypatch, chunk_rows: int): + gen = torch.Generator().manual_seed(0) + self.router = torch.randn(HIDDEN, NUM_EXPERTS, generator=gen, dtype=torch.float64) + self.chunk_rows = chunk_rows + self.calls, self.windows = [], [] + monkeypatch.setattr(M, "_MOE_MAX_ROWS", chunk_rows) + monkeypatch.setattr(M, "_MOE_MAX_ROWS_TP", chunk_rows) + monkeypatch.setattr(M, "_MOE_PREFILL_GROUPED_MIN_ROWS", 0) # the runner path + monkeypatch.setattr(M, "_ADP_WINDOW_MAX_ROWS", 2 * chunk_rows) + monkeypatch.setattr(M, "cublas_mm", lambda a, b, *args, **kwargs: a @ b) + monkeypatch.setattr(M, "renorm_moe_routing_op", self.routing) + # Scales in the `[hidden / 128, rows]` layout, which grouped chunks concatenate. + monkeypatch.setattr( + M, "fp8_quantize_1x128", lambda t, use_ue8m0: (t, t.new_zeros(HIDDEN // 128, len(t))) + ) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", self.runner) + monkeypatch.setattr(M, "silu_and_mul_rows", lambda gug, inter: gug[:, :inter]) + # The shared expert and its gate are replicated and checked elsewhere. + monkeypatch.setattr(M, "fused_sigmoid_gate_mul_add", lambda routed, gate, shared: routed) + # `_mtp_moe_block` forks the shared expert onto a side stream; the host runs it in order. + monkeypatch.setattr(torch.cuda, "current_stream", lambda *args, **kwargs: _HOST_STREAM) + monkeypatch.setattr(torch.cuda, "stream", lambda stream: contextlib.nullcontext()) + + def route(self, x): + top_logits, top_ids = torch.topk(x @ self.router, TOPK, dim=-1) + return top_ids, torch.softmax(top_logits, dim=-1) + + def routing(self, logits, topk, output_dtype): + assert topk == TOPK and logits.shape[1] == NUM_EXPERTS + top_logits, top_ids = torch.topk(logits, topk, dim=-1) + return top_ids, torch.softmax(top_logits, dim=-1) + + def runner(self, **kw): + assert kw["num_experts"] == NUM_EXPERTS and kw["intermediate_size"] == 640 + x = kw["hidden_states"] + assert 0 < x.shape[0] <= 2 * self.chunk_rows + self.calls.append(x.shape[0]) + start, count = kw["local_expert_offset"], kw["local_num_experts"] + self.windows.append((start, count)) + return _expert_out(x, kw["topk_ids"], kw["topk_weights"], start, count) + + def views(self, core): + core._side_stream = _HOST_STREAM + return SimpleNamespace( + router=self.router, + experts=W.expert_window(core.w, core.dims, NUM_LAYERS), + sh_gug=torch.zeros(HIDDEN, 2 * 640 + 8, dtype=torch.float64), + sh_dn=torch.zeros(640, HIDDEN, dtype=torch.float64), + ) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "counts, draft", + [ + ([3, 7], False), + ([0, 0, 5, 3], False), + ([4, 4, 4, 4], False), + ([3, 7, 1, 4], True), + ([1, 2, 1, 3], True), + ([4, 4, 4, 4], True), + ], + ids=[ + "dep2-trunk", + "dep4-trunk-two-ranks-without-rows", + "dep4-trunk-uniform", + "dep4-draft-extend", + "dep4-draft-decode-with-a-dummy-rank", + "dep4-draft-captured-batch", + ], +) +def test_the_attention_dp_round_trip_returns_each_ranks_rows(counts, draft, monkeypatch) -> None: + """Each rank gathers the group's rows, runs its window over the real rows only (a padding + row's zero logits would route to the lowest experts) and reduce-scatters, with the same + collectives on every rank (ragged when counts differ), getting back its own rows' MoE.""" + world = len(counts) + moe = _HostMoe(monkeypatch, chunk_rows=5) + gen = torch.Generator().manual_seed(1) + xs = [torch.randn(n, HIDDEN, generator=gen, dtype=torch.float64) for n in counts] + rows, n = max(counts), sum(counts) + split = min(counts) != rows + if split: + gathered = torch.cat(xs, 0) + starts = [sum(counts[:r]) for r in range(world)] + real = torch.arange(n) + else: + gathered = torch.cat([_pad(x, rows) for x in xs], 0) + starts = [r * rows for r in range(world)] + real = torch.cat([torch.arange(r * rows, r * rows + c) for r, c in enumerate(counts)]) + real_rows = gathered[real] + full = _expert_out(gathered, *moe.route(gathered), 0, NUM_EXPERTS) + state = {} + + def fake_allgather(t, sizes, group, process_group): + assert sizes == (list(counts) if split else None) + assert group == list(range(world)) and process_group is None + state["collectives"].append(("allgather", tuple(t.shape))) + x = xs[state["rank"]] + torch.testing.assert_close(t, x if split else _pad(x, rows), rtol=0, atol=0) + return gathered + + def fake_reducescatter(t, sizes, group, process_group): + assert sizes == (list(counts) if split else None) + assert group == list(range(world)) and process_group is None + state["collectives"].append(("reducescatter", tuple(t.shape))) + state["partial"] = t + r = state["rank"] + return full[starts[r] : starts[r] + (counts[r] if split else rows)] + + def no_tp_sum(t): + raise AssertionError("the MoE under attention DP went through the TP sum") + + monkeypatch.setattr(M, "allgather", fake_allgather) + monkeypatch.setattr(M, "reducescatter", fake_reducescatter) + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + per_rank = [] + for rank in range(world): + core = _core(f"dep{world}", rank, _mtp() if draft else None) + core.expert_sum, core.tp_sum = core._expert_sum_op(), no_tp_sum + state.update(rank=rank, collectives=[]) + moe.calls, moe.windows = [], [] + if draft: + pad_rows = M._draft_pad_rows(core, counts, counts[rank]) + out = M._mtp_moe_block(core, moe.views(core), xs[rank], pad_rows) + else: + md = SimpleNamespace(all_rank_num_tokens=counts) + pad_rows = core._dp_pad_rows(md, counts[rank]) + trunk = W.expert_window(core.w, core.dims, 0) + window = lambda t: M._routed_experts(core, t, moe.router, trunk) # noqa: E731 + out = core.expert_sum(window, xs[rank], pad_rows) + count = NUM_EXPERTS // world + assert moe.calls == [min(10, n - lo) for lo in range(0, n, 10)] + assert set(moe.windows) == {(count * rank, count)} + torch.testing.assert_close( + state["partial"][real], + _expert_out(real_rows, *moe.route(real_rows), count * rank, count), + ) + assert tuple(out.shape) == (counts[rank], HIDDEN) + torch.testing.assert_close(out, _expert_out(xs[rank], *moe.route(xs[rank]), 0, NUM_EXPERTS)) + # Each rank hands the gather its own rows: the ragged form's shapes differ by rank. + assert state["collectives"][0] == ("allgather", (counts[rank] if split else rows, HIDDEN)) + per_rank.append(state["collectives"][1:]) + assert all(c == per_rank[0] for c in per_rank), per_rank + assert per_rank[0] == [("reducescatter", (n if split else world * rows, HIDDEN))] + + +class _HostA2a: + """The NVLink MoE all-to-all at one rank: `dispatch` hands it each source's rows routed into + its window (padded with -1 routes); `combine` checks its partials and returns its rows' MoE.""" + + def __init__(self, xs, routes, rank): + self.xs, self.routes, self.rank = xs, routes, rank + self.world, self.cap = len(xs), max(len(x) for x in xs) + self.max_num_tokens_per_rank = 1 << 20 + self.calls = [] + + def dispatch(self, x, sf, ids, w, counts): + self.calls.append("dispatch") + assert sf is None and counts == [len(t) for t in self.xs] + own_ids, own_w = self.routes[self.rank] + torch.testing.assert_close(x, self.xs[self.rank], rtol=0, atol=0) + # The router runs over chunks of the rows, so the weights match to rounding. + assert torch.equal(ids, own_ids), "not routed on its own rows" + torch.testing.assert_close(w, own_w.float()) # fp32 on the wire, as the stock MoE's + lo, count = self.rank * (NUM_EXPERTS // self.world), NUM_EXPERTS // self.world + rows, rids, rw = [], [], [] + for src, (sx, (sids, sw)) in enumerate(zip(self.xs, self.routes)): + hit = ((sids >= lo) & (sids < lo + count)).any(1) + pad = self.cap - int(hit.sum()) + rows.append(torch.cat([sx[hit], sx.new_zeros(pad, HIDDEN)])) + rids.append(torch.cat([sids[hit], sids.new_full((pad, TOPK), -1)])) + rw.append(torch.cat([sw[hit], sw.new_zeros(pad, TOPK)])) + self.got = torch.cat(rows), torch.cat(rids), torch.cat(rw), lo, count + return self.got[0], None, self.got[1], self.got[2] + + def combine(self, partials): + self.calls.append("combine") + rows, ids, w, lo, count = self.got + torch.testing.assert_close(partials, _expert_out(rows, ids, w, lo, count)) + own = self.xs[self.rank] + return _expert_out(own, *self.routes[self.rank], 0, NUM_EXPERTS) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "counts, draft", + [([3, 7], False), ([0, 5, 2, 4], False), ([4, 4, 4, 4], True)], + ids=["dep2-trunk-ragged", "dep4-trunk-a-rank-without-rows", "dep4-draft-captured-batch"], +) +def test_the_attention_dp_alltoall_round_trip(counts, draft, monkeypatch) -> None: + """`TRTLLM_MODELING_V2_MOE_ALLTOALL=1`: each rank routes its own rows through the all-to-all + and returns what the combine sums; no all-gather, reduce-scatter or TP sum.""" + world = len(counts) + moe = _HostMoe(monkeypatch, chunk_rows=5) + gen = torch.Generator().manual_seed(2) + xs = [torch.randn(n, HIDDEN, generator=gen, dtype=torch.float64) for n in counts] + routes = [moe.route(x) for x in xs] + + def forbidden(*args, **kwargs): + raise AssertionError("the all-to-all round trip took the default collectives") + + monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "1") + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + for name in ("allgather", "reducescatter"): + monkeypatch.setattr(M, name, forbidden) + for rank in range(world): + core = _core(f"dep{world}", rank, _mtp() if draft else None) + a2a = _HostA2a(xs, routes, rank) + monkeypatch.setattr(M, "_build_moe_a2a", lambda mapping, max_tokens: a2a) + core.expert_sum, core.tp_sum = core._expert_sum_op(), forbidden + assert core.moe_a2a is a2a + if draft: + pad_rows = M._draft_pad_rows(core, counts, counts[rank]) + out = M._mtp_moe_block(core, moe.views(core), xs[rank], pad_rows) + else: + md = SimpleNamespace(all_rank_num_tokens=counts) + run = M._WindowRun(core, moe.router, W.expert_window(core.w, core.dims, 0)) + out = core.expert_sum(run, xs[rank], core._dp_pad_rows(md, counts[rank])) + assert a2a.calls == ["dispatch", "combine"] + torch.testing.assert_close(out, _expert_out(xs[rank], *routes[rank], 0, NUM_EXPERTS)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "accepted, rows_per_request, index_share, dp_size, want", + [ + ([1, 3, 2, 1], 3, False, 1, [0, 5, 7, 9]), + ([1, 3, 2, 1] * 4, 3, False, 1, [r * 3 + a - 1 for r, a in enumerate([1, 3, 2, 1] * 4)]), + ([1, 3, 2, 1], 1, False, 1, None), + ([1, 3, 2, 1], 3, True, 1, None), # IndexShare's capture keeps every row + ([1, 3, 2, 1], 3, False, 2, None), # attention DP pads by the worker's counts + (None, 3, False, 1, None), + ], + ids=["draft-extend", "ht-batch", "not-extend", "index-share", "attention-dp", "no-counts"], +) +def test_a_draft_extend_step_continues_from_each_requests_last_accepted_row( + accepted, rows_per_request, index_share, dp_size, want +) -> None: + """Request r's draft-extend outputs are read from row `num_accepted[r] - 1` of its + `rows_per_request` (the golden token counts as accepted); None means every row.""" + counts = None if accepted is None else torch.tensor(accepted, dtype=torch.int32) + md = SimpleNamespace(qsa_mtp_num_accepted=counts) + requests = 4 if accepted is None else len(accepted) + got = M._draft_continue_rows( + SimpleNamespace(dp_size=dp_size), md, requests, rows_per_request, index_share + ) + assert (None if got is None else got.tolist()) == want + + +# ------------------------------------------- sequence-parallel prefill at attention TP --- + + +def _sp_target(rank: int, world: int, comm: bool = False, ce=None): + target = object.__new__(M.PrefillTarget) + target._sequence_parallel = (rank, world, list(range(world)), None) + target._ce_exchange = ce + target._comm_stream = target._comm_fork = target._comm_join = target._sp_rows = None + if comm: + target._comm_stream = torch.cuda.Stream(priority=-1) + target._comm_fork, target._comm_join = torch.cuda.Event(), torch.cuda.Event() + return target + + +def _sp_share(monkeypatch, num_tokens, world, rank, want, configured=True): + """From `_SP_MIN_ROWS` rows that divide evenly a rank keeps its consecutive `T / world` rows, + gathered and scattered uniformly over the attention-TP group; otherwise there is no split.""" + target = _sp_target(rank, world) + if not configured: + target._sequence_parallel = None + calls = [] + monkeypatch.setattr( + M, "allgather", lambda t, sizes, group, pg: calls.append(("ag", sizes, group)) or t + ) + monkeypatch.setattr( + M, "reducescatter", lambda t, sizes, group, pg: calls.append(("rs", sizes, group)) or t + ) + sp = target._sp_split(num_tokens) + if want is None: + assert sp is None + return + lo, hi, gather, scatter = sp + assert (lo, hi) == want + gather(torch.zeros(1)) + scatter(torch.zeros(1)) + assert calls == [("ag", None, list(range(world))), ("rs", None, list(range(world)))] + + +def _sp_moe_own_rows(monkeypatch, rows, step, padded): + """`_routed_experts_sp` routes (padding rows to `_NO_EXPERT`) and quantizes only this rank's + rows; the gathered codes, scales, ids and weights reach the runner exactly, in row order.""" + hidden, topk = 384, 10 + core = SimpleNamespace(topk=topk, num_experts=512, moe_inter=640, moe_max_rows=step) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 0, 256) + gen = torch.Generator().manual_seed(rows) + x = torch.randn(rows, hidden, generator=gen).to(torch.bfloat16) + ids = torch.randint(0, 512, (rows, topk), generator=gen, dtype=torch.int32) + weights = torch.rand(rows, topk, generator=gen).to(torch.bfloat16) + codes = (torch.randn(rows, hidden, generator=gen) * 8).to(torch.float8_e4m3fn) + scales = torch.rand(hidden // 128, rows, generator=gen) + monkeypatch.setattr(M, "cublas_mm", lambda a, b, *args: ("logits", a.shape[0])) + monkeypatch.setattr( + M, "renorm_moe_routing_op", lambda logits, topk, output_dtype: (ids.clone(), weights) + ) + monkeypatch.setattr(M, "fp8_quantize_1x128", lambda t, use_ue8m0: (codes, scales)) + calls = [] + monkeypatch.setattr( + M, + "fp8_block_scale_moe_runner", + lambda **kw: calls.append(kw) or torch.zeros(kw["hidden_states"].shape[0], 4), + ) + mask = (torch.arange(rows) >= rows - padded)[:, None] if padded else None + want_ids = ids.clone() + want_ids[rows - padded :] = M._NO_EXPERT + gathers = [] + out = M._routed_experts_sp( + core, x, "router", experts, lambda t: gathers.append(t.dtype) or t.clone(), mask + ) + assert gathers == [torch.bfloat16, torch.bfloat16] + assert tuple(out.shape) == (rows, 4) + assert [c["hidden_states"].shape[0] for c in calls] == [ + min(step, rows - lo) for lo in range(0, rows, step) + ] + got_codes = torch.cat([c["hidden_states"] for c in calls]) + assert torch.equal(got_codes.view(torch.uint8), codes.view(torch.uint8)) + assert torch.equal(torch.cat([c["hidden_states_scale"] for c in calls], 1), scales) + assert all(c["hidden_states_scale"].is_contiguous() for c in calls) + assert torch.equal(torch.cat([c["topk_ids"] for c in calls]), want_ids) + assert torch.equal(torch.cat([c["topk_weights"] for c in calls]), weights) + assert all(c["topk_ids"].dtype == torch.int32 for c in calls) + + +def _sp_ce_scatter(monkeypatch, rank): + """A copy-engine two-rank scatter sends the peer's rows and adds its partial (the same bits in + either order) without NCCL; `CopyEngineExchange.create` builds only a one-node pair.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm.ce_exchange__sp_pair import ( # noqa: E501 + CopyEngineExchange, + ) + + num_tokens, hidden = 4096, 64 + rows = num_tokens // 2 + gen = torch.Generator().manual_seed(3) + partial = [torch.randn(num_tokens, hidden, generator=gen).to(torch.bfloat16) for _ in range(2)] + sent = [] + + class Exchange: + def exchange(self, x): + sent.append(x) + return partial[1 - rank][rank * rows : (rank + 1) * rows].clone() + + monkeypatch.setattr(M, "reducescatter", lambda *a: pytest.fail("NCCL reduce-scatter called")) + lo, hi, _, scatter = _sp_target(rank, 2, ce=Exchange())._sp_split(num_tokens) + got = scatter(partial[rank]) + assert (lo, hi) == (rank * rows, (rank + 1) * rows) + assert torch.equal(sent[0], partial[rank][(1 - rank) * rows : (2 - rank) * rows]) + assert torch.equal(got, partial[0][lo:hi] + partial[1][lo:hi]) + assert torch.equal(got, partial[1][lo:hi] + partial[0][lo:hi]) + four = SimpleNamespace(tp_size=4, gpus_per_node=4, tp_rank=0) + across = SimpleNamespace(tp_size=2, gpus_per_node=1, tp_rank=0) + assert CopyEngineExchange.create(four, 1 << 20) is None + assert CopyEngineExchange.create(across, 1 << 20) is None + + +def _sp_project(monkeypatch, rank, overlap): + """`_project_overlapped` projects this rank's rows while the comm stream gathers the others', + each into its own rows of one output; without the comm stream the block gathers first.""" + dev = torch.device("cuda") + gen = torch.Generator(device=dev).manual_seed(11 + rank) + rows, hidden, width = 512, 256, 384 + shares = [ + torch.randn(rows, hidden, generator=gen, device=dev).to(torch.bfloat16) for _ in range(2) + ] + w = (torch.randn(width, hidden, generator=gen, device=dev) * 0.1).to(torch.bfloat16).t() + target = _sp_target(rank, 2, comm=overlap) + gathered = [] + + def gather(t): + assert t.data_ptr() == shares[rank].data_ptr() + gathered.append(torch.cuda.current_stream()) + return torch.cat(shares) + + target._sp_rows = (rank * rows, (rank + 1) * rows, gather) + assert target._overlap_projection() is overlap + if overlap: + out = target._project_overlapped(shares[rank], w) + torch.cuda.synchronize() + assert gathered == [target._comm_stream], "the gather runs on the comm stream" + assert tuple(out.shape) == (2 * rows, width) + want = torch.cat(shares).float() @ w.float() + torch.testing.assert_close(out.float(), want, rtol=2e-2, atol=2e-2) + else: + assert torch.equal(target._block_rows(shares[rank]), torch.cat(shares)) + target._sp_rows = None + assert not target._overlap_projection() + assert target._block_rows(shares[rank]) is shares[rank] + + +def _sp_ce_project_reduce(monkeypatch, rank): + """With the copy-engine pair exchange, `_project_overlapped` and `_reduce_overlapped` exchange + on the comm stream and match NCCL bitwise; the reduction sums both ranks' products.""" + dev = torch.device("cuda") + gen = torch.Generator(device=dev).manual_seed(23) + rows, hidden, width = 384, 256, 320 + shares = [ + torch.randn(rows, hidden, generator=gen, device=dev).to(torch.bfloat16) for _ in range(2) + ] + a = [ + torch.randn(2 * rows, hidden, generator=gen, device=dev).to(torch.bfloat16) + for _ in range(2) + ] + w = (torch.randn(width, hidden, generator=gen, device=dev) * 0.1).to(torch.bfloat16).t() + sent = [torch.mm(a[r][(1 - r) * rows : (2 - r) * rows], w) for r in range(2)] + calls = [] + + class Exchange: + def exchange(self, x): + assert torch.cuda.current_stream() == with_ce._comm_stream + if torch.equal(x, shares[rank]): + calls.append("rows") + return shares[1 - rank].clone() + assert torch.equal(x, sent[rank]) + calls.append("partials") + return sent[1 - rank].clone() + + def gather(t): + return torch.cat(shares) if torch.equal(t, shares[rank]) else torch.cat(sent) + + with_ce, nccl = _sp_target(rank, 2, comm=True, ce=Exchange()), _sp_target(rank, 2, comm=True) + for t in (with_ce, nccl): + t._sp_rows = (rank * rows, (rank + 1) * rows, gather) + assert with_ce._overlap_reduce() + got = with_ce._project_overlapped(shares[rank], w) + want = nccl._project_overlapped(shares[rank], w) + red = with_ce._reduce_overlapped(a[rank], w) + red_want = nccl._reduce_overlapped(a[rank], w) + torch.cuda.synchronize() + assert calls == ["rows", "partials"] + assert torch.equal(got, want) and torch.equal(red, red_want) + full = sum(torch.mm(a[r], w).float() for r in range(2)) + torch.testing.assert_close( + red.float(), full[rank * rows : (rank + 1) * rows], rtol=2e-2, atol=2e-2 + ) + for world, comm in ((4, True), (2, False)): # four ranks, or no comm stream: one GEMM + target = _sp_target(rank, world, comm=comm) + target._sp_rows = with_ce._sp_rows + assert not target._overlap_reduce() + + +_cpu = pytest.mark.cpu_only +_SP_CASES = [ + *( + pytest.param(_sp_share, dict(num_tokens=t, world=w, rank=r, want=want, configured=c), + marks=_cpu, id=f"share-{t}-tp{w}-rank{r}{'' if c else '-off'}") + for t, w, r, want, c in [ + (16384, 2, 1, (8192, 16384), True), + (4096, 2, 0, (0, 2048), True), + (4095, 2, 0, None, True), # an odd eager step keeps every row + (2048, 2, 0, None, True), # below `_SP_MIN_ROWS` + (8192, 4, 3, (6144, 8192), True), + (16384, 2, 0, None, False), # one rank, attention DP, or the switch off + ] + ), + *( + pytest.param(_sp_moe_own_rows, dict(rows=rows, step=step, padded=padded), marks=_cpu, + id=f"moe-own-rows-{rows}-step{step}-pad{padded}") + for rows, step, padded in [(96, 8192, 0), (96, 40, 5), (7, 8192, 3)] + ), + *( + pytest.param(_sp_ce_scatter, dict(rank=r), marks=_cpu, id=f"ce-scatter-rank{r}") + for r in (0, 1) + ), + *( + pytest.param(_sp_project, dict(rank=r, overlap=o), marks=_needs_gpu, + id=f"project-rank{r}-{'overlap' if o else 'gather'}") + for r, o in [(0, True), (1, True), (1, False)] + ), + *( + pytest.param(_sp_ce_project_reduce, dict(rank=r), marks=_needs_gpu, + id=f"ce-project-reduce-rank{r}") + for r in (0, 1) + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _SP_CASES) +def test_a_sequence_parallel_prefill_step(check, kwargs, monkeypatch) -> None: + """Sequence-parallel prefill at TP2 / TP4: row shares, own-row MoE, overlapped projections.""" + check(monkeypatch, **kwargs) + + +# ------------------------------------------------- the split context selection --- + + +def _selection_balance(monkeypatch, chunks, starts, world): + """`_selection_cuts` ascend from 0 to the step's rows, interior cuts on the index scores' + 64-row tile, each rank's summed row cost (position + 1) within one tile of the even share.""" + cuts = M._selection_cuts(chunks, starts, world) + rows = sum(chunks) + assert len(cuts) == world + 1 and cuts[0] == 0 and cuts[-1] == rows + assert all(a <= b for a, b in zip(cuts, cuts[1:])) + assert all(c % M._SELECTION_CUT_ALIGN == 0 for c in cuts[1:-1]) + cost = [s0 + j + 1 for c, s0 in zip(chunks, starts) for j in range(c)] + parts = [sum(cost[a:b]) for a, b in zip(cuts, cuts[1:])] + even = sum(cost) / world + assert max(abs(x - even) for x in parts) <= M._SELECTION_CUT_ALIGN * max(cost) + 1, parts + + +def _selection_lands_rows(monkeypatch, chunks, starts, world, balanced, pieces): + """`_split_context_selection`: each rank selects only its share of the context rows, gathered + in `pieces` on the comm stream, and every rank ends bitwise as one rank selecting them all.""" + dev = torch.device("cuda" if pieces else "cpu") + lo, width = 5, 2051 + rows = sum(chunks) + cuts = M._selection_cuts(chunks, starts, world) if balanced else None + bounds = cuts or [min(r * -(-rows // world), rows) for r in range(world + 1)] + pipeline = None + if pieces: + pipeline = (pieces, torch.cuda.Stream(priority=-1), torch.cuda.Event(), torch.cuda.Event()) + + def select_rows(a, b, context_rows, out): + assert context_rows + out.copy_( + torch.arange(a, b, dtype=torch.int32, device=dev)[:, None] * 4099 + + torch.arange(width, dtype=torch.int32, device=dev) + ) + + want = torch.full((lo + rows + 3, width), -1, dtype=torch.int32, device=dev) + select_rows(lo, lo + rows, True, want[lo : lo + rows]) + buffers = {} # (rank, piece) -> the rank's buffer + + def run(rank, gather_of): + scored, count = [], [0] + + def gather(mine, sizes, group, pg): + if pipeline is not None: + assert torch.cuda.current_stream() == pipeline[1], "a piece gathers on comm" + count[0] += 1 + return gather_of(rank, count[0] - 1, mine) + + def record(a, b, context_rows, out): + scored.append((a, b)) + select_rows(a, b, context_rows, out) + + monkeypatch.setattr(M, "allgather", gather) + sel = torch.full_like(want, -1) + M._split_context_selection( + (rank, world, list(range(world)), None), record, sel, lo, lo + rows, pipeline, cuts + ) + if pieces: + torch.cuda.synchronize() + return sel, scored + + for rank in range(world): # each rank's own buffers (the stand-in returns a gather's shape) + _, scored = run( + rank, lambda r, j, mine: torch.cat([buffers.setdefault((r, j), mine.clone())] * world) + ) + assert scored[0][0] == lo + bounds[rank] and scored[-1][1] == lo + bounds[rank + 1] + assert all(b == a2 for (_, b), (a2, _) in zip(scored, scored[1:])), scored + assert all(b.dtype == torch.bfloat16 and b.shape[1] == 2 * width for b in buffers.values()) + if not pieces: + per = max(b - a for a, b in zip(bounds, bounds[1:])) + assert all(b.shape[0] == per for b in buffers.values()) + for rank in range(world): + sel, _ = run(rank, lambda r, j, mine: torch.cat([buffers[(q, j)] for q in range(world)])) + assert torch.equal(sel, want), rank + + +_SELECTION_CASES = [ + *( + pytest.param(_selection_balance, dict(chunks=c, starts=s, world=w), marks=_cpu, + id=f"balance-{len(c)}x{c[0]}-tp{w}") + for c, s, w in [ + ([32766], [88064], 2), + ([2728] * 12, [30000 + 9000 * i for i in range(12)], 2), + ([2046] * 16, [88064] * 16, 2), + ([32766], [88064], 4), + ([5000, 3, 7000], [100000, 5, 0], 2), + ([1024], [0], 2), + ] + ), + *( + pytest.param(_selection_lands_rows, dict(chunks=c, starts=s, world=w, balanced=b, + pieces=0), marks=_cpu, id=f"lands-{sum(c)}-tp{w}-{'balanced' if b else 'even'}") + for c, s, w, b in [ + ([16384], [0], 2, False), + ([4093], [0], 2, False), + ([1025], [0], 4, False), + ([9000, 4000], [60000, 120], 2, True), + ([9000, 4000], [60000, 120], 4, True), + ] + ), + *( + pytest.param(_selection_lands_rows, dict(chunks=c, starts=s, world=2, balanced=b, + pieces=p), marks=_needs_gpu, id=f"pipelined-{sum(c)}-{p}-pieces") + for c, s, b, p in [([32766], [88064], True, 4), ([16382], [0], False, 4)] + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _SELECTION_CASES) +def test_a_tp_ranks_context_selection_is_its_balanced_share(check, kwargs, monkeypatch) -> None: + """An attention-TP rank selects its cost-balanced share of the context rows, then gathers.""" + check(monkeypatch, **kwargs) + + +# ----------------------------------------------------------------------- the pools --- + +ATTN_LAYERS = [i for i in range(NUM_LAYERS) if i % 4 == 3] +GDN_LAYERS = [i for i in range(NUM_LAYERS) if i % 4 != 3] +PLE_LAYER = 1 # `ple_layer_ids: [2]` is one-based +MTP_LAYER = NUM_LAYERS +TOKENS_PER_BLOCK = 32 # `kv_cache_config.tokens_per_block`'s default, which the runtime keeps +MAX_BATCH_SIZE = 4 + + +_ALL = dict(heads_kv=2, kv_head_start=0, num_k_heads=16, k_head_start=0, num_v_heads=48, + v_head_start=0, conv_dim=10240) # fmt: skip + + +def _tp2(r): + return dict(heads_kv=1, kv_head_start=r, num_k_heads=8, k_head_start=8 * r, num_v_heads=24, + v_head_start=24 * r, conv_dim=5120) # fmt: skip + + +def _tp4(r): + return dict(heads_kv=1, kv_head_start=r // 2, num_k_heads=4, k_head_start=4 * r, + num_v_heads=12, v_head_start=12 * r, conv_dim=2560) # fmt: skip + + +#: Per member and rank: KV heads and the first's global index, GDN key and value heads and their +#: first, and the conv channels. TP4 repeats each KV head on two neighbouring ranks. +LAYOUT = { + **{(m, r): _ALL for m in ("tp1", "dep2", "dep4") for r in range(_world(m))}, + **{(m, r): _tp2(r) for m in ("tp2", "tep2") for r in range(2)}, + **{(m, r): _tp4(r) for m in ("tp4", "tep4") for r in range(4)}, +} + +COMMON = dict( + tokens_per_block=TOKENS_PER_BLOCK, + head_dim=256, + index_kv_heads=1, + index_head_dim=128, + conv_kernel=4, + head_k_dim=128, + head_v_dim=128, + stream=4 * 2560, + ple_state_len=(4 - 1) * 3, + ngram_context_len=3 - 1, + dtype=torch.bfloat16, +) + +#: K/V storage per KV-cache algorithm: (pool dtype, innermost extent, whether the QSA index and +#: position pages share one side page). A 32-token bf16 index page is 8192 bytes: an FP8 K page +#: of one head, an NVFP4 one of two, never a bf16 K/V page (16384 bytes a head). +KV_STORAGE = { + None: (torch.bfloat16, 256, False), + QuantAlgo.FP8: (torch.float8_e4m3fn, 256, True), + QuantAlgo.NVFP4: (torch.uint8, 128, True), +} +KV_IDS = {None: "kv_bf16", QuantAlgo.FP8: "kv_fp8", QuantAlgo.NVFP4: "kv_nvfp4"} +_ALGOS = list(KV_STORAGE) + + +def _geometry(member: str, rank: int, kv_cache_quant_algo=None) -> P.PoolGeometry: + return P.pool_geometry( + Qwen4ExpTextConfig(**_TEXT), + _mapping(member, rank), + tokens_per_block=TOKENS_PER_BLOCK, + kv_cache_quant_algo=kv_cache_quant_algo, + dtype=torch.bfloat16, + ) + + +def _meta(shape, dtype): + return torch.empty(shape, dtype=dtype, device="meta") + + +class _MetaManager: + """The stock manager's views on meta, sized by a geometry, with the QSA planes as own roles or + slices of one packed side page. Empty `gdn_layers` stands for a separate draft manager.""" + + PAGES = 64 + + def __init__(self, g: P.PoolGeometry, mapping: Mapping, layers, gdn_layers=GDN_LAYERS): + self.g, self.mapping, self.max_batch_size = g, mapping, MAX_BATCH_SIZE + self.attn_layers, self.gdn_layers = list(layers), list(gdn_layers) + mapped = sorted([*self.attn_layers, *self.gdn_layers]) + self.layer_offsets = {layer: local for local, layer in enumerate(mapped)} + self.conv_section_dims = list(g.conv_sections) + self.slots = MAX_BATCH_SIZE + P.reserved_dummy_slots(mapping) + self.ssm_dtype, self.scale_pages_per_slot = torch.bfloat16, 1 + tpb = g.tokens_per_block + if g.qsa_side_planes: + index, position = g.index_page_bytes, g.position_page_bytes + side = _meta([self.PAGES, len(self.attn_layers) * index + position + 16], torch.uint8) + self._index = { + layer: side[:, k * index : (k + 1) * index] + .view(torch.bfloat16) + .view(self.PAGES, tpb, g.index_kv_heads, g.index_head_dim) + for k, layer in enumerate(self.attn_layers) + } + start = len(self.attn_layers) * index + self._position = ( + side[:, start : start + position].view(torch.int32).view(self.PAGES, tpb, 3) + ) + else: + self._index = { + layer: _meta([self.PAGES, tpb, g.index_kv_heads, g.index_head_dim], torch.bfloat16) + for layer in self.attn_layers + } + self._position = _meta([self.PAGES, tpb, 3], torch.int32) + + def get_buffers(self, layer, kv_layout="NHD"): + assert kv_layout == "HND" + g = self.g + return _meta([self.PAGES, 2, g.heads_kv, g.tokens_per_block, g.kv_width], g.kv_dtype) + + def get_block_scale_buffers(self, layer, kv_layout="NHD"): + assert kv_layout == "HND" + g = self.g + page = [g.heads_kv, g.tokens_per_block, g.head_dim // P.NVFP4_BLOCK] + return _meta([self.PAGES, 2, *page], torch.uint8) + + def get_index_k_buffer(self, layer): + return self._index[layer] + + def get_qsa_position_buffer(self): + return self._position + + def get_conv_states(self, layer): + return _meta([self.slots, self.g.conv_dim, self.g.conv_kernel - 1], torch.bfloat16) + + def get_ssm_states(self, layer): + g = self.g + return _meta([self.slots, g.num_v_heads, g.head_v_dim, g.head_k_dim], self.ssm_dtype) + + def ple_layer_cache(self, layer): + if not self.gdn_layers: + return None + g = self.g + return ( + _meta([self.slots, g.stream, g.ple_state_len], torch.bfloat16), + _meta([self.slots, g.ngram_context_len], torch.int64), + ) + + +#: Every rank under bf16 K/V; each quantized K/V storage at one member per attention layout (the +#: storage sets the pool's dtype, width and side-page packing alone, the rank its heads alone). +_POOL_CASES = [(m, r, None) for m, r in _RANKS] + [ + (m, r, a) for m, r in LAYOUT_RANKS for a in (QuantAlgo.FP8, QuantAlgo.NVFP4) +] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, rank, algo", _POOL_CASES, ids=[f"{m}-rank{r}-{KV_IDS[a]}" for m, r, a in _POOL_CASES] +) +def test_each_rank_binds_the_pools_of_its_layout(member, rank, algo): + """`pool_geometry` is `LAYOUT` and a rank's KV head matches the stock loader's split. + `take_pools` binds a stock-like manager; `take_draft_pools` takes the draft-only manager under + attention TP (unless a cache transceiver forbids it), else the trunk's, refusing the other.""" + g, mapping = _geometry(member, rank, algo), _mapping(member, rank) + kv_dtype, kv_width, side_planes = KV_STORAGE[algo] + assert g._asdict() == dict( + COMMON, + **LAYOUT[member, rank], + kv_dtype=kv_dtype, + kv_width=kv_width, + qsa_side_planes=side_planes, + ) + assert g.conv_sections == (g.num_k_heads * 128, g.num_k_heads * 128, g.num_v_heads * 128) + assert g.kv_block_scales == (algo == QuantAlgo.NVFP4) + heads, rows_per_head = 2, 4 + weight = torch.arange(heads * rows_per_head, dtype=torch.float32).reshape(-1, 1) + tp = P.attention_tp_size(mapping) + local = duplicate_kv_weight(weight, heads, tp).chunk(tp)[rank if tp > 1 else 0] + start, stop = g.kv_head_start * rows_per_head, (g.kv_head_start + g.heads_kv) * rows_per_head + assert torch.equal(local, weight[start:stop]) + + trunk = _MetaManager(g, mapping, ATTN_LAYERS) + pools = P.take_pools(trunk, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert sorted(pools.kv) == ATTN_LAYERS and sorted(pools.conv) == GDN_LAYERS + first = pools.kv[ATTN_LAYERS[0]] + assert tuple(first.k.shape[1:]) == (g.heads_kv, TOKENS_PER_BLOCK, g.kv_width) + assert (first.k_scale is not None) == (algo == QuantAlgo.NVFP4) + + spec = MTPDecodingConfig(max_draft_len=3, num_nextn_predict_layers=1) + separate = P.draft_kv_cache_is_separate(mapping) + assert separate == ( + not mapping.enable_attention_dp and should_use_separate_draft_kv_cache(spec) + ) + assert P.draft_kv_cache_is_separate(mapping, spec) == separate + spec._allow_separate_draft_kv_cache = False # what a cache transceiver makes the executor do + assert not should_use_separate_draft_kv_cache(spec) + assert not P.draft_kv_cache_is_separate(mapping, spec) + draft_alone = _MetaManager(g, mapping, [MTP_LAYER], []) + trunk_and_draft = _MetaManager(g, mapping, [*ATTN_LAYERS, MTP_LAYER]) + for sep in sorted({separate, False}): # the engine's choice, and a disaggregated engine's + cm, wrong = (draft_alone, trunk_and_draft) if sep else (trunk_and_draft, draft_alone) + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=sep) + assert draft.pool_idx == cm.layer_offsets[MTP_LAYER] + assert tuple(draft.kv.v.shape[1:]) == (g.heads_kv, TOKENS_PER_BLOCK, g.kv_width) + with pytest.raises(AssertionError, match="draft layer"): + P.take_draft_pools(wrong, g, MTP_LAYER, separate=sep) + with pytest.raises(AssertionError, match="does not map"): + P.take_draft_pools(trunk, g, MTP_LAYER, separate=sep) + + +# ------------------------------------------------ the stock manager, on GPU --- + + +@contextlib.contextmanager +def _manager(member: str, rank: int, kv_cache_quant_algo=None, spec_config=None, is_draft=False): + """The trunk manager (or with `is_draft` the separate draft KV cache) the engine builds on + this rank, with the core's cache config: bf16 recurrent state, no block reuse, manager V2.""" + # Imported here: the stock factory pulls in the executor, which the cpu_only cases skip. + from tensorrt_llm._torch.pyexecutor._util import ( + _create_kv_cache_manager, + get_kv_cache_manager_cls, + ) + from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype + + mapping = _mapping(member, rank) + model_config = ModelConfig( + pretrained_config=Qwen4ExpTextConfig(**_TEXT), + mapping=mapping, + quant_config=QuantConfig( + quant_algo=QuantAlgo.FP8_BLOCK_SCALES, kv_cache_quant_algo=kv_cache_quant_algo + ), + sparse_attention_config=QSASparseAttentionConfig(), + spec_config=spec_config, + ) + # What the loader does with `kv_cache_config.mamba_ssm_cache_dtype: bfloat16`. + validate_and_set_mamba_ssm_cache_dtype(model_config, "bfloat16") + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + mamba_ssm_cache_dtype="bfloat16", + max_tokens=4096, + use_kv_cache_manager_v2=True, + ) + cls = get_kv_cache_manager_cls(model_config, kv_cache_config) + cm = _create_kv_cache_manager( + None, + cls, + mapping, + kv_cache_config, + tokens_per_block=TOKENS_PER_BLOCK, + max_seq_len=2048, + max_batch_size=MAX_BATCH_SIZE, + spec_config=spec_config, + sparse_attention_config=model_config.sparse_attention_config, + max_num_tokens=2048, + max_beam_width=1, + kv_connector_manager=None, + model_config=model_config, + dtype=torch.bfloat16, + is_draft=is_draft, + max_cuda_graph_batch_size=MAX_BATCH_SIZE, + ) + try: + yield cm + finally: + cm.shutdown() + + +def _stock_trunk(member, rank, algo): + """Each trunk view passes the contract; the manager's per-rank numbers are the geometry's.""" + from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import Qwen4ExpHybridCacheManager + from tensorrt_llm.bindings import DataType + + binding = {None: DataType.BF16, QuantAlgo.FP8: DataType.FP8, QuantAlgo.NVFP4: DataType.NVFP4} + g, mapping = _geometry(member, rank, algo), _mapping(member, rank) + with _manager(member, rank, algo) as cm: + assert type(cm) is Qwen4ExpHybridCacheManager + assert cm.dtype == binding[algo] + assert cm.ssm_state_dtype == torch.bfloat16 + pools = P.take_pools(cm, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert {cm.num_kv_heads_per_layer[cm.layer_offsets[i]] for i in ATTN_LAYERS} == {g.heads_kv} + assert (cm.qsa_index_kv_heads, cm.qsa_index_dim) == (g.index_kv_heads, g.index_head_dim) + assert list(cm.conv_state_shape) == [g.conv_dim, g.conv_kernel - 1] + assert list(cm.ssm_state_shape) == [g.num_v_heads, g.head_v_dim, g.head_k_dim] + assert (cm.qsa_side_planes_size is not None) == g.qsa_side_planes + assert cm._num_reserved_dummy_slots == P.reserved_dummy_slots(mapping) + assert len(set(pools.pool_idx.values())) == len(ATTN_LAYERS) + # One slot space for every QSA view of the layers, one for the recurrent and PLE state. + slots = {view.shape[0] for view in (pools.position_cache, *pools.index_cache.values())} + slots |= {planes.k.shape[0] for planes in pools.kv.values()} + assert len(slots) == 1 + assert {t.shape[0] for t in (*pools.conv.values(), *pools.ssm.values())} == { + pools.ple_state[0].shape[0] + } + assert MTP_LAYER not in cm.layer_offsets + + +def _stock_mtp(member, rank, max_draft_len): + """With MTP the verify buffers (conv windows, GDN replay history) hold this rank's heads at + width `max_draft_len + 1`; the trunk manager maps the draft layer only under attention DP.""" + g, mapping = _geometry(member, rank), _mapping(member, rank) + spec = MTPDecodingConfig(max_draft_len=max_draft_len, num_nextn_predict_layers=1) + with _manager(member, rank, spec_config=spec) as cm: + pools = P.take_pools(cm, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert cm.is_speculative() + assert cm._num_reserved_dummy_slots == P.reserved_dummy_slots(mapping) + ssm0 = pools.ssm[GDN_LAYERS[0]] + caches = {i: cm.mamba_layer_cache(i) for i in GDN_LAYERS} + for i, cache in caches.items(): + P.check_step_buffers( + g, cache.intermediate_conv_window, pools.conv[i], cm.intermediate_state_indices + ) + assert cache.intermediate_ssm is None, "the cached replay keeps no per-step state" + assert not cm.use_gdn_flashinfer_ring_replay + meta = cm.get_replay_state_update_metadata() + P.check_replay_state( + meta.replay_history_size, + meta.replay_step_width, + meta.prev_num_accepted_tokens, + meta.cache_buf_idx, + ssm0, + ) + assert meta.replay_step_width == max_draft_len + 1 + P.check_verify_width(meta.replay_step_width) + for i, cache in caches.items(): + P.check_replay_history( + g, + i, + meta.replay_history_size, + ssm0.shape[0], + cache.old_x, + cache.old_B, + cache.old_dt, + ) + separate = P.draft_kv_cache_is_separate(mapping) + assert (MTP_LAYER in cm.layer_offsets) == (not separate) + if not separate: + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=False) + assert draft.position_cache.data_ptr() == pools.position_cache.data_ptr() + + +def _stock_draft(member, rank, algo): + """Under attention TP the separate draft manager holds the draft layer alone at trunk shapes.""" + g = _geometry(member, rank, algo) + spec = MTPDecodingConfig(max_draft_len=3, num_nextn_predict_layers=1) + with _manager(member, rank, algo, spec_config=spec, is_draft=True) as cm: + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=True) + assert cm.ple_layer_cache(PLE_LAYER) is None + assert cm.local_num_mamba_layers == 0 + assert cm.num_kv_heads_per_layer[draft.pool_idx] == g.heads_kv + assert (cm.qsa_side_planes_size is not None) == g.qsa_side_planes + assert draft.kv.k.shape[0] == draft.index_cache.shape[0] == draft.position_cache.shape[0] + + +_STOCK_CASES = [ + *( + pytest.param(_stock_trunk, m, r, dict(algo=a), id=f"trunk-{m}-rank{r}-{KV_IDS[a]}") + for m, r in LAYOUT_RANKS + for a in _ALGOS + ), + # Each attention layout at three drafts, and the widest verify under attention TP and DP. + *( + pytest.param(_stock_mtp, m, r, dict(max_draft_len=d), id=f"mtp{d}-{m}-rank{r}") + for m, r, d in [*((m, r, 3) for m, r in LAYOUT_RANKS), ("tp4", 3, 7), ("dep4", 3, 7)] + ), + *( + pytest.param(_stock_draft, m, r, dict(algo=a), id=f"draft-kv-{m}-rank{r}-{KV_IDS[a]}") + for m, r in LAYOUT_RANKS + if m != "dep4" + for a in _ALGOS + ), +] + + +@_needs_gpu +@pytest.mark.parametrize("check, member, rank, kwargs", _STOCK_CASES) +def test_the_stock_manager_serves_the_layout(check, member, rank, kwargs, monkeypatch): + """The stock `Qwen4ExpHybridCacheManager`, built as the engine does, passes the contract.""" + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + check(member, rank, **kwargs) + + +# ------------------------------------------------------ the decode step's runtime paths --- + + +def _runtime_hooks(monkeypatch): + from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks + + hooks = M._runtime_hooks() + assert isinstance(hooks.decode_prologue, M.SpecStepMetadataPrologue) + assert replace(hooks, decode_prologue=None) == RuntimeHooks(loop_on_execution_stream=True) + + +def _decode_stream(monkeypatch): + """A generation-only step of up to `_DECODE_HIGH_PRIORITY_MAX_ROWS` rows runs on the core's + high-priority decode stream, ordered after the caller's work and before its next, eager and + captured; a larger step, a prefill step or a core without the stream stays on the caller's.""" + hi = torch.cuda.Stream(priority=-100) + assert hi.priority < torch.cuda.Stream().priority + core = SimpleNamespace(_decode_stream=hi) + caller = torch.cuda.current_stream() + seen = [] + src = torch.full((1 << 20,), 1.0, device="cuda") + + def step(scale): + seen.append(torch.cuda.current_stream()) + return src * scale + + src.fill_(3.0) # the caller's pending write is ordered before the step + out = M._on_decode_stream(core, 3, step, 2.0) + assert seen[-1] == hi and torch.cuda.current_stream() == caller + assert torch.equal(out, torch.full_like(src, 6.0)) + # Captured: the waits become graph edges and a replay reads the refilled source. + side = torch.cuda.Stream() + side.wait_stream(caller) + with torch.cuda.stream(side): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=side): + captured = M._on_decode_stream(core, 3, step, 5.0) + caller.wait_stream(side) + src.fill_(1.5) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, torch.full_like(src, 7.5)) + target = M.DecodeTarget.__new__(M.DecodeTarget) + target.forward = lambda *a, **k: seen.append(torch.cuda.current_stream()) + target.mtp_forward = lambda *a, **k: seen.append(torch.cuda.current_stream()) + prefill = SimpleNamespace(forward=target.forward, mtp_forward=target.mtp_forward) + rows = M._DECODE_HIGH_PRIORITY_MAX_ROWS + for chosen, n, want in ((target, rows, hi), (target, rows + 1, caller), (prefill, 3, caller)): + ids = torch.zeros(n, dtype=torch.int32, device="cuda") + core.__dict__.update(_step_begun=True, _select_target=lambda md, t=chosen: t) + M.Qwen4ExpUnifiedModelingV2.forward(core, None, input_ids=ids) + assert seen[-1] == want, (n, chosen) + M.MTPLayer.forward(SimpleNamespace(core=core), None, None, ids, None, None, None) + assert seen[-1] == want, (n, chosen) + M._on_decode_stream(SimpleNamespace(), 3, step, 1.0) + assert seen[-1] == caller, "a core without the stream runs on the caller's" + + +class _ShardHead: + """A COLUMN-parallel `lm_head` stand-in recording each call's `gather_output`.""" + + def __init__(self, padding_size: int = 0) -> None: + from tensorrt_llm._torch.modules.linear import TensorParallelMode + + self.tp_mode = TensorParallelMode.COLUMN + self.gather_output = True + self.padding_size = padding_size + self.calls = [] + + def __call__(self, x, gather_output=True): + self.calls.append(gather_output) + return "shard" + + +def _draft_head_shard(monkeypatch, case, sharded): + """With the worker's TP key exchange a greedy draft pass keeps this rank's vocab shard of the + logits; rejection sampling, guidance, excess rows, a padded vocab or no exchange gather them.""" + monkeypatch.setattr(M, "_hc_site", lambda *args: (None, torch.zeros(3, 8), None)) + core = SimpleNamespace(_mtp=SimpleNamespace(fhc_norm=None, fhc_first=None, fhc_up=None)) + head = _ShardHead(padding_size=16 if case == "padded_vocab" else 0) + worker = SimpleNamespace(key_exchange=SimpleNamespace(MAX_ROWS=8), guided_decoder=None) + if case == "no_exchange": + worker.key_exchange = None + if case == "guided": + worker.guided_decoder = object() + if case == "rows_past_exchange": + worker.key_exchange.MAX_ROWS = 2 + layer = M.MTPLayer(core, logits_processor=SimpleNamespace(forward=lambda *args: "gathered")) + layer.worker = worker + layer.spec_metadata = SimpleNamespace( + wants_advanced_draft_sampling=case == "rejection_sampling" + ) + out = layer.shared_head(torch.zeros(3, 32), head, None, True) + assert out == ("shard" if sharded else "gathered") + assert head.calls == ([False] if sharded else []) + + +_RUNTIME_CASES = [ + pytest.param(_runtime_hooks, {}, marks=_cpu, id="runtime-hooks"), + pytest.param(_decode_stream, {}, marks=_needs_gpu, id="decode-stream"), + *( + pytest.param(_draft_head_shard, dict(case=c, sharded=c == "greedy"), marks=_cpu, + id=f"draft-head-{c}") + for c in ("greedy", "rejection_sampling", "no_exchange", "guided", "rows_past_exchange", + "padded_vocab") + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _RUNTIME_CASES) +def test_the_decode_steps_runtime_paths(check, kwargs, monkeypatch) -> None: + """The runtime hooks, the high-priority decode stream and the draft head's vocab-shard path.""" + check(monkeypatch, **kwargs) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py new file mode 100644 index 000000000000..a090c410605e --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU test for the FP8 unified target's `SpeculativeRollback`: after a simulated verify step, +every pool and plane is bitwise that of a run over the accepted prefixes only, compared in full +so a write to a slot the step does not own fails too.""" + +from __future__ import annotations + +import itertools + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( # noqa: E501 + plane_table as _plane_table, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( # noqa: E501 + ngram_token_history, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.modeling import ( # noqa: E501 + SpeculativeRollback, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_index_store import ( # noqa: E501 + qsa_paged_index_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.spec_rollback_begin__verify import ( # noqa: E501 + spec_rollback_begin, +) + +__extra_import_path__ = ["../../.."] +from _validating import certifying # noqa: E402,F401 — path above; fixture by name + +assert torch.cuda.is_available(), "the rollback handler test requires a CUDA device" + +DEV = "cuda" +EOS, VOCAB, NGRAM = 248044, 248320, 3 +KERNEL, DILATION, CHANNELS = 4, 3, 64 +STATE_LEN = (KERNEL - 1) * DILATION +TPB, INDEX_DIM, LAYERS, COMPRESS = 32, 128, 3, 4 +SLOTS, PAGES, BLOCKS_PER_SEQ = 24, 48, 4 + + +class _Batch: + """One verify step: fresh context requests of `ctx_lens` rows, then generation requests of + `p` rows at `gen_starts` cached tokens, in `slots`. Generation request `s` reuses request + `dup_of[s]`'s block-table row, as the CUDA-graph padding dummy does.""" + + def __init__(self, seed, ctx_lens, gen_starts, p, slots, dup_of=None): + g = torch.Generator().manual_seed(seed) + self.ctx_lens, self.num_contexts, self.p = list(ctx_lens), len(ctx_lens), p + self.n = len(gen_starts) + self.slots = torch.tensor(slots, dtype=torch.int32, device=DEV) + assert self.slots.shape[0] == self.num_contexts + self.n + lengths = self.ctx_lens + [p] * self.n + starts = [0] * self.num_contexts + list(gen_starts) + perm = torch.randperm(PAGES, generator=g) + table = torch.reshape(perm[: len(lengths) * BLOCKS_PER_SEQ], [len(lengths), -1]) + for dup, original in (dup_of or {}).items(): + table[self.num_contexts + dup] = table[self.num_contexts + original] + self.block_table = table.to(DEV, torch.int32) + req, logical = [], [] + for r, (length, start) in enumerate(zip(lengths, starts)): + req += [r] * length + logical += range(start, start + length) + self.rows = len(req) + self.prefix = sum(self.ctx_lens) + self.req_idx = torch.tensor(req, dtype=torch.int32, device=DEV) + self.logical = torch.tensor(logical, dtype=torch.int64, device=DEV) + self.carry = torch.tensor(_carried(self), dtype=torch.bool, device=DEV) + self.lengths = lengths + self.qsl = _qsl(lengths) + + # Every pre-step pool and plane is random, so a stray write shows. + self.ngram0 = torch.randint(0, VOCAB, (SLOTS, NGRAM - 1), generator=g).to(DEV) + self.conv0 = torch.randn(SLOTS, CHANNELS, STATE_LEN, generator=g).to(DEV, torch.bfloat16) + self.index0 = [ + torch.randn(PAGES, TPB, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.position0 = torch.randint(0, 1 << 20, (PAGES, TPB, 3), generator=g).to( + DEV, torch.int32 + ) + self.ids = torch.randint(0, VOCAB, (self.rows,), generator=g).to(DEV, torch.int32) + self.x = torch.randn(self.rows, CHANNELS, generator=g).to(DEV, torch.bfloat16) + self.keys = [ + torch.randn(self.rows, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.anchor_keys = [ + torch.randn(self.rows, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.coords = torch.randint(0, 1 << 20, (self.rows, 3), generator=g).to(DEV, torch.int32) + self.ngram, self.conv = self.ngram0.clone(), self.conv0.clone() + self.index, self.position = [t.clone() for t in self.index0], self.position0.clone() + self.accepted = torch.ones(self.num_contexts + self.n, dtype=torch.int32, device=DEV) + + def reset(self): + self.ngram.copy_(self.ngram0) + self.conv.copy_(self.conv0) + for live, pre in zip(self.index, self.index0): + live.copy_(pre) + self.position.copy_(self.position0) + + def set_accepted(self, gen_accepted): + assert len(gen_accepted) == self.n + assert all(1 <= a <= self.p for a in gen_accepted) + self.gen_accepted = list(gen_accepted) + values = [1] * self.num_contexts + list(gen_accepted) + self.accepted.copy_(torch.tensor(values, dtype=torch.int32)) + + +def _qsl(lengths): + return torch.tensor([0, *itertools.accumulate(lengths)], dtype=torch.int64, device=DEV) + + +def _conv_state(conv, x, lengths, slots, carry, publish_end=None): + """Stand-in for `ple_gate_conv_inject_norm`'s conv state: per sequence below `publish_end`, the + last `STATE_LEN` columns of `[carried state or zeros | its input]`, by capturable slices.""" + start = 0 + for s, length in enumerate(lengths): + if publish_end is not None and s >= publish_end: + break + slot = slots[s : s + 1].long() + prev = conv[slot][0] if carry[s] else torch.zeros_like(conv[0]) + window = torch.cat([prev, x[start : start + length].t()], 1) + conv[slot] = window[:, -STATE_LEN:].unsqueeze(0) + start += length + + +def _carried(b): + """Per sequence, whether it carries a state (generation) or starts fresh (context).""" + return [False] * b.num_contexts + [True] * b.n + + +def _write_planes(b, layer, plane, position, keep=slice(None)): + """The index store over rows `keep`, then a compression stand-in overwriting anchor slots.""" + req_idx, logical, anchor_keys = b.req_idx[keep], b.logical[keep], b.anchor_keys[layer][keep] + keys, coords = b.keys[layer][keep], b.coords[keep] + qsa_paged_index_store(keys, coords, plane, position, req_idx, logical, b.block_table, TPB) + pages = b.block_table[req_idx.long(), logical // TPB].long() + within = logical % TPB + anchor = ((logical + 1) % COMPRESS == 0).reshape(-1, 1, 1) + plane[pages, within] = torch.where(anchor, anchor_keys, plane[pages, within]) + + +def _verify_step(b, rollback, fused, begin_position=False): + """The forward's writes over all `p` rows, staged at its call sites, then the commit. Under + `fused` the generation conv rows stay unpublished and the commit rebuilds them from the pool.""" + rollback.discard() + rollback.begin( + num_contexts=b.num_contexts, + prefix_tokens=b.prefix, + num_gens=b.n, + rows_per_gen=b.p, + slots=b.slots[b.num_contexts :], + request_indices=b.req_idx[b.prefix :], + logical_positions=b.logical[b.prefix :], + block_table=b.block_table, + tokens_per_block=TPB, + fused=fused, + **(dict(position_cache=b.position) if begin_position else {}), + ) + rollback.stage_ple_verify(b.ngram, b.conv, b.ids) + ngram_token_history(b.ids, b.ngram, b.slots, b.carry, b.qsl, EOS, NGRAM) + rollback.stage_ple_conv_input(b.x) + _conv_state(b.conv, b.x, b.lengths, b.slots, _carried(b), b.num_contexts if fused else None) + for layer in range(LAYERS): + rollback.stage_index(b.position) + _write_planes(b, layer, b.index[layer], b.position) + rollback.commit_speculative_states(b.accepted, b.slots, b.num_contexts) + + +def _reference(b): + """The same writes from the pre-step state over context rows and accepted prefixes only.""" + keep = list(range(b.prefix)) + lengths = list(b.ctx_lens) + for s, a in enumerate(b.gen_accepted): + first = b.prefix + s * b.p + keep += range(first, first + a) + lengths.append(a) + keep = torch.tensor(keep, dtype=torch.long, device=DEV) + ngram, conv = b.ngram0.clone(), b.conv0.clone() + index = [t.clone() for t in b.index0] + position = b.position0.clone() + ngram_token_history(b.ids[keep], ngram, b.slots, b.carry, _qsl(lengths), EOS, NGRAM) + _conv_state(conv, b.x[keep], lengths, b.slots, _carried(b)) + for layer in range(LAYERS): + _write_planes(b, layer, index[layer], position, keep) + return ngram, conv, index, position + + +def _bits(t): + return t.view(torch.int16) if t.dtype == torch.bfloat16 else t + + +def _assert_state(b, expected, what): + ngram, conv, index, position = expected + assert torch.equal(b.ngram, ngram), f"{what}: n-gram pool" + assert torch.equal(_bits(b.conv), _bits(conv)), f"{what}: conv pool" + for layer in range(LAYERS): + assert torch.equal(_bits(b.index[layer]), _bits(index[layer])), f"{what}: index {layer}" + assert torch.equal(b.position, position), f"{what}: position plane" + + +# (ctx_lens, gen_starts, p, gen_accepted): positions straddle pages and land on compression +# anchors; p is 4 (MTP-3) behind contexts, and 2 and 8 (max_draft_len 1 and 7) in decode alone. +STEPS = { + "mixed": ((7, 3), (12, 29, 95), 4, (2, 1, 3)), + "decode_p2": ((), (3, 31, 64), 2, (1, 2, 1)), + "decode_p8": ((), (0, 27, 60, 90, 101, 5, 44, 119), 8, (1, 2, 3, 4, 5, 6, 7, 8)), +} +#: Acceptance counts one captured decode_p8 step is replayed with (bonus token included). +_REPLAYS = [(1,) * 8, (8, 7, 6, 5, 4, 3, 2, 1), (2, 8, 1, 7, 3, 6, 4, 5), (8,) * 8] + + +def _slots(num): + return list(range(3, 3 + 2 * num, 2)) + + +#: Handler forms: torch staging with batched plane gather; `fused` (one-launch stage and commit); +#: `fused_begin`, opened with the position plane so one launch stages all, as in a captured decode. +MODES = ["batched", "fused", "fused_begin"] + + +def _mode_args(mode): + """`(fused, begin_position)` of a handler form.""" + return mode != "batched", mode == "fused_begin" + + +def _rollback(b): + rollback = SpeculativeRollback() + rollback.set_index_planes(b.index) + return rollback + + +def _eager(case, mode): + ctx_lens, gen_starts, p, gen_accepted = STEPS[case] + b = _Batch(11, ctx_lens, gen_starts, p, _slots(len(ctx_lens) + len(gen_starts))) + b.set_accepted(gen_accepted) + _verify_step(b, _rollback(b), *_mode_args(mode)) + _assert_state(b, _reference(b), case) + + +def _captured(case, mode): + """The step and commit captured once, replayed with new acceptances over reset state.""" + ctx_lens, gen_starts, p, _ = STEPS[case] + b = _Batch(18, ctx_lens, gen_starts, p, _slots(len(gen_starts))) + rollback = _rollback(b) + b.set_accepted((p,) * len(gen_starts)) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + _verify_step(b, rollback, *_mode_args(mode)) + torch.cuda.current_stream().wait_stream(side) + b.reset() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _verify_step(b, rollback, *_mode_args(mode)) + for accepted in _REPLAYS: + b.reset() + b.set_accepted(accepted) + graph.replay() + torch.cuda.synchronize() + _assert_state(b, _reference(b), f"replay {accepted}") + + +def _padded(case, mode): + """Requests 2 and 3 are the CUDA-graph padding dummy request 1 is: one slot, one block-table + row, written concurrently with no defined winner, so only the unique request is compared.""" + gen_starts, p = (5, 40, 40, 40), 4 + b = _Batch(17, (), gen_starts, p, [3, 7, 7, 7], dup_of={2: 1, 3: 1}) + b.set_accepted((2, 3, 3, 3)) + _verify_step(b, _rollback(b), *_mode_args(mode)) + pages = b.block_table[0].long() + expected = _reference(_single(b, 0)) + assert torch.equal(b.ngram[3], expected[0][3]) + assert torch.equal(_bits(b.conv[3]), _bits(expected[1][3])) + for layer in range(LAYERS): + assert torch.equal(_bits(b.index[layer][pages]), _bits(expected[2][layer][pages])), ( + f"index {layer}" + ) + assert torch.equal(b.position[pages], expected[3][pages]) + + +def _single(b, s): + """`b` restricted to its generation request `s`, same pre-step state.""" + one = _Batch.__new__(_Batch) + one.__dict__.update(b.__dict__) + rows = torch.arange(b.prefix + s * b.p, b.prefix + (s + 1) * b.p, device=DEV) + one.ctx_lens, one.num_contexts, one.n, one.prefix = [], 0, 1, 0 + one.lengths = [b.p] + one.slots = b.slots[b.num_contexts + s : b.num_contexts + s + 1] + one.carry = b.carry[b.num_contexts + s : b.num_contexts + s + 1] + one.qsl = _qsl(one.lengths) + one.ids, one.x = b.ids[rows], b.x[rows] + one.keys = [k[rows] for k in b.keys] + one.anchor_keys = [k[rows] for k in b.anchor_keys] + one.coords = b.coords[rows] + one.logical = b.logical[rows] + one.req_idx = torch.zeros_like(b.req_idx[rows]) + one.block_table = b.block_table[b.num_contexts + s : b.num_contexts + s + 1] + one.gen_accepted = [b.gen_accepted[s]] + return one + + +_RUNS = ( + [(_eager, case, mode) for case in STEPS for mode in MODES] + + [(_captured, "decode_p8", mode) for mode in ("fused", "fused_begin")] + + [(_padded, "dup", "fused")] +) + + +@pytest.mark.parametrize( + "run,case,mode", _RUNS, ids=[f"{r.__name__[1:]}-{c}-{m}" for r, c, m in _RUNS] +) +def test_commit_leaves_the_accepted_prefix_state(run, case, mode): + """After partial acceptance, the n-gram and conv pools and every index and position plane + are bitwise those of a run over the context rows and the accepted prefixes only.""" + run(case, mode) + + +@pytest.mark.parametrize( + "cell", spec_rollback_begin.CELLS, ids=[c.why[:40] for c in spec_rollback_begin.CELLS] +) +def test_certified_cells(cell, certifying): # noqa: F811 — fixture by name + """Bitwise the composed torch reference, eagerly and in a capture replayed at new slots.""" + certifying(spec_rollback_begin) + n, p = cell.spec["requests"], cell.spec["p"] + starts = [(37 * s + 5) % 100 for s in range(n)] + b = _Batch(23 + n * p, (), starts, p, _slots(n)) + planes = [torch.randn_like(t) for t in b.index0] * (cell.spec["planes"] // LAYERS) + ptrs, strides = _plane_table(planes) + args = (b.req_idx, b.logical, b.block_table, TPB) + got = spec_rollback_begin(*args, ptrs, strides, b.position0) + want = spec_rollback_begin.reference(*args, planes, b.position0) + for g, w in zip(got, want): + assert torch.equal(_bits(g), _bits(w)) + if not cell.spec["capture"]: + return + logical = b.logical.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = spec_rollback_begin( + b.req_idx, logical, b.block_table, TPB, ptrs, strides, b.position0 + ) + logical.add_(3) + graph.replay() + torch.cuda.synchronize() + want = spec_rollback_begin.reference( + b.req_idx, logical, b.block_table, TPB, planes, b.position0 + ) + for g, w in zip(outs, want): + assert torch.equal(_bits(g), _bits(w)), "replay at new slots" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py new file mode 100644 index 000000000000..1474dbea4830 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py @@ -0,0 +1,1321 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's Gated-DeltaNet entries over their certified cells, and +for launches no cell reaches. Engine-sized state pools are reserved virtual ranges mapped under +the touched slots only (`_SparsePage`); every other byte of their pages is checked unchanged.""" + +from __future__ import annotations + +import types +import warnings +import zlib + +import cuda.bindings.driver as cuda_drv +import pytest +import torch +import torch.nn.functional as F +from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import should_use_cp_host +from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + +import tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.causal_conv1d_token_major__verify as ctm # noqa: E501 +import tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_cached_replay_cute__verify as cr # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( # noqa: E501 + rms_norm_gated_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( # noqa: E501 + HEAD_K_DIM, + HEAD_V_DIM, + NUM_K_HEADS, + NUM_V_HEADS, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _m, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + low_m_bf16_mm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.causal_conv1d_token_major__verify import ( # noqa: E501 + causal_conv1d_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.flashinfer_chunk_gated_delta_rule import ( # noqa: E501 + INPLACE_MAX_TOKENS, + flashinfer_chunk_gated_delta_rule, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_in_proj_conv__verify import ( # noqa: E501 + fused_tactic, + gdn_in_proj_conv, +) +from tensorrt_llm._torch.modules.fla.cache_manager import ( + MIN_REPLAY_HISTORY_SIZE, + ReplayStateUpdateMetadata, + _advance_replay_state, +) +from tensorrt_llm._torch.modules.fla.cached_replay import ( + CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + commit_gdn_cached_replay_history_layers, +) +from tensorrt_llm._torch.modules.fla.cached_replay import ( + fused_recurrent_gated_delta_rule_cached_replay_update as stock_replay, +) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import _build_replay_work_items_triton + +__extra_import_path__ = ["../../.."] +from _validating import ULP, certifying # noqa: E402,F401 — path above; fixture by name + +assert torch.cuda.is_available(), "the Gated-DeltaNet entries require a CUDA device" + +DEV = "cuda" +#: Key heads, value heads, key and value head dims of the whole layer (tp1, a dep4 rank). +H, HV, K, V = NUM_K_HEADS, NUM_V_HEADS, HEAD_K_DIM, HEAD_V_DIM + + +# ------------------------------------------------- reserved pages, mapped where touched --- + + +def _scattered(n: int, slots: int) -> list[int]: + """`n` distinct slots: both ends, then a stride-37 walk (coprime to every `slots - 2` here).""" + rest = [(i * 37 + 4) % (slots - 2) + 1 for i in range(max(n - 2, 0))] + picked = ([slots - 1, 0] + rest)[:n] + assert len(set(picked)) == n, (n, slots) + return picked + + +def _check(result): + err, *value = result + assert err == cuda_drv.CUresult.CUDA_SUCCESS, err + return value[0] if len(value) == 1 else value + + +#: Array-interface type of each pool dtype; bf16 has none, so it is read as int16 and viewed. +_TYPESTR = {torch.float32: (" torch.Tensor: + typestr, carrier = _TYPESTR[self.dtype] + es = self.es + t = torch.as_tensor( + _ArrayInterface(self.va + es * offset, shape, [es * s for s in strides], typestr), + device=DEV, + ) + assert t.dtype == carrier + return t.view(self.dtype) + + def close(self) -> None: + torch.cuda.synchronize() + for ptr, size, handle in self.maps: + _check(cuda_drv.cuMemUnmap(ptr, size)) + _check(cuda_drv.cuMemRelease(handle)) + _check(cuda_drv.cuMemAddressFree(cuda_drv.CUdeviceptr(self.va), self.size)) + + +class _EnginePool: + """The stock manager's coalesced state page (`layers` `[v_heads, head_v, head_k]` states a + slot), viewed at the last layer's sub-page. Mapped under each touched slot's page plus the next + slot's first layer, and under the view's base (torch reads the device off it).""" + + def __init__(self, slots, layers, used, v_heads, head_v, head_k, dtype, init): + layer = v_heads * head_v * head_k + stride = layers * layer + total = slots * stride + base = (layers - 1) * layer + self.used = sorted(set(used)) + self.spans = [(s * stride, min((s + 1) * stride + layer, total)) for s in self.used] + self.page = _SparsePage(total, [*self.spans, (base, base + layer)], dtype) + self.own, self.layer = base, layer + self.pool = self.page.view( + base, (slots, v_heads, head_v, head_k), (stride, head_v * head_k, head_k, 1) + ) + self.flat = [self.page.view(lo, (hi - lo,), (1,)) for lo, hi in self.spans] + for f in self.flat: + init(f) + torch.cuda.synchronize() + self.before = [f.clone() for f in self.flat] + + def dense(self, slots: torch.Tensor): + """A dense copy of the touched slots and `slots` renumbered into it (-1 stays).""" + index = {s: i for i, s in enumerate(self.used)} + compact = torch.stack([self.pool[s] for s in self.used]).contiguous() + renamed = [index[s] if s >= 0 else -1 for s in slots.tolist()] + return compact, torch.tensor(renamed, dtype=slots.dtype, device=DEV) + + def touched(self) -> torch.Tensor: + return torch.stack([self.pool[s] for s in self.used]) + + def assert_rest_unchanged(self) -> None: + """Every mapped byte outside this layer's state of each touched slot is unchanged.""" + torch.cuda.synchronize() + lo, hi = self.own, self.own + self.layer + for f, b, s in zip(self.flat, self.before, self.used): + assert torch.equal(f[:lo], b[:lo]) and torch.equal(f[hi:], b[hi:]), ( + f"slot {s}: the call wrote outside its state" + ) + + def close(self) -> None: + del self.pool, self.flat + self.page.close() + + +# ------------------------------------------------ flashinfer_chunk_gated_delta_rule --- + +#: The Triton chunk op's band (rtol bf16 default; atol 2e-3 on `o`, 2e-2 on states): this op is +#: the same class of sum against fp64; a value head reading the wrong key head lands far outside. +CHUNK_RTOL, CHUNK_ATOL_OUTPUT, CHUNK_ATOL_STATE = 1.6e-2, 2e-3, 2e-2 +#: Rows past `out[:, :num_tokens]` in a `slice` cell's wider output buffer; they must not change. +_CHUNK_OUTPUT_TAIL = 16 + + +def _chunk_compare(out: tuple, ref: tuple) -> None: + """`(o, final_state, pool)` against `reference`'s triple; `None` must match.""" + for got, want, atol in zip(out, ref, (CHUNK_ATOL_OUTPUT, CHUNK_ATOL_STATE, CHUNK_ATOL_STATE)): + assert (got is None) == (want is None), (got is None, want is None) + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape, ( + (got.dtype, got.shape), + (want.dtype, want.shape), + ) + assert torch.isfinite(got).all() + torch.testing.assert_close(got.float(), want.float(), rtol=CHUNK_RTOL, atol=atol) + + +def _flashinfer_path(spec) -> str: + """The FlashInfer kernel the call reaches: `inplace` for a `has_initial_state` call of at most + `INPLACE_MAX_TOKENS` tokens, else `cp` when FlashInfer's heuristic picks it, else `chunked`.""" + if "initial" in spec and sum(spec["seqlens"]) <= INPLACE_MAX_TOKENS: + return "inplace" + device = torch.device(DEV) + cp = should_use_cp_host( + len(spec["seqlens"]) * spec["geometry"][1], + get_device_sm_count(device), + get_device_name(device), + device_capability=get_compute_capability(device), + ) + return "cp" if cp else "chunked" + + +def _chunk_build(spec, seed): + """`(args, engine)`: inputs in a GDN post-conv step's domain (unit-norm q/k, g < 0 in log + space, beta a sigmoid), and the `_EnginePool` behind the pool or None.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + num_k_heads, num_v_heads, head_k_dim, head_v_dim = spec["geometry"] + seqlens = spec["seqlens"] + total = sum(seqlens) + q = torch.randn(1, total, num_k_heads, head_k_dim, generator=gen, device=DEV) + k = torch.randn(1, total, num_k_heads, head_k_dim, generator=gen, device=DEV) + q = q / torch.sqrt(q.pow(2).sum(-1, keepdim=True) + 1e-6) + k = k / torch.sqrt(k.pow(2).sum(-1, keepdim=True) + 1e-6) + q, k = q.bfloat16(), k.bfloat16() + v = torch.randn( + 1, total, num_v_heads, head_v_dim, generator=gen, device=DEV, dtype=torch.bfloat16 + ) + a_log = torch.log(torch.rand(num_v_heads, generator=gen, device=DEV) * 15 + 1) + x = torch.randn(1, total, num_v_heads, generator=gen, device=DEV) + torch.randn( + num_v_heads, generator=gen, device=DEV + ) + g = (-torch.exp(a_log) * torch.nn.functional.softplus(x)).float() + beta = torch.rand(1, total, num_v_heads, generator=gen, device=DEV).sigmoid().float() + engine = None + if spec.get("page_layers"): + engine = _EnginePool( + spec["slots"], + spec["page_layers"], + spec["indices"], + num_v_heads, + head_v_dim, + head_k_dim, + torch.bfloat16, + lambda f: f.copy_(torch.randn(f.numel(), generator=gen, device=DEV) * 0.1), + ) + pool = engine.pool + else: + pool = ( + torch.randn( + spec["slots"], num_v_heads, head_v_dim, head_k_dim, generator=gen, device=DEV + ) + * 0.1 + ).to(spec.get("pool_dtype", torch.float32)) + cu = torch.tensor([0] + torch.tensor(seqlens).cumsum(0).tolist(), dtype=torch.long, device=DEV) + indices = torch.tensor(spec["indices"], dtype=torch.int32, device=DEV) + output = None + if spec.get("output") == "buffer": + output = torch.full( + (1, total, num_v_heads, head_v_dim), -7.0, device=DEV, dtype=torch.bfloat16 + ) + elif spec.get("output") == "slice": + output = torch.full( + (1, total + _CHUNK_OUTPUT_TAIL, num_v_heads, head_v_dim), + -7.0, + device=DEV, + dtype=torch.bfloat16, + )[:, :total] + extra = {} + if "initial" in spec: + extra["has_initial_state"] = torch.tensor(spec["initial"], dtype=torch.bool, device=DEV) + if spec.get("linear_gate"): + extra["linear_gate"] = True + g = torch.exp(g) + args = dict( + **extra, + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=None, + initial_state=pool, + initial_state_indices=indices, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=output, + state_workspace=None, + ) + return args, engine + + +def _check_flashinfer_chunk_gated_delta_rule(cell) -> None: + args, engine = _chunk_build(cell.spec, seed=zlib.crc32(cell.why.encode())) + try: + if engine is None: + pool = args["initial_state"] + pool_before = pool.clone() + ref = flashinfer_chunk_gated_delta_rule.reference(**args) + else: + compact, renamed = engine.dense(args["initial_state_indices"]) + ref = flashinfer_chunk_gated_delta_rule.reference( + **dict(args, initial_state=compact, initial_state_indices=renamed) + ) + output = args["output"] + full = None if output is None else output._base + tail_before = None if full is None else full[:, output.shape[1] :].clone() + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + o, final_state = flashinfer_chunk_gated_delta_rule(**args) + # FlashInfer falls back from a CP heuristic match with a RuntimeWarning, so a CP + # cell certifies the CP pipeline only if none was raised. + fallback = [str(w.message) for w in caught if "falling back to non-CP" in str(w.message)] + path = "chunked" if fallback else _flashinfer_path(cell.spec) + + got_pool = pool if engine is None else engine.touched() + _chunk_compare((o, final_state, got_pool), ref) + if "fi_path" in cell.spec: + assert path == cell.spec["fi_path"], (path, fallback) + if output is not None: + assert o.data_ptr() == output.data_ptr(), "output= must be written in place" + if tail_before is not None: + assert torch.equal(full[:, output.shape[1] :], tail_before), "rows past output written" + if engine is None: + named = set(args["initial_state_indices"].tolist()) + unnamed = [s for s in range(pool.shape[0]) if s not in named] + assert torch.equal(pool[unnamed], pool_before[unnamed]), "unnamed slots written" + else: + engine.assert_rest_unchanged() + finally: + if engine is not None: + engine.close() + + +# ------------------------------------------------------- the cached-replay test harness --- + +#: `replay_history_size` = max(MIN_REPLAY_HISTORY_SIZE, T): the minimum at every T <= 8. +HIST = MIN_REPLAY_HISTORY_SIZE +BF16_ULP = ULP[torch.bfloat16] + +#: This checkpoint's GDN `A_log` / `dt_bias` statistics: mean, standard deviation, range. +A_LOG_STATS = (1.006, 1.487, (-7.125, 5.0625)) +DT_BIAS_STATS = (-4.411, 1.732, (-11.4375, 2.53125)) + +#: Element gate of the bf16 members (output, old_u, old_k), in bf16 ulps of their block's absmax: +#: the kernel rounds q, k and the history-mixing coefficients to bf16 before its dots. +_ELEMENT_ULPS = 4.0 +#: Relative-RMS gate of the output, in bf16 ulps. +_OUTPUT_RMS_ULPS = 1.5 +#: Element gate of an fp32 checkpoint state, relative to its (slot, head) absmax: the commit +#: splits each history term into bf16 hi and lo parts (~16 bits), up to 16 terms within 2^-17. +_STATE_TOL = 2.0**-13 +#: Element gate of a bf16 checkpoint state: that fp32 commit rounded once, as the reference's is, +#: so the two are equal or adjacent, at most 2 ulps of the head's absmax apart. +_BF16_STATE_ULPS = 2.0 +#: Gate of the log-decays, |dG| <= atol + rtol * |G|: G enters only as exp(G_t - G_j), atol is an +#: eighth of a bf16 ulp of the u it scales, rtol covers the cumsum where |G| reaches thousands. +_G_ATOL, _G_RTOL = 2.0**-11, 2.0**-18 + + +def _gate(name, got, want, element, rms=None): + """Every element of `got` within `element` of its block's absmax in `want` (a block is the + last two dims, so a near-silent element's rounding does not read as relative noise).""" + block = got.shape[-2] * got.shape[-1] + g, w = got.float().reshape(-1, block), want.float().reshape(-1, block) + keep = ~torch.isnan(w).any(1) + g, w = g[keep], w[keep] + scale = w.abs().amax(1, keepdim=True).clamp_min(torch.finfo(torch.float32).tiny) + ratio = ((g - w).abs() / scale).nan_to_num(nan=float("inf")) + worst = ratio.max().item() if ratio.numel() else 0.0 + assert worst <= element, ( + f"{name}: an element is {worst:.3e} of its block's absmax from the reference " + f"(gate {element:.3e})" + ) + if rms is not None and w.numel(): + rel = ((g - w).pow(2).mean().sqrt() / w.pow(2).mean().sqrt().clamp_min(1e-30)).item() + assert rel <= rms, f"{name}: relative RMS {rel:.3e} exceeds {rms:.3e}" + + +def _replay_compare(out: tuple, ref: tuple) -> None: + """`(returned, ssm_states, old_u, old_k, old_G)` against `reference`'s five by the gates above; + a member `None` on both sides is skipped (the exact twin compares the output alone).""" + names = ("output", "ssm_states", "old_u", "old_k", "old_G") + assert len(out) == len(ref) == len(names) + for name, got, want in zip(names, out, ref): + assert (got is None) == (want is None), f"{name}: present on one side only" + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape, ( + name, + (got.dtype, tuple(got.shape)), + (want.dtype, tuple(want.shape)), + ) + if name == "old_G": + torch.testing.assert_close(got, want, rtol=_G_RTOL, atol=_G_ATOL, msg=name) + elif name == "ssm_states": + bf16 = got.dtype == torch.bfloat16 + _gate(name, got, want, _BF16_STATE_ULPS * BF16_ULP if bf16 else _STATE_TOL) + else: + rms = _OUTPUT_RMS_ULPS * BF16_ULP if name == "output" else None + _gate(name, got, want, _ELEMENT_ULPS * BF16_ULP, rms) + + +def _seed(why): + return zlib.crc32(why.encode()) + + +def _gating(gen, layers, hv=HV): + """Per-layer A_log / dt_bias around the checkpoint's statistics; head 0 pinned to the corner + that forgets everything every token (g ~ -400), head 1 to the one that hardly decays.""" + (a_mean, a_std, a_range), (d_mean, d_std, d_range) = A_LOG_STATS, DT_BIAS_STATS + a_log = torch.randn(layers, hv, generator=gen, device=DEV) * a_std + a_mean + dt_bias = torch.randn(layers, hv, generator=gen, device=DEV) * d_std + d_mean + a_log, dt_bias = a_log.clamp(*a_range), dt_bias.clamp(*d_range) + a_log[:, 0], dt_bias[:, 0] = a_range[1], d_range[1] + a_log[:, 1], dt_bias[:, 1] = a_range[0], d_range[0] + return a_log.contiguous(), dt_bias.contiguous() + + +def _unit(x): + return (x / x.norm(dim=-1, keepdim=True)).to(torch.bfloat16) + + +class _ReplayEnginePage: + """The stock page with the bench's `layers` on top, mapped under each touched slot's page, the + next slot's first sub-page (where an overrun lands) and slot 0's page (the views' base).""" + + def __init__(self, slots, page_layers, layers, used, hv, dtype, gen): + state = hv * V * K + stride = page_layers * state + total = slots * stride + self.state, self.stride, self.layers = state, stride, layers + self.top = (page_layers - layers) * state + self.used = sorted(set(used)) + self.mapped = sorted(set(self.used) | {0}) + self.spans = [(s * stride, min((s + 1) * stride + state, total)) for s in self.mapped] + self.page = _SparsePage(total, self.spans, dtype) + self.pools = [ + self.page.view(self.top + j * state, (slots, hv, V, K), (stride, V * K, K, 1)) + for j in range(layers) + ] + self.flat = [self.page.view(lo, (hi - lo,), (1,)) for lo, hi in self.spans] + for f in self.flat: + f.normal_(0.0, 0.5, generator=gen) + torch.cuda.synchronize() + self.before = [f.clone() for f in self.flat] + + def assert_rest_unchanged(self) -> None: + """Every mapped byte outside the touched slots' own sub-pages is what it was.""" + torch.cuda.synchronize() + lo, hi = self.top, self.top + self.layers * self.state + for s, f, b in zip(self.mapped, self.flat, self.before): + if s in self.used: + same = torch.equal(f[:lo], b[:lo]) and torch.equal(f[hi:], b[hi:]) + else: + same = torch.equal(f, b) + assert same, f"slot {s}: the page was written outside the cell's sub-pages" + + def close(self) -> None: + del self.pools, self.flat, self.before + self.page.close() + + +class _ReplayBench: + """`layers` GDN layers' state as `MambaHybridCacheManagerV2` lays it out, driven as the engine + drives it (work items, op per layer, all-layer commit, `_advance_replay_state`). `exact=True` + adds the exact twin: fp32 `reference` over a never-wrapping history advanced by `p += accepted` + alone. Over an engine page both see a dense copy of the touched slots (`position` maps them).""" + + def __init__(self, spec, seed, exact=True): + self.spec = spec + self.path, self.n, self.t = spec["path"], spec["n"], spec["t"] + self.h, self.hv = spec.get("H", H), spec.get("HV", HV) + assert spec.get("history_size", HIST) == HIST, spec + self.width = 2 * self.h * K + self.hv * V + self.layers = spec.get("layers", 2) + self.slots = spec.get("slots", 64) + self.steps = spec.get("steps", 8) + self.gen = torch.Generator(device=DEV).manual_seed(seed) + self.cpu = torch.Generator().manual_seed(seed) + n, t, L, S = self.n, self.t, self.layers, self.slots + h, hv = self.h, self.hv + page_layers = spec.get("page_layers") + + if page_layers: + chosen = torch.tensor(_scattered(n, S)) + else: + chosen = torch.randperm(S, generator=self.cpu)[:n] + chosen = chosen.to(torch.int32) + live_rows = torch.ones(n, dtype=torch.bool) + dup = tuple(spec.get("dup", ())) + if dup: + chosen[list(dup)] = int(chosen[dup[0]]) + # The padding rows' shared slot is nobody's state: the bookkeeping never names it. + live_rows[list(dup)] = False + self.state_indices = chosen.to(DEV) + self.live = self.state_indices[live_rows.to(DEV)].clone() + + state_dtype = spec.get("state_dtype", torch.float32) + self.page = self.idx = self.position = None + if page_layers: + self.page = _ReplayEnginePage( + S, page_layers, L, chosen.tolist(), hv, state_dtype, self.gen + ) + self.pool = self.page.pools + self.layer_stride, self.slot_stride = self.page.state, self.page.stride + self.idx = torch.tensor(self.page.used, dtype=torch.long, device=DEV) + self.position = torch.full((S,), -1, dtype=torch.long, device=DEV) + self.position[self.idx] = torch.arange(self.idx.numel(), device=DEV) + else: + # [slots, layers, HV, V, K]: the affine layout whose strides the all-layer commit takes. + self.backing = (self._randn(S, L, hv, V, K) * 0.5).to(state_dtype) + self.pool = [self.backing[:, i] for i in range(L)] + self.layer_stride, self.slot_stride = self.backing.stride(1), self.backing.stride(0) + self.old_x = torch.zeros(L, S, 2, HIST, hv, V, device=DEV, dtype=torch.bfloat16) + self.old_B = torch.zeros(L, S, 2, HIST, h, K, device=DEV, dtype=torch.bfloat16) + self.old_dt = torch.zeros(L, S, 2, hv, HIST, device=DEV) + self.old_dA = self._randn(L, S, 2, hv, HIST) # handed over as old_beta; never read + self.pnat = torch.zeros(S, dtype=torch.int32, device=DEV) + self.buf = torch.zeros(S, dtype=torch.int32, device=DEV) + self.a_log, self.dt_bias = _gating(self.gen, L, hv) + if spec.get("start", "warm") == "warm": + self.old_x.copy_(self._randn(L, S, 2, HIST, hv, V) * 0.5) + self.old_B.copy_(_unit(self._randn(L, S, 2, HIST, h, K))) + raw_a = self._randn(L, S, 2, hv, HIST) + g = -torch.exp(self.a_log)[:, None, None, :, None] * F.softplus( + raw_a + self.dt_bias[:, None, None, :, None] + ) + self.old_dt.copy_(g.cumsum(-1)) + pnat = torch.randint(0, HIST + 1, (S,), generator=self.cpu, dtype=torch.int32) + live = self.live.long().cpu() + pnat[live[0]] = HIST # a wrap at step 0 + if live.numel() > 1: + pnat[live[1]] = 0 # an append at step 0 + self.pnat.copy_(pnat) + self.buf.copy_(torch.randint(0, 2, (S,), generator=self.cpu, dtype=torch.int32)) + self.meta = ReplayStateUpdateMetadata(self.pnat, self.buf, t, HIST) + self.work_items = self.n_writes = None + if self.path == "all_layer": + # The engine hands out work items exactly from the all-layer commit's minimum batch. + assert n >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, n + self.work_items = torch.zeros(n, 4, dtype=torch.int32, device=DEV) + self.n_writes = torch.zeros(1, dtype=torch.int32, device=DEV) + + self.exact = exact + if exact: + self.depth = HIST + (self.steps + 1) * t + self.x_pool = [self.compact(p).clone() for p in self.pool] + rows = self.x_pool[0].shape[0] + self.x_u = torch.zeros(L, rows, 1, self.depth, hv, V, device=DEV) + self.x_k = torch.zeros(L, rows, 1, self.depth, h, K, device=DEV) + self.x_G = torch.zeros(L, rows, 1, hv, self.depth, device=DEV) + self.x_pnat = self.compact(self.pnat).clone() + self.x_buf = torch.zeros_like(self.x_pnat) + for s in self.live.tolist(): + p, b = int(self.pnat[s]), int(self.buf[s]) + c = s if self.position is None else int(self.position[s]) + self.x_u[:, c, 0, :p] = self.old_x[:, s, b, :p].float() + self.x_k[:, c, 0, :p] = self.old_B[:, s, b, :p].float() + self.x_G[:, c, 0, :, :p] = self.old_dt[:, s, b, :, :p] + + def _randn(self, *shape, dtype=torch.float32): + return torch.randn(*shape, generator=self.gen, device=DEV, dtype=dtype) + + def compact(self, per_slot: torch.Tensor) -> torch.Tensor: + """A per-slot tensor as the reference sees it: over a page, the touched slots' rows.""" + return per_slot if self.idx is None else per_slot[self.idx] + + def compact_indices(self, slots: torch.Tensor) -> torch.Tensor: + """Slot ids as rows of `compact`'s output (negative ids stay negative).""" + if self.position is None: + return slots + renamed = torch.where(slots >= 0, self.position[slots.long().clamp(min=0)], -1) + return renamed.to(slots.dtype) + + def close(self) -> None: + if self.page is not None: + self.pool = [] + self.page.close() + + def inputs(self): + """One layer's post-conv `[q|k|v]` rows and raw `a`/`b`, as the verify step passes them.""" + n, t, h, hv = self.n, self.t, self.h, self.hv + packed = self._randn(n * t, self.width, dtype=torch.bfloat16) + rows = packed.view(n, t, self.width) + q = rows[..., : h * K].view(n, t, h, K) + k = rows[..., h * K : 2 * h * K].view(n, t, h, K) + v = rows[..., 2 * h * K :].view(n, t, hv, V) + a = self._randn(n, t, hv, dtype=torch.bfloat16) + b = self._randn(n, t, hv, dtype=torch.bfloat16) + return dict(q=q, k=k, v=v, a=a, b=b, packed=packed) + + def kwargs(self, layer, act): + return dict( + q=act["q"], + k=act["k"], + v=act["v"], + g=act["a"], + beta=act["b"], + ssm_states=self.pool[layer], + state_indices=self.state_indices, + old_u=self.old_x[layer], + old_k=self.old_B[layer], + old_G=self.old_dt[layer], + old_beta=self.old_dA[layer], + cache_buf_idx=self.buf, + prev_num_accepted_tokens=self.pnat, + history_size=HIST, + scale=None, + use_qk_l2norm_in_kernel=True, + A_log=self.a_log[layer], + dt_bias=self.dt_bias[layer], + launch_with_pdl=False, + replay_work_items=self.work_items, + n_writes=self.n_writes, + packed_qkv=act["packed"], + use_all_layer_commit=self.path == "all_layer", + output=None, + ) + + def exact_kwargs(self, layer, kw): + return dict( + kw, + ssm_states=self.x_pool[layer], + state_indices=self.compact_indices(kw["state_indices"]), + old_u=self.x_u[layer], + old_k=self.x_k[layer], + old_G=self.x_G[layer], + cache_buf_idx=self.x_buf, + prev_num_accepted_tokens=self.x_pnat, + history_size=self.depth, + replay_work_items=None, + n_writes=None, + use_all_layer_commit=False, + output=None, + ) + + def adopt_exact(self, layer, ref): + assert torch.equal(ref[1], self.x_pool[layer]), "the twin must never wrap" + self.x_u[layer].copy_(ref[2]) + self.x_k[layer].copy_(ref[3]) + self.x_G[layer].copy_(ref[4]) + + def state(self, layer): + return self.pool[layer], self.old_x[layer], self.old_B[layer], self.old_dt[layer] + + def observed(self, layer): + """`state(layer)` through `compact`, as `reference`'s results are laid out.""" + return tuple(self.compact(x) for x in self.state(layer)) + + def prepare(self): + if self.path == "all_layer": + self.work_items.zero_() + _build_replay_work_items_triton( + self.live, self.pnat, self.buf, self.work_items, self.n_writes, self.t, HIST + ) + + def commit(self): + if self.path == "all_layer": + commit_gdn_cached_replay_history_layers( + ssm_states=self.pool[0], + ssm_state_num_layers=self.layers, + ssm_state_layer_stride=self.layer_stride, + ssm_state_slot_stride=self.slot_stride, + old_u=self.old_x, + old_k=self.old_B, + old_G=self.old_dt, + replay_work_items=self.work_items, + n_writes=self.n_writes, + history_size=HIST, + ) + + def draw_acceptance(self): + """Accepted tokens per live row, bonus token included, by `spec["accept"]`.""" + mode = self.spec.get("accept", "random") + rows = self.live.numel() + if mode == "full": + return torch.full((rows,), self.t, dtype=torch.int32, device=DEV) + assert mode == "random", mode + return torch.randint(1, self.t + 1, (rows,), generator=self.cpu, dtype=torch.int32).to(DEV) + + def accept(self, accepted): + _advance_replay_state(self.meta, self.live, accepted) + if self.exact: + self.x_pnat[self.compact_indices(self.live).long()] += accepted + + def writes(self): + """This step's append mask `[slots, 2, HIST]`, wrapping slot ids, and exact-fill count.""" + s = self.live.long() + p, b = self.pnat[s].long(), self.buf[s].long() + wrap = p + self.t > HIST + half, start = torch.where(wrap, 1 - b, b), torch.where(wrap, 0, p) + mask = torch.zeros(self.slots, 2, HIST, dtype=torch.bool, device=DEV) + for t in range(self.t): + mask[s, half, start + t] = True + return mask, s[wrap], int((p + self.t == HIST).sum()) + + +# -------------------------------------------------------------- gdn_cached_replay_cute__verify --- +# Each cell drives consecutive verify steps through `_ReplayBench`, holding every call to the +# entry's `reference` on its own inputs and every output to the exact twin. + + +_grc_op = cr.gdn_cached_replay_cute +GRC_K, GRC_V = cr.K, cr.V + + +def _grc_gates(bench, act, strided): + """The raw a / b `[N, T, HV]`: contiguous, or the same values as column views of the rank's + `[N*T, conv + z + 2 HV]` input projection, as the target hands them over.""" + if not strided: + return act["a"], act["b"] + n, t, hv = bench.n, bench.t, bench.hv + conv, z = 2 * bench.h * GRC_K + hv * GRC_V, hv * GRC_V + proj = torch.randn(n * t, conv + z + 2 * hv, device=DEV, dtype=torch.bfloat16) + a = proj[:, conv + z : conv + z + hv] + b = proj[:, conv + z + hv :] + a.copy_(act["a"].reshape(n * t, hv)) + b.copy_(act["b"].reshape(n * t, hv)) + a, b = a.reshape(n, t, hv), b.reshape(n, t, hv) + assert a.stride() == (t * proj.shape[1], proj.shape[1], 1), a.stride() + return a, b + + +def _grc_reference(bench, kw): + """The entry's `reference` on a call's arguments, every per-slot buffer through `compact`.""" + if bench.idx is None: + return _grc_op.reference(**kw) + per_slot = ( + "ssm_states", + "old_u", + "old_k", + "old_G", + "old_beta", + "cache_buf_idx", + "prev_num_accepted_tokens", + ) + compact = {name: bench.compact(kw[name]) for name in per_slot} + return _grc_op.reference( + **dict(kw, **compact, state_indices=bench.compact_indices(kw["state_indices"])) + ) + + +def _grc_drive(cell, bench) -> None: + strided = cell.spec.get("strided_gates", False) + wraps = appends = 0 + for _ in range(bench.steps): + mask, wrapped, _ = bench.writes() + wraps += wrapped.numel() + appends += bench.live.numel() - wrapped.numel() + bench.prepare() + after_commit = [] + for layer in range(bench.layers): + act = bench.inputs() + kw = bench.kwargs(layer, act) + kw["g"], kw["beta"] = _grc_gates(bench, act, strided) + _, u, k, G = bench.state(layer) + before = dict(u=u.clone(), k=k.clone(), G=G.clone(), dA=bench.old_dA[layer].clone()) + ref = _grc_reference(bench, kw) + exact = _grc_op.reference(**bench.exact_kwargs(layer, kw)) + committed = None + if bench.path == "all_layer": + committed = _grc_reference( + bench, + dict(kw, replay_work_items=None, n_writes=None, use_all_layer_commit=False), + ) + got = _grc_op(**kw) + _replay_compare((got, *bench.observed(layer)), ref) + _, u, k, G = bench.state(layer) + assert torch.equal(bench.old_dA[layer], before["dA"]), "old_beta is never written" + assert torch.equal(u[~mask], before["u"][~mask]), "old_u written outside the append" + assert torch.equal(k[~mask], before["k"][~mask]), "old_k written outside the append" + Gt, Gt0 = G.transpose(-1, -2), before["G"].transpose(-1, -2) + assert torch.equal(Gt[~mask], Gt0[~mask]), "old_G written outside the append" + bench.adopt_exact(layer, exact) + after_commit.append((got, exact, committed)) + bench.commit() + for layer, (got, exact, committed) in enumerate(after_commit): + if committed is not None: + # The partitioned call plus the manager's commit equals the in-kernel commit. + _replay_compare((got, *bench.observed(layer)), committed) + _replay_compare((got, None, None, None, None), (exact[0], None, None, None, None)) + if bench.page is not None: + bench.page.assert_rest_unchanged() + bench.accept(bench.draw_acceptance()) + assert wraps > 0 and appends > 0, f"the cell never wrapped or appended ({wraps}, {appends})" + + +def _grc_drive_padding(cell) -> None: + """Trailing rows sharing one slot, as CUDA-graph padding does: the unique rows and every + other slot match the reference run on the unique rows alone.""" + spec = {k_: v_ for k_, v_ in cell.spec.items() if k_ not in ("page_layers",)} + spec["slots"] = 24 + bench = _ReplayBench(spec, seed=_seed(cell.why), exact=False) + dup = spec["dup"] + unique = bench.n - len(dup) + shared = int(bench.state_indices[unique]) + others = torch.ones(bench.slots, dtype=torch.bool, device=DEV) + others[shared] = False + for _ in range(bench.steps): + for layer in range(bench.layers): + act = bench.inputs() + kw = bench.kwargs(layer, act) + kw["g"], kw["beta"] = _grc_gates(bench, act, spec.get("strided_gates", False)) + alone = dict( + kw, + q=kw["q"][:unique], + k=kw["k"][:unique], + v=kw["v"][:unique], + g=kw["g"][:unique], + beta=kw["beta"][:unique], + state_indices=kw["state_indices"][:unique], + packed_qkv=act["packed"][: unique * bench.t], + ) + ref = _grc_op.reference(**alone) + got = _grc_op(**kw) + pool, u, k, G = bench.state(layer) + _replay_compare( + (got[:unique], pool[others], u[others], k[others], G[others]), + (ref[0], *(x[others] for x in ref[1:])), + ) + bench.accept(bench.draw_acceptance()) + + +def _check_gdn_cached_replay_cute(cell) -> None: + if "dup" in cell.spec: + _grc_drive_padding(cell) + else: + bench = _ReplayBench(cell.spec, seed=_seed(cell.why)) + try: + _grc_drive(cell, bench) + finally: + bench.close() + + +#: The geometries the target calls the entry at: a tp4/tep4 rank's heads, a dep4 rank's. +_GRC_GEOMETRIES = {"tep4": (cr.RANK_K_HEADS, cr.RANK_V_HEADS), "dep4": (16, 48)} + + +def _grc_rank_spec(geometry, **spec): + h, hv = _GRC_GEOMETRIES[geometry] + return dict(state_dtype=torch.bfloat16, H=h, HV=hv, **spec) + + +def _variant_replay_graph(n, path, geometry, pdl) -> None: + """Two replay layers (and the all-layer commit when partitioned) captured once and replayed on + new activations with acceptance between: bitwise the same steps run eagerly on twin state.""" + t = 4 + spec = _grc_rank_spec(geometry, path=path, n=n, t=t, layers=2, slots=40) + seed = 9100 + n + eager, graph, warm = (_ReplayBench(spec, seed, exact=False) for _ in range(3)) + acts = [warm.inputs() for _ in range(2)] + gates = [_grc_gates(warm, act, True) for act in acts] + + def step(b): + b.prepare() + outs = [] + for layer in range(2): + kw = b.kwargs(layer, acts[layer]) + kw["g"], kw["beta"] = gates[layer] + kw["launch_with_pdl"] = pdl + outs.append(_grc_op(**kw)) + b.commit() + return outs + + step(warm) + torch.cuda.synchronize() + captured = torch.cuda.CUDAGraph() + with torch.cuda.graph(captured): + outs_graph = step(graph) + refill = torch.Generator(device=DEV).manual_seed(seed + 1) + for _ in range(3): + for act, (a, b) in zip(acts, gates): + for x in (act["packed"], a, b): + x.copy_(torch.randn(x.shape, generator=refill, device=DEV, dtype=x.dtype)) + outs_eager = step(eager) + captured.replay() + torch.cuda.synchronize() + for e, g in zip(outs_eager, outs_graph): + assert torch.equal(e, g), "replayed output differs from eager" + for name in ("backing", "old_x", "old_B", "old_dt", "pnat", "buf"): + assert torch.equal(getattr(eager, name), getattr(graph, name)), f"{name} differs" + accepted = eager.draw_acceptance() + eager.accept(accepted) + graph.accept(accepted) + + +def _variant_ht_verify(n) -> None: + """A tp2/tep2 rank's HT verify step as `_gdn_verify` launches it (`_GDN_REPLAY_HT_LAUNCH`, + gated norm folded, the conv too at few requests) bitwise the plain chain on twin state. A + folded conv leaves the pool to the acceptance promote (the plain advance: the last window).""" + core = types.SimpleNamespace(num_k_heads=8, num_v_heads=24) + t, h, hv = 3, 8, 24 + assert _m._gdn_verify_folds_norm(core, n, t) + conv_fold = _m._gdn_verify_folds_conv(core, n) + pdl, mapping = _m._gdn_replay_launch(core, n) + assert (pdl, mapping) != (False, None), "every HT bucket changes the launch" + spec = dict( + state_dtype=torch.bfloat16, H=h, HV=hv, path="all_layer", n=n, t=t, layers=1, slots=n + 8 + ) + width = 2 * h * GRC_K + hv * GRC_V + gen = torch.Generator(device=DEV).manual_seed(5100 + n) + proj = torch.randn(n * t, width + hv * GRC_V + 2 * hv, generator=gen, device=DEV).to( + torch.bfloat16 + ) + x = proj[:, :width] + z = proj[:, width : width + hv * GRC_V].reshape(n * t, hv, GRC_V) + weight = (torch.randn(width, 4, generator=gen, device=DEV) * 0.3).to(torch.bfloat16) + norm_w = (1 + 0.1 * torch.randn(GRC_V, generator=gen, device=DEV)).to(torch.bfloat16) + pool0 = (torch.randn(n + 8, width, 3, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + rows = torch.arange(n, dtype=torch.int32, device=DEV).flip(0).contiguous() + results = [] + for target in (False, True): + bench = _ReplayBench(spec, 7700 + n, exact=False) + act = bench.inputs() + a, b = _grc_gates(bench, act, True) + pool = pool0.clone() + window = torch.zeros(n, t, width, 3, dtype=torch.bfloat16, device=DEV) + bench.prepare() + kw = bench.kwargs(0, act) + kw.update(g=a, beta=b) + if target and conv_fold: + fold = dict(x=x, weight=weight, pool=pool, window=window, window_rows=rows) + qkv = x + kw.update(packed_qkv=None, conv_fold=fold) + else: + qkv = causal_conv1d_token_major( + x, n, pool, weight, bench.state_indices, window, rows, -1 + ) + kw.update(packed_qkv=qkv) + q, k, v = torch.split(qkv, [h * GRC_K, h * GRC_K, hv * GRC_V], 1) + kw.update( + q=q.reshape(n, t, h, GRC_K), k=k.reshape(n, t, h, GRC_K), v=v.reshape(n, t, hv, GRC_V) + ) + if target: + kw.update(launch_with_pdl=pdl, gated_norm=dict(z=z, weight=norm_w, eps=1e-6)) + if mapping is not None: + kw["config"] = mapping + out = _grc_op(**kw) + if not target: + out = rms_norm_gated_token_major( + torch.reshape(out, [-1, GRC_V]), + z, + weight=norm_w, + eps=1e-6, + fp8_scale=None, + gate_activation="sigmoid", + ) + bench.commit() + torch.cuda.synchronize() + out = torch.reshape(out, [n * t, hv * GRC_V]) + results.append(([out.clone()] + [y.clone() for y in bench.state(0)], window, pool, bench)) + (ref, win_ref, pool_ref, bench_ref), (got, win_got, pool_got, bench_got) = results + for name, y, w in zip(("normed out", "state", "u", "k", "G"), got, ref): + assert torch.equal(y, w), f"{name}: n={n} launch {pdl, mapping}, conv fold {conv_fold}" + assert torch.equal(win_got, win_ref), "snapshot windows" + if conv_fold: + assert torch.equal(pool_got, pool0), "the fold must not advance the pool" + live = bench_ref.state_indices.long() + assert torch.equal(pool_ref[live], win_ref[rows.long(), t - 1]), "advance = last window" + else: + assert torch.equal(pool_got, pool_ref), "the conv pool" + for bench in (bench_ref, bench_got): + bench.close() + + +# ------------------------------------------ DecodeTarget._gdn_verify's replay by rank geometry --- + + +GVR_K = GVR_V = 128 +GVR_HIST, GVR_T, GVR_CONV_W, GVR_PAGE_LAYERS, GVR_LAYERS = 16, 4, 4, 36, 2 +#: A rank's GDN (key, value) heads per member. +GVR_GEOM = {"tep4": (4, 12), "tep2": (8, 24), "dep4": (16, 48)} + + +def _gvr_build(member: str, n: int, seed: int): + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + h, hv = GVR_GEOM[member] + key_dim, value_dim = h * GVR_K, hv * GVR_V + conv_dim = 2 * key_dim + value_dim + split = [conv_dim, value_dim, hv, hv] + slots = n + 3 + page = ( + torch.randn(slots, GVR_PAGE_LAYERS, hv, GVR_V, GVR_K, generator=g, device=DEV) * 0.05 + ).to(bf) + conv_page = torch.randn( + slots, GVR_PAGE_LAYERS, conv_dim, GVR_CONV_W - 1, generator=g, device=DEV + ) + conv_page = conv_page.to(bf) + old, int_conv, layers, projs = {}, {}, [], [] + for i in range(GVR_LAYERS): + u = (torch.randn(slots, 2, GVR_HIST, hv, GVR_V, generator=g, device=DEV) * 0.1).to(bf) + k = torch.randn(slots, 2, GVR_HIST, h, GVR_K, generator=g, device=DEV) + k = (k / k.norm(dim=-1, keepdim=True)).to(bf) + G = -torch.cumsum(torch.rand(slots, 2, hv, GVR_HIST, generator=g, device=DEV) * 0.3, -1) + old[i] = (u, k, G.contiguous(), torch.zeros_like(G)) + int_conv[i] = torch.zeros(n, GVR_T, conv_dim, GVR_CONV_W - 1, device=DEV, dtype=bf) + layers.append( + types.SimpleNamespace( + conv_w=(torch.randn(conv_dim, GVR_CONV_W, generator=g, device=DEV) * 0.3).to(bf), + a_log=(torch.randn(hv, generator=g, device=DEV) * 1.5 + 1.0).clamp(-7, 5), + dt_bias=(torch.randn(hv, generator=g, device=DEV) * 1.7 - 4.4).clamp(-11, 2.5), + ) + ) + projs.append(torch.randn(n * GVR_T, sum(split), generator=g, device=DEV).to(bf)) + gc = torch.Generator().manual_seed(seed) + cache_buf_idx = torch.randint(0, 2, (slots,), generator=gc, dtype=torch.int32).to(DEV) + # Odd slots hold 13..15 pending tokens (they wrap: p + 4 > 16), even ones 1..8 (they + # append); the step's rows start at slot 1, so every batch has a wrapping row. + pending = torch.tensor( + [(13 + (s // 2) % 3) if s % 2 else (1 + s % 8) for s in range(slots)], dtype=torch.int32 + ).to(DEV) + replay = _m._Replay(old, cache_buf_idx, pending, GVR_HIST, GVR_T) + pools = types.SimpleNamespace( + ssm={i: page[:, i] for i in range(GVR_LAYERS)}, + conv={i: conv_page[:, i] for i in range(GVR_LAYERS)}, + ) + core = types.SimpleNamespace( + conv_dim=conv_dim, + key_dim=key_dim, + value_dim=value_dim, + num_k_heads=h, + head_k_dim=GVR_K, + num_v_heads=hv, + head_v_dim=GVR_V, + _bound=types.SimpleNamespace(pools=pools, int_conv=int_conv, replay=replay), + ) + rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) # slot 0 untouched + verify = (torch.arange(n, dtype=torch.int32, device=DEV), None, None, False) + + def run(): + outs = [] + for i in range(GVR_LAYERS): + mq, _, a, b = torch.split(projs[i], split, 1) + outs.append(_m._gdn_verify(core, mq, a, b, i, layers[i], rows, n, GVR_T, verify)) + torch.cuda.synchronize() + return outs + + state = dict(page=page, conv=conv_page, old=old, int_conv=int_conv) + return run, state + + +def _gvr_rel(x, y): + x, y = x.float(), y.float() + return float((x - y).norm() / y.norm().clamp_min(1e-30)) + + +def _gvr_stock_replay(**kw): + """The engine's stock Triton cached replay in the CuTe entry's place: contiguous gate + columns, no CuTe launch mapping (`_gdn_verify` folds nothing at these batches).""" + assert "conv_fold" not in kw and "gated_norm" not in kw + kw.pop("config", None) + return stock_replay(**dict(kw, g=kw["g"].contiguous(), beta=kw["beta"].contiguous())) + + +@pytest.mark.parametrize("member,n", [("tep4", 1), ("tep2", 4), ("dep4", 8)]) +def test_gdn_verify_agrees_with_the_engines_stock_replay(member, n, monkeypatch): + """`_gdn_verify` over two MTP-3 layers at every rank geometry against the same layers over the + engine's stock Triton replay on twin state: outputs, checkpoint page, history and conv agree + to the replays' arithmetic (one fp32 recurrence, another dot order); other slots stay put.""" + run_c, st_c = _gvr_build(member, n, seed=500 + n) + page0 = st_c["page"].clone() + outs_c = run_c() + monkeypatch.setattr(_m, "gdn_cached_replay_cute", _gvr_stock_replay) + run_t, st_t = _gvr_build(member, n, seed=500 + n) + outs_t = run_t() + for oc, o_t in zip(outs_c, outs_t): + assert torch.isfinite(oc).all() + assert _gvr_rel(oc, o_t) < 4e-3, _gvr_rel(oc, o_t) + live = slice(1, n + 1) + assert _gvr_rel(st_c["page"][live], st_t["page"][live]) < 4e-3 + assert torch.equal(st_c["page"][0], page0[0]) and torch.equal( + st_c["page"][n + 1 :], page0[n + 1 :] + ) + assert not torch.equal(st_c["page"][live], page0[live]), "no row wrapped and committed" + assert torch.equal(st_c["conv"], st_t["conv"]), "the conv is the same kernel on both paths" + for i in range(GVR_LAYERS): + assert torch.equal(st_c["int_conv"][i], st_t["int_conv"][i]) + for x, y in zip(st_c["old"][i][:3], st_t["old"][i][:3]): + assert _gvr_rel(x[live], y[live]) < 4e-3 + assert torch.equal(x[0], y[0]) + + +# ------------------------------------------- causal_conv1d_token_major__verify --- + + +def _ctm_build(rows, p, seed): + """A tep4 rank's verify conv operands: the q|k|v column view of a [rows * p, 4128] + projection, one layer's view of a 36-layer conv page, the snapshot buffer.""" + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + conv_dim, width, slots = 2560, 4, rows + 5 + proj = torch.randn(rows * p, 4128, generator=g, device=DEV).to(bf) + page = torch.randn(slots, 36, conv_dim, width - 1, generator=g, device=DEV).to(bf) + weight = (torch.randn(conv_dim, width, generator=g, device=DEV) * 0.3).to(bf) + window = torch.zeros(rows + 2, p, conv_dim, width - 1, device=DEV, dtype=bf) + slot_ids = torch.arange(2, rows + 2, dtype=torch.int32, device=DEV) + window_rows = torch.arange(rows, dtype=torch.int32, device=DEV) + return proj, page, weight, window, slot_ids, window_rows + + +def _check_causal_conv1d_token_major(cell) -> None: + rows, p, wait = cell.spec["rows"], cell.spec["p"], cell.spec.get("wait", False) + proj, page, weight, window, slot_ids, window_rows = _ctm_build(rows, p, _seed(cell.why)) + page_ref, window_ref = page.clone(), window.clone() + x = proj[:, :2560] + causal_conv1d_token_major.is_valid(x, rows, page[:, 7], weight, slot_ids, window, window_rows) + out = causal_conv1d_token_major( + x, rows, page[:, 7], weight, slot_ids, window, window_rows, -1, True, wait + ) + ref = causal_conv1d_token_major.reference( + x, rows, page_ref[:, 7], weight, slot_ids, window_ref, window_rows, -1 + ) + torch.cuda.synchronize() + assert out.is_contiguous() and out.shape == (rows * p, 2560) + assert torch.equal(out, ref), "output" + assert torch.equal(page, page_ref), "the advanced pool" + assert torch.equal(window, window_ref), "the snapshots" + + +def _variant_conv_seqs(rows, p, monkeypatch) -> None: + """The verify conv's multi-sequence kernel (`_AUTO_SEQS` sequences a program, a tep2 rank's + large batches) bitwise the one-sequence launch, with a pad slot and a ragged sequence count.""" + seqs = ctm._AUTO_SEQS + g = torch.Generator(device=DEV).manual_seed(rows * 31 + p * 7 + seqs) + bf = torch.bfloat16 + conv_dim, slots_n = 5120, rows + 7 + proj = torch.randn(rows * p, 8240, generator=g, device=DEV).to(bf) + page = torch.randn(slots_n, 12, conv_dim, 3, generator=g, device=DEV).to(bf) + weight = (torch.randn(conv_dim, 4, generator=g, device=DEV) * 0.3).to(bf) + window = torch.randn(rows + 3, p, conv_dim, 3, generator=g, device=DEV).to(bf) + slot_ids = torch.randperm(slots_n, generator=g, device=DEV)[:rows].to(torch.int32) + slot_ids[rows // 2] = -1 # a padded sequence: nothing of it is read or written + window_rows = torch.randperm(rows + 3, generator=g, device=DEV)[:rows].to(torch.int32) + x = proj[:, :conv_dim] + + def run(n_seqs, pg, win): + monkeypatch.setattr(ctm, "_SEQS", n_seqs) + return ctm.causal_conv1d_token_major( + x, rows, pg[:, 5], weight, slot_ids, win, window_rows, -1, True, _m._GDN_CONV_PDL_WAIT + ) + + # a padded sequence's output rows are never written (torch.empty): compare the live ones + keep = torch.repeat_interleave(slot_ids != -1, p) + page_ref, win_ref = page.clone(), window.clone() + ref = run(1, page_ref, win_ref) + page_new, win_new = page.clone(), window.clone() + out = run(seqs, page_new, win_new) + torch.cuda.synchronize() + assert torch.equal(out[keep], ref[keep]), "output" + assert torch.equal(page_new, page_ref), "the advanced pool" + assert torch.equal(win_new, win_ref), "the snapshots" + + +# ------------------------------------------------- gdn_in_proj_conv__verify / norm_out_proj --- + + +def _ipc_operands(n: int, p: int, seed: int): + """A tep4 verify step's rows, input projection, conv taps, strided conv pool and snapshots.""" + g = torch.Generator(device=DEV).manual_seed(seed) + rows = n * p + x = (0.5 * torch.randn(rows, 2560, device=DEV, generator=g)).to(torch.bfloat16) + w = (0.03 * torch.randn(4128, 2560, device=DEV, generator=g)).to(torch.bfloat16) + cw = (0.4 * torch.randn(2560, 4, device=DEV, generator=g)).to(torch.bfloat16) + page = torch.randn(9, 3, 2560, 3, device=DEV, generator=g).to(torch.bfloat16) + slots = torch.tensor([7 - 2 * i for i in range(n)], dtype=torch.int32, device=DEV) + win = torch.randn(8, p, 2560, 3, device=DEV, generator=g).to(torch.bfloat16) + wrows = torch.tensor([i + 1 for i in range(n)], dtype=torch.int32, device=DEV) + return x, w, cw, page, slots, win, wrows + + +def _check_gdn_in_proj_conv(cell) -> None: + """Projection, conv, snapshots and pool bitwise the two-launch chain and within the entry's + fp32 reference; a captured launch replays on new rows.""" + n, p = cell.spec["requests"], cell.spec["p"] + x, w, cw, page, slots, win, wrows = _ipc_operands(n, p, zlib.crc32(cell.why.encode())) + tac = fused_tactic(n * p) + assert tac is not None + + def two_launch(x, page, win): + proj = low_m_bf16_mm(x, w.t(), True) + conv = causal_conv1d_token_major( + proj[:, :2560], n, page[:, 1], cw, slots, win, wrows, -1, True, True + ) + return proj, conv + + page_ref, win_ref = page.clone(), win.clone() + proj_ref, conv_ref = two_launch(x, page_ref, win_ref) + page_new, win_new = page.clone(), win.clone() + proj, conv = gdn_in_proj_conv( + x, w.t(), cw, page_new[:, 1], slots, win_new, wrows, -1, p, tac, True + ) + torch.cuda.synchronize() + for got, want, what in ( + (proj, proj_ref, "projection"), + (conv, conv_ref, "conv"), + (win_new, win_ref, "snapshots"), + (page_new, page_ref, "pool page"), + ): + assert torch.equal(got.view(torch.int16), want.view(torch.int16)), what + # The entry's own fp32 reference: the projection within bf16 rounding of a 2560-term + # sum, the conv within a bf16 ulp of the activations it reads. + rp, rc = gdn_in_proj_conv.reference( + x, w.t(), cw, page.clone()[:, 1], slots, win.clone(), wrows, -1, p, tac, True + ) + torch.testing.assert_close(proj.float(), rp.float(), atol=0.06, rtol=0.02) + torch.testing.assert_close(conv.float(), rc.float(), atol=0.06, rtol=0.02) + if not cell.spec["capture"]: + return + sx = x.clone() + graph = torch.cuda.CUDAGraph() + page_g, win_g = page.clone(), win.clone() + with torch.cuda.graph(graph): + gp, gc = gdn_in_proj_conv( + sx, w.t(), cw, page_g[:, 1], slots, win_g, wrows, -1, p, tac, True + ) + x2 = (0.5 * torch.randn_like(x.float())).to(torch.bfloat16) + sx.copy_(x2) + page_g.copy_(page) + win_g.copy_(win) + graph.replay() + torch.cuda.synchronize() + page_r, win_r = page.clone(), win.clone() + rp, rc = two_launch(x2, page_r, win_r) + assert torch.equal(gp, rp) and torch.equal(gc, rc) and torch.equal(page_g, page_r) + + +# ------------------------------------------------------------------------------ the tests --- + +#: Every Gated-DeltaNet entry of the target with its checker: `CASES` is each (entry, cell). +_ENTRIES = ( + (flashinfer_chunk_gated_delta_rule, _check_flashinfer_chunk_gated_delta_rule), + (_grc_op, _check_gdn_cached_replay_cute), + (causal_conv1d_token_major, _check_causal_conv1d_token_major), + (gdn_in_proj_conv, _check_gdn_in_proj_conv), +) +CASES = [(op, check, cell) for op, check in _ENTRIES for cell in op.CELLS] + + +@pytest.mark.parametrize( + "entry,check,cell", + CASES, + ids=[f"{type(op).__name__}-{i}" for op, _ in _ENTRIES for i in range(len(op.CELLS))], +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + """Every certified cell of every entry against the entry's own reference.""" + certifying(entry) + check(cell) + + +@pytest.mark.parametrize("t", range(2, _m._MAX_VERIFY_WIDTH + 1), ids=lambda t: f"T{t}") +def test_gdn_cached_replay_every_verify_width_is_the_recurrence(t) -> None: + """The cached replay at every verify width over consecutive steps from warm histories with + random acceptance, against its reference and the exact twin.""" + spec = _grc_rank_spec( + "tep4", path="small", n=2, t=t, steps=6, layers=1, slots=24, strided_gates=True + ) + cell = types.SimpleNamespace(why=f"tep4 small n=2 T={t}", spec=spec) + bench = _ReplayBench(spec, seed=_seed(cell.why)) + try: + _grc_drive(cell, bench) + finally: + bench.close() + + +#: `(id, helper, args)`: a launch the cells do not reach, against its plain launch. +_VARIANTS = ( + [("replay_graph-tep4-n1-pdl", _variant_replay_graph, (1, "small", "tep4", True))] + + [("replay_graph-dep4-n16", _variant_replay_graph, (16, "all_layer", "dep4", False))] + + [(f"ht_verify-n{n}", _variant_ht_verify, (n,)) for n in (16, 24, 32, 64, 128)] + + [(f"conv_seqs-{r}x{p}", _variant_conv_seqs, (r, p)) for r, p in ((37, 4), (128, 3))] +) + + +@pytest.mark.parametrize("helper,args", [v[1:] for v in _VARIANTS], ids=[v[0] for v in _VARIANTS]) +def test_launch_variants_are_bitwise_the_plain_launch(helper, args, monkeypatch) -> None: + """Launches no cell reaches, each bitwise its plain counterpart.""" + if helper is _variant_conv_seqs: + helper(*args, monkeypatch) + else: + helper(*args) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py new file mode 100644 index 000000000000..b62cb88f648e --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -0,0 +1,789 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The FP8 unified target's vision path: the tower's two entries over their certified cells, the +tower against HuggingFace on the checkpoint's weights, request admission and the multimodal splice. +""" + +from __future__ import annotations + +import contextlib +import functools +import json +import zlib +from types import SimpleNamespace +from typing import List, Sequence, Tuple + +import numpy as np +import pytest +import safetensors +import torch +import torch.nn.functional as F +from utils.llm_data import llm_models_root + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + vision as V, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as W, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.vision_rope_2d_packed_qk import ( # noqa: E501 + HEAD_DIM, + NUM_HEADS, + vision_rope_2d_packed_qk, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.vision.triton_pos_embed_interpolate import ( # noqa: E501 + triton_pos_embed_interpolate, +) +from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig +from tensorrt_llm.inputs.multimodal import ( + MULTIMODAL_ENCODER_ITEM_METADATA_KEY, + MultimodalItemMetadata, + MultimodalParams, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + ULP, + assert_within_ulp, + certifying, +) + +CHECKPOINT = "Qwen3.8-Flash-Next-FP8" + +Shell = M.ModelingV2Qwen38FlashNextFp8Sm103Unified + +_VISION = dict( + deepstack_visual_indexes=[], depth=27, dtype="bfloat16", hidden_act="gelu_pytorch_tanh", + hidden_size=1152, in_channels=3, intermediate_size=4304, num_heads=16, + num_position_embeddings=2304, out_hidden_size=2560, patch_size=16, spatial_merge_size=2, + temporal_patch_size=2, +) # fmt: skip +HIDDEN_OUT = 2560 + +_needs_gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device") + + +def _dims(vcfg) -> SimpleNamespace: + """The vision widths of `weights.rank_dims` (the tower is replicated: every rank's).""" + return SimpleNamespace( + hidden=HIDDEN_OUT, dtype=torch.bfloat16, v_depth=vcfg.depth, v_hidden=vcfg.hidden_size, + v_inter=vcfg.intermediate_size, v_channels=vcfg.in_channels, + v_tpatch=vcfg.temporal_patch_size, v_patch=vcfg.patch_size, + v_pos_rows=vcfg.num_position_embeddings, + v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, + ) # fmt: skip + + +def _vision_entries(): + return [entry for entry in W.MODEL_WEIGHTS.WEIGHTS if entry.name.startswith("vis_")] + + +def _tower_over_the_weight_table() -> V.VisionTower: + """The tower over the table's `vis_*` weights on meta; `embed` only supplies the dtype.""" + vcfg = Qwen4ExpVisionConfig(**_VISION) + d = _dims(vcfg) + w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="meta")} + for entry in _vision_entries(): + for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): + w[key] = torch.empty(*entry.shape(d), dtype=entry.dtype or d.dtype, device="meta") + return V.VisionTower(w, vcfg) + + +MERGE = 2 +THETA = 10000.0 + + +def _num_tokens(grid) -> int: + return sum(t * h * w for t, h, w in grid) + + +def _spatial_merge_positions(grid: Sequence[Sequence[int]], merge: int) -> List[Tuple[int, int]]: + """Per-token `(row, col)` in spatial-merge order, by plain nested loops.""" + positions: List[Tuple[int, int]] = [] + for frames, height, width in grid: + frame = [ + (block_row * merge + r, block_col * merge + c) + for block_row in range(height // merge) + for block_col in range(width // merge) + for r in range(merge) + for c in range(merge) + ] + positions.extend(frame * frames) + return positions + + +def _angle_table(grid_thw: torch.Tensor, merge: int, theta: float, head_dim: int, device="cuda"): + """fp32 `cos` / `sin` `[T, head_dim // 2]`, height then width, rounded once from fp64.""" + rope_dim = head_dim // 2 + inv_freq = 1.0 / (theta ** (torch.arange(0, rope_dim, 2, dtype=torch.float64) / rope_dim)) + pos = torch.tensor(_spatial_merge_positions(grid_thw.tolist(), merge), dtype=torch.float64) + angles = (pos[:, :, None] * inv_freq).flatten(1).to(device=device, dtype=torch.float32) + return angles.cos().contiguous(), angles.sin().contiguous() + + +HIDDEN = 3 + +IMAGE, VIDEO = 1, 2 + + +class _Tower: + """Stub `VisionTower`: every row of item `i` of modality `m` holds `100 * m + i`.""" + + merge = MERGE + + def __init__(self): + self.calls = 0 + + def __call__(self, pixel_values, grid_thw): + self.calls += 1 + modality = int(pixel_values[0, 0]) + return torch.cat( + [ + torch.full((t * h * w // (MERGE * MERGE), HIDDEN), float(100 * modality + item)) + for item, (t, h, w) in enumerate(grid_thw.tolist()) + ] + ) + + +def _media(modality, grids): + """One `multimodal_data` entry; `pixel_values` only carries the stub's modality.""" + patches = sum(t * h * w for t, h, w in grids) + key = "pixel_values" if modality == IMAGE else "pixel_values_videos" + grid_key = "image_grid_thw" if modality == IMAGE else "video_grid_thw" + return {key: torch.full((patches, 1), float(modality)), grid_key: torch.tensor(grids)} + + +def _params(image=None, video=None, order=None, runtime=None, mrope=True): + data = {} + if mrope: + data["mrope_config"] = {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)} + if image is not None: + data["image"] = _media(IMAGE, image) + if video is not None: + data["video"] = _media(VIDEO, video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +# Two videos of 4 and 8 rows, two images of 4 and 1. +VIDEOS = [(2, 2, 4), (2, 4, 4)] +IMAGES = [(1, 4, 4), (1, 2, 2)] +MIXED_ORDER = [("video", 0), ("image", 0), ("video", 1), ("image", 1)] +MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] + + +class _StepBegun(Exception): + """Raised by the stub core's `begin_step`, so the shell's forward stops there.""" + + +def _forward_to_begin_step(tower, kwargs): + """The `vision` argument the shell's forward hands `begin_step` for a step with `kwargs`.""" + seen = {} + + def begin_step(attn_metadata, input_ids, position_ids, **step): + seen.update(step) + raise _StepBegun + + shell = SimpleNamespace(model=SimpleNamespace(begin_step=begin_step), vision_tower=tower) + with pytest.raises(_StepBegun): + Shell.forward(shell, None, torch.zeros(1, dtype=torch.int32), None, **kwargs) + return seen["vision"] + + +TEXT_TOKENS = 5 + + +@functools.cache +def _assembled_tower() -> V.VisionTower: + return _tower_over_the_weight_table() + + +def _request_media(modality, grids): + """A payload whose pixel tensor has the assembled tower's patch width and no storage.""" + patches = sum(t * h * w for t, h, w in grids) + key = "pixel_values" if modality == "image" else "pixel_values_videos" + grid_key = "image_grid_thw" if modality == "image" else "video_grid_thw" + return { + key: torch.empty(patches, _assembled_tower().pixel_width, device="meta"), + grid_key: torch.tensor(grids), + } + + +def _request(image=None, video=None, order=None, **overrides): + """Placeholders plus `TEXT_TOKENS` text tokens, with the mrope ids and placeholder cumsum.""" + rows = sum(t * h * w for t, h, w in (image or []) + (video or [])) // (MERGE * MERGE) + prompt_len = rows + TEXT_TOKENS + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, prompt_len, dtype=torch.int32)}, + "multimodal_embed_mask_cumsum": torch.cat( + [torch.arange(1, rows + 1), torch.full((TEXT_TOKENS,), rows)] + ), + } + if image is not None: + data["image"] = _request_media("image", image) + if video is not None: + data["video"] = _request_media("video", video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + fields = {"py_multimodal_data": data, "prompt_len": prompt_len} + fields.update(overrides) + return SimpleNamespace(**fields) + + +def _whole_op_reference(q, k, grid_thw, spatial_merge_size, theta=THETA): + """`VisionRope2d.reference` verbatim: the whole op from the grid, as fresh `(q_out, k_out)`.""" + rope_dim = q.shape[-1] // 2 + axis_dim = rope_dim // 2 + positions = _spatial_merge_positions(grid_thw.tolist(), spatial_merge_size) + inverse = torch.tensor( + [theta ** (-(2.0 * j) / rope_dim) for j in range(axis_dim)], dtype=torch.float64 + ) + pos = torch.tensor(positions, dtype=torch.float64).reshape(-1, 2) + angles = (pos[:, :, None] * inverse).flatten(1).to(device=q.device, dtype=torch.float32) + rotor = torch.polar(torch.ones_like(angles), angles).unsqueeze(-2) + + def rotate(x: torch.Tensor) -> torch.Tensor: + fp32 = x.float() + rotated = torch.complex(fp32[..., :rope_dim], fp32[..., rope_dim:]) * rotor + return torch.cat((rotated.real, rotated.imag), dim=-1).to(x.dtype) + + return rotate(q), rotate(k) + + +def _packed_qkv(tokens: int, dtype: torch.dtype, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randn(tokens, 3 * NUM_HEADS, HEAD_DIM, generator=g, device="cuda").to(dtype) + + +def _qk(qkv: torch.Tensor) -> torch.Tensor: + qk = qkv[:, : 2 * NUM_HEADS] + assert qk.stride() == (3 * NUM_HEADS * HEAD_DIM, HEAD_DIM, 1) and not qk.is_contiguous() + return qk + + +def _mismatch(got: torch.Tensor, want: torch.Tensor) -> Tuple[int, float]: + """(elements differing in their bits, largest absolute difference).""" + differing = (got.reshape(-1).view(torch.int16) != want.reshape(-1).view(torch.int16)).sum() + return int(differing.item()), (got.float() - want.float()).abs().max().item() + + +def _check_rope(cell) -> None: + spec = cell.spec + grid_thw = torch.tensor([list(row) for row in spec["grid"]], dtype=spec["grid_dtype"]) + tokens = _num_tokens(spec["grid"]) + qkv = _packed_qkv(tokens, spec["dtype"], seed=zlib.crc32(cell.why.encode()) % 2**31) + before = qkv.clone() + cos, sin = _angle_table(grid_thw, MERGE, THETA, HEAD_DIM) + qk = _qk(qkv) + + want = vision_rope_2d_packed_qk.reference(qk, cos, sin) + assert torch.equal(qkv, before), "reference must not mutate its input" + out = vision_rope_2d_packed_qk(qk, cos, sin) + torch.cuda.synchronize() + + assert out is qk, "the op returns the view it rotated" + assert torch.equal(qkv[:, 2 * NUM_HEADS :], before[:, 2 * NUM_HEADS :]), "v was written" + assert not torch.equal(qk, before[:, : 2 * NUM_HEADS]), "q|k was not rotated" + + differing, max_dev = _mismatch(qk, want) + assert differing == 0, f"{differing} elements differ from reference (max {max_dev:.3e})" + + q_whole, k_whole = _whole_op_reference( + before[:, :NUM_HEADS], before[:, NUM_HEADS : 2 * NUM_HEADS], grid_thw, MERGE, THETA + ) + for name, got, ref in ( + ("q", qkv[:, :NUM_HEADS], q_whole), + ("k", qkv[:, NUM_HEADS : 2 * NUM_HEADS], k_whole), + ): + differing, max_dev = _mismatch(got, ref) + assert differing == 0, ( + f"{name}: {differing} elements differ from the whole op (max {max_dev:.3e})" + ) + + +#: (height, width) in pixels of the two test images: grids 1x28x42 and 1x14x20. +_IMAGE_SIZES = ((448, 672), (224, 320)) +#: Four 256x384 frames: one video item, grid 2x16x24 (two temporal groups). +_VIDEO_FRAMES, _VIDEO_SIZE = 4, (256, 384) +#: SDPA backends of the bf16 HF forward; their spread defines the bf16 error envelope. +_HF_BACKENDS = ("MATH", "EFFICIENT_ATTENTION", "FLASH_ATTENTION", "CUDNN_ATTENTION") +#: Allowed factor over the worst bf16 HF forward: the backends differ among themselves by up to +#: 1.5x on one item, and the tower is one more sample of that spread. +_BAND = 1.25 + + +def _picture(height: int, width: int, seed: int) -> np.ndarray: + """A deterministic `[H, W, 3]` uint8 picture with both low and high spatial frequencies.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:height, 0:width] / max(height, width) + rgb = np.stack( + [np.sin(6 * x + seed), np.cos(5 * y - seed), np.sin(3 * (x + y) + 0.5 * seed)], -1 + ) + rgb = 0.5 + 0.5 * rgb + for _ in range(4): + top, left = rng.integers(0, height // 2), rng.integers(0, width // 2) + rgb[top : top + height // 4, left : left + width // 4] = rng.random(3) + rgb += 0.04 * rng.standard_normal(rgb.shape) + return (np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8) + + +@contextlib.contextmanager +def _ieee_fp32(): + """IEEE fp32 for the reference: GEMMs and the patch conv off TF32, SDPA on its math backend.""" + from torch.nn.attention import SDPBackend, sdpa_kernel + + matmul, conv = torch.backends.cuda.matmul, torch.backends.cudnn.conv + previous = (matmul.fp32_precision, conv.fp32_precision) + matmul.fp32_precision = "ieee" + conv.fp32_precision = "ieee" + try: + with sdpa_kernel(SDPBackend.MATH): + yield + finally: + matmul.fp32_precision, conv.fp32_precision = previous + + +def _stats(out: torch.Tensor, ref: torch.Tensor) -> dict: + """Max abs error, max abs error over max |ref| ("max rel"), relative L2 and cosine.""" + o, r = out.double().flatten(), ref.double().flatten() + diff = (o - r).abs() + return dict( + max_abs=diff.max().item(), + max_rel=(diff.max() / r.abs().max()).item(), + rel_l2=(torch.linalg.vector_norm(o - r) / torch.linalg.vector_norm(r)).item(), + cos=torch.nn.functional.cosine_similarity(o, r, dim=0).item(), + ) + + +@pytest.fixture(scope="module") +def checkpoint(): + root = llm_models_root() + path = None if root is None else root / CHECKPOINT + if path is None or not (path / "model.safetensors.index.json").is_file(): + pytest.skip(f"{CHECKPOINT} is not under LLM_MODELS_ROOT") + return path + + +@pytest.fixture(scope="module") +def models(checkpoint): + if not torch.cuda.is_available(): + pytest.skip("needs a CUDA device") + return _load_models(checkpoint) + + +@pytest.fixture(scope="module") +def media(checkpoint): + return _load_media(checkpoint) + + +@pytest.fixture(scope="module") +def references(models, media): + return _reference_outputs(models, media) + + +def _load_models(checkpoint): + """`(tower, hf32, hf16)`: this target's tower over the table's `vis_*` sources and HF's + `Qwen3_5VisionModel` in fp32 and bf16, all on the checkpoint's same vision tensors.""" + from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5VisionConfig + from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5VisionModel + + vision = json.loads((checkpoint / "config.json").read_text())["vision_config"] + assert vision["deepstack_visual_indexes"] == [], "the reference has no deepstack mergers" + vcfg = Qwen4ExpVisionConfig(**{k: v for k, v in vision.items() if k != "model_type"}) + d = _dims(vcfg) + index = json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + + # Each `vis_*` parameter is one whole checkpoint tensor: the tower is replicated and bf16. + sources = {} + for entry in _vision_entries(): + assert entry.transform is None and entry.cut is None, entry.name + for key, i, prefix in W.MODEL_WEIGHTS._targets(entry, d): + [(ckpt_key, at, cut)] = entry.sources(d, i, prefix) + assert at is None and cut is None, (key, ckpt_key) + sources[key] = (ckpt_key, entry.shape(d), entry.dtype or d.dtype) + visual = {k for k in index if k.startswith("model.visual.")} + assert {ckpt_key for ckpt_key, _, _ in sources.values()} == visual, ( + "the table's vis_* sources are not exactly the checkpoint's model.visual.* tensors" + ) + assert len(sources) == len(visual) + + tensors = {} + for shard in sorted({index[k] for k in visual}): + with safetensors.safe_open(str(checkpoint / shard), framework="pt", device="cpu") as f: + for key in f.keys(): + if key in visual: + tensors[key] = f.get_tensor(key) + w = {} + for key, (ckpt_key, shape, dtype) in sources.items(): + tensor = tensors[ckpt_key] + assert tuple(tensor.shape) == tuple(shape) and tensor.dtype == dtype, key + w[key] = tensor.cuda() + w["embed"] = torch.empty(1, d.hidden, dtype=torch.bfloat16, device="cuda") + tower = V.VisionTower(w, vcfg) + + fields = ( + "depth hidden_size hidden_act intermediate_size num_heads in_channels patch_size " + "spatial_merge_size temporal_patch_size out_hidden_size num_position_embeddings" + ).split() + hf_cfg = Qwen3_5VisionConfig(**{k: vision[k] for k in fields}) + hf_cfg._attn_implementation = "sdpa" + state = {k.removeprefix("model.visual."): t for k, t in tensors.items()} + out = [] + for dtype in (torch.float32, torch.bfloat16): + with torch.device("cuda"): + hf = Qwen3_5VisionModel(hf_cfg) + hf.load_state_dict(state, strict=True) + # Keep the rotary's `inv_freq` fp32 as `from_pretrained` does; `.to(bf16)` rounds it. + inv_freq = hf.rotary_pos_emb.inv_freq.clone() + hf = hf.to(dtype).eval() + hf.rotary_pos_emb.inv_freq = inv_freq + out.append(hf) + return tower, out[0], out[1] + + +def _load_media(checkpoint): + """`{name: (pixel_values, grid_thw)}` from the checkpoint's processors for two images and a + four-frame video, the pixels rounded to bf16 once so every model reads the same values.""" + from PIL import Image + from transformers import AutoImageProcessor, AutoVideoProcessor + + images = AutoImageProcessor.from_pretrained(str(checkpoint)) + videos = AutoVideoProcessor.from_pretrained(str(checkpoint)) + out = {} + for n, (height, width) in enumerate(_IMAGE_SIZES): + batch = images(images=[Image.fromarray(_picture(height, width, n))], return_tensors="pt") + out[f"image{n}"] = (batch["pixel_values"], batch["image_grid_thw"]) + frames = np.stack([_picture(*_VIDEO_SIZE, 10 + f) for f in range(_VIDEO_FRAMES)]) + batch = videos(videos=[frames], return_tensors="pt", do_sample_frames=False) + out["video"] = (batch["pixel_values_videos"], batch["video_grid_thw"]) + assert [grid.tolist() for _, grid in out.values()] == [ + [[1, 28, 42]], + [[1, 14, 20]], + [[2, 16, 24]], + ] + return { + name: (px.to(device="cuda", dtype=torch.bfloat16), grid) for name, (px, grid) in out.items() + } + + +def _reference_outputs(models, media): + """Per media item `(ref, envelope)`: the fp32 HF reference `[rows, 2560]`, and the worst + of each statistic (`_stats`) the bf16 HF forward reaches against it over `_HF_BACKENDS`.""" + from torch.nn.attention import SDPBackend, sdpa_kernel + + _, hf32, hf16 = models + out = {} + with torch.inference_mode(): + for name, (px, grid) in media.items(): + with _ieee_fp32(): + ref = hf32(px.float(), grid_thw=grid).pooler_output + floors = {} + for backend in _HF_BACKENDS: + with sdpa_kernel(getattr(SDPBackend, backend)): + floors[backend] = _stats(hf16(px, grid_thw=grid).pooler_output, ref) + envelope = { + key: max(st[key] for st in floors.values()) for key in ("max_abs", "rel_l2") + } + out[name] = (ref, envelope) + return out + + +def _check(name: str, out: torch.Tensor, ref: torch.Tensor, envelope: dict) -> dict: + """`out` within `_BAND` of the worst bf16 HF forward on every gated statistic.""" + tower = _stats(out, ref) + assert out.dtype == torch.bfloat16 and out.shape == ref.shape + assert torch.isfinite(out).all() + for key in ("rel_l2", "max_abs"): + assert tower[key] <= _BAND * envelope[key], (name, key, tower, envelope) + return tower + + +def _hf_single(name): + def run(tower, media, references, monkeypatch): + px, grid = media[name] + with torch.inference_mode(): + out = tower(px, grid) + assert out.shape == (int(grid.prod(-1).sum()) // tower.merge**2, HIDDEN_OUT) + _check(name, out, *references[name]) + + return run + + +def _hf_pair(cap=None): + """Both images in one call (or one per call under `cap`), each its own attention segment.""" + + def run(tower, media, references, monkeypatch): + (px0, grid0), (px1, grid1) = media["image0"], media["image1"] + if cap is not None: + assert V._frame_calls([(1, 28, 42), (1, 14, 20)], cap) == [[(1, 28, 42)], [(1, 14, 20)]] + monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", cap) + with torch.inference_mode(): + out = tower(torch.cat([px0, px1]), torch.cat([grid0, grid1])) + split = int(grid0.prod()) // tower.merge**2 + _check("image0 in a pair", out[:split], *references["image0"]) + _check("image1 in a pair", out[split:], *references["image1"]) + + return run + + +def _hf_video_in_two_calls(tower, media, references, monkeypatch): + """The per-call ceiling lowered to one 384-patch frame: the video runs as two calls.""" + px, grid = media["video"] + assert V._frame_calls([(2, 16, 24)], 384) == [[(1, 16, 24)], [(1, 16, 24)]] + monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", 384) + with torch.inference_mode(): + out = tower(px, grid) + _check("video in two calls", out, *references["video"]) + + +_HF_CASES = { + "image0": _hf_single("image0"), + "image1": _hf_single("image1"), + "video": _hf_single("video"), + "image_pair": _hf_pair(), + "image_pair_two_calls": _hf_pair(cap=1176), + "video_two_calls": _hf_video_in_two_calls, +} + + +@_needs_gpu +@pytest.mark.parametrize("case", list(_HF_CASES)) +def test_the_tower_is_huggingfaces_vision_model(models, media, references, case, monkeypatch): + """Each case lands within the bf16 HF forward's own error of the fp32 HF reference.""" + _HF_CASES[case](models[0], media, references, monkeypatch) + + +NUM_GRID = 48 +VISION_HIDDEN = 1152 + + +def _pos_embed_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """4 bf16 ulp per element, 1 ulp relative RMS: the op rounds its corner weights to bf16 and + multiply-adds in bf16, against the exact value (measured worst 3.5 / 0.75).""" + assert out.dtype == torch.bfloat16, out.dtype + assert_within_ulp(out.float(), ref, element_ulp=4.0, rms_ulp=1.0, ulp=ULP[torch.bfloat16]) + + +def _pos_table(seed: int, num_grid: int = NUM_GRID, hidden: int = VISION_HIDDEN) -> torch.Tensor: + """A smooth `[num_grid**2, hidden]` bf16 table, as a learned 2-D position table is.""" + g = torch.Generator(device="cuda").manual_seed(seed) + coarse = torch.randn(1, hidden, 12, 12, generator=g, device="cuda") + up = F.interpolate(coarse, size=(num_grid, num_grid), mode="bicubic", align_corners=True) + up = up[0].permute(1, 2, 0) + 0.1 * torch.randn( + num_grid, num_grid, hidden, generator=g, device="cuda" + ) + return up.reshape(num_grid * num_grid, hidden).to(torch.bfloat16).contiguous() + + +def _resize64(table, t, h, w, m_size, *, align=True, raster=False, swap=False): + """torch's bilinear resize in fp64, in merge order unless `raster`.""" + num_grid = int(round(table.shape[0] ** 0.5)) + grid = table.double().view(num_grid, num_grid, -1) + if swap: + grid = grid.transpose(0, 1) + src = grid.permute(2, 0, 1).unsqueeze(0).contiguous() + out = F.interpolate(src, size=(h, w), mode="bilinear", align_corners=align)[0].permute(1, 2, 0) + if raster: + rows = out.reshape(h * w, -1) + else: + rows = ( + out.reshape(h // m_size, m_size, w // m_size, m_size, -1) + .permute(0, 2, 1, 3, 4) + .reshape(h * w, -1) + ) + return rows.repeat(t, 1) if t > 1 else rows + + +def _pos_embed_args(spec, table): + return (table, *(spec[k] for k in ("t", "h", "w", "num_grid_per_side", "m_size", "dtype"))) + + +def _check_pos_embed(cell) -> None: + spec = cell.spec + t, h, w, m_size = spec["t"], spec["h"], spec["w"], spec["m_size"] + table = _pos_table( + zlib.crc32(cell.why.encode()) % 2**31, spec["num_grid_per_side"], spec["hidden"] + ) + before = table.clone() + + out = triton_pos_embed_interpolate(*_pos_embed_args(spec, table)) + assert out.shape == (t * h * w, spec["hidden"]) and out.dtype == spec["dtype"] + assert torch.equal(table, before) and out.data_ptr() != table.data_ptr() + + ref = triton_pos_embed_interpolate.reference(*_pos_embed_args(spec, table)) + _pos_embed_compare(out, ref) + + # An independent fp64 implementation confirms the reference. + torch.testing.assert_close(ref, _resize64(table, t, h, w, m_size).float()) + + # The band refuses each wrong variant of the function. + for wrong in (dict(align=False), dict(raster=True), dict(swap=True)): + with pytest.raises(AssertionError): + _pos_embed_compare(out, _resize64(table, t, h, w, m_size, **wrong).float()) + + +_ENTRIES = ( + (vision_rope_2d_packed_qk, _check_rope), + (triton_pos_embed_interpolate, _check_pos_embed), +) +CASES = [(op, check, cell) for op, check in _ENTRIES for cell in op.CELLS] + + +@_needs_gpu +@pytest.mark.parametrize( + "entry,check,cell", + CASES, + ids=[f"{type(op).__name__}-{i}" for op, _ in _ENTRIES for i in range(len(op.CELLS))], +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + """Every certified cell of the tower's two entries against the entry's own reference.""" + certifying(entry) + check(cell) + + +def _text(data): + return lambda: SimpleNamespace(py_multimodal_data=data) + + +def _req(edit=None, **media): + """`_request(**media)`, then `edit` on its `py_multimodal_data`.""" + + def build(): + request = _request(**media) + if edit is not None: + edit(request.py_multimodal_data) + return request + + return build + + +def _grid(value): + return lambda d: d["image"].update(image_grid_thw=torch.tensor(value)) + + +def _pixels(rows, extra=0): + def edit(d): + width = _assembled_tower().pixel_width + extra + d["video"]["pixel_values_videos"] = torch.empty(rows, width, device="meta") + + return edit + + +def _positions(*shape): + return lambda d: d["mrope_config"].update(mrope_position_ids=torch.zeros(*shape)) + + +def _cumsum(d): + d["multimodal_embed_mask_cumsum"] = torch.arange(1, 14) + + +_MIXED = dict(image=IMAGES, video=VIDEOS, order=MIXED_ORDER) +_TILE = [(1, 4, 4)] +#: Served. The per-call ceiling of 32768 patches bounds a frame, not a whole video. +_SERVED = [ + ("image", _req(image=IMAGES)), + ("video", _req(video=VIDEOS)), + ("mixed", _req(**_MIXED)), + ("frame_at_ceiling", _req(image=[(1, 128, 256)])), + ("long_video", _req(video=[(33, 32, 32)])), + ("no_cumsum", _req(lambda d: d.pop("multimodal_embed_mask_cumsum"), **_MIXED)), +] +#: Text, and position ids without media (a warmup request's): admitted without the tower. +_POSITIONS_ONLY = {"mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8)}} +_TEXT = [("text", _text(None)), ("positions_only", _text(_POSITIONS_ONLY))] +#: Refused: what the tower or the splice would assert on, or read wrongly. +_REFUSED = [ + ("frame_past_ceiling", _req(video=[(1, 192, 192)]), + r"video 0 has frames of 192x192 = 36864 patches \(3072x3072 pixels\).*at most 32768"), + ("grid_not_merge_blocks", _req(_grid([(1, 4, 5)]), image=_TILE), "spatial_merge_size 2"), + ("grid_zero", _req(_grid([(0, 4, 4)]), image=_TILE), "every entry must be positive"), + ("grid_1d", _req(_grid([1, 4, 4]), image=_TILE), r"one \(t, h, w\) row per image"), + ("pixel_rows", _req(_pixels(5), video=VIDEOS), "describe 48 patch rows"), + ("pixel_width", _req(_pixels(48, 1), video=VIDEOS), "48 patch rows of"), + ("no_grid", _req(lambda d: d["video"].pop("video_grid_thw"), video=VIDEOS), + "this target reads 'pixel_values_videos'"), + ("positions_short", _req(_positions(3, 1, 40), video=VIDEOS), "one position per prompt token"), + ("positions_2d", _req(_positions(1, 17), video=VIDEOS), r"got \(1, 17\)"), + ("no_positions", _req(lambda d: d.pop("mrope_config"), video=VIDEOS), + "mrope_position_ids.*got None"), + ("no_manifest", _req(image=IMAGES, video=VIDEOS), "needs item metadata"), + ("short_manifest", _req(**dict(_MIXED, order=MIXED_ORDER[:3])), + r"lists image items \[0\]; the payload carries 2"), + ("unknown_item", _req(**dict(_MIXED, order=MIXED_ORDER + [("audio", 0)])), + "is not an item of this request"), + ("placeholder_count", _req(_cumsum, **_MIXED), "holds 13 image/video placeholder tokens.*17"), +] # fmt: skip +_ADMISSIONS = ( + [(i, b, None, True) for i, b in _SERVED] + + [(i, b, None, False) for i, b in _TEXT] + + [(i, b, r, True) for i, b, r in _REFUSED] +) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "build,refusal,reads_tower", [a[1:] for a in _ADMISSIONS], ids=[a[0] for a in _ADMISSIONS] +) +def test_admission_serves_what_the_tower_serves(build, refusal, reads_tower): + """`validate_request` admits text without reading the tower and the media it serves, and + refuses what the tower or the splice would assert on, naming the item and the bound.""" + shell = SimpleNamespace(vision_tower=_assembled_tower() if reads_tower else None) + if refusal is None: + Shell.validate_request(shell, build()) + else: + with pytest.raises(ValueError, match=refusal): + Shell.validate_request(shell, build()) + + +def _chunk(cached, in_chunk): + return SimpleNamespace( + num_cached_mm_tokens=cached, + num_mm_tokens_in_chunk=in_chunk, + total_embeds_in_request=len(MIXED_ROWS), + ) + + +#: `(id, _params kwargs, placeholder rows, rows that land there (None: no media), tower calls)`. +#: The chunked cases take prompt-order rows 3..9 of the mixed request, or none. +_SPLICES = [ + ("text", dict(), 0, None, 0), + ("image", dict(image=IMAGES), 5, [100.0] * 4 + [101.0], 1), + ("video", dict(video=VIDEOS), 12, [200.0] * 4 + [201.0] * 8, 1), + ("mixed", _MIXED, len(MIXED_ROWS), MIXED_ROWS, 2), + ("chunk", dict(_MIXED, runtime=_chunk(3, 6)), 6, MIXED_ROWS[3:9], 2), + ("chunk_without_placeholders", dict(_MIXED, runtime=_chunk(len(MIXED_ROWS), 0)), 0, None, 0), +] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "params,num_rows,want,calls", [s[1:] for s in _SPLICES], ids=[s[0] for s in _SPLICES] +) +def test_the_splice_puts_each_items_rows_on_its_placeholders(params, num_rows, want, calls): + """The tower's rows reach `begin_step` with the placeholder indices, and the splice writes them + over the placeholders in prompt order and zeroes the padding rows after the step.""" + tower = _Tower() + indices = torch.arange(1, 1 + num_rows, dtype=torch.int64) + kwargs = { + "multimodal_params": [_params(**params)], + "mm_token_indices": indices, + "resource_manager": object(), + } + vision = _forward_to_begin_step(tower, kwargs) + assert tower.calls == calls + total = num_rows + 2 + h = torch.full((total + 3, HIDDEN), -1.0) + M.PrefillTarget._splice( + SimpleNamespace(_step=SimpleNamespace(vision=vision, num_tokens=total)), h + ) + assert torch.count_nonzero(h[total:]) == 0, "padding rows must be zeroed" + if want is None: + assert vision is None and torch.all(h[:total] == -1.0) + return + assert vision[1] is indices + assert h[1 : 1 + num_rows, 0].tolist() == want + assert torch.all(h[0] == -1.0) and torch.all(h[total - 1] == -1.0) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py new file mode 100644 index 000000000000..5f50e3703bd2 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -0,0 +1,600 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The unified target's per-rank weight table against the FP8 checkpoint, on the CPU from its +index and safetensors headers: each rank's shard, each member's coverage and `load`'s operands.""" + +from __future__ import annotations + +import collections +import json +import re +import struct +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from utils.llm_data import llm_models_root + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as WT, +) +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig +from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight +from tensorrt_llm.mapping import Mapping + +CHECKPOINT = "Qwen3.8-Flash-Next-FP8" + +#: Each member's mapping as the LLM API's parallel arguments build it. +MEMBERS = { + "tp1": dict(world_size=1, tp_size=1), + "tp2": dict(world_size=2, tp_size=2), + "tep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2), + "dep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2, enable_attention_dp=True), + "tp4": dict(world_size=4, tp_size=4), + "tep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4), + "dep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4, enable_attention_dp=True), +} +NUM_EXPERTS = 512 +MTP_LAYER = 48 + +_STORED_DTYPES = {"BF16": torch.bfloat16, "F32": torch.float32, "F8_E4M3": torch.float8_e4m3fn, + "I64": torch.int64} # fmt: skip + +#: A routed (trunk) or draft (MTP) expert's tensors. +_EXPERT_TENSOR = re.compile(r"\.mlp\.experts\.\d+\.(gate|up|down)_proj\.(weight|weight_scale_inv)$") + + +@pytest.fixture(scope="module") +def checkpoint() -> Path: + root = llm_models_root() + path = None if root is None else root / CHECKPOINT + if path is None or not (path / "model.safetensors.index.json").is_file(): + pytest.skip(f"{CHECKPOINT} is not under LLM_MODELS_ROOT") + return path + + +@pytest.fixture(scope="module") +def index(checkpoint) -> dict[str, str]: + return json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + + +@pytest.fixture(scope="module") +def stored(checkpoint, index) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: + """Every checkpoint tensor's stored shape and dtype, from the index and the headers.""" + specs = {} + for file in sorted(set(index.values())): + with open(checkpoint / file, "rb") as fh: + (length,) = struct.unpack(" Mapping: + return Mapping(rank=rank, gpus_per_node=4, pp_size=1, **MEMBERS[member]) + + +def _dims(configs, member: str, rank: int, mtp: bool = True): + text, vision = configs + return WT.rank_dims(text, vision, _mapping(member, rank), mtp_enabled=mtp, dtype=torch.bfloat16) + + +def _world(member: str) -> int: + return MEMBERS[member]["world_size"] + + +def _attention_tp(member: str) -> int: + layout = MEMBERS[member] + return 1 if layout.get("enable_attention_dp", False) else layout["tp_size"] + + +def _lm_head_rows(member: str, vocab: int) -> int: + """The shell's lm_head rows: vocab-parallel under attention TP, whole otherwise.""" + return vocab // _attention_tp(member) + + +def _rows_per_shard(stored, d) -> int: + shapes = {stored[key][0] for key in WT.ngram_shard_keys(d)} + assert len(shapes) == 1, shapes + return next(iter(shapes))[0] + + +def _rank_manifest(stored, d, member: str) -> dict[str, list[tuple]]: + """The rank's manifest as `load` builds it, with the n-gram table and the shell's lm_head.""" + manifest = WT.MODEL_WEIGHTS.manifest(d) + manifest["ple_table"] = WT.ngram_table_sources(d, _rows_per_shard(stored, d)) + cut, pad = WT.lm_head_share(_lm_head_rows(member, d.vocab), d.vocab, d.tp_rank) + manifest["lm_head.weight"] = [("lm_head.weight", None, cut, pad)] + return manifest + + +def _keys_and_cuts(ckpt_key, cut): + if isinstance(ckpt_key, tuple): + return list(zip(ckpt_key, cut if cut is not None else (None,) * len(ckpt_key))) + return [(ckpt_key, cut)] + + +def _cut_numel(shape: tuple[int, ...], cut) -> int: + """Elements of a `shape` tensor `cut` reads, after checking each slice lies inside it.""" + if cut is None: + return int(torch.Size(shape).numel()) + extents = list(shape) + for axis, s in enumerate(cut): + assert isinstance(s, slice) and s.step in (None, 1), (shape, cut) + start = 0 if s.start is None else s.start + stop = shape[axis] if s.stop is None else s.stop + assert 0 <= start < stop <= shape[axis], f"cut {cut} leaves the stored {shape}" + extents[axis] = stop - start + return int(torch.Size(extents).numel()) + + +class _HeaderEntry: + """A checkpoint tensor known by its header: indexing returns a meta tensor, nothing is read.""" + + def __init__(self, shape, dtype): + self.shape, self.dtype = torch.Size(shape), dtype + self.ndim = len(shape) + + def __getitem__(self, index): + return torch.empty(self.shape, dtype=self.dtype, device="meta")[index] + + +def _share(member: str, key: str) -> float: + """The fraction of `key` a rank of `member` reads: attention and Gated-DeltaNet tensors by + head over the attention TP ranks (two KV heads), the lm_head by vocabulary, the rest whole.""" + tp = _attention_tp(member) + if re.search(r"\.self_attn\.(q_proj|o_proj)\.weight$", key) or key == "lm_head.weight": + return 1 / tp + if re.search(r"\.self_attn\.(k_proj|v_proj)\.weight$", key): + return 1 / min(tp, 2) + if re.search( + r"\.linear_attn\.((in_proj_qkv|in_proj_z|in_proj_a|in_proj_b|conv1d|out_proj)\.weight" + r"|A_log|dt_bias)$", + key, + ): + return 1 / tp + return 1.0 + + +def _window(member: str, rank: int) -> range: + """The `512 / world` consecutive whole experts a rank holds, whatever the MoE split.""" + count = NUM_EXPERTS // _world(member) + return range(count * rank, count * (rank + 1)) + + +def _check_operand_shapes(stored, d, member: str) -> dict: + """Each source fills its parameter exactly; each tensor is read at the rank's share, once.""" + with torch.device("meta"): + params = dict(WT.MODEL_WEIGHTS.declare(d)) + # What `load` adds beside the table: the n-gram table and the shell's lm_head. + params["ple_table"] = torch.empty( + d.ple_shards * _rows_per_shard(stored, d), d.ple_head_dim, dtype=d.ple_table_dtype + ) + params["lm_head.weight"] = torch.empty( + _lm_head_rows(member, d.vocab), d.hidden, dtype=torch.bfloat16 + ) + entries = {key: _HeaderEntry(*spec) for key, spec in stored.items()} + manifest = _rank_manifest(stored, d, member) + assert manifest.keys() == params.keys() + read = collections.Counter() + results: dict = {} + for param_key, sources in manifest.items(): + param = params[param_key] + for ckpt_key, index, cut, transform in sources: + key_cuts = _keys_and_cuts(ckpt_key, cut) + for key, key_cut in key_cuts: + read[key] += _cut_numel(stored[key][0], key_cut) + dst = param if index is None else param[index] + # Equal stored tensors, cuts and transform give equal results: read each once. + signature = (transform, tuple((stored[k], c) for k, c in key_cuts)) + if signature not in results: + src = WT.MODEL_WEIGHTS.read(entries, ckpt_key, cut, transform, d) + results[signature] = (tuple(src.shape), src.dtype) + got = (tuple(dst.shape), dst.dtype) + assert results[signature] == got, (param_key, ckpt_key, results[signature], got) + wrong = { + key: (n, _share(member, key)) + for key, n in read.items() + if n != round(torch.Size(stored[key][0]).numel() * _share(member, key)) + } + assert not wrong, f"tensors not read at the rank's share, once: {sorted(wrong.items())[:8]}" + return params + + +_HIDDEN = 2 # the projections' input axis, shrunk: no cut touches it +_FAKE_OFFSETS = dict(q=1, k=2, v=3, idx=4, o=5, qkv=10, z=11, a=12, b=13, conv=14, a_log=15, + dtb=16, out=17) # fmt: skip + + +def _ids(tag: str, rows: int, *trailing: int) -> torch.Tensor: + """`rows` rows whose every value is `tag`'s offset plus the row's index.""" + ids = torch.arange(rows, dtype=torch.int64) + _FAKE_OFFSETS[tag] * 10**7 + return ids.view(rows, *([1] * len(trailing))).expand(rows, *trailing).contiguous() + + +def _col_ids(tag: str, cols: int) -> torch.Tensor: + return _ids(tag, cols, _HIDDEN).t().contiguous() + + +def _fake_checkpoint(stored) -> dict[str, torch.Tensor]: + """Attention layer 3, Gated-DeltaNet layer 0 and MTP attention at stored rows, 2-wide input.""" + fake = {} + + def rows_of(key): + return stored[key][0][0] + + for p in ("model.language_model.layers.3.self_attn", "mtp.layers.0.self_attn"): + for tag, name in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj")): + fake[f"{p}.{name}.weight"] = _ids(tag, rows_of(f"{p}.{name}.weight"), _HIDDEN) + key = f"{p}.indexer.index_qk_proj.weight" + fake[key] = _ids("idx", rows_of(key), _HIDDEN) + fake[f"{p}.o_proj.weight"] = _col_ids("o", stored[f"{p}.o_proj.weight"][0][1]) + p = "model.language_model.layers.0.linear_attn" + for tag, name in (("qkv", "in_proj_qkv"), ("z", "in_proj_z"), ("a", "in_proj_a"), + ("b", "in_proj_b")): # fmt: skip + fake[f"{p}.{name}.weight"] = _ids(tag, rows_of(f"{p}.{name}.weight"), _HIDDEN) + fake[f"{p}.conv1d.weight"] = _ids("conv", rows_of(f"{p}.conv1d.weight"), 1, 4) + fake[f"{p}.A_log"] = _ids("a_log", rows_of(f"{p}.A_log")) + fake[f"{p}.dt_bias"] = _ids("dtb", rows_of(f"{p}.dt_bias")) + fake[f"{p}.out_proj.weight"] = _col_ids("out", stored[f"{p}.out_proj.weight"][0][1]) + return fake + + +def _assemble(d, fake, sources) -> torch.Tensor: + """A parameter filled from its sources the way `load` fills it.""" + if len(sources) == 1 and sources[0][1] is None: + ckpt_key, _, cut, transform = sources[0] + return WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d) + parts = [ + (index, WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d)) + for ckpt_key, index, cut, transform in sources + ] + rows = max(index[0].stop for index, _ in parts) + out = torch.full((rows, *parts[0][1].shape[1:]), -1, dtype=parts[0][1].dtype) + for index, part in parts: + out[index] = part + return out + + +def _expected_rows(tag: str, starts) -> torch.Tensor: + return torch.tensor([_FAKE_OFFSETS[tag] * 10**7 + r for r in starts], dtype=torch.int64) + + +def _check_heads(stored, d, member: str, rank: int, params: dict) -> None: + """The cuts select the rank's heads in fused-operand order, the heads the stock pools hold.""" + fake = _fake_checkpoint(stored) + manifest = WT.MODEL_WEIGHTS.manifest(d) + tp = _attention_tp(member) + # Under attention DP and at tp1 each rank holds every head: the split is over one rank. + r = rank if tp > 1 else 0 + assert (d.heads_q, d.q_head_start) == (24 // tp, r * 24 // tp) + assert (d.heads_kv, d.kv_head_start) == (max(2 // tp, 1), r * 2 // tp) + assert (d.num_k_heads, d.k_head_start) == (16 // tp, r * 16 // tp) + assert (d.num_v_heads, d.v_head_start) == (48 // tp, r * 48 // tp) + assert params["l0_conv_w"].shape[0] == 2 * d.key_dim + d.value_dim + q_heads, k_heads, v_heads = (range(r * n // tp, (r + 1) * n // tp) for n in (24, 16, 48)) + kv_heads = [r * 2 // tp] if tp > 1 else [0, 1] + + def rows(heads, width, base=0): + return [base + h * width + j for h in heads for j in range(width)] + + q_rows = [h * 512 + j for h in q_heads for j in range(256)] + gate_rows = [h * 512 + 256 + j for h in q_heads for j in range(256)] + # fmt: off + expected = torch.cat(( + _expected_rows("q", q_rows), _expected_rows("k", rows(kv_heads, 256)), + _expected_rows("v", rows(kv_heads, 256)), _expected_rows("q", gate_rows), + _expected_rows("idx", range(640)), torch.zeros(128, dtype=torch.int64), + )) + # fmt: on + attn_in = { + key: _assemble(d, fake, manifest[key]) for key in ("l3_attn_in_w", "l48_mtp_attn_in_w") + } + for key, got in attn_in.items(): + assert torch.equal(got[:, 0], expected) and torch.equal(got[:, 1], expected), key + # The KV head a rank holds is the one the built-in path's `duplicate_kv_weight` gives it. + k_key = "mtp.layers.0.self_attn.k_proj.weight" + builtin_k = duplicate_kv_weight(fake[k_key], 2, tp).chunk(tp)[r] if tp > 1 else fake[k_key] + q_width = len(q_rows) + mtp_k = attn_in["l48_mtp_attn_in_w"][q_width : q_width + builtin_k.shape[0]] + assert torch.equal(mtp_k, builtin_k) + o_cols = _expected_rows("o", rows(q_heads, 256)) + for key in ("l3_o_w", "l48_mtp_o_w"): + assert torch.equal(_assemble(d, fake, manifest[key])[0], o_cols), key + + def qkv(tag): + k, v = rows(k_heads, 128), rows(v_heads, 128, 4096) + return torch.cat([_expected_rows(tag, r) for r in (k, rows(k_heads, 128, 2048), v)]) + + z, a, b = rows(v_heads, 128), v_heads, v_heads + in_proj = torch.cat([qkv("qkv")] + [_expected_rows(t, r) for t, r in zip("zab", (z, a, b))]) + assert torch.equal(_assemble(d, fake, manifest["l0_in_proj_w"])[:, 0], in_proj) + assert torch.equal(_assemble(d, fake, manifest["l0_conv_w"])[:, 0, 0], qkv("conv")) + assert torch.equal(_assemble(d, fake, manifest["l0_A_log"]), _expected_rows("a_log", v_heads)) + assert torch.equal(_assemble(d, fake, manifest["l0_dt_bias"]), _expected_rows("dtb", v_heads)) + assert torch.equal(_assemble(d, fake, manifest["l0_out_w"])[0], + _expected_rows("out", rows(v_heads, 128))) # fmt: skip + + +#: Per-expert operand (shape, dtype); block scales hold one fp32 value per 128x128 block. +_EXPERT_SHAPES = { + "fc1_w": ((1280, 2560), torch.float8_e4m3fn), + "fc1_s": ((10, 20), torch.float32), + "fc2_w": ((2560, 640), torch.float8_e4m3fn), + "fc2_s": ((20, 5), torch.float32), +} + + +def _check_expert_window(d, member: str, rank: int, params: dict) -> None: + """The rank holds `512 / world` uncut experts, trunk and draft layer alike (tp2 and tp4 too), + and `expert_window` hands the runner exactly those parameters.""" + window = _window(member, rank) + assert (d.expert_offset, d.local_experts, d.moe_inter) == (window.start, len(window), 640) + manifest = WT.MODEL_WEIGHTS.manifest(d) + for prefix, module in (("l0_", "model.language_model.layers.0.mlp"), + ("l48_mtp_", "mtp.layers.0.mlp")): # fmt: skip + for name, (shape, dtype) in _EXPERT_SHAPES.items(): + param, rows = params[f"{prefix}{name}"], manifest[f"{prefix}{name}"] + assert (tuple(param.shape), param.dtype) == ((len(window), *shape), dtype), name + assert [index for _, index, _, _ in rows] == [(i,) for i in range(len(rows))] + assert all(cut is None for _, _, cut, _ in rows), (prefix, name) + experts = [s[0] for s in manifest[f"{prefix}fc2_w"]] + assert experts == [f"{module}.experts.{e}.down_proj.weight" for e in window] + win = WT.expert_window(params, d, MTP_LAYER if prefix == "l48_mtp_" else 0) + assert (win.offset, win.count) == (window.start, len(window)) + assert win.fc1_w is params[f"{prefix}fc1_w"] and win.fc2_s is params[f"{prefix}fc2_s"] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("member, rank", [("tp1", 0), ("tp2", 1), ("tp4", 2), ("dep4", 3)]) +def test_rank_shard(stored, configs, member, rank): + """The heads each cut selects, every operand's shape and the expert window. tep is left out: + it differs from tp only in the `Mapping`'s MoE split, which no cut reads.""" + d = _dims(configs, member, rank) + params = _check_operand_shapes(stored, d, member) + _check_heads(stored, d, member, rank, params) + _check_expert_window(d, member, rank, params) + + +def _stub_model(configs, member: str, rank: int, mtp: bool): + """The shell and core that `load` reads, on meta: configs, mapping, parameters, lm_head.""" + text, vision = configs + core = SimpleNamespace( + model_config=SimpleNamespace( + pretrained_config=text, + mapping=_mapping(member, rank), + spec_config=object() if mtp else None, + torch_dtype=torch.bfloat16, + ), + vision_config=vision, + ) + d = WT.MODEL_WEIGHTS.dims(core) + with torch.device("meta"): + core.w = WT.MODEL_WEIGHTS.declare(d) + lm_head = torch.nn.Parameter( + torch.empty(_lm_head_rows(member, d.vocab), d.hidden, dtype=torch.bfloat16), + requires_grad=False, + ) + + def named_parameters(): + yield from ((f"model.w.{k}", v) for k, v in core.w.items()) + yield "lm_head.weight", lm_head + + return SimpleNamespace( + model=core, lm_head=SimpleNamespace(weight=lm_head), named_parameters=named_parameters + ) + + +#: The rank `load` runs at, end to end on meta tensors, for these members. +_LOAD_RANK = {("tp1", True): 0, ("tp2", True): 1, ("tep4", False): 3, ("dep4", True): 0} + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, mtp", + [(member, True) for member in MEMBERS] + [("tep4", False)], + ids=[f"{member}-mtp" for member in MEMBERS] + ["tep4-no-mtp"], +) +def test_member_covers_the_checkpoint(stored, configs, member, mtp, monkeypatch): + """Each tensor is read or predicted unread, never both; each expert's tensors by one rank and + the rest by every rank; the windows tile the 512 experts; and `load` fills every parameter.""" + everything = set(stored) + read_by = collections.Counter() + seen = [] + for rank in range(_world(member)): + d = _dims(configs, member, rank, mtp) + manifest = _rank_manifest(stored, d, member) + claimed = { + key + for sources in manifest.values() + for ckpt_key, _, cut, _ in sources + for key, _ in _keys_and_cuts(ckpt_key, cut) + } + unread = WT.MODEL_WEIGHTS.expected_unconsumed(d, stored.keys()) + assert claimed <= everything, ( + f"rank {rank} reads unknown {sorted(claimed - everything)[:8]}" + ) + assert not claimed & unread, f"rank {rank}: read and unread {sorted(claimed & unread)[:8]}" + assert claimed | unread == everything, ( + f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" + ) + read_by.update(claimed) + seen.extend(range(d.expert_offset, d.expert_offset + d.local_experts)) + mtp_keys = set() if mtp else {k for k in everything if k.startswith("mtp.")} + assert set(read_by) == everything - mtp_keys, "tensors no rank reads: " + str( + sorted(everything - mtp_keys - set(read_by))[:8] + ) + wrong = sorted( + k for k, n in read_by.items() if n != (1 if _EXPERT_TENSOR.search(k) else _world(member)) + ) + assert not wrong, f"tensors not read by their expected rank count: {wrong[:8]}" + assert seen == list(range(NUM_EXPERTS)) + if member in ("tp2", "tp4"): + mapping = _mapping(member, 0) + assert (mapping.moe_tp_size, mapping.moe_ep_size) == (mapping.tp_size, 1) + + if (member, mtp) not in _LOAD_RANK: + return + monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None) + monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None) + model = _stub_model(configs, member, _LOAD_RANK[member, mtp], mtp) + WT.MODEL_WEIGHTS.load(model, {key: _HeaderEntry(*spec) for key, spec in stored.items()}) + d = WT.MODEL_WEIGHTS.dims(model.model) + assert tuple(model.model.w["ple_table"].shape) == (128 * _rows_per_shard(stored, d), 160) + assert model.lm_head.weight.shape[0] == _lm_head_rows(member, 248320) + + +def _random_bytes(gen, *shape) -> torch.Tensor: + return torch.randint(0, 256, shape, generator=gen, dtype=torch.uint8) + + +def _random_scales(gen, *shape) -> torch.Tensor: + return (0.001 + torch.rand(shape, generator=gen)).to(torch.bfloat16) + + +def _expert_operands(configs, which: str) -> None: + """An expert's operands are the checkpoint's bytes in the runner's layout: FC1 `[up ; gate]` + with its block scales in that order widened exactly to fp32, FC2 as stored, trunk and MTP.""" + d = _dims(configs, "tp4", 1) + window = _window("tp4", 1) + expert = window[0] if which == "first" else window[-1] + local = expert - window.start + gen = torch.Generator().manual_seed(expert * 7 + 1) + manifest = WT.MODEL_WEIGHTS.manifest(d) + for prefix, module in (("l5_", "model.language_model.layers.5.mlp"), + ("l48_mtp_", "mtp.layers.0.mlp")): # fmt: skip + p = f"{module}.experts.{expert}" + fake = { + f"{p}.up_proj.weight": _random_bytes(gen, 640, 2560).view(torch.float8_e4m3fn), + f"{p}.gate_proj.weight": _random_bytes(gen, 640, 2560).view(torch.float8_e4m3fn), + f"{p}.down_proj.weight": _random_bytes(gen, 2560, 640).view(torch.float8_e4m3fn), + f"{p}.up_proj.weight_scale_inv": _random_scales(gen, 5, 20), + f"{p}.gate_proj.weight_scale_inv": _random_scales(gen, 5, 20), + f"{p}.down_proj.weight_scale_inv": _random_scales(gen, 20, 5), + } + + def operand(name): + (row,) = [s for s in manifest[f"{prefix}{name}"] if s[1] == (local,)] + ckpt_key, _, cut, transform = row + return WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d) + + def u8(name): + return fake[f"{p}.{name}"].view(torch.uint8) + + fc1_w, fc1_s, fc2_w, fc2_s = (operand(n) for n in ("fc1_w", "fc1_s", "fc2_w", "fc2_s")) + assert fc1_w.dtype == fc2_w.dtype == torch.float8_e4m3fn + assert fc1_s.dtype == fc2_s.dtype == torch.float32 + assert torch.equal(fc1_w.view(torch.uint8), + torch.cat((u8("up_proj.weight"), u8("gate_proj.weight")))) # fmt: skip + assert torch.equal(fc2_w.view(torch.uint8), u8("down_proj.weight")) + up_s, gate_s = (fake[f"{p}.{n}_proj.weight_scale_inv"] for n in ("up", "gate")) + assert torch.equal(fc1_s, torch.cat((up_s, gate_s)).float()) + assert torch.equal(fc2_s, fake[f"{p}.down_proj.weight_scale_inv"].float()) + assert torch.equal(fc1_s.to(torch.bfloat16), torch.cat((up_s, gate_s))) + + +def _shared_segment_hands_over(tmp_path) -> None: + """`SharedHostSegment` over a memfd: the first handle owns, a concurrent one attaches on + publish, a late one at once, and an owner gone unpublished hands the segment to the waiter.""" + import ctypes + + HS = WT._host_share + nbytes = 1 << 20 + + def segment(label="numa0"): + backing = HS.MemfdBacking(lambda pointer, nbytes, read_only: lambda: None) + return HS.SharedHostSegment("a" * 64, nbytes, label, backing, directory=str(tmp_path)) + + owner = segment() + assert owner.role == HS.OWNER + waiter = segment() + assert waiter.role == HS.PENDING and waiter.pointer == 0 + ctypes.memset(owner.pointer, 0x5A, nbytes) + owner.publish() + waiter.wait() + late = segment() + for handle in (waiter, late): + assert handle.role == HS.ATTACHED and not handle.private + assert ctypes.string_at(handle.pointer + nbytes - 8, 8) == b"Z" * 8 + other = segment("numa1") # another scope never meets this one + assert other.role == HS.OWNER + for handle in (waiter, late, other, owner): + handle.close() + # Closing the owner withdrew its descriptor: the next handle owns a fresh segment. + fresh = segment() + assert fresh.role == HS.OWNER + waiter = segment() + assert waiter.role == HS.PENDING + fresh.close() + waiter.wait(poll_seconds=0.01) + assert waiter.role == HS.OWNER + waiter.close() + + +def _shared_ngram_table_is_the_private_one(configs, tmp_path, monkeypatch) -> None: + """With `ple_shared_host`, the first core fills the segment and the second attaches without + reading the shards; both tables are host memory holding exactly the shards' bytes.""" + monkeypatch.setenv(WT._host_share.PLE_SHARED_HOST_DIR_ENV, str(tmp_path)) + HS = WT._host_share + d = _dims(configs, "tep2", 0, mtp=False) + keys = WT.ngram_shard_keys(d) + gen = torch.Generator().manual_seed(12) + shards = {key: _random_bytes(gen, 5, d.ple_head_dim).view(WT.PLE_TABLE_DTYPE) for key in keys} + device = torch.device("cuda") + cores = [] + for _ in range(2): + core = SimpleNamespace(w={"embed": torch.empty(1, device=device)}, ple_shared_host="node") + rows_per_shard = WT._declare_ngram_table(core, d, shards) + cores.append(core) + assert [c._ple_table_host.role for c in cores] == [HS.OWNER, HS.PENDING] + sources = WT.ngram_table_sources(d, rows_per_shard) + assert WT.MODEL_WEIGHTS._fill_shared_table(cores[0], sources, shards, d) == set(keys) + + class _Untouchable(dict): + def __getitem__(self, key): + raise AssertionError(f"an attached rank read {key}") + + consumed = WT.MODEL_WEIGHTS._fill_shared_table(cores[1], sources, _Untouchable(), d) + assert consumed == set(keys) and cores[1]._ple_table_host.role == HS.ATTACHED + expected = torch.cat([shards[k].view(torch.uint8) for k in keys]).to(device) + for core in cores: + segment = core._ple_table_host + assert segment.backing.resident_on_host(segment.pointer), segment.describe() + assert torch.equal(core.w["ple_table"].view(torch.uint8), expected) + for core in cores: + core._ple_table_host.close() + + +_NEEDS_GPU = pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("expert-first", marks=pytest.mark.cpu_only), + pytest.param("expert-last", marks=pytest.mark.cpu_only), + pytest.param("shared-segment", marks=pytest.mark.cpu_only), + pytest.param("shared-table", marks=_NEEDS_GPU), + ], +) +def test_load_operands(case, request, tmp_path, monkeypatch): + """What `load` fills operands from: expert codes and scales, and the shared n-gram table.""" + if case == "shared-segment": + _shared_segment_hands_over(tmp_path) + elif case == "shared-table": + _shared_ngram_table_is_the_private_one( + request.getfixturevalue("configs"), tmp_path, monkeypatch + ) + else: + _expert_operands(request.getfixturevalue("configs"), case.split("-")[1]) From 597a86264132ffbad6e53757edfea4765994ceb7 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:58:42 -0800 Subject: [PATCH 03/51] [None][chore] modeling_v2 qwen4_exp: apply clang-format and cmake-format to the FP8 target's native code Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt | 6 +++--- .../csrc/fp8BlockMoeTpb.cu | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt index 975d0627c590..a9f9a0c1eb8f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt @@ -19,9 +19,9 @@ # modeling_v2 README. add_modeling_v2_family(qwen4_exp) -# The FP8 target routes on SM103 only (its kernels need only SM90+ features). The -# hyper-connection all-reduce uses the stock all-reduce's IPC workspace, which -# exists only in multi-device builds. +# The FP8 target routes on SM103 only (its kernels need only SM90+ features). +# The hyper-connection all-reduce uses the stock all-reduce's IPC workspace, +# which exists only in multi-device builds. add_modeling_v2_target( qwen4_exp qwen3_8_flash_next_fp8__sm_103__unified ARCHS 103a diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu index 477b75b44666..d2b7bc988601 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu @@ -52,11 +52,11 @@ constexpr int kEpiWarp = 5; constexpr int kProducerWarp = 6; constexpr int kThreads = 7 * 32; constexpr int kConsumerThreads = kConsumerWarps * 32; -constexpr int kMtPerWarp = kMt / kConsumerWarps; // 4 -constexpr int kItemRowBytes = 16 * kH; // 40960: 16 FC1 rows, or 320 FC2 rows x 128 -constexpr int kStageBytes = kItemRowBytes + 128; // + the item's 20 fp32 scales +constexpr int kMtPerWarp = kMt / kConsumerWarps; // 4 +constexpr int kItemRowBytes = 16 * kH; // 40960: 16 FC1 rows, or 320 FC2 rows x 128 +constexpr int kStageBytes = kItemRowBytes + 128; // + the item's 20 fp32 scales constexpr int kMaxStages = 8; -constexpr int kMaxSlots = kRows * kK; // 80 +constexpr int kMaxSlots = kRows * kK; // 80 constexpr int kMaxBlk = 2 * kMaxSlots + kShItems; // up to two blocks a routed expert, plus the shared items constexpr int kActRow = 128 + 16; constexpr int kMaxClusters = 40; From 4de86ae8485eb3778ccc2075039d8cb30eee6271 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:17:46 -0800 Subject: [PATCH 04/51] [None][fix] modeling_v2 qwen4_exp: open the shared PLE table's rendezvous files without following symlinks The rendezvous directory (default /dev/shm) is writable by every user of the host. The lock, descriptor and temporary descriptor files are now opened with O_NOFOLLOW, created 0600 (the temporary one with O_EXCL), and read only when the calling uid owns them. Resolves bandit B108 on the default directory. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../host_share.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py index 2b56f59da34e..4242ad263c55 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py @@ -24,6 +24,7 @@ from __future__ import annotations +import contextlib import ctypes import errno import fcntl @@ -38,7 +39,9 @@ PLE_SHARED_HOST_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST" #: Where the rendezvous files live; must be a directory every sharing process sees (tmpfs). PLE_SHARED_HOST_DIR_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST_DIR" -_DEFAULT_DIR = "/dev/shm" +# Shared by every user of the host: the files are named per uid, opened with O_NOFOLLOW, created +# 0600 and read only when this uid owns them (`_open_own`). +_DEFAULT_DIR = "/dev/shm" # nosec B108 _OFF = frozenset({"", "0", "off", "false", "no", "none"}) _SCOPE = "node" @@ -244,6 +247,15 @@ def describe(self) -> str: return "memfd" +def _open_own(path: str, flags: int, mode: int = 0o600) -> int: + """`os.open` that refuses a symlink and a file another uid owns (the directory is shared).""" + fd = os.open(path, flags | os.O_NOFOLLOW | os.O_CLOEXEC, mode) + if os.fstat(fd).st_uid != os.getuid(): + os.close(fd) + raise PermissionError(f"{path} is owned by another user") + return fd + + def segment_name(key: str, scope_label: str) -> str: """The rendezvous file stem: per user, per table content, per scope.""" return f"trtllm-ple-{os.getuid()}-{scope_label}-{key[:24]}" @@ -292,7 +304,7 @@ def __init__( self._fd = -1 # Bound now: at interpreter shutdown the modules may already be torn down. self._close = os.close - self._lock_fd = os.open(self.lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600) + self._lock_fd = _open_own(self.lock_path, os.O_RDWR | os.O_CREAT) try: fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as error: @@ -316,7 +328,7 @@ def _unlock(self) -> None: def _read_descriptor(self) -> Optional[dict]: try: - with open(self.descriptor_path) as fh: + with os.fdopen(_open_own(self.descriptor_path, os.O_RDONLY)) as fh: desc = json.load(fh) except (OSError, ValueError): return None @@ -398,7 +410,9 @@ def publish(self) -> None: backing=self.backing.describe(), ) tmp = f"{self.descriptor_path}.{os.getpid()}.tmp" - with open(tmp, "w") as fh: + with contextlib.suppress(FileNotFoundError): + os.unlink(tmp) + with os.fdopen(_open_own(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "w") as fh: json.dump(desc, fh) os.replace(tmp, self.descriptor_path) self._unlock() @@ -416,7 +430,7 @@ def describe(self) -> str: def _withdraw(self) -> None: """Owner: unlink the descriptor if it still names this pid and fd, before the fd closes.""" try: - with open(self.descriptor_path) as fh: + with os.fdopen(_open_own(self.descriptor_path, os.O_RDONLY)) as fh: desc = json.load(fh) if desc.get("pid") == os.getpid() and desc.get("fd") == self._fd: os.unlink(self.descriptor_path) From 6d3973bb8f4d98bf6682ac3493d783c52963e202 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:30:30 -0800 Subject: [PATCH 05/51] [None][fix] modeling_v2 qwen4_exp: mirror wide in hc_up_gate_mean's reference, check gdn_in_proj_conv's bf16 conv operands Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py | 9 +++++++-- .../ops/ssm/gdn_in_proj_conv__verify.py | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py index bb3a664b6ee9..445a7d9faf66 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py @@ -972,9 +972,14 @@ def __call__( return hc_up_gate_mean(low, up, normed, hc_count, _BLOCK_H, wide) def reference( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> torch.Tensor: - """float64, one rounding.""" + """float64, one rounding. `wide` does not change the result.""" hidden = normed.shape[1] // hc_count gate = torch.sigmoid(F.linear(low.double(), up.double().t())).unflatten( -1, (hc_count, hidden) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py index b0486477c914..8051c6608283 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py @@ -731,6 +731,9 @@ def is_valid( assert tuple(x.shape) == (rows, K_IN) and tuple(in_proj.shape) == (K_IN, N_OUT) assert in_proj.stride() == (1, K_IN), "in_proj is the column-major [2560, 4128] operand" assert rows % rows_per_request == 0 and 1 <= rows <= tactic.token_tile, (rows, tactic) + assert conv_weight.dtype == conv_state.dtype == window.dtype == torch.bfloat16, ( + "the epilogue reads and writes the conv taps, pool and window as bf16" + ) assert tuple(conv_weight.shape) == (CONV_DIM, CONV_WIDTH) and conv_weight.is_contiguous() assert conv_state.dim() == 3 and tuple(conv_state.shape[1:]) == (CONV_DIM, CONV_WIDTH - 1) assert window.dim() == 4 and tuple(window.shape[2:]) == (CONV_DIM, CONV_WIDTH - 1) From 8b84564fa74a5efba13ba7189d6247f923d1d952 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:22:23 -0800 Subject: [PATCH 06/51] [None][fix] modeling_v2 qwen4_exp: allow one bf16 rounding in the HC inject-norm warps test Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py index 71baf8bba059..17ac60159043 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py @@ -278,7 +278,9 @@ def inputs(seed): for warps in (M._HC_INJECT_WARPS, 2): hc_w, normed_w = M._hc_inject_norm_launch(*args, warps) assert torch.equal(hc_w, hc8) - assert_within_ulp(normed_w, normed8, element_ulp=1.0, rms_ulp=0.25) + # One bf16 rounding of an element is at most 2^-7 of its magnitude, so up to 2 ulp of the + # row's largest one (B300: one element of 288 x 10240 one rounding apart, 1.14 ulp). + assert_within_ulp(normed_w, normed8, element_ulp=2.0, rms_ulp=0.25) o, args = inputs(700 + rows) hc8, normed8 = qwen4_exp_hc_inject_norm(*args) hc_l, normed_l = M._hc_inject_norm_launch(*args, 8) From 5837c73206ce2a4bede26c5ec335adb30d34dfdb Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:44:38 -0800 Subject: [PATCH 07/51] [None][fix] modeling_v2 qwen4_exp: FP8 target kernel barriers, page-size support and precompiles Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- tensorrt_llm/_torch/autotuner.py | 5 + .../qsa_index_scores_cute__prefill.py | 6 +- .../qsa_sparse_gqa_splitk_cute__decode.py | 24 ++- .../attention/qsa_topk_rows_cute__prefill.py | 3 + .../ops/attention/vision_rope_2d_packed_qk.py | 2 +- .../qwen4_exp_hc_decode_tc_first__decode.py | 21 +- .../ops/moe/fp8_block_scale_moe_runner.py | 2 +- .../ssm/flashinfer_chunk_gated_delta_rule.py | 5 +- .../ops/ssm/gdn_cached_replay_cute__verify.py | 133 +++++++++--- .../vision/triton_pos_embed_interpolate.py | 203 +++++++++++++++--- ..._moe_a2a__dp_dispatch_combine_op_matrix.py | 6 +- .../comm/test_fp8_unified_op_matrices.py | 5 + .../test_attention_ops.py | 4 +- .../test_hc_ops.py | 2 +- 14 files changed, 334 insertions(+), 87 deletions(-) diff --git a/tensorrt_llm/_torch/autotuner.py b/tensorrt_llm/_torch/autotuner.py index 794a8ebede80..7ffcaec389f0 100644 --- a/tensorrt_llm/_torch/autotuner.py +++ b/tensorrt_llm/_torch/autotuner.py @@ -1150,6 +1150,11 @@ def get(cls): cls._instance = AutoTuner() return cls._instance + @property + def active_capture(self) -> Optional['AutoTuner.TacticsCapture']: + """The `capture()` / `replay()` context currently open, or None.""" + return self._active_capture + class TacticsCapture: """Object returned by capture() that can be iterated to get all tactic combinations. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py index e01fd7b76506..537c08d55af5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py @@ -51,6 +51,8 @@ MAX_ITEMS = 64 THREADS = 32 * (EPI_WARPS + 1 + LOAD_WARPS) LOAD_THREADS = 32 * LOAD_WARPS +#: Keys a page holds: one load thread's two token rows (`__init__`'s assert). +KEYS_PER_PAGE = 2 * (LOAD_THREADS // 16) @dsl_user_op @@ -70,7 +72,7 @@ def __init__(self, *, tokens_per_block, compress_ratio): self.cr = compress_ratio self.kpp = tokens_per_block // compress_ratio # keys a page assert BM % self.kpp == 0 and BM // self.kpp <= 32 - assert self.kpp == 2 * (LOAD_THREADS // 16), "a thread's keys: two token rows of each page" + assert self.kpp == KEYS_PER_PAGE, "a thread's keys: two token rows of each page" self.pages_per_tile = BM // self.kpp @cute.jit @@ -890,7 +892,7 @@ def supports(q, index_cache, tokens_per_block, compress_ratio) -> bool: and index_cache.stride(0) % 8 == 0 and index_cache.data_ptr() % 16 == 0 and tokens_per_block % compress_ratio == 0 - and BM % (tokens_per_block // compress_ratio) == 0 + and tokens_per_block // compress_ratio == KEYS_PER_PAGE and torch.cuda.get_device_capability(q.device) == (10, 3) ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py index e85cf8190595..887f2bb0d45a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py @@ -1275,22 +1275,32 @@ def sparse_gqa_decode( ) if n_splits == 1: return out - mkey = ("merge", num_q_heads, n_splits, gated, gate_t.dim(), use_pdl) - mfn = _COMPILED.get(mkey) - if mfn is None: - mfn = cute.compile( - _MergeKernel(heads=num_q_heads, splits=n_splits, gated=gated, pdl=use_pdl), - from_dlpack(gate_t, assumed_align=16).mark_layout_dynamic(leading_dim=gate_t.dim() - 1), + + def _compile_merge(merge_gated, gate, stride): + return cute.compile( + _MergeKernel(heads=num_q_heads, splits=n_splits, gated=merge_gated, pdl=use_pdl), + from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=gate.dim() - 1), _ptr(part, cutlass.Float32), _ptr(lse, cutlass.Float32, 4), _ptr(out, cutlass.BFloat16), cutlass.Int32(rows), - cutlass.Int32(gate_stride), + cutlass.Int32(stride), cutlass.Int32(gate_hs), cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), options="--opt-level 2 --enable-tvm-ffi", ) + + mkey = ("merge", num_q_heads, n_splits, gated, gate_t.dim(), use_pdl) + mfn = _COMPILED.get(mkey) + if mfn is None: + mfn = _compile_merge(gated, gate_t, gate_stride) _COMPILED[mkey] = mfn + # A mixed step's generation rows (and a draft-extend step's) call the merge ungated at + # the geometry a decode step calls it gated: compile that flavour now as well, so it + # never compiles inside a served step. Ungated, the gate operand is `q` (3-D). + ukey = ("merge", num_q_heads, n_splits, False, q.dim(), use_pdl) + if gated and ukey not in _COMPILED: + _COMPILED[ukey] = _compile_merge(False, q, 0) mfn(gate_t, part.data_ptr(), lse.data_ptr(), out.data_ptr(), rows, gate_stride, gate_hs) return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py index 26d53f11ee6c..bdf8fb865d68 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py @@ -261,6 +261,9 @@ def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): state[4] = cutlass.Int32(1) state[5] = cutlass.Int32(0) # candidates compacted state[6] = cutlass.Int32(0) + # Thread 0's reset of the candidate counter must be visible to every warp's `_compact` + # atomic below: the `n > k` path has a barrier of its own, a row of <= k blocks does not. + cute.arch.sync_threads() n = mEnd[row] seqlen = mLen[row] visible = cutlass.Int32(mPos[row]) + 1 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py index 2b309b091c32..512ea7574cbd 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py @@ -53,7 +53,7 @@ _BLOCK_M = 8 -@triton.jit +@triton.jit(do_not_specialize=["seqlen", "seqlen_ro"]) def _rotary_neox_inplace_kernel( X, COS, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py index 6bb9b7f5755a..6560f8f6706b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py @@ -100,20 +100,6 @@ def _bulk_to_peer(dst, src, nbytes, mbar, peer, *, loc=None, ip=None): ) -@dsl_user_op -def _bulk_commit_wait_read(*, loc=None, ip=None): - """Commit this thread's bulk copies and wait until they have read their sources.""" - _asm( - T.i32(), - [], - "{ cp.async.bulk.commit_group; cp.async.bulk.wait_group.read 0; mov.u32 $0, 0; }", - "=r", - loc, - ip, - side_effects=True, - ) - - @dsl_user_op def _lds_v4_f32(addr, *, loc=None, ip=None): """16 bytes of this CTA's shared memory; side-effecting, so it stays behind the @@ -597,8 +583,11 @@ class SharedStorage: _bf16x2(injs[0], injs[1]), _bf16x2(injs[2], injs[3]), ) - if tidx == 0: - _bulk_commit_wait_read() + # No CTA may exit (releasing its shared memory) while a peer's push out of it is still + # reading it: every CTA arrives only after all pushes into it landed, so the cluster + # barrier completes once every push has read its source. + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() def _rows_of(self, owner, nreal): """Real rows of the tile that cluster CTA `owner` reduces (0 past the real rows).""" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py index 2bf7d75d6e1c..6f9915531d0a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py @@ -156,7 +156,7 @@ def _dispatch(geometry: tuple, inputs: list) -> Optional[tuple]: """`(runner, tactic)` as `torch.ops.trtllm.fp8_block_scale_moe_runner` would choose them, remembered; None while the AutoTuner tunes or captures (the custom op then runs as is).""" tuner = AutoTuner.get() - if tuner.is_tuning_mode or tuner._active_capture is not None: + if tuner.is_tuning_mode or tuner.active_capture is not None: return None shapes = tuple(None if t is None else tuple(t.shape) for t in inputs) key = (geometry, shapes, len(tuner.profiling_cache)) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py index 91dfe24f539a..f02d8d805245 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py @@ -533,7 +533,10 @@ class FlashinferChunkGatedDeltaRule(OpWrapper): 4/12 heads on each path they reach. Not certified: fp32 / fp16 pools, `scale` other than the default, other head layouts, K or V other than 128, 2 context sequences on CP at 4/12, a carried sequence on CP at 8/24, the measured slot counts of tp4, tp2 and tep2 (stand-ins - above), a call inside a CUDA graph (the target calls it eagerly). + above). The CP in-place path runs inside the breakable prefill graph + (`PrefillTarget._gdn_core_graph`), covered end to end by the `tep4_fp8_kv_breakable` CI + case; a captured call is not certified for 2-sequence CP at 4/12, a carried sequence at + 8/24, or padded rows at those head counts. """ def __call__( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py index fbd39a5303f7..e46225a50fc6 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -2166,6 +2166,96 @@ def _compile(key, kern): return fn +def _kernel_kw( + n: int, + t: int, + h: int, + hv: int, + *, + commit: bool, + pdl: bool, + l2norm: bool, + gating: bool, + g_f32: bool, + b_f32: bool, + work_items: bool, + config: Optional[dict] = None, + fold: bool = False, + norm_eps: Optional[float] = None, +) -> dict: + """The kernel's compile-time parameters for a call of `n` requests: its key in `_COMPILED`. + `fold` is a `conv_fold` call, `norm_eps` a `gated_norm` call's epsilon.""" + cfg = dict(default_config(n, t, commit)) + if config: + cfg.update(config) + if fold: + if cfg.get("tpc") not in (8, 24): + # small batches' mappings split a value head over CTAs: take the whole-head ones + whole = ( + dict(tpc=8, warps=4, pf=1, minb=5) + if n > 16 + else dict(tpc=24, warps=8, pf=1, minb=2) + ) + cfg.update(whole) + cfg.pop("early_acc", None) + cfg["fold"] = True + if norm_eps is not None: + if cfg.get("tpc") != 8 or cfg.get("warps") != 4: + cfg.update(tpc=8, warps=4) + cfg["pf"] = min(int(cfg.get("pf", 1)), 2) + cfg.setdefault("minb", 5) + cfg["norm"] = True + cfg["norm_eps"] = norm_eps + # PDL: state / history before the wait; without it, the step's own inputs first + cfg.setdefault("qk_first", not pdl) + cfg["use_wi"] = bool(cfg.pop("use_wi", True)) and work_items + return dict( + T=t, + h=h, + hv=hv, + commit=commit, + pdl=pdl, + l2norm=l2norm, + gating=gating, + g_f32=g_f32, + b_f32=b_f32, + **cfg, + ) + + +def precompile( + t: int, h: int, hv: int, max_requests: int, *, pdl: bool, partition_from: int +) -> int: + """Compile every kernel a verify step of 1..`max_requests` requests launches through the + default mapping, the way a target calls this entry: work items, and with them no in-kernel + commit, from `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 + `a` / `b`; no `config`, `conv_fold` or `gated_norm`. A compile costs 5-10 s, so a target + runs this at bind, before any step: the captured decode graphs do not reach every mapping + (padded graphs skip 9 to 15 requests, which an eager mixed step carries). Returns the + number of distinct kernels.""" + keys = set() + for n in range(1, max_requests + 1): + partitioned = n >= partition_from + kw = _kernel_kw( + n, + t, + h, + hv, + commit=not partitioned, + pdl=pdl, + l2norm=True, + gating=True, + g_f32=False, + b_f32=False, + work_items=partitioned, + ) + key = tuple(sorted(kw.items())) + if key not in keys: + keys.add(key) + _compile(key, _Replay(**kw)) + return len(keys) + + def conv_fold_supported(x, weight, pool, window) -> bool: """Whether `cached_replay_update(conv_fold=...)` takes these operands: bf16, the layouts it reads, and 16-byte alignment of every 8-channel group (vector loads / stores).""" @@ -2280,9 +2370,6 @@ def cached_replay_update( ap, dp = A_log.data_ptr(), dt_bias.data_ptr() else: ap = dp = 0 - cfg = dict(default_config(N, T, commit)) - if config: - cfg.update(config) fold_args = (0,) * 9 if conv_fold is not None: cx, cw, cs = conv_fold["x"], conv_fold["weight"], conv_fold["pool"] @@ -2297,21 +2384,12 @@ def cached_replay_update( assert conv_fold_supported(cx, cw, cs, win), ( "conv_fold operands: layout / 16-byte alignment" ) - if cfg.get("tpc") not in (8, 24): - # small batches' mappings split a value head over CTAs: take the whole-head ones - whole = ( - dict(tpc=8, warps=4, pf=1, minb=5) - if N > 16 - else dict(tpc=24, warps=8, pf=1, minb=2) - ) - cfg.update(whole) - cfg.pop("early_acc", None) - cfg["fold"] = True fold_args = ( cx.data_ptr(), cx.stride(0), cw.data_ptr(), cs.data_ptr(), cs.stride(0), win.data_ptr(), win.stride(0), win.stride(1), wrows.data_ptr(), ) # fmt: skip norm_args = (0,) * 4 + norm_eps = None if gated_norm is not None: zn, gw = gated_norm["z"], gated_norm["weight"] assert T <= 4, "the folded norm takes a warp a row" @@ -2320,30 +2398,25 @@ def cached_replay_update( ) assert gw.shape == (V,) and gw.is_contiguous() and gw.dtype == torch.bfloat16 assert zn.data_ptr() % 8 == 0 and zn.stride(0) % 4 == 0 and gw.data_ptr() % 8 == 0 - if cfg.get("tpc") != 8 or cfg.get("warps") != 4: - cfg.update(tpc=8, warps=4) - cfg["pf"] = min(int(cfg.get("pf", 1)), 2) - cfg.setdefault("minb", 5) - cfg["norm"] = True - cfg["norm_eps"] = float(gated_norm["eps"]) + norm_eps = float(gated_norm["eps"]) norm_args = (zn.data_ptr(), zn.stride(0), gw.data_ptr(), output.data_ptr()) - # PDL: state / history before the wait; without it, the step's own inputs first - cfg.setdefault("qk_first", not bool(launch_with_pdl)) - use_wi = bool(cfg.pop("use_wi", True)) and replay_work_items is not None - cfg["use_wi"] = use_wi - pdl = bool(launch_with_pdl) - kw = dict( - T=T, - h=H, - hv=HV, + kw = _kernel_kw( + N, + T, + H, + HV, commit=commit, - pdl=pdl, + pdl=bool(launch_with_pdl), l2norm=bool(use_qk_l2norm_in_kernel), gating=gating, g_f32=g.dtype == torch.float32, b_f32=beta.dtype == torch.float32, - **cfg, + work_items=replay_work_items is not None, + config=config, + fold=conv_fold is not None, + norm_eps=norm_eps, ) + use_wi = kw["use_wi"] key = tuple(sorted(kw.items())) fn = _COMPILED.get(key) if fn is None: diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py index 03f3efab6515..fa83b33e0ef4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py @@ -1,18 +1,29 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Vision position-embedding bilinear interpolation onto one image's patch grid, in -spatial-merge order, by the stock Qwen3-VL tower's Triton kernel -(`_triton_pos_embed_interpolate` in `_torch/models/modeling_qwen3vl.py`), one call per image. -The NVFP4 default target's entry; the two checkpoints share the bf16 vision tower. +"""Vision position-embedding bilinear interpolation onto one image's patch +grid, emitted in spatial-merge order, by the stock Qwen3-VL tower's fused +Triton kernel. + +`__call__` allocates the output and makes one launch of the Triton JIT kernel +`_bilinear_pos_embed_kernel`: the four corner gathers, the bilinear weights and +the merge-order row permutation in one pass. The kernel and its launch are the +ones `tensorrt_llm/_torch/models/modeling_qwen3vl.py` runs for the stock vision +towers (`Qwen3VisionModel.fast_pos_embed_interpolate`, one call per image), with +the grid sides left unspecialized so one compile serves every image. +Concatenating several images' rows is the caller's. The FP8 unified target's entry; the +NVFP4 default target keeps its own copy (the two checkpoints share the bf16 vision tower). """ from __future__ import annotations import torch +import triton +import triton.language as tl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm._torch.models.modeling_qwen3vl import _triton_pos_embed_interpolate +#: This checkpoint: num_position_embeddings 2304 (a 48 x 48 grid), hidden 1152, +#: spatial_merge_size 2. NUM_GRID = 48 HIDDEN = 1152 MERGE = 2 @@ -31,6 +42,108 @@ def _ckpt(t: int, h: int, w: int) -> dict: ) +# The grid sides vary per image: not specialized, so one compile serves every grid. +@triton.jit(do_not_specialize=["H", "W"]) +def _bilinear_pos_embed_kernel( + embed_ptr, + output_ptr, + H, + W, + h_scale, + w_scale, + NUM_GRID: tl.constexpr, + M_SIZE: tl.constexpr, + HIDDEN_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """One output row: the bilinear blend of the table's four corner rows around the + row's patch, the patch taken in spatial-merge order.""" + pid = tl.program_id(0) + total_spatial = H * W + spatial_idx = pid % total_spatial + + num_blocks_w = W // M_SIZE + block_idx = spatial_idx // (M_SIZE * M_SIZE) + local_idx = spatial_idx % (M_SIZE * M_SIZE) + br = block_idx // num_blocks_w + bc = block_idx % num_blocks_w + lr = local_idx // M_SIZE + lc = local_idx % M_SIZE + row = br * M_SIZE + lr + col = bc * M_SIZE + lc + + h_frac = row.to(tl.float32) * h_scale + w_frac = col.to(tl.float32) * w_scale + + hf = tl.math.floor(h_frac).to(tl.int32) + wf = tl.math.floor(w_frac).to(tl.int32) + hc = tl.minimum(hf + 1, NUM_GRID - 1) + wc = tl.minimum(wf + 1, NUM_GRID - 1) + + dh = h_frac - hf.to(tl.float32) + dw = w_frac - wf.to(tl.float32) + w11 = dh * dw + w10 = dh - w11 + w01 = dw - w11 + w00 = 1.0 - dh - w01 + + off00 = (hf * NUM_GRID + wf) * HIDDEN_DIM + off01 = (hf * NUM_GRID + wc) * HIDDEN_DIM + off10 = (hc * NUM_GRID + wf) * HIDDEN_DIM + off11 = (hc * NUM_GRID + wc) * HIDDEN_DIM + out_off = pid * HIDDEN_DIM + + # The corner weights rounded to the output dtype; the multiply-adds run in it. + out_dtype = output_ptr.dtype.element_ty + w00_c = w00.to(out_dtype) + w01_c = w01.to(out_dtype) + w10_c = w10.to(out_dtype) + w11_c = w11.to(out_dtype) + + for d in tl.range(0, HIDDEN_DIM, BLOCK_D): + cols = d + tl.arange(0, BLOCK_D) + mask = cols < HIDDEN_DIM + + e00 = tl.load(embed_ptr + off00 + cols, mask=mask) + e01 = tl.load(embed_ptr + off01 + cols, mask=mask) + e10 = tl.load(embed_ptr + off10 + cols, mask=mask) + e11 = tl.load(embed_ptr + off11 + cols, mask=mask) + + val = w00_c * e00 + w01_c * e01 + w10_c * e10 + w11_c * e11 + + tl.store(output_ptr + out_off + cols, val, mask=mask) + + +def _pos_embed_interpolate( + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, +) -> torch.Tensor: + """One launch over a `(t, h, w)` grid: `[t * h * w, hidden]` in spatial-merge order.""" + hidden_dim = embed_weight.shape[1] + total_out = t * h * w + output = torch.empty(total_out, hidden_dim, device=embed_weight.device, dtype=dtype) + h_scale = (num_grid_per_side - 1) / (h - 1) if h > 1 else 0.0 + w_scale = (num_grid_per_side - 1) / (w - 1) if w > 1 else 0.0 + _bilinear_pos_embed_kernel[(total_out,)]( + embed_weight, + output, + h, + w, + h_scale, + w_scale, + num_grid_per_side, + m_size, + hidden_dim, + triton.next_power_of_2(hidden_dim), + ) + return output + + class TritonPosEmbedInterpolate(OpWrapper): """Resample a square row-major `[N*N, hidden]` table onto one `(t, h, w)` patch grid; return a fresh `[t*h*w, hidden]` tensor in `dtype`. @@ -42,13 +155,18 @@ class TritonPosEmbedInterpolate(OpWrapper): blocks row-major, row-major inside a block -- which is what the patch merger's 4-to-1 reshape consumes. `t` repeats the frame's rows. - Arithmetic is the HF reference's (`fast_pos_embed_interpolate`): fp32 corner - weights rounded to `dtype`, multiply-adds in `dtype` (not an fp32 accumulation). + Arithmetic is the HF reference's (`fast_pos_embed_interpolate`): the four + corner weights are computed in fp32, rounded to `dtype`, and the + multiply-adds run in `dtype`. Not an fp32 accumulation; see `note`. + + Fusion boundary: one image's position embeddings. Adding them to the patch + embeddings, the 2-D rope, and joining images are the caller's. """ ARCHS = frozenset({Arch.SM_103}) - # The tower is replicated on every rank, so these are every member's shapes. + # spec: t, h, w, num_grid_per_side, m_size, dtype, hidden. The tower is + # replicated on every rank, so these are every member's shapes. CELLS: tuple[Cell, ...] = ( Cell(why="one 4x8 image: 32 rows, both sides below the table's 48", spec=_ckpt(1, 4, 8)), Cell(why="the 62x90 serving grid: 5580 rows, one image per call", spec=_ckpt(1, 62, 90)), @@ -60,16 +178,50 @@ class TritonPosEmbedInterpolate(OpWrapper): ) note = """ - Source: the Triton kernel `_bilinear_pos_embed_kernel` behind the stock - `_triton_pos_embed_interpolate` (not a registered op). Measured on GB300 within 4 / 1 - bf16 ulp (element / relative RMS) of the fp64 reference. `is_valid` refuses what the - kernel computes silently wrong: a strided table, a `num_grid_per_side` that does not - match the table, an empty side, `h` or `w` not divisible by `m_size`. Hard limit: - `t*h*w*hidden` below 2^31 (int32 output offset). - Not certified: tables with uncorrelated neighbouring rows (can exceed the - 4-ulp band), fp16 or fp32 tables, a `dtype` other than the table's, `m_size` - other than 2, a hidden size other than 1152, a CPU table, and CUDA-graph - capture (the vision tower is eager only). + Source: a Triton JIT kernel, compiled at first call for the device it runs + on: the stock Qwen3-VL towers' kernel (`modeling_qwen3vl`), kept here so its + grid sides can stay unspecialized. `H` and `W` are int32 and not specialized, + so every grid shares one compile (the tower's `warm_up` makes it at load; the + test holds one variant); the scales are fp32, and `NUM_GRID`, `M_SIZE`, + `HIDDEN_DIM` (1152) and `BLOCK_D` (2048) are constexprs, so another table + geometry compiles another kernel. Not a registered `torch.ops.trtllm` op. + + Accuracy, against the exact (fp64) corner-aligned bilinear value, as + row-scale element error / relative RMS in bf16 ulp (`assert_within_ulp`): + at most 3.522 / 0.752 over 50 seeds of the test's smooth table (a bicubic + upsampled normal field, neighbour cosine 0.911 against the checkpoint + table's 0.907), and at most 3.349 / 0.745 on the checkpoint's own + `model.visual.pos_embed.weight`. The test gates 4 / 1. The closest wrong + variant (align_corners=False) lands at 41.3 / 15.2 or more; raster row + order and swapped table axes above 400 / 300. + + Not certified on tables whose neighbouring rows are uncorrelated: a normal + table with independent rows reaches 4.182 / 0.879 over 20 seeds. + Averaging uncorrelated corners shrinks the output's row maximum below the + corners' magnitude, and the bf16 rounding error scales with the corners. A + learned 2-D position table is not such a table. + + Silent, refused by `is_valid` (each measured on GB300): + - a strided table: the kernel indexes `embed_weight` as dense rows of + `hidden`. A column slice of a wider buffer returned, differing from the + same values made contiguous by up to 5.05. + - `num_grid_per_side ** 2 != embed_weight.shape[0]`: the grid side is an + argument, not read off the table. 47 on the 2304-row table returned a + correctly shaped result 564 / 340 ulp off. + - an empty side (`t`, `h` or `w` of 0): returns `[0, hidden]`, no error. + - `h` or `w` not divisible by `m_size`: a 6x9 grid returned a finite + `[54, 1152]` tensor, its rows read from positions outside the grid and the + table. + + Hard limits: the output offset is `program_id * hidden` in int32, so + `t*h*w*hidden` must stay below 2^31 (1,864,135 rows at hidden 1152; the + tower's per-call cap is 32768). Positions are `row * scale` in fp32, + exact up to 2^24 rows per side. + + Not certified: fp16 or fp32 tables, a `dtype` other than the table's, + `m_size` other than 2, a hidden size other than 1152, a CPU table, and + CUDA-graph capture (the call site, the vision tower, is eager only; the + grid is a host argument, frozen at capture). """ def __call__( @@ -82,9 +234,7 @@ def __call__( m_size: int, dtype: torch.dtype, ) -> torch.Tensor: - return _triton_pos_embed_interpolate( - embed_weight, t, h, w, num_grid_per_side, m_size, dtype - ) + return _pos_embed_interpolate(embed_weight, t, h, w, num_grid_per_side, m_size, dtype) def reference( self, @@ -96,8 +246,13 @@ def reference( m_size: int, dtype: torch.dtype, ) -> torch.Tensor: - """The four-corner weighted sum in fp64 at the exact sample positions, rounded once to - fp32 (not to `dtype`, so a band in `dtype` ulp measures the op's error alone).""" + """The explicit four-corner weighted sum in fp64, returned as fp32. + + Sample positions are the exact `i * (N - 1) / (size - 1)` in fp64. + `dtype` is not applied: the result is the exact value rounded once to + fp32, so a test's band in `dtype` ulp measures the op's error alone + rather than adding the reference's own rounding to `dtype`. + """ n = num_grid_per_side hidden = embed_weight.shape[1] device = embed_weight.device @@ -151,7 +306,7 @@ def is_valid( ) assert h % m_size == 0 and w % m_size == 0, ( f"grid sides h={h}, w={w} are not whole m_size={m_size} merge blocks; " - f"the kernel's bare assert is all that stops an out-of-grid read" + f"the kernel would read rows outside the grid and the table" ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py index 4a7dd6a69dbd..819bf0d36b7c 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Certification matrix for the FP8 target's MoE all-to-all round trip (`moe_a2a_dispatch` and `moe_a2a_combine` on the stock NVLink one-sided all-to-all) at dep2 and dep4 under mpirun, eagerly -and graph-replayed. Passes without checking where the platform has no MNNVL.""" +and graph-replayed. Fails where the platform has no MNNVL (the collected test skips there first).""" import os import sys @@ -176,9 +176,9 @@ def _run_one_mpi_rank() -> int: max_rows = max(max(c.spec["counts"]) for c in entry.moe_a2a_dispatch.CELLS) STRATEGY = entry.build_strategy(mapping, max_rows) if STRATEGY is None: - print(f"[rank {RANK}] no MNNVL on this platform: nothing to certify", flush=True) + print(f"[rank {RANK}] no MNNVL on this platform: nothing certified", flush=True) MPI_RANK.MPI.Finalize() - os._exit(0) + os._exit(1) _lockstep.run_checks(MPI_RANK, list(CHECKS)) torch.cuda.synchronize() print(f"[rank {RANK}] {len(CHECKS)} checks passed over the MPI session", flush=True) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py index e602819089f9..fdee67b97583 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -40,4 +40,9 @@ def _sm103_devices(world: int) -> int: def test_op_matrix(entry: str, world: int) -> None: if _sm103_devices(world) < world: pytest.skip(f"the {entry} matrix at world {world} runs on {world} sm_103 GPUs in one node") + if entry == "moe_a2a__dp_dispatch_combine": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided + + if not NVLinkOneSided.is_platform_supported(): + pytest.skip(f"the {entry} matrix needs MNNVL, which this platform lacks") _rank_job.run(entry, world_size=world, directory=_HERE) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 5aedc026a4ca..81a2603b350d 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1465,7 +1465,7 @@ def composed(buf, pos, pos_off, kv, kv_off, seq, mapping): pytest.param( entry, cell, check, id=f"{type(entry).__name__}: {cell.why[:44]}", - marks=[pytest.mark.large] if cell.spec.get("large") else [], + marks=[pytest.mark.high_cuda_memory] if cell.spec.get("large") else [], ) for entry, cells, check in _ENTRIES for cell in cells @@ -1815,6 +1815,8 @@ def _family_index_scores_prefill(requests, monkeypatch) -> None: pos = torch.cat([torch.arange(p, p + r, dtype=torch.int32) for p, r in requests]).to(DEV) q = torch.randn(pos.numel(), 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] assert _isc.supports(q, cache, 32, 4) + assert not _isc.supports(q, cache, 64, 4) + assert _isc.supports(q, cache, 16, 2) fam = M.qsa_paged_index_scores( q, cache, table, pos, req, 32, 4, only_visible_blocks=True, context_rows=True ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py index 17ac60159043..e34b53674d5b 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py @@ -249,7 +249,7 @@ def tp_sum(x): x_lo = torch.zeros(lo, HIDDEN, dtype=torch.bfloat16, device=DEV) x16 = torch.zeros(16, HIDDEN, dtype=torch.bfloat16, device=DEV) off = SimpleNamespace(tp_sum=tp_sum) - on = SimpleNamespace(hc_ar_inject=object(), tp_sum=tp_sum) + on = SimpleNamespace(hc_ar_inject=SimpleNamespace(max_rows=1 << 30), tp_sum=tp_sum) assert not isinstance(M._tp_partial(off, x_lo), M._TpPartial) assert isinstance(M._tp_partial(on, x_lo), M._TpPartial) assert not isinstance(M._tp_partial(on, x16), M._TpPartial) From f2d0f002a918326fe40a14aea86f871a67e4065a Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:44:39 -0800 Subject: [PATCH 08/51] [None][fix] modeling_v2 qwen4_exp: decline what the FP8 target does not run, honor PDL and all-reduce settings Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 339 ++++++++++++++---- .../vision.py | 28 +- .../weights.py | 56 ++- .../test_accuracy.py | 8 + .../test_routing_and_layout.py | 102 ++++++ .../test_vision.py | 19 +- 6 files changed, 457 insertions(+), 95 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 857e48912db2..39a19d077625 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -88,7 +88,6 @@ import contextlib import math -import os from typing import TYPE_CHECKING, NamedTuple import torch @@ -245,6 +244,7 @@ from tensorrt_llm._torch.attention.backends.fmha.phased import get_spec_decoding_position_offsets from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.distributed.allreduce_helper import CustomAllReduceHelper from tensorrt_llm._torch.distributed.communicator import Distributed from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl @@ -259,10 +259,9 @@ from tensorrt_llm._torch.pyexecutor.model_loader import effective_mamba_ssm_cache_dtype from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks from tensorrt_llm._torch.speculative import get_spec_worker, should_use_separate_draft_kv_cache -from tensorrt_llm._torch.utils import is_gdn_replay_enabled +from tensorrt_llm._torch.utils import is_gdn_flashinfer_replay_enabled, is_gdn_replay_enabled from tensorrt_llm._utils import mpi_disabled -from tensorrt_llm.functional import AllReduceFusionOp -from tensorrt_llm.logger import logger +from tensorrt_llm.functional import AllReduceFusionOp, AllReduceStrategy from tensorrt_llm.quantization.mode import QuantAlgo if TYPE_CHECKING: @@ -345,6 +344,7 @@ from .ops.ssm.gdn_cached_replay_cute__verify import GEOMETRIES as _GDN_REPLAY_CUTE_GEOMETRIES from .ops.ssm.gdn_cached_replay_cute__verify import conv_fold_supported as gdn_conv_fold_supported from .ops.ssm.gdn_cached_replay_cute__verify import gdn_cached_replay_cute +from .ops.ssm.gdn_cached_replay_cute__verify import precompile as gdn_replay_precompile from .ops.ssm.gdn_in_proj_conv__verify import CONV_DIM as _GDN_IN_PROJ_CONV_DIM from .ops.ssm.gdn_in_proj_conv__verify import N_OUT as _GDN_IN_PROJ_N_OUT from .ops.ssm.gdn_in_proj_conv__verify import fused_tactic as _gdn_in_proj_conv_fused_tactic @@ -365,6 +365,41 @@ _TEXT_SHAPE = (48, 2560, 512, 24, 48, 248320) _VISION_SHAPE = (27, 1152, 16, 2, 2560, 2304, (), "gelu_pytorch_tanh") +# The checkpoint's settings that change what it computes without changing a weight's shape, +# at the values the target implements: a fine-tune of the same shape that changed one would +# load and run with wrong logits and no error. Read off the text config as the target reads +# them (`_text_settings`). +_TEXT_SETTINGS = { + # The routed experts: Renormalize routing over the top 10 and a SiLU activation. + "norm_topk_prob": True, + "num_experts_per_tok": 10, + "hidden_act": "silu", + # Attention: two KV heads of 256, no projection bias, a sigmoid output gate on every head + # (absent `attn_output_gate` reads as gated), and an output head of its own. + "num_key_value_heads": 2, + "head_dim": 256, + "attention_bias": False, + "output_gate_type": "sigmoid", + "attn_output_gate": True, + "tie_word_embeddings": False, + # Rope: a quarter of the head (64 dims), theta 1e7, interleaved mRoPE over sections + # 11 / 11 / 10, unscaled. + "rope_theta": 1e7, + "partial_rotary_factor": 0.25, + "mrope_section": (11, 11, 10), + "mrope_interleaved": True, + "rope_factor": None, + # The MTP module: one full-attention layer on the trunk's embedding that reads the end of + # the stack (`mtp_use_hidden_state_from_layer: null`). The `mtp.` fields are the `mtp` + # block's, a mapping or a config object; a field it omits reads as the value here. (Its + # own rope theta the target reads.) + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": False, + "mtp.num_hidden_layers": 1, + "mtp.layer_types": ("full_attention",), + "mtp.mtp_use_hidden_state_from_layer": None, +} + # Rows per request of the widest verify step (`max_draft_len + 1`). _MAX_VERIFY_WIDTH = _pool.MAX_VERIFY_WIDTH @@ -448,8 +483,9 @@ _ACT_TYPE_SWIGLU = 0 _PAD_SLOT_ID = -1 #: Whether a decode step's `low_m_bf16_mm` projections launch with PDL (the FlashInfer kernels -#: stream the weight before waiting on the producer; the cuBLAS branch ignores it). -_DECODE_PROJ_PDL = True +#: stream the weight before waiting on the producer; the cuBLAS branch ignores it). Off with +#: TRTLLM_ENABLE_PDL=0. +_DECODE_PROJ_PDL = get_env_enable_pdl() #: (request x value head) pairs a verify step up to which the CuTe cached replay launches with PDL #: and the conv before it triggers at its start, so the replay streams the checkpoint and history #: (nothing earlier in the step writes them) while the conv runs. @@ -459,13 +495,14 @@ _GDN_REPLAY_HT_GEOMETRY = (8, 24) _GDN_REPLAY_HT_MIN_REQUESTS = 16 #: `(max requests, pdl, CuTe replay mapping or None for its default)` for a tp2/tep2 HT verify -#: batch. The mappings (`gdn_cached_replay_cute.default_config`'s knobs) only move where a tile is -#: loaded and which CTA runs it, so every choice is bitwise the default launch. +#: batch (PDL off throughout with TRTLLM_ENABLE_PDL=0). The mappings +#: (`gdn_cached_replay_cute.default_config`'s knobs) only move where a tile is loaded and which +#: CTA runs it, so every choice is bitwise the default launch. _GDN_REPLAY_HT_LAUNCH = ( - (24, True, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), - (40, True, dict(tpc=8, warps=4, pf=1, minb=6)), - (56, True, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), - (80, True, None), + (24, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (40, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=1, minb=6)), + (56, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (80, get_env_enable_pdl(), None), (1 << 30, False, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), ) #: Requests up to which a tp2/tep2 HT verify batch folds the conv into the CuTe replay @@ -474,7 +511,8 @@ _GDN_VERIFY_CONV_FOLD_MAX_REQUESTS = 24 #: Whether a verify step's conv (`causal_conv1d_token_major`) is a PDL dependent of the input #: projection: it reads the slots and the conv pool before `griddepcontrol.wait`, the q|k|v after. -_GDN_CONV_PDL_WAIT = True +#: Off with TRTLLM_ENABLE_PDL=0. +_GDN_CONV_PDL_WAIT = get_env_enable_pdl() #: Whether a decode step's QSA index scoring (`_decode_index_selection`) launches with PDL: its #: kernels wait before their first read and the decode prologue triggers at its start. _INDEX_SCORES_PDL = get_env_enable_pdl() @@ -568,6 +606,11 @@ #: Rows from which a context chunk's selection over a wide score table takes the row top-k kernel #: (`qsa_topk_rows`) instead of the family's wide kernel; same bits. _TOPK_ROWS_MIN_ROWS = 256 +#: Elements of one score table: the index scores write their output rows and the selection +#: kernels address theirs with int32 arithmetic (row times the table's width), so a call scores +#: fewer rows than this over the table's columns: at `max_seq_len` 262144 (65568 columns) a +#: context part past 32752 rows is scored in chunks. +_SCORE_TABLE_ELEMENTS = 2**31 def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, cr, context_rows): @@ -635,14 +678,15 @@ def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, # Over an fp8 pool the launch reads its requests' pages widened into a bf16 scratch; a launch # whose scratch would pass this many bytes stays on the per-row kernel. _FP8_ROWTILE_MAX_SCRATCH = 1 << 30 # capacity: fixed: bounds a per-call scratch allocation -#: Opt-in (`1`, `true`, `yes` or `on`): a context step over an fp8 pool whose pages would widen +#: Opt-in switch (`weights.switch`): a context step over an fp8 pool whose pages would widen #: past `_FP8_ROWTILE_MAX_SCRATCH` runs one row-tile launch per group of requests that fits #: (`_rowtile_scratch_groups`) instead of the per-row kernel. Off by default because such steps' #: outputs then move by rounding. Read once, when the target is built. ROWTILE_SCRATCH_GROUPS_ENV = "TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS" -#: Under attention DP, 1 runs the MoE round trip as an all-to-all dispatch / combine -#: (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one. Needs MPI and MNNVL; -#: elsewhere the all-gather / reduce-scatter stays. Read once, when the target is built. +#: Opt-in switch (`weights.switch`): under attention DP, the MoE round trip runs as an all-to-all +#: dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one. Needs +#: MPI, attention DP and MNNVL; `check_deployment` declines it elsewhere. Read once, when the +#: target is built. MOE_ALLTOALL_ENV = "TRTLLM_MODELING_V2_MOE_ALLTOALL" # Rows per FP8 runner call over an attention-DP group's gathered rows (`_routed_experts`): one # call reads the window's weights once, bitwise the `_MOE_MAX_ROWS`-row calls; the router, its @@ -779,14 +823,16 @@ class _HcArInject(NamedTuple): workspace: torch.Tensor rank: int nranks: int + #: rows whose `nranks * rows * hidden * 2` bytes fit one of the workspace's sub-buffers + max_rows: int def _hc_ar_inject_defers(core, rows: int) -> bool: """Whether `_tp_partial` leaves a decode block's partial of `rows` rows to the next site.""" - return ( - getattr(core, "hc_ar_inject", None) is not None - and max(_HC_AR_INJECT_MIN_ROWS, _HC_ROWS_TC_MAX_ROWS + 1) <= rows < _HC_RSTD_MIN_ROWS - ) + fused = getattr(core, "hc_ar_inject", None) + return fused is not None and max( + _HC_AR_INJECT_MIN_ROWS, _HC_ROWS_TC_MAX_ROWS + 1 + ) <= rows < min(_HC_RSTD_MIN_ROWS, fused.max_rows + 1) def _tp_partial(core, x: torch.Tensor): @@ -837,6 +883,8 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self.mapping = mapping # Ranks the attention and recurrent heads are split over (1 under attention DP). self.attn_tp_size = _pool.attention_tp_size(mapping) + # The engine's all-reduce strategy for the TP sums; the fused IPC paths need AUTO. + self.allreduce_strategy = int(model_config.allreduce_strategy) # Under attention DP each rank serves its own requests; MoE runs over the TP group's rows. self.dp_size = mapping.tp_size if mapping.enable_attention_dp else 1 self.dp_rank = mapping.tp_rank @@ -1140,7 +1188,12 @@ def _precompile_draft_hc_rows(self, device) -> None: torch.cuda.synchronize(device) def post_load_weights(self) -> None: - """Derive load-time tensors and build the targets; a shell hook, not an override.""" + """Derive load-time tensors and build the targets; a shell hook, not an override. + + Reached twice per load (the shell's hook and the loader's module walk): the targets are + built once, until the shell's `load_weights` clears them for a reload.""" + if self._prefill is not None: + return w = self.w device = w["embed"].device # The delta-rule kernels read A_log and dt_bias as contiguous fp32; the checkpoint is bf16. @@ -1203,7 +1256,7 @@ def _tp_sum_op(self): """Return the attention-TP sum of block partials, bit-identical on every rank.""" if self.attn_tp_size == 1: return lambda x: x - group = allreduce_group_args(self.mapping) + group = allreduce_group_args(self.mapping, self.allreduce_strategy) def tp_sum(x: torch.Tensor) -> torch.Tensor: return allreduce( @@ -1218,7 +1271,9 @@ def tp_sum(x: torch.Tensor) -> torch.Tensor: **group, ) - if not MNNVLAllReduce.is_mnnvl(self.mapping, torch.bfloat16, explicitly_requested=True): + if self.allreduce_strategy != int(AllReduceStrategy.AUTO) or not MNNVLAllReduce.is_mnnvl( + self.mapping, torch.bfloat16, explicitly_requested=True + ): return tp_sum # A collective over the TP group: every rank builds the workspace here, in order. mnnvl = mnnvl_workspace_args( @@ -1241,14 +1296,22 @@ def _hc_ar_inject(self): and self.mapping.world_size == self.attn_tp_size and self.hidden == 2560 and self.hc_count == 4 + and self.allreduce_strategy == int(AllReduceStrategy.AUTO) + and can_access_peer(self.mapping) ): return None - group = allreduce_group_args(self.mapping) + group = allreduce_group_args(self.mapping, self.allreduce_strategy) if group["workspace"] is None: return None members = list(self.mapping.tp_group) rank = members.index(self.mapping.rank) - return _HcArInject(group["workspace"], rank, len(members)) + # `get_allreduce_workspace` sized each IPC sub-buffer `size * tp_size` with this call; + # the fused kernel's span in one is `nranks * rows * hidden * 2` bytes. + size = CustomAllReduceHelper.max_workspace_size_auto( + self.mapping.tp_size, support_deterministic=False + ) + max_rows = size * self.mapping.tp_size // (self.hidden * 2 * len(members)) + return _HcArInject(group["workspace"], rank, len(members), max_rows) def _expert_sum_op(self): """Return `expert_sum(run, x, padding)`: this rank's rows of a MoE block's output. @@ -1276,17 +1339,15 @@ def round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor gathered = allgather(x, sizes, group, process_group) return reducescatter(run(gathered), sizes, group, process_group) - a2a = os.environ.get(MOE_ALLTOALL_ENV, "0").strip().lower() in ("1", "true", "yes") - strategy = ( - None if not a2a or mpi_disabled() else _build_moe_a2a(self.mapping, self.max_num_tokens) - ) - if strategy is None: - if a2a: - logger.warning( - f"{MOE_ALLTOALL_ENV}=1 needs MPI and MNNVL; the attention-DP MoE runs the " - "all-gather / reduce-scatter round trip" - ) + if not _weights.switch(MOE_ALLTOALL_ENV): return round_trip + strategy = None if mpi_disabled() else _build_moe_a2a(self.mapping, self.max_num_tokens) + if strategy is None: + # `check_deployment` declines this deployment; reached only when routing is bypassed. + raise RuntimeError( + f"{MOE_ALLTOALL_ENV}=1 needs MPI and MNNVL, which this deployment lacks; unset it " + "to run the attention-DP MoE as the all-gather / reduce-scatter round trip" + ) self.moe_a2a = strategy def alltoall_round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: @@ -1369,10 +1430,6 @@ def _bind_cache_manager(self, md, position_ids) -> None: f"KV addressing over {len(pools)} pools is not certified; layer->pool ids " f"{sorted(pools)}" ) - assert self.quant_mode != QM_NVFP4_KV_CACHE or md.use_paged_context_fmha, ( - "an NVFP4 KV cache is read by the paged-context attention kernels only, which the " - "engine enables for chunked prefill, KV cache block reuse or speculative decoding" - ) g = _pool.pool_geometry( self.model_config.pretrained_config, self.mapping, @@ -1428,10 +1485,10 @@ def _bind_cache_manager(self, md, position_ids) -> None: [rank_pools.index_cache[i] for i in self.attn_layers] ) if self.mtp_enabled: - bound = bound._replace(**self._bind_step_buffers(cm, g, rank_pools)) + bound = bound._replace(**self._bind_step_buffers(cm, g, rank_pools, md)) self._bound = bound - def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools) -> dict: + def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, md) -> dict: """Bind the verify step's per-step buffers and return the `_Bound` fields they add. Per-row conv windows feed `update_mamba_states`; the recurrent state uses cached replay.""" @@ -1474,6 +1531,21 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools) old[i] = (cache.old_x, cache.old_B, cache.old_dt, cache.old_dA_cumsum) _pool.check_replay_history(g, i, history, ssm0.shape[0], *old[i][:3]) replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) + if ssm0.dtype == torch.bfloat16 and ssm0.is_cuda: + # The CuTe replay kernels of the default mapping a verify step launches, compiled + # here: a compile costs 5-10 s, and the padded decode graphs skip the request counts + # an eager mixed step carries. PDL up to `_GDN_REPLAY_PDL_MAX_HEAD_ROWS` + # (request x value head) pairs; a tp2/tep2 HT batch takes its own mapping, compiled + # on its first launch (`_gdn_replay_launch`). + heads = (width, self.num_k_heads, self.num_v_heads) + partition_from = CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + requests = md.max_num_requests + if heads[1:] == _GDN_REPLAY_HT_GEOMETRY: + requests = min(requests, _GDN_REPLAY_HT_MIN_REQUESTS - 1) + pdl_requests = min(requests, _GDN_REPLAY_PDL_MAX_HEAD_ROWS // self.num_v_heads) + if get_env_enable_pdl() and pdl_requests > 0: + gdn_replay_precompile(*heads, pdl_requests, pdl=True, partition_from=partition_from) + gdn_replay_precompile(*heads, requests, pdl=False, partition_from=partition_from) return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) def _bind_draft_cache_manager(self, md) -> None: @@ -2001,7 +2073,7 @@ def _gdn_replay_launch(core, n: int) -> tuple: for bound, pdl, mapping in _GDN_REPLAY_HT_LAUNCH: if n <= bound: return pdl, mapping - return n * core.num_v_heads <= _GDN_REPLAY_PDL_MAX_HEAD_ROWS, None + return get_env_enable_pdl() and n * core.num_v_heads <= _GDN_REPLAY_PDL_MAX_HEAD_ROWS, None def _draft_positions(position_ids: torch.Tensor) -> tuple: @@ -2311,6 +2383,11 @@ def _routed_experts( `pad_mask` (`[T, 1]` bool) routes a prefill graph bucket's padding rows to expert -1, so they come back zero instead of all landing on the first window's ten lowest experts.""" mm = mm or cublas_mm + if getattr(core, "dp_size", 1) > 1: + # Attention DP routes the same gathered rows on every rank, whose steps may pick another + # `mm`: `_decode_projection`'s low-M kernels and cuBLAS differ in the last bits, and a + # reordered top-k is a different MoE sum. One router kernel on every rank. + mm = cublas_mm rows = x.shape[0] if pad_mask is None and mm is _decode_projection and 0 < rows <= _MOE_FRONT_MAX_ROWS: if before_runner is not None: @@ -2766,7 +2843,7 @@ def _hc_site( fused = getattr(core, "hc_ar_inject", None) if ( fused is not None - and hc.shape[0] < _HC_RSTD_MIN_ROWS + and hc.shape[0] < min(_HC_RSTD_MIN_ROWS, fused.max_rows + 1) and _hc_ar_inject_operands_fit(hc, partial, inject, shared, gate_logits) ): warps = 2 if hc.shape[0] >= _HC_INJECT_FEW_WARPS_MIN_ROWS else _HC_INJECT_WARPS @@ -3837,10 +3914,12 @@ def _copy_positions(position_ids: torch.Tensor) -> torch.Tensor: return pos3 -def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens) -> dict: +def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens, paged_context) -> dict: """thop_attention's batch state off the prepared metadata. - Tensors are engine-owned buffers refreshed in place; ints are per-capture constants.""" + Tensors are engine-owned buffers refreshed in place; ints are per-capture constants. + `paged_context` forces the paged-context path the engine enables only for chunked prefill + or KV cache block reuse: an NVFP4 pool's context kernels exist on it alone.""" return dict( sequence_length=md.kv_lens_cuda_runtime, host_past_key_value_lengths=md.kv_lens_runtime, @@ -3860,7 +3939,7 @@ def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens) -> dic num_contexts=num_contexts, num_ctx_tokens=num_ctx_tokens, trtllm_gen_jit_warmup=md.trtllm_gen_jit_warmup, - use_paged_context_fmha=md.use_paged_context_fmha, + use_paged_context_fmha=md.use_paged_context_fmha or paged_context, beam_width=md.effective_beam_width, cache_indirection=md.cache_indirection, block_ids_per_seq=md.block_ids_per_seq, @@ -3956,12 +4035,7 @@ def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: core.mapping.tp_group_pg.boxed() if mpi_disabled() else None, ) self._selection_split = split - self._rowtile_groups = os.environ.get(ROWTILE_SCRATCH_GROUPS_ENV, "0").strip().lower() in ( - "1", - "true", - "yes", - "on", - ) + self._rowtile_groups = _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) self._sequence_parallel = split # TP sum, or reduce-scatter under SP; set by `forward`, a capture-time constant per bucket. self._block_sum = core.tp_sum @@ -4132,7 +4206,9 @@ def _check_step_contract(self, md) -> None: def step_args(self, md: TrtllmAttentionMetadata) -> dict: """Project the prepared metadata onto thop_attention's batch state.""" - return _step_args(md, md.num_contexts, md.num_ctx_tokens) + return _step_args( + md, md.num_contexts, md.num_ctx_tokens, self.core.quant_mode == QM_NVFP4_KV_CACHE + ) # --------------------------------------------------------------- blocks --- @@ -4331,28 +4407,38 @@ def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: device=proj.device, ) + # A call scores fewer than `_SCORE_TABLE_ELEMENTS / columns` rows (whole query + # tiles): a row's scores and selection read only its query, position and request, + # so the chunks select what one call would; below the bound a part is one call. + columns = block_table.shape[1] * tokens_per_block // core.compress_ratio + chunk = max(1, (_SCORE_TABLE_ELEMENTS - 1) // columns) + if chunk >= _SELECTION_CUT_ALIGN: + chunk -= chunk % _SELECTION_CUT_ALIGN + def score_and_select(lo, hi, context_rows, out): - scores = _context_index_scores( - q_index[lo:hi], - core._bound.pools.index_cache[i], - block_table, - logical[lo:hi], - req_idx[lo:hi], - tokens_per_block, - core.compress_ratio, - context_rows, - ) - _selection( - core, - scores, - visible_blocks[lo:hi], - logical[lo:hi], - seq_lens_q[lo:hi], - topk_indices[lo:hi], - topk_row_starts[lo:hi], - out, - context=context_rows, - ) + for a in range(lo, hi, chunk): + b = min(a + chunk, hi) + scores = _context_index_scores( + q_index[a:b], + core._bound.pools.index_cache[i], + block_table, + logical[a:b], + req_idx[a:b], + tokens_per_block, + core.compress_ratio, + context_rows, + ) + _selection( + core, + scores, + visible_blocks[a:b], + logical[a:b], + seq_lens_q[a:b], + topk_indices[a:b], + topk_row_starts[a:b], + out[a - lo : b - lo], + context=context_rows, + ) for lo, hi, context_rows in parts: if context_rows and self._split_context_rows(hi - lo): @@ -5542,7 +5628,12 @@ def _check_step_contract(self, md) -> None: def step_args(self, md: TrtllmAttentionMetadata) -> dict: """See PrefillTarget.step_args. Context counts are stated, not read: routing guarantees them, and a decode-only CUDA graph would freeze whatever it read.""" - return _step_args(md, num_contexts=0, num_ctx_tokens=0) + return _step_args( + md, + num_contexts=0, + num_ctx_tokens=0, + paged_context=self.core.quant_mode == QM_NVFP4_KV_CACHE, + ) def _verify_qsl(self, seq_lens, batch: int, p: int, device) -> torch.Tensor: """PLE segmentation `[0, p, ..., batch * p]` of a generation-only verify step, cached @@ -5806,8 +5897,9 @@ def _ple_block( rows, conv_input=conv_input, inject_block=inject_block, - # The three launches chain through PDL; the norm weights are read before the wait. - pdl=True, + # The three launches chain through PDL (off with TRTLLM_ENABLE_PDL=0); the norm + # weights are read before the wait. + pdl=get_env_enable_pdl(), # every sequence here is a generation request the commit rewrites publish_end=0 if from_pool else None, ) @@ -6565,6 +6657,42 @@ def forward( ) +def _field(block, name: str, default=None): + """`block`'s `name` -- a mapping's entry or an object's attribute -- or `default` where it + is absent or null.""" + value = block.get(name) if isinstance(block, dict) else getattr(block, name, None) + return default if value is None else value + + +def _text_settings(text) -> dict: + """`_TEXT_SETTINGS`' entries as `text` declares them. The rope fields are the target's + reading: the `rope_parameters` dict, or the flattened `rope_scaling` with the theta and + the partial factor at the top level.""" + rope = getattr(text, "rope_parameters", None) or getattr(text, "rope_scaling", None) or {} + settings = {name: getattr(text, name, None) for name in _TEXT_SETTINGS} + settings.update( + rope_theta=rope.get("rope_theta", getattr(text, "rope_theta", None)), + partial_rotary_factor=rope.get( + "partial_rotary_factor", getattr(text, "partial_rotary_factor", None) + ), + mrope_section=tuple(rope.get("mrope_section") or ()), + mrope_interleaved=rope.get("mrope_interleaved"), + rope_factor=rope.get("factor"), + attn_output_gate=_field(text, "attn_output_gate", True), + mtp_num_hidden_layers=_field(text, "mtp_num_hidden_layers", 1), + mtp_use_dedicated_embeddings=_field(text, "mtp_use_dedicated_embeddings", False), + ) + mtp = getattr(text, "mtp", None) + settings.update( + { + "mtp.num_hidden_layers": _field(mtp, "num_hidden_layers", 1), + "mtp.layer_types": tuple(_field(mtp, "layer_types", ("full_attention",))), + "mtp.mtp_use_hidden_state_from_layer": _field(mtp, "mtp_use_hidden_state_from_layer"), + } + ) + return settings + + class ModelingV2Qwen38FlashNextFp8Sm103Unified( DecoderModelForCausalLM[Qwen4ExpUnifiedModelingV2, PretrainedConfig] ): @@ -6612,6 +6740,30 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"text shape {shape}, vision shape {vshape}; the target is Qwen3.8-Flash-Next's " f"{_TEXT_SHAPE}, vision {_VISION_SHAPE}" ) + differs = {k: v for k, v in _text_settings(text).items() if v != _TEXT_SETTINGS[k]} + if differs: + raise NoTarget( + f"the checkpoint has Qwen3.8-Flash-Next's shape but declares {differs}; the " + f"target implements { ({k: _TEXT_SETTINGS[k] for k in differs}) }" + ) + # The opt-in switches, read here so a misspelled value fails before the build. + mapping = model_config.mapping + _weights.moe_tpb_requested() + _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) + if _weights.switch(MOE_ALLTOALL_ENV): + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + NVLinkOneSided, + ) + + if ( + mpi_disabled() + or not mapping.enable_attention_dp + or not NVLinkOneSided.is_platform_supported() + ): + raise NoTarget( + f"{MOE_ALLTOALL_ENV}=1 at {layout}: the all-to-all MoE round trip needs MPI, " + "attention DP and MNNVL on this node" + ) spec = model_config.spec_config if spec is not None: # One linear MTP chain, verified at `max_draft_len + 1` rows on the GDN cached replay. @@ -6644,6 +6796,12 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: "MTP with TRTLLM_USE_GDN_REPLAY=0: the verify step runs the GDN cached replay, " "not the legacy per-step recurrent states; leave TRTLLM_USE_GDN_REPLAY unset" ) + if is_gdn_flashinfer_replay_enabled() and spec.max_draft_len + 1 in (4, 8): + raise NoTarget( + f"TRTLLM_USE_GDN_FLASHINFER_REPLAY with max_draft_len={spec.max_draft_len}: " + "the manager lays the replay history out for the FlashInfer ring kernel, " + "which the verify step does not run" + ) prefill_graph = str(llm_args.prefill_cuda_graph_backend.value) if prefill_graph == "piecewise": raise NoTarget( @@ -6651,12 +6809,26 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: "which the target does not implement; 'breakable' runs the prefill step as a " "breakable CUDA graph" ) - if model_config.mapping.enable_attention_dp and prefill_graph == "breakable": + if mapping.enable_attention_dp and prefill_graph == "breakable": raise NoTarget( f"prefill_cuda_graph_backend='breakable' with attention DP ({layout}): the engine " "pads every rank to one prefill bucket, so a rank holding only generation " "requests is handed more rows than its batch, which the decode step refuses" ) + windows = llm_args.kv_cache_config.max_attention_window + full = llm_args.max_seq_len or model_config.max_seq_len + if windows is not None and ( + any(w < full for w in windows) if full else len(set(windows)) > 1 + ): + raise NoTarget( + f"kv_cache_config.max_attention_window={windows}; the target implements no " + "sliding window: every attention layer's window is max_seq_len" + ) + if getattr(llm_args.multimodal_config, "vision_gpu_normalize", False): + raise NoTarget( + "multimodal_config.vision_gpu_normalize: the input processor then ships uint8 " + "pixels, and the vision tower runs at the bf16 activation dtype" + ) kv_cache_quant_algo = engine_kv_cache_quant_algo(model_config, llm_args) if kv_cache_quant_algo not in (None, QuantAlgo.FP8, QuantAlgo.NVFP4): raise NoTarget( @@ -6672,6 +6844,12 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"moe_config.backend={model_config.moe_backend_requested!r}; the target runs the " f"routed experts on {', '.join(_TRUNK_MOE_BACKENDS)}" ) + if llm_args.moe_config.load_balancer is not None and mapping.moe_ep_size > 1: + raise NoTarget( + f"moe_config.load_balancer (EPLB) at {layout}: the target runs fixed contiguous " + "expert windows and registers no layer with the balancer, so the placement " + "would do nothing" + ) state_dtype = effective_mamba_ssm_cache_dtype( model_config, llm_args.kv_cache_config.mamba_ssm_cache_dtype ) @@ -6859,6 +7037,8 @@ def forward( ) def load_weights(self, weights, *args, **kwargs): + # A reload re-derives everything `post_load_weights` builds from the weights. + self.model._prefill = self.model._decode = None _weights.MODEL_WEIGHTS.load(self, weights) def post_load_weights(self): @@ -6867,6 +7047,7 @@ def post_load_weights(self): self.vision_tower = _vision.VisionTower(self.model.w, self.model.vision_config) # The CPU-only load checks build the shell over host weights; they launch nothing. if self.lm_head.weight.is_cuda: + self.vision_tower.warm_up() self._load_eager_gemm_kernels() def _load_eager_gemm_kernels(self) -> None: diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py index b4d169760aa4..f462e4ede4c7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -47,6 +47,10 @@ #: ops' certified ceiling. Calls are cut between frames, so a request is bounded per frame. VISION_CALL_MAX_PATCHES = 32768 +#: The grids `VisionTower.warm_up` runs: one patch count that is a multiple of 16 (4x4) and one +#: that is not (2x2). +WARM_UP_GRIDS = (((1, 4, 4),), ((1, 2, 2),)) + #: `mask_type` 0: bidirectional within a batch segment (causal would depend on raster order). _VISION_MASK_PADDING = 0 @@ -293,6 +297,16 @@ def __init__(self, w: dict, vision_config) -> None: # Persistent scratch `thop_attention` grows in place; not the engine's own workspace. self._ws: torch.Tensor | None = None + def warm_up(self) -> None: + """Run the tower once per `WARM_UP_GRIDS` grid on zero pixels, so every JIT kernel it + launches is compiled at load rather than on the first media request. The rope and + position-embedding kernels take the grid as unspecialized ints; the layer norm's row + count is specialized only on divisibility by 16, and the two grids cover both.""" + device = self._w.pos.device + for grid in WARM_UP_GRIDS: + rows = sum(t * h * w for t, h, w in grid) + self._run(torch.zeros(rows, self.pixel_width, dtype=self.dtype, device=device), grid) + def __call__(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: """Return the `[sum(t*h*w) / merge**2, out_hidden_size]` placeholder rows, in grid order. @@ -514,10 +528,17 @@ def in_prompt_order(merge: int, params, towers: dict) -> torch.Tensor: def validate_request(tower, request) -> None: """Admission check the executor runs on every new request, from its host-side payload only. - It refuses what the forward would assert on: an assert inside the step ends the executor.""" + It refuses what the forward would assert on: an assert inside the step ends the executor. + It also refuses precomputed image embeddings, which the tower cannot splice.""" data = request.py_multimodal_data if not data: return + if data.get("multimodal_embedding") is not None: + raise ValueError( + "this target does not splice precomputed image embeddings " + "(multimodal_data['multimodal_embedding']); send the image/video pixels " + "(multimodal_data['image'/'video']) so the vision tower runs" + ) grids = {} for modality, (pixel_key, grid_key) in MEDIA_KEYS.items(): media = data.get(modality) @@ -589,6 +610,11 @@ def _media_grid(tower, modality, media, pixel_key, grid_key): f"{pixel_key} is {tuple(pixel_values.shape)}; its grids describe {patches} patch " f"rows of {tower.pixel_width}" ) + if pixel_values.dtype != tower.dtype: + raise ValueError( + f"{pixel_key} is {pixel_values.dtype}; the vision tower runs at {tower.dtype} " + "(multimodal_config.vision_gpu_normalize ships uint8)" + ) return grid diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index 4a69b5a2db3d..ba09f891d981 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -44,6 +44,7 @@ consumed or predicted by `expected_unconsumed`. """ +import mmap import os from dataclasses import dataclass from types import SimpleNamespace @@ -65,6 +66,26 @@ Cut = tuple[slice, ...] +#: The spellings of a `TRTLLM_MODELING_V2_*` switch: these turn it on, these off. +SWITCH_ON = frozenset({"1", "true", "yes", "on"}) +SWITCH_OFF = frozenset({"0", "false", "no", "off"}) + + +def switch(env: str, on: frozenset = SWITCH_ON) -> bool: + """A `TRTLLM_MODELING_V2_*` switch, default off: unset, empty or a `SWITCH_OFF` spelling is + off, an `on` spelling on; any other value raises.""" + raw = os.environ.get(env, "") + value = raw.strip().lower() + if not value or value in SWITCH_OFF: + return False + if value in on: + return True + raise ValueError( + f"{env}={raw!r}: expected {' / '.join(sorted(on))} (on) or " + f"{' / '.join(sorted(SWITCH_OFF))} (off)" + ) + + #: Opt-in (`rr`; default off) block-major copy of every MoE layer's routed experts for #: `ops/moe/moe_tpb_decode` (attention TP4 only): expert e's 128-row intermediate block b on rank #: (e + b) % 4, plus this rank's slice of the shared expert. The whole-expert window stays for @@ -72,12 +93,13 @@ MOE_TPB_ENV = "TRTLLM_MODELING_V2_DECODE_MOE_TPB" +def moe_tpb_requested() -> bool: + """Whether `MOE_TPB_ENV` asks for the block-major copy (`rr`); another value raises.""" + return switch(MOE_TPB_ENV, on=frozenset({"rr"})) + + def moe_tpb_enabled(mapping) -> bool: - raw = os.environ.get(MOE_TPB_ENV, "").strip().lower() - if raw in ("", "0", "false", "off", "no"): - return False - assert raw == "rr", f"{MOE_TPB_ENV}={raw!r}: expected off or rr" - return mapping.tp_size == 4 and not mapping.enable_attention_dp + return moe_tpb_requested() and mapping.tp_size == 4 and not mapping.enable_attention_dp @dataclass(frozen=True) @@ -159,10 +181,15 @@ def _local_cpus(device) -> set | None: class _RegisteredHostBuffer: - """Pageable CPU memory registered (pinned, mapped) for the device. - - Unregistered before the storage is freed, since glibc returns this size to the OS with - `munmap`. Not torch's pinned allocator, which rounds the 47.7 GiB table up to 64 GiB. + """A pageable CPU mapping of its own, registered (pinned, mapped) for the device. + + Unregistered before its pages are unmapped: freeing registered pages would leave the + driver mapping memory the OS may hand out again. The mapping is advised `MADV_HUGEPAGE` + before the registration faults it in, so the table sits on transparent huge pages whatever + torch's CPU allocator does (TensorRT LLM imports torch with mimalloc's THP advice off): on + GB300 the device's random row gathers from 64 KiB pages run 2.5x (64 tokens) to 23x + (1024 tokens) slower. Not torch's pinned allocator, which rounds the 47.7 GiB table up to + 64 GiB. """ def __init__(self, nbytes: int, device=None) -> None: @@ -173,7 +200,16 @@ def __init__(self, nbytes: int, device=None) -> None: if local: os.sched_setaffinity(0, local) try: - self._storage = torch.empty(nbytes, dtype=torch.uint8) + self._mapping = mmap.mmap(-1, nbytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) + try: + self._mapping.madvise(mmap.MADV_HUGEPAGE) + except OSError as error: # a kernel without transparent huge pages + logger.warning( + f"madvise(MADV_HUGEPAGE) failed for the host-resident PLE table ({error}); " + "the device's row gathers from its base pages run several times slower" + ) + # Shares the mapping's pages, and keeps the mapping alive while it lives. + self._storage = torch.frombuffer(self._mapping, dtype=torch.uint8) self.pointer = self._storage.data_ptr() CUASSERT( cudart.cudaHostRegister( diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py index 0c589671d855..7e1a158a3c1a 100644 --- a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py @@ -159,6 +159,14 @@ def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): runtime turns reuse off. No FP8_BLOCK_SCALES row names a KV-cache algorithm, so the FP8 pool is held to the bf16 pool's row. """ + if env.get("TRTLLM_MODELING_V2_MOE_ALLTOALL"): + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + NVLinkOneSided, + ) + + # The target declines the all-to-all where it cannot run it. + if not NVLinkOneSided.is_platform_supported(): + pytest.skip("no MNNVL on this node: the all-to-all MoE round trip cannot run") kv_cache_config = KvCacheConfig( dtype=kv_dtype, free_gpu_memory_fraction=0.6, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index daf8efa583b3..6587eec4cef2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -30,7 +30,9 @@ from tensorrt_llm.llmapi import QSASparseAttentionConfig from tensorrt_llm.llmapi.llm_args import ( KvCacheConfig, + MoeConfig, MTPDecodingConfig, + MultimodalConfig, NGramDecodingConfig, TorchLlmArgs, ) @@ -97,12 +99,29 @@ def on_sm103(monkeypatch): monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + for env in ( + "TRTLLM_USE_GDN_FLASHINFER_REPLAY", + M.MOE_ALLTOALL_ENV, + M.ROWTILE_SCRATCH_GROUPS_ENV, + W.MOE_TPB_ENV, + ): + monkeypatch.delenv(env, raising=False) def _llm_args(mode="auto", **kwargs) -> TorchLlmArgs: return TorchLlmArgs(model=_DUMMY_MODEL, modeling_v2=mode, **kwargs) +# The checkpoint's `mtp` block. +_MTP_BLOCK = dict( + hybrid=True, + layer_types=["full_attention"], + mtp_use_hidden_state_from_layer=None, + num_hidden_layers=1, + rope_theta=10000000, +) + + def _qwen4_exp_config(text_overrides=None, composite_overrides=None, **vision_overrides): """Qwen3.8-Flash-Next's normalized composite config (the FP8 and NVFP4 exports alike).""" text = dict( @@ -112,6 +131,22 @@ def _qwen4_exp_config(text_overrides=None, composite_overrides=None, **vision_ov num_attention_heads=24, linear_num_value_heads=48, vocab_size=248320, + # The settings the target pins (`modeling._TEXT_SETTINGS`), as the normalized config + # carries them: rope theta and partial factor flattened to the top level. + norm_topk_prob=True, + num_experts_per_tok=10, + hidden_act="silu", + num_key_value_heads=2, + head_dim=256, + attention_bias=False, + output_gate_type="sigmoid", + tie_word_embeddings=False, + rope_theta=10000000, + partial_rotary_factor=0.25, + rope_scaling=dict(type="mrope", mrope_section=[11, 11, 10], mrope_interleaved=True), + mtp=dict(_MTP_BLOCK), + mtp_num_hidden_layers=1, + mtp_use_dedicated_embeddings=False, ) text.update(text_overrides or {}) vision = dict( @@ -276,6 +311,41 @@ def test_the_engine_choices_the_target_runs_are_admitted( {}, "torch.float16", ), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(norm_topk_prob=False))), + {}, + "'norm_topk_prob'", + ), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(rope_theta=1e6))), + {}, + "'rope_theta'", + ), + ( + dict( + member="tep4", + pretrained_config=_qwen4_exp_config( + dict(mtp=dict(_MTP_BLOCK, layer_types=["linear_attention"])) + ), + ), + {}, + "'mtp.layer_types'", + ), + ( + dict(member="tep4"), + dict(max_seq_len=8192, kv_cache_config=KvCacheConfig(max_attention_window=[4096])), + "sliding window", + ), + ( + dict(member="tep4"), + dict(moe_config=MoeConfig(load_balancer={"num_slots": 512})), + r"load_balancer \(EPLB\)", + ), + ( + dict(member="tep4"), + dict(multimodal_config=MultimodalConfig(vision_gpu_normalize=True)), + "vision_gpu_normalize", + ), ], ids=[ "dep2-mtp", @@ -293,6 +363,12 @@ def test_the_engine_choices_the_target_runs_are_admitted( "tep8", "text-shape", "fp16-model", + "other-routing-setting", + "other-rope-theta", + "other-mtp-layer", + "sliding-window", + "eplb", + "vision-gpu-normalize", ], ) def test_what_the_target_does_not_run_declines(on_sm103, config_kwargs, llm_args_kwargs, match): @@ -315,6 +391,32 @@ def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypa monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 0)) with pytest.raises(ValueError, match=r"compute capability \(10, 0\)"): modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY") + monkeypatch.setenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY", "1") + with pytest.raises(ValueError, match="TRTLLM_USE_GDN_FLASHINFER_REPLAY"): + modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) + monkeypatch.delenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY") + # An all-to-all MoE the deployment cannot run: no attention DP, or no MNNVL. + monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "1") + with pytest.raises(ValueError, match=M.MOE_ALLTOALL_ENV): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) + with mock.patch( + "tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided.NVLinkOneSided" + ".is_platform_supported", + return_value=False, + ): + with pytest.raises(ValueError, match=M.MOE_ALLTOALL_ENV): + modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) + # A misspelled switch raises rather than routing as if it were off. + monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "enabled") + with pytest.raises(ValueError, match=f"{M.MOE_ALLTOALL_ENV}='enabled'"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) + monkeypatch.delenv(M.MOE_ALLTOALL_ENV) + monkeypatch.setenv(W.MOE_TPB_ENV, "1") + with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}='1': expected rr"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) + monkeypatch.delenv(W.MOE_TPB_ENV) @pytest.mark.cpu_only diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py index b62cb88f648e..aec661fdfbca 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -203,12 +203,14 @@ def _assembled_tower() -> V.VisionTower: def _request_media(modality, grids): - """A payload whose pixel tensor has the assembled tower's patch width and no storage.""" + """A payload whose pixel tensor has the assembled tower's patch width and dtype and no + storage.""" patches = sum(t * h * w for t, h, w in grids) key = "pixel_values" if modality == "image" else "pixel_values_videos" grid_key = "image_grid_thw" if modality == "image" else "video_grid_thw" + tower = _assembled_tower() return { - key: torch.empty(patches, _assembled_tower().pixel_width, device="meta"), + key: torch.empty(patches, tower.pixel_width, dtype=tower.dtype, device="meta"), grid_key: torch.tensor(grids), } @@ -665,10 +667,12 @@ def _grid(value): return lambda d: d["image"].update(image_grid_thw=torch.tensor(value)) -def _pixels(rows, extra=0): +def _pixels(rows, extra=0, dtype=None): def edit(d): - width = _assembled_tower().pixel_width + extra - d["video"]["pixel_values_videos"] = torch.empty(rows, width, device="meta") + tower = _assembled_tower() + d["video"]["pixel_values_videos"] = torch.empty( + rows, tower.pixel_width + extra, dtype=dtype or tower.dtype, device="meta" + ) return edit @@ -704,6 +708,11 @@ def _cumsum(d): ("grid_1d", _req(_grid([1, 4, 4]), image=_TILE), r"one \(t, h, w\) row per image"), ("pixel_rows", _req(_pixels(5), video=VIDEOS), "describe 48 patch rows"), ("pixel_width", _req(_pixels(48, 1), video=VIDEOS), "48 patch rows of"), + ("pixel_dtype", _req(_pixels(48, dtype=torch.uint8), video=VIDEOS), + "torch.uint8; the vision tower runs at torch.bfloat16"), + ("precomputed_embedding", + _req(lambda d: d.update(multimodal_embedding=[torch.zeros(4, 8)]), image=_TILE), + "precomputed image embeddings"), ("no_grid", _req(lambda d: d["video"].pop("video_grid_thw"), video=VIDEOS), "this target reads 'pixel_values_videos'"), ("positions_short", _req(_positions(3, 1, 40), video=VIDEOS), "one position per prompt token"), From 99c6bd255bd6213bbca10845a4c21963a202fe1a Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:49:16 -0800 Subject: [PATCH 09/51] [None][test] modeling_v2 qwen4_exp: list the FP8 target's accuracy tests in the QA function list Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../test_lists/qa/functional/llm_function_core.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_lists/qa/functional/llm_function_core.txt b/tests/integration/test_lists/qa/functional/llm_function_core.txt index 18bdb4c2d696..5b395375c2df 100644 --- a/tests/integration/test_lists/qa/functional/llm_function_core.txt +++ b/tests/integration/test_lists/qa/functional/llm_function_core.txt @@ -813,6 +813,13 @@ accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_ac accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_mmmu TIMEOUT (90) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2 TIMEOUT (90) +# Qwen3.8-Flash-Next-FP8, modeling_v2 qwen4_exp FP8 target (sm_103 only): every accuracy test of +# the target, the CI ones included. +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4_alltoall] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) + # Dense FP4 MLA accuracy accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_mla[unchunked] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_mla[chunked] From 8be84d8018511a6d8ac4b47675fd8d204a39d2d1 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:02:16 -0800 Subject: [PATCH 10/51] [None][fix] modeling_v2 qwen4_exp: stock TRTLLM_FORCE_COMM_METHOD, shared Lamport protocol and IpcMemory, OpWrappers for every FP8 target op, family op fixes Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../allReduceFusionKernels.cu | 95 +------- .../communicationKernels/lamportComm.cuh | 129 +++++++++++ .../ops/attention/qsa_paged_index_scores.py | 54 +++-- .../ops/hc/qwen4_exp_hc_inject_norm.py | 19 +- .../ops/hc/qwen4_exp_hc_lowrank_act.py | 23 +- .../models/qwen4_exp/ops/jit_variants.py | 29 ++- .../ops/moe/renorm_moe_routing_op.py | 2 +- .../ops/ple/ple_gate_conv_inject_norm.py | 40 ++-- .../qwen4_exp/ops/ple/ple_ngram_embed.py | 32 ++- .../ops/quantization/fp8_quantize_1x128.py | 2 +- .../csrc/hcAllreduceInjectNorm.cu | 101 +------- .../modeling.py | 122 ++++------ .../qsa_index_scores_cute__prefill.py | 74 +++++- .../qsa_index_scores_groups_cute__decode.py | 82 ++++++- .../qsa_sparse_gqa_rowtile__prefill.py | 7 +- .../attention/qsa_topk_rows_cute__prefill.py | 85 ++++++- .../qsa_triton_kernels__stable_jit.py | 138 ++++++++++- .../ops/comm/ce_exchange__sp_pair.py | 113 ++++++--- .../comm/hc_allreduce_inject_norm__tp_sum.py | 148 +++++++++--- .../comm/lamport_allgather__index_split.py | 179 +++++++------- .../ops/comm/moe_a2a__dp_dispatch_combine.py | 11 +- .../ops/moe/blockwise_gemm_grouped.py | 218 ++++++++++++++++++ .../ops/moe/moe_prefill_grouped__prefill.py | 212 +++++++---------- .../ops/ssm/gdn_inforward_tail.py | 93 +++++++- ...test_modeling_v2_qsa_paged_index_scores.py | 43 ++++ ...t_modeling_v2_ple_gate_conv_inject_norm.py | 7 +- .../ple/test_modeling_v2_ple_ngram_embed.py | 5 +- .../comm/_ce_exchange__sp_pair_op_matrix.py | 18 +- .../comm/_hc_ar_inject_norm_op_matrix.py | 11 +- ...amport_allgather__index_split_op_matrix.py | 21 +- .../test_attention_ops.py | 25 +- .../test_routing_and_layout.py | 27 ++- 32 files changed, 1502 insertions(+), 663 deletions(-) create mode 100644 cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu index a70e1b604ec2..69b84226c935 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/reduceKernelUtils.cuh" #include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" +#include "tensorrt_llm/kernels/communicationKernels/lamportComm.cuh" #include "tensorrt_llm/kernels/quantization.cuh" #include @@ -64,56 +65,6 @@ struct SyncComm int flag_value; }; -template -struct LamportComm -{ - __device__ __forceinline__ LamportComm(void** workspace, int rank) - { - counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; - flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; - // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and - // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar - // MiniMax kernel writes 4-byte words. - clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; - flag_value = *flag_ptr; - auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; - clear_size = *clear_ptr; - int data_offset = flag_value % 3; - int clear_offset = (flag_value + 2) % 3; - for (int r = 0; r < NRanks; ++r) - { - data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; - } - clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; - __syncthreads(); - if (threadIdx.x == 0) - { - atomicAdd(counter_ptr, 1); - } - } - - __device__ __forceinline__ void update(int64_t new_clear_size) - { - if (blockIdx.x == 0 && threadIdx.x == 0) - { - while (*reinterpret_cast(counter_ptr) != gridDim.x) - { - } - *flag_ptr = (flag_value + 1) % 3; - *clear_ptr = new_clear_size; - *counter_ptr = 0; - } - } - - int* counter_ptr; - int* flag_ptr; - int64_t* clear_ptr; - uint8_t* data_bufs[NRanks]; - uint8_t* clear_buf; - int64_t clear_size; - int flag_value; -}; - template class Barrier { @@ -345,36 +296,6 @@ private: float4 m_gamma_val; }; -__device__ __forceinline__ bool is_neg_zero(float v) -{ - return *reinterpret_cast(&v) == 0x80000000; -} - -__device__ __forceinline__ bool is_neg_zero(float4 v) -{ - return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); -} - -__device__ __forceinline__ float4 get_neg_zero() -{ - float4 vec; -#pragma unroll - for (int i = 0; i < 4; ++i) - { - reinterpret_cast(&vec)[i] = 0x80000000; - } - return vec; -} - -__device__ __forceinline__ float4 ld_global_volatile(float4* addr) -{ - float4 val; - asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" - : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) - : "l"(addr)); - return val; -} - template __device__ __forceinline__ float4 allreduce_sum(float4* vals) { @@ -595,20 +516,6 @@ __global__ void __launch_bounds__(1024) allreduce_fusion_kernel_twoshot_sync( #endif } -int get_sm_count() -{ - static int sm_count = 0; - if (sm_count == 0) - { - int device_id; - TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); - cudaDeviceProp device_prop; - cudaGetDeviceProperties(&device_prop, device_id); - sm_count = device_prop.multiProcessorCount; - } - return sm_count; -} - template void launch_oneshot_lamport(AllReduceFusionParams const& params, cudaLaunchConfig_t& cfg) { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh b/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh new file mode 100644 index 000000000000..adf9a943cfa9 --- /dev/null +++ b/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/cudaUtils.h" + +#include +#include + +TRTLLM_NAMESPACE_BEGIN + +namespace kernels::ar_fusion +{ +//! The one-shot Lamport protocol on the IPC all-reduce workspace (`allreduce_fusion_kernel_oneshot_lamport`), for +//! every kernel that runs on that workspace between its calls: workspace[2 * NRanks + r] is rank r's buffer of three +//! sub-buffers of workspace[3 * NRanks + 1][1] bytes; with flag = workspace[3 * NRanks][2], a call writes and polls +//! sub-buffer flag % 3 and clears its own (flag + 2) % 3 back to -0.0f. +template +struct LamportComm +{ + __device__ __forceinline__ LamportComm(void** workspace, int rank) + { + counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; + flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; + // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and + // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar + // MiniMax kernel writes 4-byte words. + clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; + flag_value = *flag_ptr; + auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; + clear_size = *clear_ptr; + int data_offset = flag_value % 3; + int clear_offset = (flag_value + 2) % 3; + for (int r = 0; r < NRanks; ++r) + { + data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; + } + clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; + __syncthreads(); + if (threadIdx.x == 0) + { + atomicAdd(counter_ptr, 1); + } + } + + __device__ __forceinline__ void update(int64_t new_clear_size) + { + if (blockIdx.x == 0 && threadIdx.x == 0) + { + while (*reinterpret_cast(counter_ptr) != gridDim.x) + { + } + *flag_ptr = (flag_value + 1) % 3; + *clear_ptr = new_clear_size; + *counter_ptr = 0; + } + } + + int* counter_ptr; + int* flag_ptr; + int64_t* clear_ptr; + uint8_t* data_bufs[NRanks]; + uint8_t* clear_buf; + int64_t clear_size; + int flag_value; +}; + +//! -0.0f is the not-arrived marker: a 16-byte slot has arrived once none of its words is -0.0f. +__device__ __forceinline__ bool is_neg_zero(float v) +{ + return *reinterpret_cast(&v) == 0x80000000; +} + +__device__ __forceinline__ bool is_neg_zero(float4 v) +{ + return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); +} + +__device__ __forceinline__ float4 get_neg_zero() +{ + float4 vec; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + reinterpret_cast(&vec)[i] = 0x80000000; + } + return vec; +} + +__device__ __forceinline__ float4 ld_global_volatile(float4* addr) +{ + float4 val; + asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" + : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) + : "l"(addr)); + return val; +} + +//! The current device's SM count, read once per process (per linked library). +inline int get_sm_count() +{ + static int sm_count = 0; + if (sm_count == 0) + { + int device_id; + TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); + cudaDeviceProp device_prop; + cudaGetDeviceProperties(&device_prop, device_id); + sm_count = device_prop.multiProcessorCount; + } + return sm_count; +} +} // namespace kernels::ar_fusion + +TRTLLM_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py index 5ecd5f289901..fca40bd492b2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py @@ -516,15 +516,15 @@ def _qsa_index_scores_prefill_query_tile( launcher with `context_rows` and `only_visible_blocks`: columns at and past a row's visible bound are unspecified.""" rows = q.shape[0] - if index_cache.ndim != 4 or index_cache.shape[2] != 1 or index_cache.shape[3] != q.shape[2]: - raise ValueError("QSA index cache must have shape [pages, tokens, 1, dim]") - if block_table.ndim != 2 or block_table.dtype != torch.int32: - raise ValueError("QSA index scoring requires a two-dimensional int32 block table") - if q.dtype != index_cache.dtype: - raise ValueError("QSA index query and cache dtypes must match") - if query_positions.shape != (rows,) or request_indices.shape != (rows,): - raise ValueError("QSA index-scoring metadata must have one entry per row") - max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + max_blocks = _check_index_scores_operands( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) output = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) grid = ( triton.cdiv(rows, _PREFILL_QUERY_TILE), @@ -566,7 +566,7 @@ def _qsa_index_scores_prefill_query_tile( return output -def _qsa_index_scores_request_groups( +def _check_index_scores_operands( q: torch.Tensor, index_cache: torch.Tensor, block_table: torch.Tensor, @@ -574,15 +574,14 @@ def _qsa_index_scores_request_groups( request_indices: torch.Tensor, tokens_per_block: int, compress_ratio: int, - use_pdl: bool, - rows_per_request: int, -) -> torch.Tensor: - """One launch of the request-group kernel after the callable's own checks.""" +) -> int: + """The engine launcher's operand checks for a `[rows, heads, dim]` query; returns the + output's compressed-block count.""" if index_cache.ndim != 4 or index_cache.shape[2] != 1: raise ValueError("QSA index cache must have shape [pages, tokens, 1, dim]") if block_table.ndim != 2 or block_table.dtype != torch.int32: raise ValueError("QSA index scoring requires a two-dimensional int32 block table") - rows, num_index_heads, index_head_dim = q.shape + rows, _, index_head_dim = q.shape if index_head_dim <= 0 or index_cache.shape[3] != index_head_dim: raise ValueError("QSA index query and cache dimensions do not match") if q.dtype != index_cache.dtype: @@ -599,6 +598,31 @@ def _qsa_index_scores_request_groups( max_compressed_blocks = block_table.shape[1] * tokens_per_block // compress_ratio if max_compressed_blocks <= 0: raise ValueError("QSA index scoring requires a non-empty block table") + return max_compressed_blocks + + +def _qsa_index_scores_request_groups( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, +) -> torch.Tensor: + """One launch of the request-group kernel after the callable's own checks.""" + rows, num_index_heads, index_head_dim = q.shape + max_compressed_blocks = _check_index_scores_operands( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) output = torch.empty((rows, max_compressed_blocks), dtype=torch.float32, device=q.device) if rows == 0: return output diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py index e7e1a88fae30..31598e6c1199 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py @@ -962,7 +962,12 @@ def __call__( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: + # `num_warps` (None: 8, or 2 from `_FEW_WARPS_MIN_ROWS` rows, at hidden >= 2048; else 4) + # and `pdl_wait` (launch with PDL and wait for the predecessor before any read) set the + # Triton launch; the finalize fold's CuTe path ignores both. rows, width = hyper_input.shape hidden = width // hc_count normed = torch.empty((rows, width), dtype=hyper_input.dtype, device=hyper_input.device) @@ -1040,7 +1045,11 @@ def __call__( STREAMS_PER_PROGRAM=spp, PDL_TRIGGER=_PDL_TRIGGER, ) - if block >= 2048: + if pdl_wait: + constexprs.update(PDL_WAIT=True, launch_pdl=True) + if num_warps is not None: + warps = num_warps + elif block >= 2048: # Both warp counts compile at a geometry's first call. warps = 2 if rows >= _FEW_WARPS_MIN_ROWS else 8 warm_variants( @@ -1054,6 +1063,7 @@ def __call__( has_shared, top_k, spp, + pdl_wait, ), args, [{"num_warps": 8}, {"num_warps": 2}], @@ -1076,11 +1086,14 @@ def reference( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: """float64 throughout. The finalize (with `topk_rows`) and the combine (with `shared`) are each rounded once to `block_out`'s dtype; `hc_out` is the float64 accumulate rounded once; `normed` is the float64 norm of - that rounded residual, rounded once. No other intermediate rounding.""" + that rounded residual, rounded once. No other intermediate rounding. + `num_warps` and `pdl_wait` (launch options) are ignored.""" dtype = hyper_input.dtype rows, width = hyper_input.shape hidden = width // hc_count @@ -1115,6 +1128,8 @@ def is_valid( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> None: # The kernel indexes with raw row strides and unit column stride, and # reads `inject` / `block_out` / `shared` / the gate per row: a broadcast diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py index c058e449497f..d8158f9212a7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py @@ -94,22 +94,20 @@ class Qwen4ExpHcLowrankAct(OpWrapper): why="default stack-final mixer (no inject): one decode token", spec=dict(rows=1, inject=False, logits_width=352), ), - # The FP8 unified target's HC mixers in its captured decode graph (batches 1..128). + # The FP8 unified target's HC mixers in its captured graphs at 641..3071 rows (e.g. the + # breakable prefill buckets): fewer rows run `qwen4_exp_hc_decode_tc_first`, more the + # rstd form. Cell( - why="FP8 unified captured HC mixer, batch 2: 352-wide logits", - spec=dict(rows=2, logits_width=352, capture=True), + why="FP8 unified captured prefill HC mixer, 641 rows: 352-wide logits", + spec=dict(rows=641, logits_width=352, capture=True), ), Cell( - why="FP8 unified captured HC mixer, batch 72: 352-wide logits", - spec=dict(rows=72, logits_width=352, capture=True), + why="FP8 unified captured prefill HC mixer, 1024 rows: 352-wide logits", + spec=dict(rows=1024, logits_width=352, capture=True), ), Cell( - why="FP8 unified captured HC mixer, batch 120: 352-wide logits", - spec=dict(rows=120, logits_width=352, capture=True), - ), - Cell( - why="FP8 unified captured HC mixer, batch 128 (the largest): 352-wide", - spec=dict(rows=128, logits_width=352, capture=True), + why="FP8 unified captured prefill HC mixer, 3071 rows (the largest): 352-wide", + spec=dict(rows=3071, logits_width=352, capture=True), ), ) @@ -134,7 +132,8 @@ class Qwen4ExpHcLowrankAct(OpWrapper): Certified: bf16 outputs from fp32 logits 352 wide (the call site) at 1, 32, 1024 and 8192 rows with the injection and at 1 row without it, and 384 wide - at 128 and 8192 rows (without the injection at 8192); graph replay. + at 128 and 8192 rows (without the injection at 8192); graph replay, and + captured at 641, 1024 and 3071 rows 352 wide. Against the float64 reference: bitwise at 1, 32 and 128 rows; at 1024 and 8192 rows at most 0.99 ulp on a few `low` elements, relative L2 <= 1.6e-5. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py index 3109d8cef5d3..47fb6841536f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py @@ -19,15 +19,24 @@ _WARMED: set = set() +def _warm_key(name: str, key: Hashable) -> tuple: + """`(name, key, device)`: Triton compiles per device, so each device warms on its own.""" + device = torch.cuda.current_device() if torch.cuda.is_available() else None + return name, key, device + + def first_time(name: str, key: Hashable) -> bool: - """True once per `(name, key)` in a process; False inside a CUDA-graph capture (the key - stays unmarked, so the next eager call warms it).""" - if (name, key) in _WARMED: - return False - if torch.cuda.is_current_stream_capturing(): + """True while `(name, key)` is not yet warmed on the current device; False inside a + CUDA-graph capture (the next eager call warms it). The caller marks the key with + `mark_warmed` once its compiles succeed, so a failed compile is retried.""" + if _warm_key(name, key) in _WARMED: return False - _WARMED.add((name, key)) - return True + return not torch.cuda.is_current_stream_capturing() + + +def mark_warmed(name: str, key: Hashable) -> None: + """Record `(name, key)` as compiled on the current device.""" + _WARMED.add(_warm_key(name, key)) def compile_only(kernel, args: Sequence[Any], **kwargs: Any) -> None: @@ -45,9 +54,11 @@ def warm_variants( variants: Iterable[Mapping[str, Any]], **kwargs: Any, ) -> None: - """Compile `kernel` once per `key` for every mapping of `variants` laid over `kwargs` (the - call's constexprs and launch options).""" + """Compile `kernel` once per `key` and device for every mapping of `variants` laid over + `kwargs` (the call's constexprs and launch options); the key is marked only after every + variant compiled.""" name = getattr(getattr(kernel, "fn", None), "__name__", None) or repr(kernel) if first_time(name, key): for variant in variants: compile_only(kernel, args, **{**kwargs, **variant}) + mark_warmed(name, key) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py index 1c42339b7a81..92e97f2c1030 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py @@ -90,7 +90,7 @@ class RenormMoeRoutingOp(OpWrapper): ), # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). Cell( - why="FP8 unified captured decode: 8 rows (batch 8; dep2 2 x 4, dep4 4 x 2)", + why="FP8 unified captured attention-DP decode: 8 rows (dep2 2 x 4, dep4 4 x 2)", spec=_ckpt(8, capture=True), ), Cell( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py index 76453c5f453d..0a8f3d8a93e3 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py @@ -34,6 +34,7 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( compile_only, first_time, + mark_warmed, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, @@ -712,29 +713,28 @@ def __call__( ) # `MAX_P` follows the step's longest sequence (0 past PER_SEQ_MAX_ROWS): every value, and # the context-shaped rows kernel, compile at a geometry's first call. - if first_time( - "ple_gate_conv_inject_norm", - ( - xc.dtype, - conv_states.dtype, - carry.dtype, - state_indices.dtype, - query_start_loc.dtype, - gated.stride(0), - conv_states.stride(0), - pad_slot_id, - ksize, - dilation, - state_len, - conv_rows, - has_carry, - log_s, - pdl, - ), - ): + warm_key = ( + xc.dtype, + conv_states.dtype, + carry.dtype, + state_indices.dtype, + query_start_loc.dtype, + gated.stride(0), + conv_states.stride(0), + pad_slot_id, + ksize, + dilation, + state_len, + conv_rows, + has_carry, + log_s, + pdl, + ) + if first_time("ple_gate_conv_inject_norm", warm_key): for max_p in range(PER_SEQ_MAX_ROWS + 1): compile_only(_ple_conv_kernel, conv_args, **{**conv_constexprs, "MAX_P": max_p}) compile_only(_ple_conv_rows_kernel, rows_args, **rows_constexprs) + mark_warmed("ple_gate_conv_inject_norm", warm_key) if not per_seq: _ple_conv_rows_kernel[(triton.cdiv(rows, _ROWS_BLOCK_T), width // tile_ch)]( *rows_args, **rows_constexprs diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py index d84606cac32f..714b8ecabc1a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py @@ -31,7 +31,12 @@ import triton.language as tl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ngram_hash +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ( + ngram_hash, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + warm_variants, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, TEP4_SLOTS, @@ -52,7 +57,9 @@ #: Above this many rows per sequence (MTP verify carries at most 4) the call #: takes the context-shaped path: packed row tiles plus a separate publish. PER_SEQ_MAX_ROWS = 4 -_PER_SEQ_BLOCK_P = triton.next_power_of_2(PER_SEQ_MAX_ROWS) +#: `BLOCK_P` values a per-sequence call can pick (`next_power_of_2` of 1..PER_SEQ_MAX_ROWS), +#: all compiled at a geometry's first call. +_PER_SEQ_BLOCK_PS = (1, 2, 4) _ROWS_BLOCK_T = 32 #: The row-to-sequence search covers at least 2^8 sequences, so one compiled #: variant serves every batch up to that size. @@ -509,7 +516,7 @@ def __call__( num_warps=1, ) elif num_seqs > 0: - _ple_ngram_embed_kernel[(num_seqs,)]( + embed_args = ( input_ids, context_pool, state_indices, @@ -525,18 +532,33 @@ def __call__( table.stride(0), out.stride(0), eos_token_id, + ) + embed_constexprs = dict( NGRAM=ngram_size, HEADS_PER=heads // (ngram_size - 1), HEADS=heads, BLOCK_H=triton.next_power_of_2(heads), HEAD_DIM=head_dim, BLOCK_D=triton.next_power_of_2(head_dim), - # One compiled variant for every per-sequence row count (rows past `length` masked). - BLOCK_P=_PER_SEQ_BLOCK_P, + BLOCK_P=triton.next_power_of_2(max(max_rows, 1)), BLOCK_C=triton.next_power_of_2(ngram_size - 1), HAS_CARRY=has_initial_states is not None, num_warps=4, ) + # `BLOCK_P` follows the step's longest sequence: every value compiles at a + # geometry's first call, so a served step never compiles. + warm_variants( + _ple_ngram_embed_kernel, + ( + tuple(t.dtype for t in embed_args[:11]), + embed_args[11:], + tuple(embed_constexprs[k] for k in ("NGRAM", "HEADS", "HEAD_DIM", "HAS_CARRY")), + ), + embed_args, + [{"BLOCK_P": p} for p in _PER_SEQ_BLOCK_PS], + **embed_constexprs, + ) + _ple_ngram_embed_kernel[(num_seqs,)](*embed_args, **embed_constexprs) return out def reference( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py index b3fd4b0e0123..e3ac3a0103c7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py @@ -95,7 +95,7 @@ class Fp8Quantize1x128(OpWrapper): ), # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). Cell( - why="FP8 unified captured decode: 8 rows (batch 8; dep2 2 x 4, dep4 4 x 2)", + why="FP8 unified captured attention-DP decode: 8 rows (dep2 2 x 4, dep4 4 x 2)", spec=dict(tokens=8, **_CKPT, capture=True), ), Cell( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu index 34d0e9cf4bf4..81e83bdfc98c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/kernels/communicationKernels/lamportComm.cuh" #include "tensorrt_llm/modeling_v2/common/namespace.h" #include @@ -33,105 +34,19 @@ namespace kernels namespace hc_inject { // This kernel runs on the workspace of the stock one-shot all-reduce `allreduce_fusion_kernel_oneshot_lamport` -// (communicationKernels/allReduceFusionKernels.cu) between its calls, so LamportComm, the helpers below and the -// push / clear / poll / update steps must match that file's Lamport protocol: +// (communicationKernels/allReduceFusionKernels.cu) between its calls, so it takes that file's LamportComm and helpers +// (communicationKernels/lamportComm.cuh) and its push / clear / poll / update steps match that kernel's: // - workspace[2 * NRanks + r] is rank r's buffer of three sub-buffers of workspace[3 * NRanks + 1][1] bytes; with // flag = workspace[3 * NRanks][2], a call writes and polls sub-buffer flag % 3 and clears its own (flag + 2) % 3. // - workspace[3 * NRanks + 1][0] holds the bytes the previous call wrote, cleared as 16-byte stores of -0.0f. // - A 16-byte slot has arrived once none of its 32-bit words is -0.0f, so senders store -0.0f as +0.0f. // - Every CTA increments workspace[3 * NRanks][0] after reading the flag; once all have, CTA 0 advances the flag to // (flag + 1) % 3, stores the bytes this call wrote and resets the counter. -template -struct LamportComm -{ - __device__ __forceinline__ LamportComm(void** workspace, int rank) - { - counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; - flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; - // In bytes whatever the writer's dtype; readers round up to 16-byte stores (some writers store 4-byte words). - clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; - flag_value = *flag_ptr; - auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; - clear_size = *clear_ptr; - int data_offset = flag_value % 3; - int clear_offset = (flag_value + 2) % 3; - for (int r = 0; r < NRanks; ++r) - { - data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; - } - clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; - __syncthreads(); - if (threadIdx.x == 0) - { - atomicAdd(counter_ptr, 1); - } - } - - __device__ __forceinline__ void update(int64_t new_clear_size) - { - if (blockIdx.x == 0 && threadIdx.x == 0) - { - while (*reinterpret_cast(counter_ptr) != gridDim.x) - { - } - *flag_ptr = (flag_value + 1) % 3; - *clear_ptr = new_clear_size; - *counter_ptr = 0; - } - } - - int* counter_ptr; - int* flag_ptr; - int64_t* clear_ptr; - uint8_t* data_bufs[NRanks]; - uint8_t* clear_buf; - int64_t clear_size; - int flag_value; -}; - -__device__ __forceinline__ bool is_neg_zero(float v) -{ - return *reinterpret_cast(&v) == 0x80000000; -} - -__device__ __forceinline__ bool is_neg_zero(float4 v) -{ - return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); -} - -__device__ __forceinline__ float4 get_neg_zero() -{ - float4 vec; -#pragma unroll - for (int i = 0; i < 4; ++i) - { - reinterpret_cast(&vec)[i] = 0x80000000; - } - return vec; -} - -__device__ __forceinline__ float4 ld_global_volatile(float4* addr) -{ - float4 val; - asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" - : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) - : "l"(addr)); - return val; -} - -int get_sm_count() -{ - static int sm_count = 0; - if (sm_count == 0) - { - int device_id; - TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); - cudaDeviceProp device_prop; - cudaGetDeviceProperties(&device_prop, device_id); - sm_count = device_prop.multiProcessorCount; - } - return sm_count; -} +using ::tensorrt_llm::kernels::ar_fusion::get_neg_zero; +using ::tensorrt_llm::kernels::ar_fusion::get_sm_count; +using ::tensorrt_llm::kernels::ar_fusion::is_neg_zero; +using ::tensorrt_llm::kernels::ar_fusion::LamportComm; +using ::tensorrt_llm::kernels::ar_fusion::ld_global_volatile; } // namespace hc_inject // Thread t holds columns [8t, 8t + 8) of a row's block output and of each of its 4 streams. The Triton program runs a diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 39a19d077625..4a6501df0123 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -24,7 +24,8 @@ 128-row FP8 weight block. Under attention TP the plain sum completes the window partials. Under attention DP each rank's rows are gathered (`catalog/comm/allgather`, ragged when the counts differ), run through the window and reduce-scattered back - (`catalog/comm/reducescatter`; `_expert_sum_op`, or an all-to-all with `MOE_ALLTOALL_ENV`). + (`catalog/comm/reducescatter`; `_expert_sum_op`, or an all-to-all with + `MOE_COMM_METHOD_ENV=NVLINK_ONE_SIDED`). * **Replicated:** the router, the bf16 shared expert, the hyper-connections, PLE and the embedding run on this rank's rows. `lm_head` is the shell's: vocab-parallel under attention TP, whole under attention DP and at tp1. @@ -88,10 +89,10 @@ import contextlib import math +import os from typing import TYPE_CHECKING, NamedTuple import torch -import triton from torch import nn from transformers import PretrainedConfig @@ -186,7 +187,6 @@ _PDL_TRIGGER as _HC_INJECT_PDL_TRIGGER, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( - _hc_inject_norm_kernel, qwen4_exp_hc_inject_norm, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_rstd import ( @@ -245,7 +245,6 @@ from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.distributed.allreduce_helper import CustomAllReduceHelper -from tensorrt_llm._torch.distributed.communicator import Distributed from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.model_config import ModelConfig @@ -683,11 +682,20 @@ def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, #: (`_rowtile_scratch_groups`) instead of the per-row kernel. Off by default because such steps' #: outputs then move by rounding. Read once, when the target is built. ROWTILE_SCRATCH_GROUPS_ENV = "TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS" -#: Opt-in switch (`weights.switch`): under attention DP, the MoE round trip runs as an all-to-all -#: dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one. Needs -#: MPI, attention DP and MNNVL; `check_deployment` declines it elsewhere. Read once, when the -#: target is built. -MOE_ALLTOALL_ENV = "TRTLLM_MODELING_V2_MOE_ALLTOALL" +#: The stock MoE communication override (`communication_factory`), compared upper-cased like the +#: stock factory. Under attention DP, `NVLINK_ONE_SIDED` runs the MoE round trip as an all-to-all +#: dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one; it +#: needs MPI, attention DP and MNNVL. Unset or `ALLGATHER` keeps the all-gather / reduce-scatter +#: round trip. `check_deployment` declines any other value, and the all-to-all where it cannot +#: run. Read once, when the target is built. +MOE_COMM_METHOD_ENV = "TRTLLM_FORCE_COMM_METHOD" + + +def _moe_comm_method() -> str: + """`MOE_COMM_METHOD_ENV` upper-cased, `ALLGATHER` when unset.""" + return os.environ.get(MOE_COMM_METHOD_ENV, "ALLGATHER").upper() + + # Rows per FP8 runner call over an attention-DP group's gathered rows (`_routed_experts`): one # call reads the window's weights once, bitwise the `_MOE_MAX_ROWS`-row calls; the router, its # routing and the activation quantization still run per `_MOE_MAX_ROWS` rows. @@ -1319,8 +1327,9 @@ def _expert_sum_op(self): `run(rows)` applies this rank's expert window; `padding` comes from `_dp_pad_rows`. Attention TP: every rank holds every row and `tp_sum` completes the partial. Attention DP: all-gather the group's rows in rank order, run the window, reduce-scatter; routing - gathered rows keeps every rank's routing of a row identical. `MOE_ALLTOALL_ENV` (MPI and - MNNVL) swaps in the NVLink all-to-all, with `run` a `_WindowRun`. Unequal per-rank + gathered rows keeps every rank's routing of a row identical. + `MOE_COMM_METHOD_ENV=NVLINK_ONE_SIDED` (MPI and MNNVL) swaps in the NVLink all-to-all, + with `run` a `_WindowRun`. Unequal per-rank counts (`padding` set) use the ragged collectives so the window never sees padding rows, which would all route to the same experts and stall the group on one window.""" if self.dp_size == 1: @@ -1339,14 +1348,15 @@ def round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor gathered = allgather(x, sizes, group, process_group) return reducescatter(run(gathered), sizes, group, process_group) - if not _weights.switch(MOE_ALLTOALL_ENV): + if _moe_comm_method() != "NVLINK_ONE_SIDED": return round_trip strategy = None if mpi_disabled() else _build_moe_a2a(self.mapping, self.max_num_tokens) if strategy is None: # `check_deployment` declines this deployment; reached only when routing is bypassed. raise RuntimeError( - f"{MOE_ALLTOALL_ENV}=1 needs MPI and MNNVL, which this deployment lacks; unset it " - "to run the attention-DP MoE as the all-gather / reduce-scatter round trip" + f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED needs MPI and MNNVL, which this " + "deployment lacks; unset it to run the attention-DP MoE as the all-gather / " + "reduce-scatter round trip" ) self.moe_a2a = strategy @@ -2743,51 +2753,23 @@ def _hc_mixer(core, normed, first, up, with_inject): def _hc_inject_norm_launch( hc, block_out, inject, norm, eps, hc_count, shared, gate_logits, num_warps ): - """Launch `qwen4_exp_hc_inject_norm` (no finalize fold) at `num_warps` warps with PDL. + """Run `qwen4_exp_hc_inject_norm` (no finalize fold) at `num_warps` warps with PDL. The residual is bitwise at any warp count; the normed stream varies only with the fp32 sum-of-squares order. Programs read nothing before `griddepcontrol.wait`. Returns `(hc_out, normed)`; `hc_out` is `hc` itself without `block_out`.""" - rows, width = hc.shape - hidden = width // hc_count - has_inject = block_out is not None - has_shared = shared is not None - normed = torch.empty((rows, width), dtype=hc.dtype, device=hc.device) - hc_out = torch.empty((rows, width), dtype=hc.dtype, device=hc.device) if has_inject else hc - _hc_inject_norm_kernel[(rows * hc_count,)]( - hc, - block_out if has_inject else hc, - inject if has_inject else hc, - shared if has_shared else hc, - gate_logits if has_shared else hc, - hc, + return qwen4_exp_hc_inject_norm( hc, + block_out, + inject, norm, - hc_out, - normed, - hc.stride(0), - block_out.stride(0) if has_inject else 0, - inject.stride(0) if has_inject else 0, - shared.stride(0) if has_shared else 0, - gate_logits.stride(0) if has_shared else 0, - 0, - 0, - hc_out.stride(0), - normed.stride(0), eps, - HC_COUNT=hc_count, - HIDDEN=hidden, - BLOCK=triton.next_power_of_2(hidden), - HAS_INJECT=has_inject, - HAS_SHARED=has_shared, - TOPK=0, - STREAMS_PER_PROGRAM=1, - PDL_TRIGGER=_HC_INJECT_PDL_TRIGGER, - PDL_WAIT=_HC_INJECT_PDL_WAIT, + hc_count, + shared, + gate_logits, num_warps=num_warps, - launch_pdl=_HC_INJECT_PDL_WAIT, + pdl_wait=_HC_INJECT_PDL_WAIT, ) - return hc_out, normed def _hc_mixer_rstd(core, hc, rstd, norm, first, up, with_inject): @@ -3266,12 +3248,7 @@ def _index_split_lamport(core): return None width = core.token_topk + core.compress_ratio - 1 rows = min(_INDEX_SPLIT_LAMPORT_ROWS, core.max_num_tokens) - return _selection_lamport_build( - core.mapping.tp_rank, - core.attn_tp_size, - rows * width, - Distributed.get(core.mapping).tp_allgather, - ) + return _selection_lamport_build(core.mapping, rows * width) def _index_split_rows(split, rows: int, rows_per_seq: int) -> tuple | None: @@ -5521,18 +5498,15 @@ def mtp_forward( device=qkv.device, ) for lo, hi, context_rows in parts: - scores = qsa_paged_index_scores( - q=q_index[lo:hi], - index_cache=draft.index_cache, - block_table=block_table, - query_positions=logical[lo:hi], - request_indices=req_idx[lo:hi], - tokens_per_block=tokens_per_block, - compress_ratio=core.compress_ratio, - only_visible_blocks=True, - context_rows=context_rows, - prefill_query_tile=True, - wide_split=True, + scores = _context_index_scores( + q_index[lo:hi], + draft.index_cache, + block_table, + logical[lo:hi], + req_idx[lo:hi], + tokens_per_block, + core.compress_ratio, + context_rows, ) _selection( core, @@ -6750,7 +6724,13 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: mapping = model_config.mapping _weights.moe_tpb_requested() _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) - if _weights.switch(MOE_ALLTOALL_ENV): + comm_method = _moe_comm_method() + if comm_method not in ("ALLGATHER", "NVLINK_ONE_SIDED"): + raise NoTarget( + f"{MOE_COMM_METHOD_ENV}={comm_method} at {layout}: the target runs the MoE round " + "trip as ALLGATHER (the default) or NVLINK_ONE_SIDED" + ) + if comm_method == "NVLINK_ONE_SIDED": from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( NVLinkOneSided, ) @@ -6761,8 +6741,8 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: or not NVLinkOneSided.is_platform_supported() ): raise NoTarget( - f"{MOE_ALLTOALL_ENV}=1 at {layout}: the all-to-all MoE round trip needs MPI, " - "attention DP and MNNVL on this node" + f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at {layout}: the all-to-all MoE " + "round trip needs MPI, attention DP and MNNVL on this node" ) spec = model_config.spec_config if spec is not None: diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py index 537c08d55af5..a78b664f0edb 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py @@ -24,6 +24,11 @@ from cutlass.cute.typing import Float32, Int32, Int64 from cutlass.cutlass_dsl import dsl_user_op +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + qsa_paged_index_scores as _family_index_scores, +) + try: from cutlass.memory import SmemAllocator except ImportError: # DSL releases before the memory namespace split @@ -897,7 +902,7 @@ def supports(q, index_cache, tokens_per_block, compress_ratio) -> bool: ) -def qsa_index_scores_prefill( +def _qsa_index_scores_prefill( q, index_cache, block_table, @@ -944,3 +949,70 @@ def qsa_index_scores_prefill( max(1, min(items, sms)), ) return out + + +class QsaIndexScoresPrefill(OpWrapper): + """The family's `qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=True)` + on context rows `supports` admits: `[rows, max_blocks]` fp32, the same bits over each row's + visible columns, the columns at and past them unspecified.""" + + def __call__( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> torch.Tensor: + return _qsa_index_scores_prefill( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) + + def reference( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> torch.Tensor: + """The family entry's reference over context rows: NaN past each row's visible columns.""" + return _family_index_scores.reference( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + only_visible_blocks=True, + context_rows=True, + ) + + def is_valid( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> None: + # The kernel is built for this geometry and checks none of it. + assert supports(q, index_cache, tokens_per_block, compress_ratio), ( + "a call outside `supports`" + ) + + +qsa_index_scores_prefill = QsaIndexScoresPrefill() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py index b801c168a6c6..125124bd1bf2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py @@ -25,6 +25,10 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + qsa_paged_index_scores as _family_index_scores, +) from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from .qsa_sparse_gqa_splitk_cute__decode import ( @@ -359,7 +363,7 @@ def serves(q: torch.Tensor, index_cache: torch.Tensor, rows_per_request: int) -> ) -def index_scores_groups( +def _index_scores_groups( q: torch.Tensor, index_cache: torch.Tensor, block_table: torch.Tensor, @@ -430,3 +434,79 @@ def index_scores_groups( col_progs, ) return out + + +class QsaIndexScoresGroups(OpWrapper): + """The family's `qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=False, + rows_per_request=...)` on verify rows `serves` admits: `[rows, max_blocks]` fp32 to the last + bits over each row's visible columns, the columns at and past them unspecified.""" + + def __call__( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> torch.Tensor: + return _index_scores_groups( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + use_pdl, + rows_per_request, + ) + + def reference( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> torch.Tensor: + """The family entry's reference over verify rows: NaN past each row's visible columns. + `use_pdl` picks a launch, not a result.""" + return _family_index_scores.reference( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + only_visible_blocks=True, + context_rows=False, + use_pdl=use_pdl, + rows_per_request=rows_per_request, + ) + + def is_valid( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> None: + # The kernel is built for this geometry and checks none of it. + assert serves(q, index_cache, rows_per_request), "a call outside `serves`" + + +index_scores_groups = QsaIndexScoresGroups() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py index e0b6167e5b41..fc88da9ec6a5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py @@ -1637,6 +1637,11 @@ class QsaSparseGqaRowtile(OpWrapper): "groups), [128, 8196] table, trunk pool straddling 2^31", spec=_spec([(120000, 8192)], _TP2, shared_scores=True), ), + Cell( + why="tp4/tep4 engine position dtype: int64 query positions with int32 request " + "indices, 2049 rows from 9000, 6 q / 1 kv heads", + spec=_spec([(9000, 2049)], _TP4, position_dtype=torch.int64), + ), ) note = """ @@ -1656,7 +1661,7 @@ class QsaSparseGqaRowtile(OpWrapper): Calls allocate their plan scratch, so CUDA-graph capture is not certified (the prefill path runs eager). Deterministic. - Not certified: int64 request indices or positions; rows whose selection resolves no token + Not certified: int64 request indices; rows whose selection resolves no token (all padding, a foreign request, unmapped pages); `rows == 0`. """ diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py index bdf8fb865d68..63f4819d6bcf 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py @@ -18,6 +18,10 @@ from cutlass import const_expr from cutlass.cute.runtime import from_dlpack +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + qsa_topk_expand_decode as _family_topk, +) from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr @@ -534,7 +538,7 @@ def supports(scores, topk_indices) -> bool: ) -def qsa_topk_rows( +def _qsa_topk_rows( scores, row_ends, query_positions, @@ -610,3 +614,82 @@ def qsa_topk_rows( ) fn(keys, *ptrs, rows) return sel + + +class QsaTopkRows(OpWrapper): + """The family's `qsa_topk_expand_decode` on tables `supports` admits, the same bits: writes + `topk_indices` and returns the token selection `[rows, token_topk + compress_ratio - 1]` + int32 (`out` when given).""" + + def __call__( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + return _qsa_topk_rows( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out, + ) + + def reference( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """The family entry's reference: `(topk_indices_after, selection)`, fresh tensors.""" + return _family_topk.reference( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + def is_valid( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> None: + # The kernel is built for this width and checks none of it; past that, the family + # entry's contract and its silent failures are this kernel's. + assert supports(scores, topk_indices), "a call outside `supports`" + _family_topk.is_valid( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + +qsa_topk_rows = QsaTopkRows() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py index 182ddcdc831b..96d8198eb3c7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py @@ -19,9 +19,15 @@ import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( compile_only, first_time, + mark_warmed, +) +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_paged_kv_store, + triton_qsa_paged_sparse_gqa, ) from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber from tensorrt_llm._utils import get_sm_version @@ -166,7 +172,7 @@ def _qsa_kv_store_stable_kernel( ) -def qsa_paged_kv_store( +def _qsa_paged_kv_store( *, k: torch.Tensor, v: torch.Tensor, @@ -258,6 +264,65 @@ def qsa_paged_kv_store( ) +class QsaPagedKvStoreStable(OpWrapper): + """`triton_qsa_paged_kv_store` on `_qsa_kv_store_stable_kernel`: every row's K and V into + its request's page of the HND pool, in place. The entry `qsa_paged_kv_store` (its own + module) calls this and carries the cells.""" + + def __call__( + self, + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> None: + _qsa_paged_kv_store( + k=k, + v=v, + k_cache=k_cache, + v_cache=v_cache, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + + def reference( + self, + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(k_cache, v_cache)` after the shared launcher's store into copies (the same bits); + the inputs are not touched.""" + expected_k, expected_v = k_cache.clone(), v_cache.clone() + triton_qsa_paged_kv_store( + k=k, + v=v, + k_cache=expected_k, + v_cache=expected_v, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + return expected_k, expected_v + + +qsa_paged_kv_store = QsaPagedKvStoreStable() + + @triton.jit( do_not_specialize=[ "num_cache_pages", @@ -709,7 +774,7 @@ def can_fuse_splitk_output_gate( return num_splits > 1 -def qsa_paged_sparse_gqa( +def _qsa_paged_sparse_gqa( *, q: torch.Tensor, k_cache: torch.Tensor, @@ -878,6 +943,74 @@ def qsa_paged_sparse_gqa( return output +class QsaPagedSparseGqaStable(OpWrapper): + """`triton_qsa_paged_sparse_gqa` on this module's prefill-pair, split-K and merge kernels: + `[rows, Hq, D]` sparse GQA over each row's selected tokens. The entry `qsa_paged_sparse_gqa` + (its own module) calls this and carries the cells.""" + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, + ) -> torch.Tensor: + return _qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, + ) -> torch.Tensor: + """The shared launcher's output on the same arguments (the same bits).""" + return triton_qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + +qsa_paged_sparse_gqa = QsaPagedSparseGqaStable() + + def _pair_args( q, k_cache, @@ -1184,6 +1317,7 @@ def _warm_decode_buckets( ), **_pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio), ) + mark_warmed("qsa_stable_decode_buckets", key) def warm_decode_buckets( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py index e0a347d6462c..799e4460dad1 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py @@ -21,10 +21,18 @@ from __future__ import annotations +from typing import Sequence + import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm.logger import logger + #: Bytes kept for the two flags after the data region (one 128-byte line each). _FLAG_BYTES = 256 +HIDDEN = 2560 +#: Words of one QSA selection row: the target's `token_topk + compress_ratio - 1`. +_SELECTION_WIDTH = 2051 class _CudaArray: @@ -41,9 +49,10 @@ def __init__(self, ptr: int, nbytes: int): class CopyEngineExchange: - """`exchange(x)` -> the peer's `x` (a view of this rank's receive buffer, same shape and - dtype). `capacity` bytes a call at most. Built on the attention-TP group of a two-rank - `mapping` (IPC memory inside a node); `create` returns None where that does not hold.""" + """The pair's buffers and flags: `exchange(x)` -> the peer's `x` (`ce_exchange`, a view of + this rank's receive buffer, same shape and dtype). `capacity` bytes a call at most. Built on + the attention-TP group of a two-rank `mapping` (IPC memory inside a node); `create` returns + None where that does not hold.""" def __init__(self, mapping, capacity: int): from cuda.bindings import driver, runtime @@ -73,50 +82,88 @@ def create(cls, mapping, capacity: int): try: exchange = cls(mapping, capacity) - except Exception: # noqa: BLE001 -- keep NCCL where the copy engines cannot be reached + except (RuntimeError, OSError, MemoryError) as error: + logger.warning(f"copy-engine exchange unavailable ({error}); the pair runs NCCL") exchange = None if not all(Distributed.get(mapping).tp_allgather(exchange is not None)): return None return exchange - def _write(self, stream, addr: int, value: int) -> None: - d = self._driver - (err,) = d.cuStreamWriteValue32( - stream, - d.CUdeviceptr(addr), - value, - d.CUstreamWriteValue_flags.CU_STREAM_WRITE_VALUE_DEFAULT, + def exchange(self, x: torch.Tensor) -> torch.Tensor: + """`ce_exchange(self, x)`.""" + return ce_exchange(self, x) + + +def _write(driver, stream, addr: int, value: int) -> None: + (err,) = driver.cuStreamWriteValue32( + stream, + driver.CUdeviceptr(addr), + value, + driver.CUstreamWriteValue_flags.CU_STREAM_WRITE_VALUE_DEFAULT, + ) + assert err == driver.CUresult.CUDA_SUCCESS, err + + +def _wait(driver, stream, addr: int, value: int) -> None: + (err,) = driver.cuStreamWaitValue32( + stream, + driver.CUdeviceptr(addr), + value, + driver.CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ, + ) + assert err == driver.CUresult.CUDA_SUCCESS, err + + +class CeExchange(OpWrapper): + """The peer's `x` over `pair` (a `CopyEngineExchange`), by the module docstring's protocol on + the caller's stream: a view of this rank's receive buffer, same shape and dtype, valid until + the next exchange. Both ranks call it identically.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell(why=f"sp2: {why}", spec=dict(world=2, rows=rows, width=width, dtype=dtype)) + for dtype, width, what in ( + (torch.bfloat16, HIDDEN, "bf16 hidden rows"), + (torch.int32, _SELECTION_WIDTH, "int32 QSA selection rows"), ) - assert err == d.CUresult.CUDA_SUCCESS, err - - def _wait(self, stream, addr: int, value: int) -> None: - d = self._driver - (err,) = d.cuStreamWaitValue32( - stream, - d.CUdeviceptr(addr), - value, - d.CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ, + for rows, why in ( + (8192, f"half of a 16384-token step's {what}, the buffer's largest exchange"), + (4096, f"half of an 8192-token step's {what}, also replayed from a CUDA graph chain"), + (1024, f"half of a 2048-token chunk's {what}"), + (77, f"an odd row count of {what}"), ) - assert err == d.CUresult.CUDA_SUCCESS, err + ) - def exchange(self, x: torch.Tensor) -> torch.Tensor: + note = """ + Certified byte-exact against the peer's input, eagerly and replayed from a CUDA graph chain + with new inputs. Not certified: pairs spanning nodes, groups other than two ranks. + """ + + def __call__(self, pair: CopyEngineExchange, x: torch.Tensor) -> torch.Tensor: assert x.is_contiguous() and x.is_cuda nbytes = x.numel() * x.element_size() - assert nbytes <= self.capacity, (nbytes, self.capacity) - stream = self._driver.CUstream(torch.cuda.current_stream().cuda_stream) - r = self._runtime - self._write(stream, self._peer_ready, 1) - self._wait(stream, self._my_ready, 1) - self._write(stream, self._my_ready, 0) + assert nbytes <= pair.capacity, (nbytes, pair.capacity) + d, r = pair._driver, pair._runtime + stream = d.CUstream(torch.cuda.current_stream().cuda_stream) + _write(d, stream, pair._peer_ready, 1) + _wait(d, stream, pair._my_ready, 1) + _write(d, stream, pair._my_ready, 0) (err,) = r.cudaMemcpyAsync( - self._peer, + pair._peer, x.data_ptr(), nbytes, r.cudaMemcpyKind.cudaMemcpyDefault, r.cudaStream_t(torch.cuda.current_stream().cuda_stream), ) assert err == r.cudaError_t.cudaSuccess, err - self._write(stream, self._peer_data, 1) - self._wait(stream, self._my_data, 1) - self._write(stream, self._my_data, 0) - return self._recv[:nbytes].view(x.dtype).view(x.shape) + _write(d, stream, pair._peer_data, 1) + _wait(d, stream, pair._my_data, 1) + _write(d, stream, pair._my_data, 0) + return pair._recv[:nbytes].view(x.dtype).view(x.shape) + + def reference(self, xs_in_rank_order: Sequence[torch.Tensor], rank: int) -> torch.Tensor: + """The other rank's `x`. The documented collective exception to mirroring `__call__`.""" + return xs_in_rank_order[1 - rank] + + +ce_exchange = CeExchange() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py index 62d3e1040a5a..980dbd033db9 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py @@ -14,13 +14,38 @@ from __future__ import annotations +from typing import Sequence + import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, +) + from ... import native_ops HIDDEN = 2560 HC_COUNT = 4 +#: Decode-site row counts certified at 2 and 4 ranks, each with and without the MoE combine, +#: and why each is there. +_ROWS = ( + (33, "the smallest fused site (`_HC_AR_INJECT_MIN_ROWS`)"), + (96, "a decode batch on 4-warp programs"), + (168, "a decode batch on 4-warp programs"), + (192, "a decode batch on 4-warp programs"), + (216, "a decode batch on 4-warp programs"), + (288, "a decode batch on 4-warp programs"), + (384, "a decode batch on 4-warp programs"), + (480, "a decode batch on 4-warp programs"), + (511, "the last row count on 4-warp programs"), + (512, "the first row count on 2-warp programs"), + (576, "a decode batch on 2-warp programs"), + (640, "a decode batch on 2-warp programs"), + (1024, "a decode batch on 2-warp programs"), +) + def _aligned(t: torch.Tensor) -> bool: return t.data_ptr() % 16 == 0 and t.stride(-1) == 1 and t.stride(0) % 8 == 0 @@ -45,35 +70,100 @@ def fits(hc, partial, inject, shared=None, gate=None) -> bool: return dtypes == {torch.bfloat16} -def hc_allreduce_inject_norm( - partial: torch.Tensor, - workspace: torch.Tensor, - rank: int, - nranks: int, - hc: torch.Tensor, - inject: torch.Tensor, - norm: torch.Tensor, - eps: float, - shared: torch.Tensor | None = None, - gate: torch.Tensor | None = None, - program_warps: int = 4, -) -> tuple[torch.Tensor, torch.Tensor]: +class HcAllreduceInjectNorm(OpWrapper): """`(hc_out, normed)` of the site after a block whose per-rank output is `partial`: bitwise `_hc_inject_norm_launch(hc, tp_sum(partial), inject, norm, eps, 4, shared, gate, - program_warps)`.""" - hc_out, normed = native_ops.hc_allreduce_inject_norm( - partial, - workspace, - int(rank), - int(nranks), - hc, - inject, - shared, - gate, - norm, - float(eps), - int(program_warps), - # trigger_completion_at_end=False: release the next launch right after this kernel's wait. - False, + program_warps)`. `workspace`, `rank` and `nranks` are the stock IPC all-reduce's + (`catalog/comm/allreduce`); `fits` says which operands the kernel takes.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell( + why=f"tp{world}: {rows} rows, {why}", + spec=dict( + world=world, rows=rows, hidden=HIDDEN, hc_count=HC_COUNT, dtype=torch.bfloat16 + ), + ) + for world in (2, 4) + for rows, why in _ROWS ) - return hc_out, normed + + note = """ + Certified bitwise the IPC all-reduce followed by the Triton injection, with and without the + MoE combine, interleaved with plain all-reduces and in a replayed CUDA graph chain. + Not certified: hc_count other than 4, hidden other than 2560, groups spanning nodes. + """ + + def __call__( + self, + partial: torch.Tensor, + workspace: torch.Tensor, + rank: int, + nranks: int, + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + program_warps: int = 4, + ) -> tuple[torch.Tensor, torch.Tensor]: + hc_out, normed = native_ops.hc_allreduce_inject_norm( + partial, + workspace, + int(rank), + int(nranks), + hc, + inject, + shared, + gate, + norm, + float(eps), + int(program_warps), + # trigger_completion_at_end=False: release the next launch right after this kernel's + # wait. + False, + ) + return hc_out, normed + + def reference( + self, + partials_in_rank_order: Sequence[torch.Tensor], + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """The fp32 sum of every rank's `partial`, in rank order, rounded once, then + `qwen4_exp_hc_inject_norm`'s reference of the site. The documented collective exception + to mirroring `__call__`; `program_warps` only orders the kernel's sum of squares.""" + total = torch.zeros_like(partials_in_rank_order[0], dtype=torch.float32) + for partial in partials_in_rank_order: + total += partial.float() + block = total.to(partials_in_rank_order[0].dtype) + return qwen4_exp_hc_inject_norm.reference( + hc, block, inject[:, :HC_COUNT], norm, eps, HC_COUNT, shared, gate + ) + + def is_valid( + self, + partial: torch.Tensor, + workspace: torch.Tensor, + rank: int, + nranks: int, + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + program_warps: int = 4, + ) -> None: + assert fits(hc, partial, inject, shared, gate), "operands the kernel does not take" + assert nranks in (2, 4) and 0 <= rank < nranks, (rank, nranks) + assert program_warps in (2, 4), program_warps + + +hc_allreduce_inject_norm = HcAllreduceInjectNorm() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py index 088dd772cf53..a6eeb7949056 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py @@ -20,11 +20,13 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING, Sequence import torch import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber @@ -32,40 +34,26 @@ SENTINEL = -(2**31) _BLOCK = 1024 _PROGRAMS = 256 +#: Words of one selection row: the target's `token_topk + compress_ratio - 1`. +_SELECTION_WIDTH = 2051 -class _IpcBuffers: - """This rank's buffer and the peers' opened ones (addresses in rank order); freed and - closed with the object.""" - - def __init__(self, rank: int, local: int, peers: list): - self.rank, self.local, self.peers = rank, local, peers - - def __del__(self): - import sys - - if sys.is_finalizing(): - return - from tensorrt_llm._ipc_utils import cudart - - for q, ptr in enumerate(self.peers): - if ptr and q != self.rank: - cudart.cudaIpcCloseMemHandle(ptr) - if self.local: - cudart.cudaFree(self.local) +if TYPE_CHECKING: + from tensorrt_llm._ipc_utils import IpcMemory @dataclass(frozen=True) class SelectionLamport: - """One rank's view of the group's buffers: `buffers` the IPC allocations (kept alive here), - `state` the `[4]` int32 slot state (`state[0]` the next call's slot, `state[1]` the - previous call's share in words, `state[2]` the ticket counter), `args` the `[8]` int64 the - kernel reads: the peers' buffer addresses at [0, world), this rank's own at `rank`.""" + """One rank's view of the group's buffers: `buffers` the `IpcMemory` (kept alive here; it + frees this rank's buffer and closes the peers' with the object), `state` the `[4]` int32 + slot state (`state[0]` the next call's slot, `state[1]` the previous call's share in words, + `state[2]` the ticket counter), `args` the `[8]` int64 the kernel reads: the peers' buffer + addresses at [0, world), this rank's own at `rank`.""" rank: int world: int max_words: int - buffers: _IpcBuffers + buffers: IpcMemory state: torch.Tensor args: torch.Tensor @@ -74,49 +62,32 @@ def fits(self, words: int) -> bool: return 0 < words <= self.max_words -def build(rank: int, world: int, max_words: int, allgather) -> SelectionLamport: - """Allocate this rank's buffers and open the peers' (CUDA IPC, one node). `allgather(b)` - gathers one `bytes` a rank in rank order (e.g. `Distributed.get(mapping).tp_allgather`): a - collective, so every rank calls `build` at the same point, before any CUDA graph capture.""" - from tensorrt_llm._ipc_utils import ( - _ipc_mem_handle_from_bytes, - _ipc_mem_handle_to_bytes, - _raise_if_error, - cuda, - cudart, - ) +def build(mapping, max_words: int) -> SelectionLamport: + """Allocate this rank's buffers and open the peers' over `mapping`'s TP group (CUDA IPC, + one node, `IpcMemory`). A TP collective: every rank calls `build` at the same point, before + any CUDA graph capture.""" + from tensorrt_llm._ipc_utils import IpcMemory, cuda + from tensorrt_llm._torch.distributed.communicator import Distributed + rank, world = mapping.tp_rank, mapping.tp_size assert 2 <= world <= 4, "the kernel addresses 2 to 4 ranks" - nbytes = -(-(SLOTS * max_words * 4) // (1 << 21)) * (1 << 21) - err, local = cudart.cudaMalloc(nbytes) - _raise_if_error(err) - (err,) = cuda.cuMemsetD32(local, SENTINEL & 0xFFFFFFFF, nbytes // 4) - _raise_if_error(err) - err, handle = cudart.cudaIpcGetMemHandle(local) - _raise_if_error(err) - handles = allgather(_ipc_mem_handle_to_bytes(handle)) - assert len(handles) == world, (len(handles), world) - peers = [] - for q, h in enumerate(handles): - if q == rank: - peers.append(int(local)) - continue - err, ptr = cudart.cudaIpcOpenMemHandle( - _ipc_mem_handle_from_bytes(h), cudart.cudaIpcMemLazyEnablePeerAccess - ) - _raise_if_error(err) - peers.append(int(ptr)) + ipc = IpcMemory(mapping, SLOTS * max_words * 4, open_ipc=True) + assert ipc.open_ipc, "the TP group spans nodes: no CUDA IPC" + (err,) = cuda.cuMemsetD32(ipc.local_ptr, SENTINEL & 0xFFFFFFFF, SLOTS * max_words) + if err != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"cuMemsetD32 of the selection buffers: {err!r}") dev = torch.device("cuda", torch.cuda.current_device()) state = torch.zeros(4, dtype=torch.int32, device=dev) + peers = [int(ptr) for ptr in ipc.peer_ptrs] args = torch.tensor(peers + [0] * (8 - world), dtype=torch.int64, device=dev) torch.cuda.synchronize() # every rank's buffer is filled and opened before any rank's first push - allgather(b"") + Distributed.get(mapping).tp_allgather(b"") return SelectionLamport( rank=rank, world=world, max_words=max_words, - buffers=_IpcBuffers(rank, int(local), peers), + buffers=ipc, state=state, args=args, ) @@ -215,38 +186,74 @@ def _lamport_allgather_kernel( tl.store(out, v, mask=m) -def lamport_allgather( - src: torch.Tensor, lam: SelectionLamport, dst_rows: torch.Tensor | None = None -) -> torch.Tensor: +class LamportAllgather(OpWrapper): """`src`'s rows from every rank of `lam`'s group concatenated in rank order (the NCCL `allgather` with equal shares): `src` is this rank's contiguous int32 `[rows, width]` share. Returns a fresh `[world * rows, width]` int32 tensor. With `dst_rows` (int32 `[world * rows]`, a permutation, the same on every rank) the gathered row `j` is stored at - row `dst_rows[j]` instead (`out[dst_rows] = gathered`), in the same launch.""" - assert src.dtype == torch.int32 and src.is_contiguous() and src.ndim == 2 - rows, width = src.shape - share = rows * width - assert lam.fits(share * lam.world), (share, lam.world, lam.max_words) - dst = torch.empty([lam.world * rows, width], dtype=torch.int32, device=src.device) - if dst_rows is not None: - assert dst_rows.dtype == torch.int32 and dst_rows.is_contiguous() - assert dst_rows.numel() == lam.world * rows - pdl = get_env_enable_pdl() - _lamport_allgather_kernel[(_PROGRAMS,)]( - src, - dst, - lam.args, - lam.state, - share, - lam.rank, - lam.max_words, - dst if dst_rows is None else dst_rows, - width, - WORLD=lam.world, - BLOCK=_BLOCK, - ROW_MAP=dst_rows is not None, - USE_PDL=pdl, - num_warps=4, - launch_pdl=pdl, + row `dst_rows[j]` instead (`out[dst_rows] = gathered`), in the same launch. `lam` comes from + `build`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell( + why=f"tep{world}: a served step of {requests} requests, 12 trunk calls at " + f"{3 * requests} rows and 2 draft calls at {requests}", + spec=dict(world=world, requests=requests, width=_SELECTION_WIDTH, dtype=torch.int32), + ) + for world in (2, 4) + for requests in (16, 32, 64, 128, 192) ) - return dst + + note = """ + Certified bitwise NCCL's gather over a long mixed sequence of each cell's calls (the three + slots rotating), with and without the row map, eagerly and replayed from a CUDA graph. + Not certified: groups spanning nodes, world sizes other than 2 and 4, widths other than + the target's selection rows. + """ + + def __call__( + self, src: torch.Tensor, lam: SelectionLamport, dst_rows: torch.Tensor | None = None + ) -> torch.Tensor: + assert src.dtype == torch.int32 and src.is_contiguous() and src.ndim == 2 + rows, width = src.shape + share = rows * width + assert lam.fits(share * lam.world), (share, lam.world, lam.max_words) + dst = torch.empty([lam.world * rows, width], dtype=torch.int32, device=src.device) + if dst_rows is not None: + assert dst_rows.dtype == torch.int32 and dst_rows.is_contiguous() + assert dst_rows.numel() == lam.world * rows + pdl = get_env_enable_pdl() + _lamport_allgather_kernel[(_PROGRAMS,)]( + src, + dst, + lam.args, + lam.state, + share, + lam.rank, + lam.max_words, + dst if dst_rows is None else dst_rows, + width, + WORLD=lam.world, + BLOCK=_BLOCK, + ROW_MAP=dst_rows is not None, + USE_PDL=pdl, + num_warps=4, + launch_pdl=pdl, + ) + return dst + + def reference( + self, srcs_in_rank_order: Sequence[torch.Tensor], dst_rows: torch.Tensor | None = None + ) -> torch.Tensor: + """Every rank's `src` concatenated in rank order, its row `j` at row `dst_rows[j]` with + `dst_rows`. The documented collective exception to mirroring `__call__`.""" + gathered = torch.cat(list(srcs_in_rank_order)) + if dst_rows is None: + return gathered + out = torch.empty_like(gathered) + out[dst_rows.long()] = gathered + return out + + +lamport_allgather = LamportAllgather() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py index cbed53aae03e..f5a0a05b2da7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """The attention-DP MoE round trip as an all-to-all (`NVLinkOneSided`), run when -`TRTLLM_MODELING_V2_MOE_ALLTOALL=1` (`modeling.MOE_ALLTOALL_ENV`) in place of the default -all-gather / reduce-scatter. `moe_a2a_dispatch` sends every rank the rows with a top-k expert in -its whole-expert window, with their top-k ids and weights; `moe_a2a_combine` returns each -partial to the row's rank and sums them there. The strategy (`build_strategy`) holds the MNNVL -workspace and the state between a dispatch and its combine: build it once, pass it to every call.""" +`TRTLLM_FORCE_COMM_METHOD=NVLINK_ONE_SIDED` (`modeling.MOE_COMM_METHOD_ENV`) in place of the +default all-gather / reduce-scatter. `moe_a2a_dispatch` sends every rank the rows with a top-k +expert in its whole-expert window, with their top-k ids and weights; `moe_a2a_combine` returns +each partial to the row's rank and sums them there. The strategy (`build_strategy`) holds the +MNNVL workspace and the state between a dispatch and its combine: build it once, pass it to every +call.""" from __future__ import annotations diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py index 0c11293a3316..3b33c577d312 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py @@ -72,11 +72,20 @@ import cutlass.pipeline as pipeline import cutlass.utils as utils import cutlass.utils.blackwell_helpers as sm100_utils +import torch from cutlass._mlir.dialects import llvm from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import from_dlpack, make_ptr from cutlass.cutlass_dsl import T, dsl_user_op from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper, true_fp32_matmul + +from .fp8_block_scale_moe_runner import dequantize_block_weight, requantize_1x128 + +#: Scale block edge: 1x128 SFA, 128x128 SFB. +BLOCK = 128 + @dsl_user_op def _div_rn(a, b, *, loc=None, ip=None): @@ -1856,3 +1865,212 @@ def wrapper_grouped_quant( num_tiles=num_tiles, c_scale=c_scale, ) + + +_GEMMS: dict = {} + + +def _gmem_ptr(t: torch.Tensor, dtype, align: int = 16): + """A global-memory CuTe pointer at `t`'s data.""" + return make_ptr(dtype, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + +def _grouped_reference(a, w, sfa, sfw, rows, group, ntiles) -> torch.Tensor: + """fp32 `[rows, n]`: tile `t < ntiles[0]` of `rows / len(group)` rows times its group's + dequantized weight; the rows of skipped tiles NaN (unspecified).""" + _, n, k = w.shape + m_tile = rows // group.numel() + act = ( + a[:rows].float().reshape(rows, k // BLOCK, BLOCK) + * sfa[:, :rows].t().float().reshape(rows, k // BLOCK, 1) + ).reshape(rows, k) + out = torch.full((rows, n), float("nan"), dtype=torch.float32, device=a.device) + with true_fp32_matmul(): + for t, e in enumerate(group[: int(ntiles[0])].tolist()): + r = slice(t * m_tile, (t + 1) * m_tile) + out[r] = act[r] @ dequantize_block_weight(w[e], sfw[e]).t() + return out + + +class BlockwiseGemmGrouped(OpWrapper): + """`c[r, n] = sum_k (sfa * a)[r, k] (sfw * w)[group[r // M_tile], n, k]` over `tiles` M tiles + (`M_tile` the tactic's MMA M) of which the first `ntiles[0]` (device scalar) are live: `a` + `[tiles * M_tile, k]` e4m3, `sfa` `[k / 128, sf_m]` fp32 (M-major), `w` `[groups, n, k]` e4m3, + `sfw` `[groups, n / 128, k / 128]` fp32, `c` `[tiles * M_tile, n]` written in place; `tactic` + is `(mma_tiler_mn, cluster_shape_mn)`. Compiled once per `c` dtype and tactic.""" + + def __call__( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + c: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> None: + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + tiler, cluster = tactic + c3 = c.view(tiles, tiler[0], n).permute(1, 2, 0) + key = (c.dtype, tactic) + fn = _GEMMS.get(key) + if fn is None: + gemm = Sm100BlockwiseGemmKernel(mma_tiler_mn=tiler, cluster_shape_mn=cluster) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + fn = cute.compile( + gemm.wrapper_grouped, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + _gmem_ptr(a, cutlass.Float8E4M3FN), + _gmem_ptr(w, cutlass.Float8E4M3FN), + _gmem_ptr(sfa, cutlass.Float32), + _gmem_ptr(sfw, cutlass.Float32), + from_dlpack(c3).mark_layout_dynamic(leading_dim=1), + _gmem_ptr(group, cutlass.Int32, 4), + _gmem_ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + c3, + group.data_ptr(), + ntiles.data_ptr(), + ) + + def reference( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + c: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> torch.Tensor: + """`c` after the call, as a fresh tensor of its dtype (fp32 GEMMs over the dequantized + operands, rounded once); the rows of skipped tiles NaN (unspecified). `sf_m` and + `tactic` pick a layout and a launch, not a result.""" + return _grouped_reference(a, w, sfa, sfw, c.shape[0], group[:tiles], ntiles).to(c.dtype) + + +blockwise_gemm_grouped = BlockwiseGemmGrouped() + + +class BlockwiseGemmGroupedQuant(OpWrapper): + """`blockwise_gemm_grouped` with the 1x128 e4m3 round trip in the epilogue (`epi_quant`): + `cq[r, n]` e4m3 codes and `cs[n // 128, r]` fp32 scales (row-major `[n / 128, sf_m]`) + instead of `c`. Compiled once per tactic.""" + + def __call__( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + cq: torch.Tensor, + cs: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> None: + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + key = ("q", tactic) + fn = _GEMMS.get(key) + if fn is None: + tiler, cluster = tactic + gemm = Sm100BlockwiseGemmKernel( + mma_tiler_mn=tiler, cluster_shape_mn=cluster, epi_quant=True + ) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + fn = cute.compile( + gemm.wrapper_grouped_quant, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + _gmem_ptr(a, cutlass.Float8E4M3FN), + _gmem_ptr(w, cutlass.Float8E4M3FN), + _gmem_ptr(sfa, cutlass.Float32), + _gmem_ptr(sfw, cutlass.Float32), + _gmem_ptr(cq, cutlass.Float8E4M3FN), + from_dlpack(cs).mark_layout_dynamic(leading_dim=1), + _gmem_ptr(group, cutlass.Int32, 4), + _gmem_ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + cq.data_ptr(), + cs, + group.data_ptr(), + ntiles.data_ptr(), + ) + + def reference( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + cq: torch.Tensor, + cs: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> torch.Tensor: + """The values the call's codes and scales encode, `cq.float() * cs[n // 128, r]`, as + fp32 `[rows, n]`: `blockwise_gemm_grouped.reference`'s fp32 product through the MoE + runner's 1x128 e4m3 round trip; the rows of skipped tiles NaN (unspecified). `sf_m` and + `tactic` pick a layout and a launch, not a result.""" + return requantize_1x128( + _grouped_reference(a, w, sfa, sfw, cq.shape[0], group[:tiles], ntiles) + ) + + +blockwise_gemm_grouped_quant = BlockwiseGemmGroupedQuant() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py index 953053858c71..9b848414c84f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py @@ -15,6 +15,11 @@ import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper + +from .blockwise_gemm_grouped import blockwise_gemm_grouped, blockwise_gemm_grouped_quant +from .fp8_block_scale_moe_runner import fp8_block_scale_moe_runner + HIDDEN = 2560 INTER = 640 FC1_ROWS = 2 * INTER @@ -336,134 +341,6 @@ def _finalize_pipe_kernel( tl.store(out_ptr + t * H + lc, acc.to(tl.bfloat16), mask=lc < H) -_GEMMS: dict = {} - - -def _grouped_gemm(a, w, sfa, sfw, c, group, ntiles, tiles, sf_m, tactic): - """c[r, n] = sum_k a[r, k] w[group(r // M_TILE), n, k] with a's 1x128 (M-major `sfa[kb, r]`) - and w's 128x128 fp32 scales, over `tiles` M tiles of which the first `ntiles[0]` are live.""" - import cutlass - import cutlass.cute as cute - from cutlass.cute.runtime import from_dlpack, make_ptr - - from .blockwise_gemm_grouped import Sm100BlockwiseGemmKernel - - groups, n, k = w.shape - sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) - tiler, cluster = tactic - c3 = c.view(tiles, M_TILE, n).permute(1, 2, 0) - key = (c.dtype, tactic) - fn = _GEMMS.get(key) - if fn is None: - gemm = Sm100BlockwiseGemmKernel(mma_tiler_mn=tiler, cluster_shape_mn=cluster) - mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) - - def ptr(t, dt, align=16): - return make_ptr(dt, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) - - fn = cute.compile( - gemm.wrapper_grouped, - tiles, - n, - k, - sf_m, - sf_n, - sf_k, - groups, - ptr(a, cutlass.Float8E4M3FN), - ptr(w, cutlass.Float8E4M3FN), - ptr(sfa, cutlass.Float32), - ptr(sfw, cutlass.Float32), - from_dlpack(c3).mark_layout_dynamic(leading_dim=1), - ptr(group, cutlass.Int32, 4), - ptr(ntiles, cutlass.Int32, 4), - max_active_clusters=mac, - stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), - options="--opt-level 2 --enable-tvm-ffi", - ) - _GEMMS[key] = fn - fn( - tiles, - n, - k, - sf_m, - sf_n, - sf_k, - groups, - a.data_ptr(), - w.data_ptr(), - sfa.data_ptr(), - sfw.data_ptr(), - c3, - group.data_ptr(), - ntiles.data_ptr(), - ) - - -def _grouped_gemm_q(a, w, sfa, sfw, cq, cs, group, ntiles, tiles, sf_m, tactic): - """`_grouped_gemm` with the 1x128 e4m3 round trip in the epilogue (`epi_quant`): `cq[r, n]` - e4m3 codes, `cs[n // 128, r]` fp32 scales (row-major `[n / 128, sf_m]`).""" - import cutlass - import cutlass.cute as cute - from cutlass.cute.runtime import from_dlpack, make_ptr - - from .blockwise_gemm_grouped import Sm100BlockwiseGemmKernel - - groups, n, k = w.shape - sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) - key = ("q", tactic) - fn = _GEMMS.get(key) - if fn is None: - tiler, cluster = tactic - gemm = Sm100BlockwiseGemmKernel( - mma_tiler_mn=tiler, cluster_shape_mn=cluster, epi_quant=True - ) - mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) - - def ptr(t, dt, align=16): - return make_ptr(dt, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) - - fn = cute.compile( - gemm.wrapper_grouped_quant, - tiles, - n, - k, - sf_m, - sf_n, - sf_k, - groups, - ptr(a, cutlass.Float8E4M3FN), - ptr(w, cutlass.Float8E4M3FN), - ptr(sfa, cutlass.Float32), - ptr(sfw, cutlass.Float32), - ptr(cq, cutlass.Float8E4M3FN), - from_dlpack(cs).mark_layout_dynamic(leading_dim=1), - ptr(group, cutlass.Int32, 4), - ptr(ntiles, cutlass.Int32, 4), - max_active_clusters=mac, - stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), - options="--opt-level 2 --enable-tvm-ffi", - ) - _GEMMS[key] = fn - fn( - tiles, - n, - k, - sf_m, - sf_n, - sf_k, - groups, - a.data_ptr(), - w.data_ptr(), - sfa.data_ptr(), - sfw.data_ptr(), - cq.data_ptr(), - cs, - group.data_ptr(), - ntiles.data_ptr(), - ) - - def tile_bound(rows: int, local: int, topk: int = TOPK) -> int: """M tiles that hold any routing of `rows` rows over a `local`-expert window.""" return -(-rows * min(topk, local) // M_TILE) + local @@ -552,7 +429,7 @@ def permute(): def fc1g(): if epiq: - _grouped_gemm_q( + blockwise_gemm_grouped_quant( a1, fc1, sfa.view(HIDDEN // BLOCK, m), @@ -566,7 +443,7 @@ def fc1g(): t1, ) else: - _grouped_gemm( + blockwise_gemm_grouped( a1, fc1, sfa.view(HIDDEN // BLOCK, m), fc1_scale, c1, group, ntiles, tiles, m, t1 ) @@ -613,7 +490,7 @@ def act(): ) def fc2g(): - _grouped_gemm( + blockwise_gemm_grouped( a2.view(torch.float8_e4m3fn), fc2, sfa2, fc2_scale, c2, group, ntiles, tiles, m, t2 ) @@ -659,7 +536,7 @@ def finalize_rows(r0, r1): ] -def moe_prefill_grouped( +def _moe_prefill_grouped( xq: torch.Tensor, xsf: torch.Tensor, ids: torch.Tensor, @@ -688,6 +565,77 @@ def moe_prefill_grouped( return out +class MoePrefillGrouped(OpWrapper): + """This window's partial of a prefill chunk's routed experts, pre-routed, `[rows, HIDDEN]` + bf16 (see `_moe_prefill_grouped` for the operands). `fin_first = (r0, r1)` (rows at either + end) finalizes those rows, calls `after_first(out)`, then finalizes the rest.""" + + def __call__( + self, + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, + ) -> torch.Tensor: + return _moe_prefill_grouped( + xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics, fin_first, + after_first, + ) # fmt: skip + + def reference( + self, + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, + ) -> torch.Tensor: + """`fp8_block_scale_moe_runner`'s pre-routed reference over the same window. `tactics` + and `fin_first` pick launches, not a result; `after_first` is not called.""" + local = fc1.shape[0] + return fp8_block_scale_moe_runner.reference( + routing_logits=None, + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=fc1, + gemm1_weights_scale=fc1_scale, + gemm2_weights=fc2, + gemm2_weights_scale=fc2_scale, + # The routing space only bounds the runner's `is_valid`; the reference reads the window. + num_experts=offset + local, + top_k=TOPK, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=INTER, + local_expert_offset=offset, + local_num_experts=local, + routed_scaling_factor=None, + routing_method_type=1, + topk_weights=wts, + topk_ids=ids, + ) + + +moe_prefill_grouped = MoePrefillGrouped() + + _READY: set = set() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py index 733c9e9471e9..5c931a08ada1 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py @@ -17,6 +17,8 @@ import triton import triton.language as tl +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper + # The tail's token, sequence, slot and stage-row lists are scalar-read views. @triton.jit(do_not_specialize_on_alignment=["tok_ptr", "cu_ptr", "slot_ptr", "row_ptr"]) @@ -83,7 +85,7 @@ def _gdn_tail_kernel( tl.store(dst, state.to(stage_ptr.dtype.element_ty)) -def gdn_inforward_tail( +def _gdn_inforward_tail( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, @@ -181,7 +183,7 @@ def _conv_swap_kernel( tl.store(live, tl.load(x_ptr + rows[None, :] * stride_xt + c[:, None], mask=m), mask=m) -def gdn_inforward_conv_swap( +def _gdn_inforward_conv_swap( conv: torch.Tensor, stage: torch.Tensor, x: torch.Tensor, @@ -209,3 +211,90 @@ def gdn_inforward_conv_swap( conv.stride(0), conv.stride(1), stage.stride(0), stage.stride(1), x.stride(0), W=width, BW=triton.next_power_of_2(width), BLOCK=block, num_warps=4, ) # fmt: skip + + +class GdnInforwardTail(OpWrapper): + """The split requests' tails of gated delta rule, in place (see `_gdn_inforward_tail`).""" + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, + ) -> None: + _gdn_inforward_tail( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(stage, out)` after `gdn_inforward_tail_reference` writes copies of them; the inputs + are not touched.""" + expected_stage, expected_out = stage.clone(), out.clone() + gdn_inforward_tail_reference( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, expected_stage, stage_rows, + expected_out, + ) # fmt: skip + return expected_stage, expected_out + + +gdn_inforward_tail = GdnInforwardTail() + + +class GdnInforwardConvSwap(OpWrapper): + """The split requests' conv windows staged and replaced, in place (see + `_gdn_inforward_conv_swap`).""" + + def __call__( + self, + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, + ) -> None: + _gdn_inforward_conv_swap(conv, stage, x, tail_slots, stage_rows, conv_win) + + def reference( + self, + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(conv, stage)` after the swap, as fresh tensors; the inputs are not touched.""" + expected_conv, expected_stage = conv.clone(), stage.clone() + _, chans, width = conv.shape + for j, (slot, row) in enumerate(zip(tail_slots.tolist(), stage_rows.tolist())): + expected_stage[row] = conv[slot] + rows = conv_win[j * width : (j + 1) * width].long() + expected_conv[slot] = x[rows, :chans].t() + return expected_conv, expected_stage + + +gdn_inforward_conv_swap = GdnInforwardConvSwap() diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py index f20bb36722a2..6bd0734141e0 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py @@ -502,6 +502,49 @@ def test_rejects_metadata_shorter_than_the_rows(only_visible_blocks) -> None: op(**args) +@pytest.mark.parametrize( + "bad, match", + [ + (dict(tokens_per_block=0), "positive pages"), + (dict(compress_ratio=1), "compression ratio"), + ("float_positions", "integer storage"), + ("cpu_table", "one CUDA device"), + ("empty_table", "non-empty block table"), + ], + ids=["tokens_per_block-0", "compress_ratio-1", "float-positions", "cpu-table", "empty-table"], +) +def test_prefill_query_tile_rejects_what_the_engine_rejects(bad, match) -> None: + """The query-tiled launch refuses the operands the engine's launcher refuses.""" + gen = torch.Generator(device=DEV).manual_seed(1703) + rows, tpb, cr = _entry._DECODE_ROW_LIMIT + 44, 32, 4 + cache = (torch.randn(16, tpb, 1, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + q = (torch.randn(rows, 4, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + args = dict( + q=q, + index_cache=cache, + block_table=torch.arange(16, dtype=torch.int32, device=DEV).view(1, 16), + query_positions=torch.arange(rows, dtype=torch.int64, device=DEV), + request_indices=torch.zeros(rows, dtype=torch.int32, device=DEV), + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=True, + ) + assert _entry._takes_prefill_query_tile(q, True, True, False) + if bad == "float_positions": + args["query_positions"] = args["query_positions"].float() + elif bad == "cpu_table": + args["block_table"] = args["block_table"].cpu() + elif bad == "empty_table": + args["block_table"] = args["block_table"][:, :0] + else: + args.update(bad) + with pytest.raises(ValueError, match=match): + _entry._qsa_index_scores_prefill_query_tile( + **{k: v for k, v in args.items() if k not in ("only_visible_blocks", "context_rows")} + ) + + @pytest.mark.parametrize( "chunks", [((67584, 8190),), ((5000, 300), (20000, 1700)), ((131072, 4093),)], diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py index 044675e3f75e..fcb6c32779f2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py @@ -296,13 +296,16 @@ def test_pdl_launches_are_the_plain_launches(rows_per_seq) -> None: hc_out, normed = op(*args, pdl=True) assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) assert torch.equal(args[7], want[2]) - fresh = _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq) + # Capture over other inputs, then refill every operand with `want`'s before the replay. + fresh = _build(dict(lengths=(rows_per_seq,) * 4), seed=40 + rows_per_seq) pool = fresh[7].clone() fresh[7] = pool graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): g_out, g_normed = op(*fresh, pdl=True) - pool.copy_(_build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq)[7]) + for dst, src in zip(fresh, _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq)): + if isinstance(dst, torch.Tensor): + dst.copy_(src) graph.replay() torch.cuda.synchronize() assert torch.equal(g_out, want[0]) and torch.equal(g_normed, want[1]) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py index 4311ef2aee9a..79e4d25e4c7b 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py @@ -224,8 +224,9 @@ def test_cuda_graph_capture_and_replay(lengths, pool_slots) -> None: assert not torch.equal(args[1], pool_before) -def test_per_sequence_rows_compile_one_variant() -> None: - """1 to 4 rows a sequence launch one compiled variant per carry flag, matching the reference.""" +def test_per_sequence_rows_compile_at_first_call() -> None: + """1 to 4 rows a sequence match the reference, and every `BLOCK_P` variant compiles at the + first call per carry flag: later row counts add no compiled variant.""" kernel = _embed_module._ple_ngram_embed_kernel dev = torch.cuda.current_device() seen = {} diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py index 117253966a5d..459ca994f84f 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py @@ -44,23 +44,23 @@ def fill(t, r, it): else: t.copy_(torch.randn(t.shape, generator=g, device=dev).to(t.dtype)) - # check_create: the buffer holds the largest exchange, 8192 int32 rows of 2051. + # check_create: the buffer holds the largest cell, 8192 int32 rows of 2051. capacity = 72 << 20 ce = ce_mod.CopyEngineExchange.create(mapping, capacity) assert ce is not None, f"rank {rank}: the exchange was not built" - # check_eager - for it in range(24): - rows = (8192, 1024, 77, 4096)[it % 4] - dtype = torch.int32 if it % 3 == 2 else torch.bfloat16 - width = 2051 if dtype == torch.int32 else HIDDEN - x = torch.empty(rows, width, dtype=dtype, device=dev) + # check_eager: every certified cell, three rounds interleaved. + cells = [c.spec for c in ce_mod.ce_exchange.CELLS if c.spec["world"] == world] + assert cells, f"no cell is certified at world size {world}" + for it in range(3 * len(cells)): + spec = cells[it % len(cells)] + x = torch.empty(spec["rows"], spec["width"], dtype=spec["dtype"], device=dev) fill(x, rank, it) - got = ce.exchange(x).clone() + got = ce_mod.ce_exchange(ce, x).clone() want = torch.empty_like(x) fill(want, peer, it) torch.cuda.synchronize() - assert torch.equal(got, want), f"rank {rank}: eager exchange {it} differs" + assert torch.equal(got, want), f"rank {rank}: eager exchange {it} ({spec}) differs" comm.Barrier() # check_graph diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py index b6588f80923d..74ffa0569ccc 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py @@ -18,8 +18,9 @@ HC, HIDDEN = 4, 2560 WIDTH = HC * HIDDEN EPS = 1e-6 -#: Decode-site row counts on both sides of the switch to 2-warp programs at 512 rows. -ROWS = (33, 96, 168, 192, 216, 288, 384, 480, 511, 512, 576, 640, 1024) +#: Decode-site row counts on both sides of the switch to 2-warp programs at 512 rows: the entry's +#: CELLS at this world, set by `_run_one_rank`. +ROWS: tuple = () _MODELING_V2_TESTS = Path(__file__).resolve().parents[4] modeling = fused_mod = tp_sum = MPI_RANK = WS = None @@ -175,7 +176,7 @@ def chain(kinds): def _run_one_rank() -> int: - global modeling, fused_mod, tp_sum, MPI_RANK, RANK, WORLD, WS + global modeling, fused_mod, tp_sum, MPI_RANK, RANK, WORLD, WS, ROWS os.environ.pop("TLLM_DISABLE_MPI", None) os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) @@ -194,6 +195,10 @@ def _run_one_rank() -> int: from tensorrt_llm.functional import AllReduceFusionOp modeling, fused_mod = _modeling, _fused_mod + ROWS = tuple( + c.spec["rows"] for c in fused_mod.hc_allreduce_inject_norm.CELLS if c.spec["world"] == WORLD + ) + assert ROWS, f"no cell is certified at world size {WORLD}" group = entry.allreduce_group_args(MPI_RANK.mapping) WS = group["workspace"] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py index a93e4965d9e3..eac810b4f4db 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py @@ -16,9 +16,10 @@ DEADLINE_S = 900 WIDTH = 2051 -#: Requests a step (each rank scores `requests / world` of them): tep2 / tep4 HT batches. -REQUESTS = (16, 32, 64, 128, 192) -MAX_ROWS = 3 * max(REQUESTS) +#: Requests a step (each rank scores `requests / world` of them): the entry's CELLS at this world +#: (tep2 / tep4 HT batches), set by `_run_one_rank`. +REQUESTS: tuple = () +MAX_ROWS = 0 _MODELING_V2_TESTS = Path(__file__).resolve().parents[4] lam = gather = MPI_RANK = None @@ -156,7 +157,7 @@ def check_row_map_stores_each_gathered_row_at_its_row() -> None: def _run_one_rank() -> int: - global lam, gather, MPI_RANK, RANK, WORLD + global lam, gather, MPI_RANK, RANK, WORLD, REQUESTS, MAX_ROWS os.environ.pop("TLLM_DISABLE_MPI", None) os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) @@ -168,11 +169,15 @@ def _run_one_rank() -> int: lamport_allgather__index_split as entry, ) - # The buffers' handles travel over MPI (the engine passes its TP communicator); the NCCL - # reference needs a torch process group. - lam = entry.build(RANK, WORLD, MAX_ROWS * WIDTH, MPI_RANK.comm.allgather) - _ensure_process_group() gather = entry.lamport_allgather + cells = [c.spec for c in gather.CELLS if c.spec["world"] == WORLD] + assert cells and all(s["width"] == WIDTH for s in cells), f"no cell at world {WORLD}" + REQUESTS = tuple(s["requests"] for s in cells) + MAX_ROWS = 3 * max(REQUESTS) + # The buffers' handles travel over MPI (the engine's TP communicator); the NCCL reference + # needs a torch process group. + lam = entry.build(MPI_RANK.mapping, MAX_ROWS * WIDTH) + _ensure_process_group() code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) torch.cuda.synchronize() sys.stdout.flush() diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 81a2603b350d..b82cc22ec8f9 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1361,7 +1361,7 @@ def _rtl_build(spec, seed): q=qkv[:, :q_heads], k_cache=st.k, v_cache=st.v, block_table=st.table, selected_tokens=selected, request_indices=st.rows.to(spec["request_dtype"]), tokens_per_block=spec["tokens_per_block"], softmax_scale=d**-0.5, - query_positions=st.positions.to(spec["request_dtype"]), + query_positions=st.positions.to(spec.get("position_dtype", spec["request_dtype"])), compress_ratio=spec["compress_ratio"], ) # fmt: skip @@ -1802,9 +1802,10 @@ def _family_index_scores_groups(contexts, monkeypatch) -> None: assert torch.equal(captured[inside], eager[inside]), "the replay differs from an eager call" -def _family_index_scores_prefill(requests, monkeypatch) -> None: - """The tcgen05 context-row index scores, `requests` = ((prefix, rows), ...): bitwise the - family's kernel over every row's visible columns, and replayable.""" +def _family_index_scores_prefill(case, monkeypatch) -> None: + """The tcgen05 context-row index scores, `case` = (((prefix, rows), ...), position dtype): + bitwise the family's kernel over every row's visible columns, and replayable.""" + requests, pos_dtype = case need = [-(-(p + r) // 32) for p, r in requests] gen, cache, table, slot_of = _index_pool( len(requests), need, sum(r for _, r in requests), 12000, 64 @@ -1812,7 +1813,7 @@ def _family_index_scores_prefill(requests, monkeypatch) -> None: req = torch.cat( [torch.full((r,), int(slot_of[i]), dtype=torch.int32) for i, (_, r) in enumerate(requests)] ).to(DEV) - pos = torch.cat([torch.arange(p, p + r, dtype=torch.int32) for p, r in requests]).to(DEV) + pos = torch.cat([torch.arange(p, p + r, dtype=pos_dtype) for p, r in requests]).to(DEV) q = torch.randn(pos.numel(), 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] assert _isc.supports(q, cache, 32, 4) assert not _isc.supports(q, cache, 64, 4) @@ -1857,12 +1858,14 @@ def _family_topk_rows(case, monkeypatch) -> None: ) ), *( - pytest.param(_family_index_scores_prefill, requests, id=f"index scores prefill {name}") - for name, requests in ( - ("88k_one", ((88064, 1024),)), - ("three_requests", ((88064, 700), (40000, 333), (3, 300))), - ("windows_from_0", ((0, 1500),)), - ("131k_250k", ((131072, 320), (250000, 64))), + pytest.param(_family_index_scores_prefill, (requests, pos_dtype), + id=f"index scores prefill {name}") + for name, requests, pos_dtype in ( + ("88k_one", ((88064, 1024),), torch.int32), + ("three_requests", ((88064, 700), (40000, 333), (3, 300)), torch.int32), + ("windows_from_0", ((0, 1500),), torch.int32), + ("131k_250k", ((131072, 320), (250000, 64)), torch.int32), + ("three_requests_int64", ((88064, 700), (40000, 333), (3, 300)), torch.int64), ) ), *( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index 6587eec4cef2..83a6d5f38989 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -101,7 +101,7 @@ def on_sm103(monkeypatch): monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) for env in ( "TRTLLM_USE_GDN_FLASHINFER_REPLAY", - M.MOE_ALLTOALL_ENV, + M.MOE_COMM_METHOD_ENV, M.ROWTILE_SCRATCH_GROUPS_ENV, W.MOE_TPB_ENV, ): @@ -398,21 +398,24 @@ def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypa modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) monkeypatch.delenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY") # An all-to-all MoE the deployment cannot run: no attention DP, or no MNNVL. - monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "1") - with pytest.raises(ValueError, match=M.MOE_ALLTOALL_ENV): + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "nvlink_one_sided") + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED"): modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) with mock.patch( "tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided.NVLinkOneSided" ".is_platform_supported", return_value=False, ): - with pytest.raises(ValueError, match=M.MOE_ALLTOALL_ENV): + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED"): modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) - # A misspelled switch raises rather than routing as if it were off. - monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "enabled") - with pytest.raises(ValueError, match=f"{M.MOE_ALLTOALL_ENV}='enabled'"): - modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) - monkeypatch.delenv(M.MOE_ALLTOALL_ENV) + # A stock method the target does not run declines rather than routing as the default. + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "NVLINK_TWO_SIDED") + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_TWO_SIDED"): + modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) + # `ALLGATHER`, in any case, is the default round trip. + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "allgather") + _admitted(_routed_config("dep4")) + monkeypatch.delenv(M.MOE_COMM_METHOD_ENV) monkeypatch.setenv(W.MOE_TPB_ENV, "1") with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}='1': expected rr"): modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) @@ -853,8 +856,8 @@ def combine(self, partials): ids=["dep2-trunk-ragged", "dep4-trunk-a-rank-without-rows", "dep4-draft-captured-batch"], ) def test_the_attention_dp_alltoall_round_trip(counts, draft, monkeypatch) -> None: - """`TRTLLM_MODELING_V2_MOE_ALLTOALL=1`: each rank routes its own rows through the all-to-all - and returns what the combine sums; no all-gather, reduce-scatter or TP sum.""" + """`TRTLLM_FORCE_COMM_METHOD=NVLINK_ONE_SIDED`: each rank routes its own rows through the + all-to-all and returns what the combine sums; no all-gather, reduce-scatter or TP sum.""" world = len(counts) moe = _HostMoe(monkeypatch, chunk_rows=5) gen = torch.Generator().manual_seed(2) @@ -864,7 +867,7 @@ def test_the_attention_dp_alltoall_round_trip(counts, draft, monkeypatch) -> Non def forbidden(*args, **kwargs): raise AssertionError("the all-to-all round trip took the default collectives") - monkeypatch.setenv(M.MOE_ALLTOALL_ENV, "1") + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "NVLINK_ONE_SIDED") monkeypatch.setattr(M, "mpi_disabled", lambda: False) for name in ("allgather", "reducescatter"): monkeypatch.setattr(M, name, forbidden) From c8e1b128795f1848603d9c2a2e56de4f558e1188 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:02:17 -0800 Subject: [PATCH 11/51] [None][test] modeling_v2 qwen4_exp: FP8 target long-context and tp1 accuracy tests, measured acceptance-length gates Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../test_accuracy.py | 137 +++++++++++++++--- .../references/acceptance_length.yaml | 19 +++ .../defs/accuracy/references/gsm8k.yaml | 6 + .../qa/functional/llm_function_core.txt | 2 + .../test_lists/test-db/l0_b300.yml | 2 + .../test_lists/test-db/l0_gb300.yml | 7 +- 6 files changed, 152 insertions(+), 21 deletions(-) diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py index 7e1a158a3c1a..3edb23b070d8 100644 --- a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py @@ -12,9 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Whole-model GSM8K gates for the qwen3_8_flash_next_fp8__sm_103__unified target on sm_103.""" +"""Whole-model gates for the qwen3_8_flash_next_fp8__sm_103__unified target on sm_103. + +GSM8K at tp1, tep4, dep4 and dep2 disaggregated; long-context pass keys at tep4. +""" import pytest +from transformers import AutoTokenizer from tensorrt_llm import LLM from tensorrt_llm._torch.modeling_v2 import MODELING_V2_VALIDATE_ENV @@ -25,13 +29,26 @@ MambaStateConfig, MoeConfig, MTPDecodingConfig, + QSASparseAttentionConfig, + SamplingParams, ) from tensorrt_llm.quantization import QuantAlgo from .....conftest import llm_models_root -from ....accuracy_core import GSM8K, LlmapiAccuracyTestHarness, compute_acceptance_length +from ....accuracy_core import ( + GSM8K, + LlmapiAccuracyTestHarness, + assert_acceptance_length, + compute_acceptance_length, +) from ....test_disaggregated_serving import launch_disaggregated_llm, run_accuracy_test from ..._env import scoped_env_overrides +from ..qwen3_8_flash_next_nvfp4__sm_103__default.test_accuracy import ( + _PASSKEY_LENGTHS, + _PASSKEY_MAX_TOKENS, + _acceptance_length_from_requests, + _passkey_prompts, +) skip_not_sm103 = pytest.mark.skipif( get_sm_version() != 103, reason="modeling_v2 targets in this batch are certified on sm_103 only" @@ -59,6 +76,36 @@ def _disagg_server_config() -> dict: } +def _passkey_acceptance(llm, prompts, label: str, draft_len: int) -> float: + """Pass-key retrieval on ``llm`` and the requests' mean acceptance length. + + Every key must be stated back, and two identical requests far apart in one batch (different + slots) must give the same tokens, which fails if a request's selection leaks into another + slot. Needs no iteration statistics: the requests carry their own speculative metrics. + """ + texts = [prompt for prompt, _ in prompts] + [prompts[len(prompts) // 2][0]] + outputs = llm.generate( + texts, + SamplingParams(max_tokens=_PASSKEY_MAX_TOKENS, temperature=0, return_perf_metrics=True), + use_tqdm=False, + ) + missed = [ + key + for (_, key), output in zip(prompts, outputs) + if key not in output.outputs[0].text[:_PASSKEY_MAX_TOKENS] + ] + assert not missed, ( + f"{label}: pass keys not stated back: {missed} of {[key for _, key in prompts]}" + ) + twins = outputs[len(prompts) // 2], outputs[-1] + assert twins[0].outputs[0].token_ids == twins[1].outputs[0].token_ids, ( + f"{label}: the same prompt in two slots gave different tokens" + ) + acceptance_length = _acceptance_length_from_requests(outputs, draft_len) + print(f"[AL] {label} acceptance_length = {acceptance_length:.3f}") + return acceptance_length + + _TEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=False) _DEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=True) @@ -67,7 +114,7 @@ def _disagg_server_config() -> dict: pytest.param(_TEP4, "fp8", True, {}, id="tep4_fp8_kv_breakable"), pytest.param(_DEP4, "auto", False, {}, id="dep4"), pytest.param( - _DEP4, "auto", False, {"TRTLLM_MODELING_V2_MOE_ALLTOALL": "1"}, id="dep4_alltoall" + _DEP4, "auto", False, {"TRTLLM_FORCE_COMM_METHOD": "NVLINK_ONE_SIDED"}, id="dep4_alltoall" ), ] @@ -117,9 +164,10 @@ class TestModelingV2Qwen38FlashNextFp8Sm103Unified(LlmapiAccuracyTestHarness): # One-model MTP-Eagle with the checkpoint's single MTP module replayed three times. MTP3 = MTPDecodingConfig(max_draft_len=3) - # Acceptance length reads 1.0 for a collapsed draft path and 1 + max_draft_len when every - # draft is accepted; at or above the midpoint, more than half the drafts are accepted. - MTP3_COLLAPSE_FLOOR = 1.0 + MTP3.max_draft_len / 2 + # The acceptance-length floors in references/acceptance_length.yaml: gsm8k over bf16 KV + # (`mtp3`) and over an FP8 pool (`mtp3_fp8_kv`), and the long-context pass keys. + ACCEPTANCE_KEY_MTP3 = "ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3" + ACCEPTANCE_KEY_LONG_CONTEXT = "ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_long_context" # The breakable prefill CUDA graph, up to max_num_tokens. BREAKABLE_PREFILL = dict( @@ -127,11 +175,14 @@ class TestModelingV2Qwen38FlashNextFp8Sm103Unified(LlmapiAccuracyTestHarness): prefill_capture_num_tokens=[512, 1024, 2048, 4096, 8192], ) - def _gsm8k(self, mocker, llm, label: str) -> None: + def _gsm8k(self, mocker, llm, label: str, key: str | None = None) -> None: """gsm8k against the engine's FP8_BLOCK_SCALES row of references/gsm8k.yaml. - Under speculation also the acceptance length: rejection sampling keeps the score of a - miscomputed draft layer, so only the acceptance length sees it. + Under speculation also the acceptance length, held to the floor under `key` in + references/acceptance_length.yaml: rejection sampling keeps the score of a miscomputed + draft layer, so only the acceptance length sees it. It comes from the engine's + iteration statistics, so the engine needs ``enable_iter_perf_stats`` and + ``max_stats_len=-1`` (the default queue keeps only the last 1000 iterations). """ assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES, ( f"{label}: the engine resolved quant_algo {llm.args.quant_config.quant_algo}; the " @@ -143,10 +194,7 @@ def _gsm8k(self, mocker, llm, label: str) -> None: return acceptance_length = compute_acceptance_length(llm) print(f"[AL] {label} acceptance_length = {acceptance_length:.3f}") - assert acceptance_length >= self.MTP3_COLLAPSE_FLOOR, ( - f"{label}: acceptance length {acceptance_length:.3f} is below " - f"{self.MTP3_COLLAPSE_FLOOR}: the draft path is rejected more often than accepted" - ) + assert_acceptance_length(key, acceptance_length) @skip_not_sm103 @pytest.mark.skip_less_device(4) @@ -156,10 +204,10 @@ def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): """Three drafts with block reuse and chunked prefill, against the FP8_BLOCK_SCALES MTP row. Block reuse needs the Gated-DeltaNet and PLE snapshots (every 256 tokens); without them the - runtime turns reuse off. No FP8_BLOCK_SCALES row names a KV-cache algorithm, so the FP8 - pool is held to the bf16 pool's row. + runtime turns reuse off. An FP8 pool reads the FP8_BLOCK_SCALES row with an FP8 KV cache + and its own acceptance-length floor (`mtp3_fp8_kv`). """ - if env.get("TRTLLM_MODELING_V2_MOE_ALLTOALL"): + if env.get("TRTLLM_FORCE_COMM_METHOD") == "NVLINK_ONE_SIDED": from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( NVLinkOneSided, ) @@ -175,6 +223,11 @@ def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), ) engine = dict(self.ENGINE, enable_chunked_prefill=True, kv_cache_config=kv_cache_config) + key = ( + self.ACCEPTANCE_KEY_MTP3 + if kv_dtype == "auto" + else f"{self.ACCEPTANCE_KEY_MTP3}_{kv_dtype}_kv" + ) if breakable: engine.update(self.BREAKABLE_PREFILL) with ( @@ -183,6 +236,7 @@ def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): self.MODEL_PATH, speculative_config=self.MTP3, enable_iter_perf_stats=True, + max_stats_len=-1, **engine, **parallel, modeling_v2="require", @@ -194,9 +248,40 @@ def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): f"kv_cache_config.dtype={kv_dtype} resolved kv_cache_quant_algo " f"{llm.args.quant_config.kv_cache_quant_algo}" ) - # Only the reference lookup reads this; the engine is built over the FP8 pool. - llm.args.quant_config.kv_cache_quant_algo = None - self._gsm8k(mocker, llm, request.node.callspec.id) + self._gsm8k(mocker, llm, request.node.callspec.id, key) + + @skip_not_sm103 + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + def test_mtp3_long_context(self): + """tep4 with three drafts and IndexShare: pass keys over 3k-6k tokens. + + The prompts are past the QSA indexer's 2048-token budget, so the target's decode steps and + its draft layer's take the sparse attention path, which gsm8k never leaves, and IndexShare + makes the draft loop reuse the selection of its first step. A miscomputed draft path keeps + the text correct and only lowers the acceptance length, which is held to its floor in + references/acceptance_length.yaml. + """ + tokenizer = AutoTokenizer.from_pretrained(self.MODEL_PATH) + prompts = _passkey_prompts(tokenizer, _PASSKEY_LENGTHS) + with ( + scoped_env_overrides(_VALIDATE) as rank_env, + LLM( + self.MODEL_PATH, + speculative_config=self.MTP3, + sparse_attention_config=QSASparseAttentionConfig( + index_share_for_mtp_iteration=True + ), + **self.ENGINE, + **_TEP4, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + acceptance_length = _passkey_acceptance( + llm, prompts, "tep4 mtp3 long context", self.MTP3.max_draft_len + ) + assert_acceptance_length(self.ACCEPTANCE_KEY_LONG_CONTEXT, acceptance_length) @skip_not_sm103 @pytest.mark.timeout(5400) @@ -257,3 +342,17 @@ def test_disaggregated_dep2(self, mocker): extra_evaluator_kwargs={GSM8K: self.EXTRA_EVALUATOR_KWARGS}, timeout=3000, ) + + @skip_not_sm103 + def test_tp1_gsm8k(self, mocker): + """One rank without speculation, against the FP8_BLOCK_SCALES row.""" + with ( + scoped_env_overrides(_VALIDATE) as rank_env, + LLM( + self.MODEL_PATH, + **self.ENGINE, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + self._gsm8k(mocker, llm, "tp1") diff --git a/tests/integration/defs/accuracy/references/acceptance_length.yaml b/tests/integration/defs/accuracy/references/acceptance_length.yaml index f30517a4599f..97b798463943 100644 --- a/tests/integration/defs/accuracy/references/acceptance_length.yaml +++ b/tests/integration/defs/accuracy/references/acceptance_length.yaml @@ -283,3 +283,22 @@ ModelingV2Qwen38FlashNextNvfp4Sm103Default::mtp3_long_context: # 2.5% below ref_al, as for tp1_mtp3_long_context, whose deliberately wrong # reused selection scores 3.760. min_al: 3.9 +# Read by TestModelingV2Qwen38FlashNextFp8Sm103Unified (the qwen4_exp FP8 target's +# test_accuracy.py): three drafts of the checkpoint's single MTP module, so acceptance length +# lives in [1.0, 4.0]. modeling_v2's own readings on GB300 (TRTLLM_POPULATE_ACCEPTANCE_LENGTH=1), +# min_al 95% of each. +# test_mtp3_gsm8k[dep4] and [dep4_alltoall], over gsm8k: bf16 KV under attention DP; dep4 read +# 3.394 and dep4_alltoall 3.375. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3: + ref_al: 3.394 + min_al: 3.224 +# test_mtp3_gsm8k[tep4_fp8_kv_breakable], over gsm8k: an FP8 KV pool with the breakable prefill +# graph. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_fp8_kv: + ref_al: 3.274 + min_al: 3.110 +# test_mtp3_long_context, over the 3k-6k-token pass keys at tep4 (the sparse attention path): +# IndexShare on, bf16 KV. Every draft of a pass-key answer is accepted. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_long_context: + ref_al: 4.000 + min_al: 3.800 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index 8981defa6c5c..0e4d94084381 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -318,6 +318,12 @@ Qwen/Qwen3.8-Flash-Next: - quant_algo: FP8_BLOCK_SCALES spec_dec_algo: MTP accuracy: 95.11 + # FP8 KV cache over the checkpoint's bf16 KV, measured on the modeling_v2 FP8 target: TEP4 on + # GB300, MTP draft length 3, the breakable prefill graph, bf16 Mamba SSM cache. + - quant_algo: FP8_BLOCK_SCALES + kv_cache_quant_algo: FP8 + spec_dec_algo: MTP + accuracy: 95.15 - quant_algo: MIXED_PRECISION accuracy: 96.36 - quant_algo: MIXED_PRECISION diff --git a/tests/integration/test_lists/qa/functional/llm_function_core.txt b/tests/integration/test_lists/qa/functional/llm_function_core.txt index 5b395375c2df..3be578a7dfe4 100644 --- a/tests/integration/test_lists/qa/functional/llm_function_core.txt +++ b/tests/integration/test_lists/qa/functional/llm_function_core.txt @@ -815,7 +815,9 @@ accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_ac # Qwen3.8-Flash-Next-FP8, modeling_v2 qwen4_exp FP8 target (sm_103 only): every accuracy test of # the target, the CI ones included. +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_tp1_gsm8k TIMEOUT (60) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_long_context TIMEOUT (60) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4_alltoall] TIMEOUT (60) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 008afc6b6e8d..911bc9308ff7 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -180,6 +180,8 @@ l0_b300: - unittest/_torch/modeling_v2/models/qwen4_exp/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified TIMEOUT (30) + # qwen4_exp FP8 target at tp1, pre-merge: gsm8k without speculation. + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_tp1_gsm8k TIMEOUT (60) - unittest/_torch/modeling_v2/models/minimax_m3/test_shared_kernel_contracts.py --import-mode=importlib TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_gb300.yml b/tests/integration/test_lists/test-db/l0_gb300.yml index 1cec7166955f..8f4587b96a04 100644 --- a/tests/integration/test_lists/test-db/l0_gb300.yml +++ b/tests/integration/test_lists/test-db/l0_gb300.yml @@ -165,10 +165,13 @@ l0_gb300: - unittest/_torch/attention/multi_gpu/test_gqa_kv_replica.py - unittest/_torch/multi_gpu/test_mnnvl_fused_allreduce_norm.py - unittest/_torch/multi_gpu/test_ll_exchange.py - # qwen4_exp FP8 target, pre-merge: its 2- and 4-rank collective matrices (l0_b300 skips them) - # and tep4 MTP3 over an FP8 KV pool. dep4 and dep2 disaggregated run post-merge. + # qwen4_exp FP8 target, pre-merge: its 2- and 4-rank collective matrices (l0_b300 skips them), + # tep4 MTP3 over an FP8 KV pool, and tep4 MTP3 with IndexShare over 3k-6k-token pass keys (the + # sparse attention path) with their acceptance-length floor. dep4 and dep2 disaggregated run + # post-merge. - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_long_context TIMEOUT (60) # K3 collectives and decode-path exchanges, four ranks in one NVLink domain. - condition: From 2c9de0a7fc881336da73c8aa0b86457cd74bdf00 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:13:38 -0800 Subject: [PATCH 12/51] [None][chore] modeling_v2 qwen4_exp fp8: drop the shared host PLE table TRTLLM_MODELING_V2_PLE_SHARED_HOST mapped one host copy of the PLE n-gram table per node through a rendezvous in /dev/shm and pidfd_getfd. It needs ptrace rights over the owner, holds a lock for the whole weight load and keys the table by sampled rows. Remove it: the FP8 target keeps one host-resident copy per rank, as the NVFP4 target does. A family-level replacement that works for both targets follows separately. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> (cherry picked from commit 7adf9730f2636959be4e0f6dd399112281697876) --- .../host_share.py | 457 ------------------ .../modeling.py | 21 +- .../weights.py | 236 +-------- .../test_weights.py | 90 +--- 4 files changed, 11 insertions(+), 793 deletions(-) delete mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py deleted file mode 100644 index 4242ad263c55..000000000000 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/host_share.py +++ /dev/null @@ -1,457 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""One host copy of a read-only table per host, shared by every rank there. - -With `PLE_SHARED_HOST_ENV` set, the ranks of one host map one allocation of the PLE n-gram table -instead of one copy each, across processes and servers (a disaggregated node's context and -generation servers share no communicator). The memory comes from a `Backing`: the engine's -`weights._VmmHostBacking` is a CUDA VMM host allocation exported as a POSIX fd, because on GB300 -registered shmem pages gather several times slower; `MemfdBacking` serves the CPU tests. - -Processes meet through `.lock` (an `flock`) and `.json` (the published descriptor: -owner pid and start time, fd number, size) in `PLE_SHARED_HOST_DIR_ENV` (default `/dev/shm`): - -* The first process to take the lock while no live descriptor exists is the **owner**: it creates - and fills the allocation, then publishes the descriptor and releases the lock. -* Every other process **attaches** by duplicating the owner's fd (`pidfd_getfd`, else - `/proc//fd/`) and mapping it, or keeps a private, unpublished copy if neither works. -* A process arriving while the owner fills is **pending**: it loads its other weights and waits - on the lock only at the table. An owner's death releases the lock; the waiter becomes owner. - -The allocation lives while any process maps it, so a crash leaks nothing. The bytes and the -gather match the per-rank copy, so output is bitwise identical. -""" - -from __future__ import annotations - -import contextlib -import ctypes -import errno -import fcntl -import hashlib -import json -import os -import platform -import time -from typing import Callable, Optional, Protocol - -#: `node` (one copy per host); unset, `0`, `off`, `false` or `no` keep one copy per rank. -PLE_SHARED_HOST_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST" -#: Where the rendezvous files live; must be a directory every sharing process sees (tmpfs). -PLE_SHARED_HOST_DIR_ENV = "TRTLLM_MODELING_V2_PLE_SHARED_HOST_DIR" -# Shared by every user of the host: the files are named per uid, opened with O_NOFOLLOW, created -# 0600 and read only when this uid owns them (`_open_own`). -_DEFAULT_DIR = "/dev/shm" # nosec B108 -_OFF = frozenset({"", "0", "off", "false", "no", "none"}) -_SCOPE = "node" - -OWNER = "owner" -ATTACHED = "attached" -PENDING = "pending" - -# mmap(2) / mbind(2) constants (Linux, identical on aarch64 and x86_64). -_PROT_READ = 0x1 -_PROT_WRITE = 0x2 -_MAP_SHARED = 0x01 -_MPOL_PREFERRED = 1 -_SYS_MBIND = {"aarch64": 235, "x86_64": 237} -# pidfd_open(2) / pidfd_getfd(2): the generic syscall table, shared by aarch64 and x86_64. -_SYS_PIDFD_OPEN = 434 -_SYS_PIDFD_GETFD = 438 - -_libc = ctypes.CDLL(None, use_errno=True) -_libc.mmap.restype = ctypes.c_void_p -_libc.mmap.argtypes = [ - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int, - ctypes.c_int, - ctypes.c_int, - ctypes.c_long, -] -_libc.munmap.restype = ctypes.c_int -_libc.munmap.argtypes = [ctypes.c_void_p, ctypes.c_size_t] -_libc.syscall.restype = ctypes.c_long -_MAP_FAILED = ctypes.c_void_p(-1).value - - -def shared_host_scope() -> Optional[str]: - """`node`, or None (one copy per rank) from `PLE_SHARED_HOST_ENV`.""" - value = os.environ.get(PLE_SHARED_HOST_ENV, "").strip().lower() - if value in _OFF: - return None - if value != _SCOPE: - raise ValueError(f"{PLE_SHARED_HOST_ENV}={value!r}: expected {_SCOPE!r} or off") - return _SCOPE - - -def _parse_cpulist(text: str) -> set: - cpus = set() - for part in filter(None, text.strip().split(",")): - lo, _, hi = part.partition("-") - cpus.update(range(int(lo), int(hi or lo) + 1)) - return cpus - - -def numa_node_of_cpus(cpus: set, sysfs: str = "/sys/devices/system/node") -> Optional[int]: - """The NUMA node whose CPU list holds the lowest of `cpus`, or None when sysfs says nothing.""" - if not cpus: - return None - first = min(cpus) - try: - names = os.listdir(sysfs) - except OSError: - return None - for name in sorted(names): - if not (name.startswith("node") and name[4:].isdigit()): - continue - try: - with open(os.path.join(sysfs, name, "cpulist")) as fh: - if first in _parse_cpulist(fh.read()): - return int(name[4:]) - except OSError: - continue - return None - - -def _proc_start_time(pid: int) -> Optional[int]: - """`pid`'s start time in clock ticks, None if gone; with the pid, it survives pid reuse.""" - try: - with open(f"/proc/{pid}/stat") as fh: - stat = fh.read() - except OSError: - return None - # The command name (field 2) may hold spaces and parentheses: split after its last ')'. - return int(stat[stat.rindex(")") + 2 :].split()[19]) - - -def _dup_from(pid: int, fd: int) -> int: - """Duplicate `pid`'s fd `fd`: `pidfd_getfd` (needs ptrace-attach rights over `pid`), else - reopen `/proc//fd/` (enough for a memfd, not for a CUDA export).""" - pidfd = _libc.syscall(ctypes.c_long(_SYS_PIDFD_OPEN), ctypes.c_int(pid), ctypes.c_uint(0)) - if pidfd >= 0: - try: - local = _libc.syscall( - ctypes.c_long(_SYS_PIDFD_GETFD), - ctypes.c_int(pidfd), - ctypes.c_int(fd), - ctypes.c_uint(0), - ) - if local >= 0: - return int(local) - err = ctypes.get_errno() - finally: - os.close(pidfd) - else: - err = ctypes.get_errno() - try: - return os.open(f"/proc/{pid}/fd/{fd}", os.O_RDWR | os.O_CLOEXEC) - except OSError as error: - raise OSError( - error.errno, - f"cannot duplicate fd {fd} of pid {pid}: pidfd_getfd errno {err} " - f"({os.strerror(err)}), /proc reopen {error.strerror}", - ) from None - - -class Backing(Protocol): - """Where a segment's bytes live: `create` returns the device-read address and an fd another - process duplicates and passes to `attach`; `release` undoes either.""" - - def create(self, nbytes: int) -> tuple[int, int]: ... - - def attach(self, fd: int, nbytes: int) -> int: ... - - def release(self, address: int, nbytes: int) -> None: ... - - def describe(self) -> str: ... - - -#: `register(host_pointer, nbytes, read_only) -> unregister()`: `MemfdBacking`'s device side. -Register = Callable[[int, int, bool], Callable[[], None]] - - -def _prefer_node(address: int, nbytes: int, node: int) -> bool: - """Best effort `mbind(MPOL_PREFERRED, node)` over a mapping.""" - number = _SYS_MBIND.get(platform.machine()) - if number is None or node < 0: - return False - words = node // 64 + 1 - mask = (ctypes.c_ulong * words)() - mask[node // 64] = 1 << (node % 64) - rc = _libc.syscall( - ctypes.c_long(number), - ctypes.c_void_p(address), - ctypes.c_ulong(nbytes), - ctypes.c_int(_MPOL_PREFERRED), - mask, - ctypes.c_ulong(words * 64 + 1), - ctypes.c_uint(0), - ) - return rc == 0 - - -def _mmap(fd: int, nbytes: int) -> int: - address = _libc.mmap(None, nbytes, _PROT_READ | _PROT_WRITE, _MAP_SHARED, fd, 0) - if address in (None, _MAP_FAILED): - err = ctypes.get_errno() - raise OSError(err, f"mmap of {nbytes} bytes failed: {os.strerror(err)}") - return int(address) - - -class MemfdBacking: - """Shared memory (a sealed memfd) mapped in every process and handed to `register`.""" - - def __init__(self, register: Register, node: Optional[int] = None) -> None: - self._register = register - self.node = node - self._unregister: dict[int, Callable[[], None]] = {} - self._munmap = _libc.munmap - - def create(self, nbytes: int) -> tuple[int, int]: - fd = os.memfd_create("trtllm-ple", os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING) - try: - os.ftruncate(fd, nbytes) - # Nobody may resize what everyone maps (a shrink would SIGBUS every reader). - fcntl.fcntl(fd, fcntl.F_ADD_SEALS, fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_GROW) - address = _mmap(fd, nbytes) - except BaseException: - os.close(fd) - raise - if self.node is not None: - _prefer_node(address, nbytes, self.node) - self._map(address, nbytes) - return address, fd - - def attach(self, fd: int, nbytes: int) -> int: - if os.fstat(fd).st_size != nbytes: - raise ValueError(f"the shared memfd is {os.fstat(fd).st_size} bytes, not {nbytes}") - address = _mmap(fd, nbytes) - self._map(address, nbytes) - return address - - def _map(self, address: int, nbytes: int) -> None: - try: - self._unregister[address] = self._register(address, nbytes, False) - except BaseException: - self._munmap(address, nbytes) - raise - - def release(self, address: int, nbytes: int) -> None: - unregister = self._unregister.pop(address, None) - if unregister is not None: - unregister() - self._munmap(address, nbytes) - - def describe(self) -> str: - return "memfd" - - -def _open_own(path: str, flags: int, mode: int = 0o600) -> int: - """`os.open` that refuses a symlink and a file another uid owns (the directory is shared).""" - fd = os.open(path, flags | os.O_NOFOLLOW | os.O_CLOEXEC, mode) - if os.fstat(fd).st_uid != os.getuid(): - os.close(fd) - raise PermissionError(f"{path} is owned by another user") - return fd - - -def segment_name(key: str, scope_label: str) -> str: - """The rendezvous file stem: per user, per table content, per scope.""" - return f"trtllm-ple-{os.getuid()}-{scope_label}-{key[:24]}" - - -def table_key(*parts) -> str: - """A content key over `parts` (bytes or anything `str`-able).""" - digest = hashlib.sha256() - for part in parts: - digest.update(part if isinstance(part, (bytes, bytearray)) else str(part).encode()) - digest.update(b"\0") - return digest.hexdigest() - - -class SharedHostSegment: - """One process's handle on the scope's shared allocation. - - `role` is `OWNER` (fills, then `publish`es), `ATTACHED` (bytes ready) or `PENDING` (`wait` - first). `pointer` is the device address (0 while pending); `private` marks an owner that could - not attach to a live segment and keeps its copy to itself.""" - - def __init__( - self, - key: str, - nbytes: int, - scope_label: str, - backing: Backing, - directory: Optional[str] = None, - log: Callable[[str], None] = print, - ) -> None: - assert nbytes > 0, nbytes - self.key = key - self.nbytes = nbytes - self.scope_label = scope_label - self.backing = backing - self._log = log - directory = directory or os.environ.get(PLE_SHARED_HOST_DIR_ENV, _DEFAULT_DIR) - stem = os.path.join(directory, segment_name(key, scope_label)) - self.lock_path = stem + ".lock" - self.descriptor_path = stem + ".json" - self.role = PENDING - self.private = False - self.pointer = 0 - self.owner_pid = 0 - self.wait_seconds = 0.0 - self._fd = -1 - # Bound now: at interpreter shutdown the modules may already be torn down. - self._close = os.close - self._lock_fd = _open_own(self.lock_path, os.O_RDWR | os.O_CREAT) - try: - fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - if error.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): - raise - return # PENDING: someone holds the lock (an owner filling, or a quick reader) - self._resolve_locked() - - def _resolve_locked(self) -> None: - """With the lock held: attach to a live published segment, or become its owner.""" - if self._try_attach(): - self._unlock() - return - self._create() - - def _unlock(self) -> None: - if self._lock_fd >= 0: - fcntl.flock(self._lock_fd, fcntl.LOCK_UN) - os.close(self._lock_fd) - self._lock_fd = -1 - - def _read_descriptor(self) -> Optional[dict]: - try: - with os.fdopen(_open_own(self.descriptor_path, os.O_RDONLY)) as fh: - desc = json.load(fh) - except (OSError, ValueError): - return None - if desc.get("key") != self.key or desc.get("nbytes") != self.nbytes: - return None - if _proc_start_time(int(desc["pid"])) != desc.get("start"): - return None # the owner is gone (or its pid was reused) - return desc - - def _try_attach(self) -> bool: - desc = self._read_descriptor() - if desc is None: - return False - try: - fd = _dup_from(int(desc["pid"]), int(desc["fd"])) - except OSError as error: - self._log(f"ple shared host: {error}; this process keeps a private copy") - self.private = True - return False - try: - self.pointer = self.backing.attach(fd, self.nbytes) - except Exception as error: - self._log( - f"ple shared host: attaching pid {desc['pid']}'s segment failed ({error}); " - f"this process keeps a private copy" - ) - self.private = True - return False - finally: - # The mapping holds the allocation; the duplicate is no longer needed. - os.close(fd) - self.role = ATTACHED - self.owner_pid = int(desc["pid"]) - return True - - def _create(self) -> None: - """Become the owner of a new, unfilled allocation; the lock stays held until `publish`.""" - self.pointer, self._fd = self.backing.create(self.nbytes) - self.role = OWNER - self.owner_pid = os.getpid() - - def wait(self, poll_seconds: float = 0.2, report_seconds: float = 30.0) -> None: - """Pending -> attached (the owner published) or owner (it never will: it died, or the - lock holder was only reading). Returns at once if not pending.""" - if self.role != PENDING: - return - t0 = time.monotonic() - next_report = t0 + report_seconds - while True: - try: - fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except OSError as error: - if error.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): - raise - now = time.monotonic() - if now >= next_report: - self._log( - f"ple shared host: waiting {now - t0:.0f} s for the {self.scope_label} " - f"segment's owner to fill it ({self.lock_path})" - ) - next_report = now + report_seconds - time.sleep(poll_seconds) - self.wait_seconds = time.monotonic() - t0 - self._resolve_locked() - - def publish(self) -> None: - """Owner, after the bytes are complete (device writes synchronized): make the segment - attachable and release the waiters. A private owner only releases the lock.""" - assert self.role == OWNER and self._fd >= 0, self.role - if not self.private: - desc = dict( - key=self.key, - nbytes=self.nbytes, - pid=os.getpid(), - start=_proc_start_time(os.getpid()), - fd=self._fd, - scope=self.scope_label, - backing=self.backing.describe(), - ) - tmp = f"{self.descriptor_path}.{os.getpid()}.tmp" - with contextlib.suppress(FileNotFoundError): - os.unlink(tmp) - with os.fdopen(_open_own(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "w") as fh: - json.dump(desc, fh) - os.replace(tmp, self.descriptor_path) - self._unlock() - - def describe(self) -> str: - extra = "" - if self.role == OWNER: - extra = ", private copy" if self.private else "" - elif self.role == ATTACHED: - extra = f" to pid {self.owner_pid}'s segment" + ( - f", waited {self.wait_seconds:.1f} s" if self.wait_seconds else "" - ) - return f"shared per {self.scope_label} ({self.backing.describe()}, {self.role}{extra})" - - def _withdraw(self) -> None: - """Owner: unlink the descriptor if it still names this pid and fd, before the fd closes.""" - try: - with os.fdopen(_open_own(self.descriptor_path, os.O_RDONLY)) as fh: - desc = json.load(fh) - if desc.get("pid") == os.getpid() and desc.get("fd") == self._fd: - os.unlink(self.descriptor_path) - except (OSError, ValueError, TypeError): - pass - - def close(self) -> None: - if self.role == OWNER and self._fd >= 0 and not self.private: - self._withdraw() - pointer, self.pointer = self.pointer, 0 - if pointer: - self.backing.release(pointer, self.nbytes) - if self._fd >= 0: - fd, self._fd = self._fd, -1 - self._close(fd) - if self._lock_fd >= 0: - fd, self._lock_fd = self._lock_fd, -1 - self._close(fd) - - def __del__(self) -> None: - try: - self.close() - except Exception: - pass diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 4a6501df0123..98d02b7be9e6 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -50,7 +50,7 @@ attention core and GDN recurrence) each write rows `[:n]` of a fixed output and zero rows `[n:]`. **Host offload.** The PLE n-gram table lives in device-addressable host memory, one copy per -rank, or one per host with `weights.PLE_SHARED_HOST_ENV` (`host_share.py`). +rank. **One-model MTP-Eagle.** With a `speculative_config` the core also holds the MTP module (`MTPLayer`), replayed `max_draft_len` times by the spec worker; the trunk runs every generation @@ -1012,8 +1012,6 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig eos = cfg.eos_token_id self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) self.ple_table_dtype = _weights.PLE_TABLE_DTYPE - # One host copy per host, not per rank (`host_share.py`). - self.ple_shared_host = _weights.ple_shared_host_scope() # Host memory behind the offloaded table; `ple_table` is a non-owning device view. self._ple_table_host = None @@ -1399,18 +1397,11 @@ def _index_rope_table(self, theta: float, device) -> torch.Tensor: def _check_ple_residency(self) -> None: """Assert via the driver that the n-gram table is host-resident; outputs cannot show it.""" table = self.w["ple_table"] - segment = self._ple_table_host - if hasattr(segment, "backing"): - # A VMM mapping: the allocation's own properties say where its pages are. - assert segment.backing.resident_on_host(segment.pointer), ( - f"the shared n-gram table ({segment.describe()}) is not host NUMA memory" - ) - else: - kind = _weights.memory_kind(table.data_ptr()) - assert kind == _weights.CUDA_MEMORY_TYPE_HOST, ( - f"the n-gram table was declared in host memory, but the driver reports " - f"cudaMemoryType {kind} for the pointer the forward gathers from" - ) + kind = _weights.memory_kind(table.data_ptr()) + assert kind == _weights.CUDA_MEMORY_TYPE_HOST, ( + f"the n-gram table was declared in host memory, but the driver reports " + f"cudaMemoryType {kind} for the pointer the forward gathers from" + ) # ---------------------------------------------------------------- bind --- diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index ba09f891d981..6f6a23b28ff7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -51,17 +51,13 @@ from typing import Any, Callable, NamedTuple import torch -from cuda.bindings import driver as cuda_driver from cuda.bindings import runtime as cudart from torch import nn -from tensorrt_llm._dlpack_utils import pack_strided_memory from tensorrt_llm._torch.pyexecutor.config_utils import mamba_effective_tp_size from tensorrt_llm._utils import CUASSERT from tensorrt_llm.logger import logger -from . import host_share as _host_share - ALL_LAYERS = object() Cut = tuple[slice, ...] @@ -268,207 +264,6 @@ def _device_view(host_pointer: int, rows: int, cols: int, dtype, device) -> torc return table -#: One host copy of the n-gram table per host (`node`), mapped by every rank there; default off. -PLE_SHARED_HOST_ENV = _host_share.PLE_SHARED_HOST_ENV - - -def ple_shared_host_scope() -> str | None: - """`node` or None, from `PLE_SHARED_HOST_ENV`; read once into `core.ple_shared_host`.""" - return _host_share.shared_host_scope() - - -def _device_numa_node(device) -> int | None: - """NUMA node of `device`'s local CPUs (ignoring this process's affinity), or None.""" - try: - props = torch.cuda.get_device_properties(device) - base = ( - f"/sys/bus/pci/devices/{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:" - f"{props.pci_device_id:02x}.0" - ) - with open(f"{base}/local_cpulist") as fh: - cpus = _host_share._parse_cpulist(fh.read()) - except (OSError, AttributeError, RuntimeError, ValueError): - return None - return _host_share.numa_node_of_cpus(cpus) - - -def _cu(result): - err, *values = result - if err != cuda_driver.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"CUDA driver call failed: {err}") - return values[0] if len(values) == 1 else tuple(values) - - -class _VmmHostBacking: - """`host_share.Backing` on CUDA VMM memory on host NUMA node `node`, shared by POSIX fd. - - Not `cudaHostRegister` on shared memory: device gathers from such pages are over 10x - slower than from a private registered buffer. - """ - - def __init__(self, device_index: int, node: int) -> None: - self.device_index = device_index - self.node = node - self._maps: dict[int, tuple[int, int]] = {} - - def _prop(self): - D = cuda_driver - prop = D.CUmemAllocationProp() - prop.type = D.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED - prop.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA - prop.location.id = self.node - prop.requestedHandleTypes = ( - D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR - ) - return prop - - def granularity(self) -> int: - option = cuda_driver.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED - return int(_cu(cuda_driver.cuMemGetAllocationGranularity(self._prop(), option))) - - def _map(self, handle, size: int, read_only: bool) -> int: - D = cuda_driver - va = _cu(D.cuMemAddressReserve(size, self.granularity(), 0, 0)) - try: - _cu(D.cuMemMap(va, size, 0, handle, 0)) - desc = D.CUmemAccessDesc() - desc.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE - desc.location.id = self.device_index - desc.flags = ( - D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READ - if read_only - else D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - ) - _cu(D.cuMemSetAccess(va, size, [desc], 1)) - except BaseException: - D.cuMemUnmap(va, size) - D.cuMemAddressFree(va, size) - raise - self._maps[int(va)] = (handle, size) - return int(va) - - def _size(self, nbytes: int) -> int: - granularity = self.granularity() - return -(-nbytes // granularity) * granularity - - def create(self, nbytes: int) -> tuple[int, int]: - D = cuda_driver - size = self._size(nbytes) - handle = _cu(D.cuMemCreate(size, self._prop(), 0)) - try: - va = self._map(handle, size, read_only=False) - except BaseException: - D.cuMemRelease(handle) - raise - try: - fd = _cu( - D.cuMemExportToShareableHandle( - handle, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0 - ) - ) - except BaseException: - self.release(va, nbytes) - raise - return va, int(fd) - - def attach(self, fd: int, nbytes: int) -> int: - D = cuda_driver - handle = _cu( - D.cuMemImportFromShareableHandle( - fd, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR - ) - ) - size = self._size(nbytes) - try: - # The owner's node, not this device's: under the `node` scope they may differ. - self.node = int(_cu(D.cuMemGetAllocationPropertiesFromHandle(handle)).location.id) - try: - return self._map(handle, size, read_only=True) - except RuntimeError: - return self._map(handle, size, read_only=False) - except BaseException: - D.cuMemRelease(handle) - raise - - def release(self, address: int, nbytes: int) -> None: - handle, size = self._maps.pop(address) - D = cuda_driver - D.cuMemUnmap(address, size) - D.cuMemAddressFree(address, size) - D.cuMemRelease(handle) - - def resident_on_host(self, address: int) -> bool: - prop = _cu(cuda_driver.cuMemGetAllocationPropertiesFromHandle(self._maps[address][0])) - return prop.location.type == cuda_driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA - - def describe(self) -> str: - return f"vmm host numa {self.node}" - - -def _view_at(pointer: int, rows: int, cols: int, dtype, device) -> torch.Tensor: - """`[rows, cols]` view of a VMM mapping via DLPack, which names the device explicitly.""" - index = device.index if device.index is not None else torch.cuda.current_device() - flat = pack_strided_memory(pointer, rows * cols, rows * cols, 1, torch.uint8, index) - table = flat.view(rows, cols).view(dtype) - assert table.data_ptr() == pointer and table.device == torch.device("cuda", index) - return table - - -#: Rows sampled at each shard's head and tail: the shared segment's key and an attacher's check. -_SAMPLE_ROWS = 2 - - -def _ngram_samples(weights, keys, rows_per_shard: int) -> list[tuple[int, bytes]]: - samples = [] - for s, key in enumerate(keys): - for local in (0, rows_per_shard - _SAMPLE_ROWS): - rows = _read(weights[key], (slice(local, local + _SAMPLE_ROWS),)) - data = rows.contiguous().view(torch.uint8).cpu().numpy().tobytes() - width = len(data) // _SAMPLE_ROWS - for i in range(_SAMPLE_ROWS): - samples.append((s * rows_per_shard + local + i, data[i * width : (i + 1) * width])) - return samples - - -def _shared_host_table(rows: int, cols: int, dtype, device, weights, keys, rows_per_shard): - """`(table, segment)` over the scope's `host_share.SharedHostSegment`. - - Owner: mapped, empty until load fills and publishes it. Attached: ready. Pending: a meta - placeholder until `_fill_shared_table`. Without VMM host NUMA: `_host_resident_empty`. - """ - index = device.index if device.index is not None else torch.cuda.current_device() - node = _device_numa_node(device) - backing = _VmmHostBacking(index, 0 if node is None else node) - try: - backing.granularity() - except RuntimeError as error: - logger.warning( - f"ple shared host: no VMM host NUMA allocation on this device ({error}); " - "keeping one copy per rank" - ) - return _host_resident_empty(rows, cols, dtype, device) - samples = _ngram_samples(weights, keys, rows_per_shard) - key = _host_share.table_key( - "qwen4_exp ple n-gram table", rows, cols, dtype, *(b for _, b in samples) - ) - segment = _host_share.SharedHostSegment(key, rows * cols, "host", backing) - segment.samples = samples - if segment.role == _host_share.PENDING: - return torch.empty(rows, cols, dtype=dtype, device="meta"), segment - return _view_at(segment.pointer, rows, cols, dtype, device), segment - - -def _check_samples(segment, table: torch.Tensor) -> None: - """An attached segment's sampled rows are the checkpoint's: shard order and content.""" - rows = torch.tensor([row for row, _ in segment.samples], device=table.device) - got = table.view(torch.uint8)[rows].cpu() - for (row, expected), bytes_ in zip(segment.samples, got): - assert bytes(bytes_.numpy()) == expected, ( - f"the shared n-gram table ({segment.describe()}) differs from the checkpoint at " - f"row {row}" - ) - - # ------------------------------------------------------------------ the table -- _LM = "model.language_model" @@ -1461,24 +1256,6 @@ def fill(self, param: nn.Parameter, sources, weights, d: SimpleNamespace) -> set consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) return consumed - def _fill_shared_table(self, core, sources, weights, d: SimpleNamespace) -> set: - """Fill (owner), wait for (pending) or check (attached) the shared `ple_table`.""" - segment = core._ple_table_host - table = core.w["ple_table"] - rows, cols = table.shape - if segment.role == _host_share.PENDING: - segment.wait() - view = _view_at(segment.pointer, rows, cols, table.dtype, core.w["embed"].device) - core.w["ple_table"] = nn.Parameter(view, requires_grad=False) - assert core.w["ple_table"].data_ptr() == view.data_ptr() - if segment.role == _host_share.OWNER: - consumed = self.fill(core.w["ple_table"], sources, weights, d) - torch.cuda.synchronize() - segment.publish() - return consumed - _check_samples(segment, core.w["ple_table"]) - return {ckpt for ckpt, _, _, _ in sources} - def load(self, model, weights) -> None: """Fill every declared parameter and assert checkpoint coverage both ways.""" core = model.model @@ -1493,12 +1270,8 @@ def load(self, model, weights) -> None: "manifest/parameter drift", set(manifest.keys()) ^ set(core.w.keys()), ) - shared = isinstance(getattr(core, "_ple_table_host", None), _host_share.SharedHostSegment) for param_key, sources in manifest.items(): - if shared and param_key == "ple_table": - consumed |= self._fill_shared_table(core, sources, weights, d) - else: - consumed |= self.fill(core.w[param_key], sources, weights, d) + consumed |= self.fill(core.w[param_key], sources, weights, d) if param_key.endswith(_RELEASE_AFTER): torch.cuda.empty_cache() @@ -1742,8 +1515,7 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: The shards are contiguous row chunks (global row `r` is shard `r // rows_per_shard`), so the flat buffer takes the hash's global rows. On CUDA it is host memory the device - addresses (the scope's shared segment with `core.ple_shared_host`); otherwise an ordinary - allocation on the weights' device. + addresses; otherwise an ordinary allocation on the weights' device. """ keys = ngram_shard_keys(d) for key in keys: @@ -1769,10 +1541,6 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: if device.type != "cuda": table = torch.empty(rows, head_dim, dtype=dtype, device=device) core._ple_table_host = None - elif getattr(core, "ple_shared_host", None): - table, core._ple_table_host = _shared_host_table( - rows, head_dim, dtype, device, weights, keys, rows_per_shard - ) else: table, core._ple_table_host = _host_resident_empty(rows, head_dim, dtype, device) core.w["ple_table"] = nn.Parameter(table, requires_grad=False) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py index 5f50e3703bd2..0208bcc279b2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -504,97 +504,13 @@ def u8(name): assert torch.equal(fc1_s.to(torch.bfloat16), torch.cat((up_s, gate_s))) -def _shared_segment_hands_over(tmp_path) -> None: - """`SharedHostSegment` over a memfd: the first handle owns, a concurrent one attaches on - publish, a late one at once, and an owner gone unpublished hands the segment to the waiter.""" - import ctypes - - HS = WT._host_share - nbytes = 1 << 20 - - def segment(label="numa0"): - backing = HS.MemfdBacking(lambda pointer, nbytes, read_only: lambda: None) - return HS.SharedHostSegment("a" * 64, nbytes, label, backing, directory=str(tmp_path)) - - owner = segment() - assert owner.role == HS.OWNER - waiter = segment() - assert waiter.role == HS.PENDING and waiter.pointer == 0 - ctypes.memset(owner.pointer, 0x5A, nbytes) - owner.publish() - waiter.wait() - late = segment() - for handle in (waiter, late): - assert handle.role == HS.ATTACHED and not handle.private - assert ctypes.string_at(handle.pointer + nbytes - 8, 8) == b"Z" * 8 - other = segment("numa1") # another scope never meets this one - assert other.role == HS.OWNER - for handle in (waiter, late, other, owner): - handle.close() - # Closing the owner withdrew its descriptor: the next handle owns a fresh segment. - fresh = segment() - assert fresh.role == HS.OWNER - waiter = segment() - assert waiter.role == HS.PENDING - fresh.close() - waiter.wait(poll_seconds=0.01) - assert waiter.role == HS.OWNER - waiter.close() - - -def _shared_ngram_table_is_the_private_one(configs, tmp_path, monkeypatch) -> None: - """With `ple_shared_host`, the first core fills the segment and the second attaches without - reading the shards; both tables are host memory holding exactly the shards' bytes.""" - monkeypatch.setenv(WT._host_share.PLE_SHARED_HOST_DIR_ENV, str(tmp_path)) - HS = WT._host_share - d = _dims(configs, "tep2", 0, mtp=False) - keys = WT.ngram_shard_keys(d) - gen = torch.Generator().manual_seed(12) - shards = {key: _random_bytes(gen, 5, d.ple_head_dim).view(WT.PLE_TABLE_DTYPE) for key in keys} - device = torch.device("cuda") - cores = [] - for _ in range(2): - core = SimpleNamespace(w={"embed": torch.empty(1, device=device)}, ple_shared_host="node") - rows_per_shard = WT._declare_ngram_table(core, d, shards) - cores.append(core) - assert [c._ple_table_host.role for c in cores] == [HS.OWNER, HS.PENDING] - sources = WT.ngram_table_sources(d, rows_per_shard) - assert WT.MODEL_WEIGHTS._fill_shared_table(cores[0], sources, shards, d) == set(keys) - - class _Untouchable(dict): - def __getitem__(self, key): - raise AssertionError(f"an attached rank read {key}") - - consumed = WT.MODEL_WEIGHTS._fill_shared_table(cores[1], sources, _Untouchable(), d) - assert consumed == set(keys) and cores[1]._ple_table_host.role == HS.ATTACHED - expected = torch.cat([shards[k].view(torch.uint8) for k in keys]).to(device) - for core in cores: - segment = core._ple_table_host - assert segment.backing.resident_on_host(segment.pointer), segment.describe() - assert torch.equal(core.w["ple_table"].view(torch.uint8), expected) - for core in cores: - core._ple_table_host.close() - - -_NEEDS_GPU = pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") - - @pytest.mark.parametrize( "case", [ pytest.param("expert-first", marks=pytest.mark.cpu_only), pytest.param("expert-last", marks=pytest.mark.cpu_only), - pytest.param("shared-segment", marks=pytest.mark.cpu_only), - pytest.param("shared-table", marks=_NEEDS_GPU), ], ) -def test_load_operands(case, request, tmp_path, monkeypatch): - """What `load` fills operands from: expert codes and scales, and the shared n-gram table.""" - if case == "shared-segment": - _shared_segment_hands_over(tmp_path) - elif case == "shared-table": - _shared_ngram_table_is_the_private_one( - request.getfixturevalue("configs"), tmp_path, monkeypatch - ) - else: - _expert_operands(request.getfixturevalue("configs"), case.split("-")[1]) +def test_load_operands(case, request): + """What `load` fills the routed-expert operands from: codes and scales.""" + _expert_operands(request.getfixturevalue("configs"), case.split("-")[1]) From 049ea99102d7c371332630966119bd7bdade39f9 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:19:08 -0800 Subject: [PATCH 13/51] [None][feat] modeling_v2 qwen4_exp: PLE host table per rank, per server or per node The PLE n-gram table (47.7 GiB of FP8 rows on Qwen3.8-Flash-Next) is host-resident with one copy per rank. qwen4_exp/ple_host_table.py stores it as TRTLLM_MODELING_V2_PLE_HOST_TABLE says, for both targets: * rank (default): one private copy per rank, as before. * server: the rows are cut into 8 fixed chunks; under attention TP each rank stores only its chunks (tp 2: 4, tp 4: 2), the n-gram gather reads those rows and writes zeros for the others (ple_ngram_embed's new row_window), and the target sums the gathered rows over the TP group. Every element has one nonzero term, so the sum is exact. One copy per server; no inter-process state. * node: each chunk is one CUDA VMM allocation on host NUMA memory. The first process of the host to bind the chunk's abstract Unix socket creates and fills it and passes its POSIX fd to the others with SCM_RIGHTS (qwen4_exp/chunk_broker.py: peers of another uid are refused through SO_PEERCRED, an owner that exits or aborts hands the chunk to its waiters, nothing is written to the filesystem). Every rank maps all chunks back to back and gathers locally. A rank fills the chunks of its share first, so the ranks of a server load the table in parallel; servers on one node (disaggregated context and generation) share it. The table is keyed by the checkpoint directory's safetensors files (names, sizes, modification times), and attached chunks are checked against the checkpoint's rows. server needs attention TP over a size dividing 8, else it runs as rank. When a rank cannot take part in node (no VMM host NUMA allocation, no checkpoint identity, no owner within the timeout), every rank of the server runs server instead, or rank. The FP8 target keeps its PLE front on the main stream when the rows are summed. Tests: the chunk layout, the broker across processes (publish, late attach, owner exit), the four storage outcomes against the checkpoint's bytes, and row-window shares that sum bitwise to the whole table's output on both gather kernels. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> (cherry picked from commit 66985845cc04b8fd9d21cd8f01c94438e74e4136) --- .../models/qwen4_exp/chunk_broker.py | 154 +++++ .../qwen4_exp/ops/ple/ple_ngram_embed.py | 56 +- .../models/qwen4_exp/ple_host_table.py | 530 ++++++++++++++++++ .../modeling.py | 35 +- .../weights.py | 53 +- .../modeling.py | 56 +- .../weights.py | 56 +- .../ple/test_modeling_v2_ple_ngram_embed.py | 27 + .../test_weights.py | 197 +++++++ 9 files changed, 1120 insertions(+), 44 deletions(-) create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py new file mode 100644 index 000000000000..e081217ac82f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hand one process's exportable allocation to the other processes of the host that ask for it. + +A segment is named by a key. The first process to bind the key's abstract Unix socket +(`\\0trtllm-ple--`) owns it: it creates and fills the allocation, then serves the +allocation's file descriptor to every process that connects (`SCM_RIGHTS`; a peer of another +uid is refused through `SO_PEERCRED`). A process that cannot bind connects and waits. When the +owner exits or aborts before publishing, the kernel closes its sockets and the waiter tries to +own the segment itself. Nothing is written to the filesystem, so a crash leaves nothing behind. + +Abstract socket names belong to a network namespace: processes in different namespaces do not +meet, and each keeps its own copy. +""" + +from __future__ import annotations + +import errno +import json +import os +import socket +import struct +import threading +import time +from typing import Callable, Optional + +OWNER = "owner" +ATTACHED = "attached" + +_CRED = struct.Struct("3i") # struct ucred: pid, uid, gid +_POLL_SECONDS = 1.0 + + +def socket_name(key: str) -> str: + """The abstract socket name of `key` (per uid).""" + return f"\0trtllm-ple-{os.getuid()}-{key}" + + +class OwnedSegment: + """The owner's side: serves `publish`ed `(fd, meta)` to every same-uid peer, until `close`.""" + + def __init__(self, server: socket.socket, key: str) -> None: + self.key = key + self._server = server + self._ready = threading.Event() + self._closed = threading.Event() + self._fd = -1 + self._meta: dict = {} + self._answers: list[threading.Thread] = [] + self._thread = threading.Thread( + target=self._serve, name=f"ple-broker-{key[:12]}", daemon=True + ) + self._thread.start() + + def publish(self, fd: int, meta: dict) -> None: + """Make the allocation available. `fd` stays owned by the caller and must stay open.""" + self._fd = fd + self._meta = dict(meta) + self._ready.set() + + def close(self) -> None: + """Stop serving; peers still waiting see the connection close and retry.""" + self._closed.set() + try: + # Closing alone does not wake the thread blocked in `accept`; shutting down does. + self._server.shutdown(socket.SHUT_RDWR) + except OSError: + pass + self._server.close() + for thread in [self._thread, *self._answers]: + thread.join(_POLL_SECONDS * 5) + + def _serve(self) -> None: + while not self._closed.is_set(): + try: + conn, _ = self._server.accept() + except OSError: + return + answer = threading.Thread(target=self._answer, args=(conn,), daemon=True) + self._answers = [t for t in self._answers if t.is_alive()] + [answer] + answer.start() + + def _answer(self, conn: socket.socket) -> None: + with conn: + try: + _, uid, _ = _CRED.unpack( + conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, _CRED.size) + ) + if uid != os.getuid(): + return + while not self._ready.wait(_POLL_SECONDS): + if self._closed.is_set(): + return + if self._closed.is_set(): + return + socket.send_fds(conn, [json.dumps(self._meta).encode()], [self._fd]) + except OSError: + return + + +def try_own(key: str) -> Optional[OwnedSegment]: + """Own `key` if no process does (non-blocking), else None.""" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) + try: + server.bind(socket_name(key)) + server.listen(64) + except OSError as error: + server.close() + if error.errno == errno.EADDRINUSE: + return None + raise + return OwnedSegment(server, key) + + +def acquire( + key: str, timeout_s: float, log: Callable[[str], None] = print, report_s: float = 60.0 +) -> tuple: + """`(OWNER, OwnedSegment)` when no process owns `key`, else `(ATTACHED, fd, meta)` once + the owner publishes; an owner that exits or aborts first hands `key` to its waiters. + Raises `TimeoutError` when an owner has not published within `timeout_s`.""" + name = socket_name(key) + deadline = time.monotonic() + timeout_s + next_report = time.monotonic() + report_s + while True: + owned = try_own(key) + if owned is not None: + return OWNER, owned + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) + try: + conn.connect(name) + except (ConnectionRefusedError, FileNotFoundError): + # The owner just exited, or is between `bind` and `listen`. + conn.close() + time.sleep(0.05) + continue + got: Optional[tuple] = None + try: + while got is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"no allocation for {key!r} within {timeout_s:.0f} s") + conn.settimeout(min(remaining, report_s)) + try: + got = socket.recv_fds(conn, 4096, 1) + except socket.timeout: + if time.monotonic() >= next_report: + log(f"ple host table: waiting for the owner of {key!r} to publish it") + next_report = time.monotonic() + report_s + finally: + conn.close() + msg, fds, _, _ = got + if fds: + return ATTACHED, fds[0], json.loads(msg.decode()) + # The connection closed without a descriptor: the owner exited or aborted. Retry. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py index 714b8ecabc1a..ba3f8a97ec75 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py @@ -66,7 +66,9 @@ _MIN_LOG_S = 8 -@triton.jit +# `row_lo`, `row_hi`, `row_base`: the gather reads table rows `[row_lo, row_hi)` from the view whose +# first row is `row_base` and writes zeros for the others (a rank's share of a row-sharded table). +@triton.jit(do_not_specialize=["row_lo", "row_hi", "row_base"]) def _ple_ngram_embed_kernel( ids_ptr, pool_ptr, @@ -83,6 +85,9 @@ def _ple_ngram_embed_kernel( stride_table, stride_out, eos, + row_lo, + row_hi, + row_base, NGRAM: tl.constexpr, HEADS_PER: tl.constexpr, HEADS: tl.constexpr, @@ -135,7 +140,12 @@ def _ple_ngram_embed_kernel( d = tl.arange(0, BLOCK_D) m3 = tmask[:, None, None] & hmask[None, :, None] & (d < HEAD_DIM)[None, None, :] - q = tl.load(table_ptr + rows[:, :, None] * stride_table + d[None, None, :], mask=m3, other=0.0) + own = ((rows >= row_lo) & (rows < row_hi))[:, :, None] + q = tl.load( + table_ptr + (rows - row_base)[:, :, None] * stride_table + d[None, None, :], + mask=m3 & own, + other=0.0, + ) val = q.to(tl.float32) * tl.load(scale_ptr).to(tl.float32) cols = heads[None, :, None] * HEAD_DIM + d[None, None, :] tl.store( @@ -173,7 +183,7 @@ def _seq_of_rows(qsl_ptr, t, num_seqs, LOG_S: tl.constexpr): return s -@triton.jit +@triton.jit(do_not_specialize=["row_lo", "row_hi", "row_base"]) def _ple_ngram_embed_rows_kernel( ids_ptr, pool_ptr, @@ -192,6 +202,9 @@ def _ple_ngram_embed_rows_kernel( stride_table, stride_out, eos, + row_lo, + row_hi, + row_base, NGRAM: tl.constexpr, HEADS_PER: tl.constexpr, HEADS: tl.constexpr, @@ -249,7 +262,12 @@ def _ple_ngram_embed_rows_kernel( scale = tl.load(scale_ptr).to(tl.float32) for h in tl.static_range(HEADS): row_h = tl.sum(tl.where((heads == h)[None, :], rows, 0), axis=1) - q = tl.load(table_ptr + row_h[:, None] * stride_table + d[None, :], mask=m2, other=0.0) + own = ((row_h >= row_lo) & (row_h < row_hi))[:, None] + q = tl.load( + table_ptr + (row_h - row_base)[:, None] * stride_table + d[None, :], + mask=m2 & own, + other=0.0, + ) val = q.to(tl.float32) * scale tl.store( out_ptr + t[:, None] * stride_out + (h * HEAD_DIM + d)[None, :], @@ -310,7 +328,10 @@ class PleNgramEmbed(OpWrapper): out[t, h * D : (h + 1) * D] = to(scale.dtype)(float(table[row[t, h]]) * float(scale)) - with `D = table.shape[1]` -- one rounding of the scaled fp8 value. + with `D = table.shape[1]` -- one rounding of the scaled fp8 value. With + `row_window = (lo, hi, base)` the view `table` holds global rows from `base` + on: rows in `[lo, hi)` are read from `table[row - base]` and the others + give zeros, so a row-sharded table's shares sum to the whole table's output. Then, **in place**, `context_pool[slot]` becomes the last `ngram_size - 1` ids of the stream (a zero-row sequence republishes its carry). Unnamed slots are untouched. @@ -461,8 +482,10 @@ def __call__( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> torch.Tensor: num_tokens = input_ids.shape[0] + row_lo, row_hi, row_base = row_window or (0, table.shape[0], 0) num_seqs = state_indices.shape[0] ngram_size = layer_multipliers.numel() heads = head_vocab_sizes.numel() @@ -491,6 +514,9 @@ def __call__( table.stride(0), out.stride(0), eos_token_id, + row_lo, + row_hi, + row_base, NGRAM=ngram_size, HEADS_PER=heads // (ngram_size - 1), HEADS=heads, @@ -532,6 +558,9 @@ def __call__( table.stride(0), out.stride(0), eos_token_id, + row_lo, + row_hi, + row_base, ) embed_constexprs = dict( NGRAM=ngram_size, @@ -575,6 +604,7 @@ def reference( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> tuple[torch.Tensor, torch.Tensor]: """`(out, context_pool)` as the call leaves them, from the scalar references of the two entries it fuses -- `ngram_token_history` @@ -598,7 +628,14 @@ def reference( rows = ngram_hash.reference( window, layer_multipliers, head_vocab_sizes, head_offsets, eos_token_id, 1 ).reshape(window.shape[0], -1) - gathered = table[rows.reshape(-1)].double().reshape(rows.shape[0], -1) + flat = rows.reshape(-1) + lo, hi, base = row_window or (0, table.shape[0], 0) + own = (flat >= lo) & (flat < hi) + picked = torch.zeros( + (flat.numel(), table.shape[1]), dtype=torch.float64, device=table.device + ) + picked[own] = table[flat[own] - base].double() + gathered = picked.reshape(rows.shape[0], -1) out = (gathered * scale.double().reshape(())).to(scale.dtype) return out, pool @@ -616,6 +653,7 @@ def is_valid( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> None: num_seqs = state_indices.shape[0] ngram_size = layer_multipliers.numel() @@ -643,6 +681,12 @@ def is_valid( assert table.dim() == 2 and table.stride(1) == 1, "table [rows, D], unit column stride" assert scale.numel() == 1, "one per-tensor scale" assert max_rows >= 1, "max_rows >= 1" + if row_window is not None: + lo, hi, base = row_window + assert 0 <= base <= lo <= hi and hi - base <= table.shape[0], ( + f"row_window {row_window}: rows [lo, hi) must lie in the view of " + f"{table.shape[0]} rows starting at row `base`" + ) ple_ngram_embed = PleNgramEmbed() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py new file mode 100644 index 000000000000..7cf949aa89a6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Where the PLE n-gram table lives in host memory, for both Qwen3.8-Flash-Next targets. + +The table's rows are cut into `CHUNKS` fixed chunks (`chunk_rows`); a rank of a server of +`world` ranks is responsible for chunks `[rank * CHUNKS / world, (rank + 1) * CHUNKS / world)`. +`PLE_HOST_TABLE_ENV` picks how the rows are stored and looked up: + +* `rank` (default): every rank holds the whole table in private host memory. +* `server`: under attention TP every rank holds only the rows of its chunks; the n-gram gather + reads those rows and writes zeros for the others (`window`), and the caller sums the gathered + rows over the attention-TP group (`needs_sum`). One copy per server, no inter-process state. + Exact: every element has one nonzero term. +* `node`: every chunk is one CUDA VMM allocation on host NUMA memory, shared by every process of + the host through `chunk_broker`; every rank maps all chunks back to back and gathers locally. + A rank fills the chunks it is responsible for and attaches to the others. One copy per node. + +`server` needs attention TP with a size that divides `CHUNKS`, else it runs as `rank`. When a +rank cannot take part in `node` (no VMM host NUMA allocation, no checkpoint identity, no owner +within the timeout), every rank of the server runs `server` instead, or `rank`. +""" + +from __future__ import annotations + +import glob +import hashlib +import math +import os +from typing import Callable, Optional + +import torch +from cuda.bindings import driver as cuda_driver +from cuda.bindings import runtime as cudart + +from tensorrt_llm._dlpack_utils import pack_strided_memory +from tensorrt_llm._utils import CUASSERT +from tensorrt_llm.logger import logger + +from . import chunk_broker as _broker + +#: `rank`, `server` or `node` (see the module docstring); default `rank`. +PLE_HOST_TABLE_ENV = "TRTLLM_MODELING_V2_PLE_HOST_TABLE" +RANK = "rank" +SERVER = "server" +NODE = "node" +_MODES = (RANK, SERVER, NODE) + +#: Fixed chunks of the table; every server size that divides it maps whole chunks. +CHUNKS = 8 +#: Chunk byte sizes are multiples of this: the VMM host NUMA granularity, so `node` can map the +#: chunks back to back as one table. +CHUNK_ALIGN = 2 << 20 +#: How long a rank waits for another process to publish a chunk before giving up on `node`. +_ATTACH_TIMEOUT_S = 3600.0 +_CUDA_MEMORY_TYPE_HOST = 1 + +Copy = Callable[[torch.Tensor, int, int, int], None] +Read = Callable[[int, int], torch.Tensor] +Private = Callable[[int, int, torch.dtype, torch.device], tuple] + + +def host_table_mode() -> str: + """The requested mode from `PLE_HOST_TABLE_ENV`.""" + value = os.environ.get(PLE_HOST_TABLE_ENV, RANK).strip().lower() or RANK + if value not in _MODES: + raise ValueError(f"{PLE_HOST_TABLE_ENV}={value!r}: expected one of {', '.join(_MODES)}") + return value + + +def chunk_rows(rows: int, cols: int, align: int = CHUNK_ALIGN) -> int: + """Rows of one chunk: an even split of `rows` over `CHUNKS`, rounded up so a chunk's bytes + (`cols` one-byte columns a row) are a multiple of `align`.""" + unit = align // math.gcd(align, cols) + per = -(-rows // CHUNKS) + return -(-per // unit) * unit + + +def chunk_range(k: int, rows: int, cols: int, align: int = CHUNK_ALIGN) -> tuple[int, int]: + """Global rows `[lo, hi)` of chunk `k` (empty past the table's end).""" + per = chunk_rows(rows, cols, align) + return min(k * per, rows), min((k + 1) * per, rows) + + +def owned_chunks(rank: int, world: int) -> range: + """The chunks rank `rank` of `world` is responsible for.""" + assert world > 0 and CHUNKS % world == 0, (rank, world) + per = CHUNKS // world + return range(rank * per, (rank + 1) * per) + + +def owned_rows(rank: int, world: int, rows: int, cols: int, align: int = CHUNK_ALIGN) -> tuple: + """Global rows `[lo, hi)` of rank `rank`'s chunks.""" + chunks = owned_chunks(rank, world) + return chunk_range(chunks[0], rows, cols, align)[0], chunk_range(chunks[-1], rows, cols, align)[ + 1 + ] + + +def checkpoint_identity(checkpoint_dir: Optional[str], *parts) -> Optional[str]: + """A key for the table's bytes: the checkpoint directory, its safetensors files' names, sizes + and modification times, and `parts`; None when the directory or its files cannot be read.""" + if not checkpoint_dir: + return None + root = os.path.realpath(checkpoint_dir) + try: + files = sorted(glob.glob(os.path.join(root, "*.safetensors"))) + stats = [(os.path.basename(f), os.stat(f)) for f in files] + except OSError: + return None + if not stats: + return None + digest = hashlib.sha256(root.encode()) + for name, st in stats: + digest.update(f"\0{name}:{st.st_size}:{st.st_mtime_ns}".encode()) + for part in parts: + digest.update(f"\0{part}".encode()) + return digest.hexdigest()[:32] + + +def shard_rows(weights, keys, rows_per_shard: int, read_entry) -> tuple[Copy, Read]: + """`(copy, read)` over a table stored as equal contiguous row shards `keys`. + + `copy(dst, lo, hi, base)` writes global rows `[lo, hi)` into `dst`, whose row 0 is global row + `base`, one shard at a time; `read(lo, hi)` returns those rows. `read_entry(entry, cut)` reads + rows `cut` of a checkpoint entry.""" + + def pieces(lo: int, hi: int): + s = lo // rows_per_shard + while lo < hi: + a = lo - s * rows_per_shard + b = min(hi - s * rows_per_shard, rows_per_shard) + yield lo, read_entry(weights[keys[s]], (slice(a, b),)) + lo += b - a + s += 1 + + def copy(dst: torch.Tensor, lo: int, hi: int, base: int) -> None: + for start, src in pieces(lo, hi): + dst[start - base : start - base + src.shape[0]].copy_(src.to(dst.device)) + + def read(lo: int, hi: int) -> torch.Tensor: + return torch.cat([src for _, src in pieces(lo, hi)]) + + return copy, read + + +def _parse_cpulist(text: str) -> set: + cpus = set() + for part in filter(None, text.strip().split(",")): + lo, _, hi = part.partition("-") + cpus.update(range(int(lo), int(hi or lo) + 1)) + return cpus + + +def device_numa_node(device) -> Optional[int]: + """The NUMA node of `device`'s local CPUs, or None.""" + try: + props = torch.cuda.get_device_properties(device) + pci = f"{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:{props.pci_device_id:02x}.0" + with open(f"/sys/bus/pci/devices/{pci}/local_cpulist") as fh: + first = min(_parse_cpulist(fh.read())) + for name in sorted(os.listdir("/sys/devices/system/node")): + if name.startswith("node") and name[4:].isdigit(): + with open(f"/sys/devices/system/node/{name}/cpulist") as fh: + if first in _parse_cpulist(fh.read()): + return int(name[4:]) + except (OSError, AttributeError, RuntimeError, ValueError): + return None + return None + + +def _cu(result): + err, *values = result + if err != cuda_driver.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"CUDA driver call failed: {err}") + return values[0] if len(values) == 1 else tuple(values) + + +class _VmmChunks: + """`count` chunk allocations of `chunk_bytes` on host NUMA memory, mapped back to back for + one device, each created here or imported from a POSIX fd another process exported.""" + + def __init__(self, device_index: int, chunk_bytes: int, count: int, node: int) -> None: + self.device_index = device_index + self.chunk_bytes = chunk_bytes + self.count = count + self.node = node + granularity = int( + _cu( + cuda_driver.cuMemGetAllocationGranularity( + self._prop(node), + cuda_driver.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED, + ) + ) + ) + if chunk_bytes % granularity: + raise RuntimeError( + f"chunks of {chunk_bytes} bytes are not a multiple of the {granularity}-byte " + "VMM host NUMA granularity" + ) + self.va = int(_cu(cuda_driver.cuMemAddressReserve(count * chunk_bytes, granularity, 0, 0))) + self._handles: dict[int, object] = {} + + @staticmethod + def _prop(node: int): + D = cuda_driver + prop = D.CUmemAllocationProp() + prop.type = D.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + prop.location.id = node + prop.requestedHandleTypes = ( + D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + return prop + + def _map(self, k: int, handle, read_only: bool) -> None: + D = cuda_driver + address = self.va + k * self.chunk_bytes + _cu(D.cuMemMap(address, self.chunk_bytes, 0, handle, 0)) + desc = D.CUmemAccessDesc() + desc.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = self.device_index + desc.flags = ( + D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READ + if read_only + else D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + ) + try: + _cu(D.cuMemSetAccess(address, self.chunk_bytes, [desc], 1)) + except RuntimeError: + if not read_only: + raise + desc.flags = D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + _cu(D.cuMemSetAccess(address, self.chunk_bytes, [desc], 1)) + self._handles[k] = handle + + def create(self, k: int) -> None: + handle = _cu(cuda_driver.cuMemCreate(self.chunk_bytes, self._prop(self.node), 0)) + try: + self._map(k, handle, read_only=False) + except BaseException: + cuda_driver.cuMemRelease(handle) + raise + + def export(self, k: int) -> int: + D = cuda_driver + return int( + _cu( + D.cuMemExportToShareableHandle( + self._handles[k], + D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, + 0, + ) + ) + ) + + def attach(self, k: int, fd: int) -> None: + D = cuda_driver + handle = _cu( + D.cuMemImportFromShareableHandle( + fd, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + ) + try: + self._map(k, handle, read_only=True) + except BaseException: + D.cuMemRelease(handle) + raise + + def complete(self) -> bool: + return len(self._handles) == self.count + + def view(self, rows: int, cols: int, dtype, device) -> torch.Tensor: + index = device.index if device.index is not None else torch.cuda.current_device() + flat = pack_strided_memory(self.va, rows * cols, rows * cols, 1, torch.uint8, index) + return flat.view(rows, cols).view(dtype) + + def resident_on_host(self) -> bool: + host = cuda_driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + return all( + _cu(cuda_driver.cuMemGetAllocationPropertiesFromHandle(h)).location.type == host + for h in self._handles.values() + ) + + def close(self) -> None: + D = cuda_driver + for k, handle in self._handles.items(): + D.cuMemUnmap(self.va + k * self.chunk_bytes, self.chunk_bytes) + D.cuMemRelease(handle) + self._handles.clear() + if self.va: + D.cuMemAddressFree(self.va, self.count * self.chunk_bytes) + self.va = 0 + + +def _all_agree(ok: bool) -> bool: + """Whether every rank of the server reports `ok` (a collective: every rank calls it).""" + from tensorrt_llm._utils import mpi_allgather, mpi_disabled + + if not mpi_disabled(): + result = mpi_allgather(bool(ok)) + return all(result) if isinstance(result, list) else bool(result) + import torch.distributed as dist + + if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1: + return ok + flag = torch.tensor([1 if ok else 0], dtype=torch.int32, device="cuda") + dist.all_reduce(flag, op=dist.ReduceOp.MIN) + return bool(flag.item()) + + +class PleHostTable: + """One rank's storage of the PLE n-gram table (`rows` x `cols` one-byte elements). + + After `fill`: `table` is the view the gather reads, `window` its `row_window` (None for the + whole table), `needs_sum` whether the caller sums the gathered rows over the attention-TP + group, and `mode` the mode that runs. `rank` / `world` are this rank's place in the TP group + and `dp` the attention-DP size; `private(rows, cols, dtype, device)` returns + `(view, owner)` of the target's private host allocation.""" + + def __init__( + self, + mode: str, + rows: int, + cols: int, + dtype: torch.dtype, + device: torch.device, + *, + rank: int, + world: int, + dp: int, + private: Private, + identity: Optional[str] = None, + align: int = CHUNK_ALIGN, + log: Callable[[str], None] = logger.info, + ) -> None: + assert mode in _MODES, mode + assert torch.empty(0, dtype=dtype).element_size() == 1, dtype + self.requested = mode + self.rows, self.cols, self.dtype, self.device = rows, cols, dtype, device + self.rank, self.world, self.dp = rank, world, dp + self.align = align + self.identity = identity + self._private = private + self._log = log + self.mode = RANK + self.window: Optional[tuple[int, int, int]] = None + self.table: Optional[torch.Tensor] = None + self._owner = None + self._vmm: Optional[_VmmChunks] = None + self._segments: dict[int, _broker.OwnedSegment] = {} + self._fds: list[int] = [] + self._node_error: Optional[str] = None + self._attached = 0 + if device.type != "cuda": + self.table = torch.empty(rows, cols, dtype=dtype, device=device) + elif mode == NODE: + self._declare_node() + else: + self.mode = SERVER if mode == SERVER and self._shards() else RANK + if mode == SERVER and self.mode == RANK and world > 1: + self._log( + f"ple host table: {SERVER} needs attention TP over a size that divides " + f"{CHUNKS} (tp {world}, attention DP {dp}); keeping one copy per rank" + ) + self._declare_private() + + @property + def needs_sum(self) -> bool: + return self.mode == SERVER + + def _shards(self) -> bool: + return self.dp == 1 and self.world > 1 and CHUNKS % self.world == 0 + + def _key(self, k: int) -> str: + return f"{self.identity}-c{k}" + + # -------------------------------------------------------------------- declare -- + + def _declare_private(self) -> None: + if self.mode == SERVER: + lo, hi = owned_rows(self.rank, self.world, self.rows, self.cols, self.align) + self.window = (lo, hi, lo) + else: + lo, hi = 0, self.rows + if hi > lo: + self.table, self._owner = self._private(hi - lo, self.cols, self.dtype, self.device) + else: + self.table = torch.empty(0, self.cols, dtype=self.dtype, device=self.device) + + def _declare_node(self) -> None: + """Map nothing yet; own (create) the chunks this rank is responsible for if no process + does. A failure is kept for `fill`, where every rank agrees on the mode.""" + if self.identity is None: + self._node_error = "no checkpoint identity" + return + try: + chunk_bytes = chunk_rows(self.rows, self.cols, self.align) * self.cols + node = device_numa_node(self.device) + index = ( + self.device.index if self.device.index is not None else torch.cuda.current_device() + ) + self._vmm = _VmmChunks(index, chunk_bytes, CHUNKS, 0 if node is None else node) + if CHUNKS % self.world == 0: + for k in owned_chunks(self.rank, self.world): + segment = _broker.try_own(self._key(k)) + if segment is not None: + self._vmm.create(k) + self._segments[k] = segment + except (RuntimeError, OSError) as error: + self._node_error = str(error) + self._abort_node() + + # ----------------------------------------------------------------------- fill -- + + def fill(self, copy: Copy, read: Read) -> None: + """Fill (and in `node`, share and gather) the rows this rank stores; ends with `table`, + `window` and `mode` set. Every rank of the server calls it.""" + if self.device.type != "cuda": + copy(self.table, 0, self.rows, 0) + return + if self.requested != NODE: + lo, hi, base = self.window or (0, self.rows, 0) + if hi > lo: + copy(self.table, lo, hi, base) + return + ok = self._node_error is None and self._fill_node(copy, read) + if _all_agree(ok): + self.mode = NODE + return + reason = self._node_error or "another rank of this server could not share the table" + self.mode = SERVER if self._shards() else RANK + self._log(f"ple host table: {NODE} unavailable ({reason}); running {self.mode}") + if ok: + # Keep the full shared view; `server` reads only this rank's rows from it. + if self.mode == SERVER: + lo, hi = owned_rows(self.rank, self.world, self.rows, self.cols, self.align) + self.window = (lo, hi, 0) + return + self._abort_node() + self._declare_private() + lo, hi, base = self.window or (0, self.rows, 0) + if hi > lo: + copy(self.table, lo, hi, base) + + def _fill_node(self, copy: Copy, read: Read) -> bool: + vmm = self._vmm + self.table = vmm.view(self.rows, self.cols, self.dtype, self.device) + try: + for k in list(self._segments): + self._fill_and_publish(k, copy) + for k in range(CHUNKS): + if k in self._segments: + continue + got = _broker.acquire(self._key(k), _ATTACH_TIMEOUT_S, self._log) + if got[0] == _broker.OWNER: + vmm.create(k) + self._segments[k] = got[1] + self._fill_and_publish(k, copy) + continue + _, fd, meta = got + try: + if meta.get("chunk_bytes") != vmm.chunk_bytes: + raise RuntimeError(f"chunk {k} is {meta.get('chunk_bytes')} bytes") + vmm.attach(k, fd) + finally: + os.close(fd) + self._attached += 1 + self._check_chunk(k, read) + except (RuntimeError, OSError, TimeoutError, AssertionError) as error: + self._node_error = str(error) + self._abort_node() + return False + return vmm.complete() + + def _fill_and_publish(self, k: int, copy: Copy) -> None: + lo, hi = chunk_range(k, self.rows, self.cols, self.align) + if hi > lo: + copy(self.table, lo, hi, 0) + torch.cuda.synchronize(self.device) + fd = self._vmm.export(k) + self._fds.append(fd) + self._segments[k].publish(fd, {"chunk": k, "chunk_bytes": self._vmm.chunk_bytes}) + + def _check_chunk(self, k: int, read: Read) -> None: + """An attached chunk's first and last rows are the checkpoint's.""" + lo, hi = chunk_range(k, self.rows, self.cols, self.align) + for row in {lo, hi - 1} if hi > lo else (): + got = self.table[row : row + 1].view(torch.uint8).cpu() + want = read(row, row + 1).contiguous().view(torch.uint8).cpu() + assert torch.equal(got, want), ( + f"shared chunk {k} differs from the checkpoint at row {row}" + ) + + def _abort_node(self) -> None: + for segment in self._segments.values(): + segment.close() + self._segments.clear() + for fd in self._fds: + os.close(fd) + self._fds.clear() + if self._vmm is not None: + self._vmm.close() + self._vmm = None + self.table = None + + # ----------------------------------------------------------------- inspection -- + + def resident_on_host(self) -> bool: + """Whether the table's pages are host memory, from the driver.""" + if self._vmm is not None: + return self._vmm.resident_on_host() + if self.table is None or self.table.numel() == 0: + return True + (attributes,) = CUASSERT(cudart.cudaPointerGetAttributes(self.table.data_ptr())) + return int(attributes.type) == _CUDA_MEMORY_TYPE_HOST + + def describe(self) -> str: + if self.mode == NODE: + return ( + f"{NODE}: {CHUNKS} shared chunks, this process created {len(self._segments)} and " + f"attached {self._attached}" + ) + if self.mode == SERVER: + lo, hi, _ = self.window + return f"{SERVER}: rows [{lo}, {hi}) of {self.rows}, summed over {self.world} ranks" + return f"{RANK}: the whole table" + + def close(self) -> None: + self._abort_node() + self._owner = None diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 98d02b7be9e6..ab1cf3cb2410 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -49,8 +49,8 @@ bookkeeping live there. The graph breaks (the vision splice, the PLE block, each layer's attention core and GDN recurrence) each write rows `[:n]` of a fixed output and zero rows `[n:]`. -**Host offload.** The PLE n-gram table lives in device-addressable host memory, one copy per -rank. +**Host offload.** The PLE n-gram table lives in device-addressable host memory: one copy per +rank, per server or per node (`qwen4_exp/ple_host_table.py`). **One-model MTP-Eagle.** With a `speculative_config` the core also holds the MTP module (`MTPLayer`), replayed `max_draft_len` times by the spec worker; the trunk runs every generation @@ -261,11 +261,13 @@ from tensorrt_llm._torch.utils import is_gdn_flashinfer_replay_enabled, is_gdn_replay_enabled from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.functional import AllReduceFusionOp, AllReduceStrategy +from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo if TYPE_CHECKING: from tensorrt_llm.llmapi.llm_args import TorchLlmArgs +from .. import ple_host_table as _ple_host from . import pool as _pool from . import vision as _vision from . import weights as _weights @@ -1012,7 +1014,9 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig eos = cfg.eos_token_id self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) self.ple_table_dtype = _weights.PLE_TABLE_DTYPE - # Host memory behind the offloaded table; `ple_table` is a non-owning device view. + # How the table is stored (`ple_host_table`); the storage is `_ple_table_host` after load. + self.ple_host_mode = _ple_host.host_table_mode() + self.checkpoint_dir = getattr(composite_config, "_name_or_path", None) self._ple_table_host = None vcfg = getattr(composite_config, "vision_config", None) @@ -1214,7 +1218,9 @@ def post_load_weights(self) -> None: for i in self.gdn_layers: w[f"l{i}_in_proj_w"][sum(self.in_proj_split) :].zero_() # An index past the table's end is a device-side assert that poisons the process. - table_rows = w["ple_table"].shape[0] + # Rows of the whole table: under `server` storage `ple_table` holds only this rank's. + host = self._ple_table_host + table_rows = host.rows if host is not None else w["ple_table"].shape[0] used_rows = int(w["ple_offs"][-1]) + int(w["ple_vocab"][-1]) assert used_rows <= table_rows, ( f"the n-gram table has {table_rows} rows but its per-head offsets and vocabulary " @@ -1396,12 +1402,12 @@ def _index_rope_table(self, theta: float, device) -> torch.Tensor: def _check_ple_residency(self) -> None: """Assert via the driver that the n-gram table is host-resident; outputs cannot show it.""" - table = self.w["ple_table"] - kind = _weights.memory_kind(table.data_ptr()) - assert kind == _weights.CUDA_MEMORY_TYPE_HOST, ( - f"the n-gram table was declared in host memory, but the driver reports " - f"cudaMemoryType {kind} for the pointer the forward gathers from" + host = self._ple_table_host + assert host.resident_on_host(), ( + f"the n-gram table ({host.describe()}) was declared in host memory, but the driver " + "reports device memory for the pointer the forward gathers from" ) + logger.info(f"ple n-gram table: {host.describe()}") # ---------------------------------------------------------------- bind --- @@ -4873,7 +4879,10 @@ def _ple_block(self, hc, hc_out, normed_out) -> None: S.ple_offs, core.eos_token_id, num_tokens, + row_window=core._ple_table_host.window, ) + if core._ple_table_host.needs_sum: + ngram = core.tp_sum(ngram) ple_gate_conv_inject_norm( hc[:num_tokens], cublas_mm(ngram, S.ple_key), @@ -5827,7 +5836,10 @@ def _ple_front(self, input_ids, ple_state, stage) -> tuple: S.ple_offs, core.eos_token_id, rows, + row_window=core._ple_table_host.window, ) + if core._ple_table_host.needs_sum: + ngram = core.tp_sum(ngram) return cublas_mm(ngram, S.ple_key), cublas_mm(ngram, S.ple_val) def _ple_block( @@ -5976,10 +5988,11 @@ def forward( hc_count = core.hc_count layers, S = self._layers, self._stack - # The PLE front reads no residual, so it runs on the PLE stream beside the first layers. + # The PLE front reads no residual, so it runs on the PLE stream beside the first layers; + # a row-sharded table's sum is a TP collective, which stays on the main stream. ple_stream = getattr(core, "_ple_stream", None) ple_front = None - if ple_stream is not None and core.ple_layer > 0: + if ple_stream is not None and core.ple_layer > 0 and not core._ple_table_host.needs_sum: caller = torch.cuda.current_stream() ple_stream.wait_stream(caller) with torch.cuda.stream(ple_stream): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index 6f6a23b28ff7..e0661fdc50fa 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -58,6 +58,8 @@ from tensorrt_llm._utils import CUASSERT from tensorrt_llm.logger import logger +from .. import ple_host_table as _ple_host + ALL_LAYERS = object() Cut = tuple[slice, ...] @@ -1271,6 +1273,9 @@ def load(self, model, weights) -> None: set(manifest.keys()) ^ set(core.w.keys()), ) for param_key, sources in manifest.items(): + if param_key == "ple_table": + consumed |= _fill_ngram_table(core, d, weights, rows_per_shard) + continue consumed |= self.fill(core.w[param_key], sources, weights, d) if param_key.endswith(_RELEASE_AFTER): torch.cuda.empty_cache() @@ -1515,7 +1520,8 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: The shards are contiguous row chunks (global row `r` is shard `r // rows_per_shard`), so the flat buffer takes the hash's global rows. On CUDA it is host memory the device - addresses; otherwise an ordinary allocation on the weights' device. + addresses, stored as `core.ple_host_mode` says (`ple_host_table`); otherwise an ordinary + allocation on the weights' device. """ keys = ngram_shard_keys(d) for key in keys: @@ -1538,14 +1544,47 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: ) device = core.w["embed"].device rows = d.ple_shards * rows_per_shard - if device.type != "cuda": - table = torch.empty(rows, head_dim, dtype=dtype, device=device) - core._ple_table_host = None - else: - table, core._ple_table_host = _host_resident_empty(rows, head_dim, dtype, device) + mode = getattr(core, "ple_host_mode", _ple_host.RANK) + identity = None + if mode == _ple_host.NODE: + identity = _ple_host.checkpoint_identity( + getattr(core, "checkpoint_dir", None), rows, head_dim, dtype, *keys + ) + mapping = getattr(core, "mapping", None) + core._ple_table_host = _ple_host.PleHostTable( + mode, + rows, + head_dim, + dtype, + device, + rank=mapping.tp_rank if mapping is not None else 0, + world=mapping.tp_size if mapping is not None else 1, + dp=getattr(core, "dp_size", 1), + private=_host_resident_empty, + identity=identity, + ) + _bind_ngram_table(core) + return rows_per_shard + + +def _bind_ngram_table(core) -> None: + """`core.w["ple_table"]` as the host table's current view (a placeholder until a `node` + table is mapped).""" + host = core._ple_table_host + table = host.table + if table is None: + table = torch.empty(0, host.cols, dtype=host.dtype, device="meta") core.w["ple_table"] = nn.Parameter(table, requires_grad=False) # Host offload relies on `nn.Parameter` sharing storage rather than copying. assert core.w["ple_table"].data_ptr() == table.data_ptr(), ( "wrapping the n-gram table in nn.Parameter moved its storage" ) - return rows_per_shard + + +def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: + """Fill this rank's rows of the n-gram table (`ple_host_table`); return the shard keys.""" + keys = ngram_shard_keys(d) + copy, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + core._ple_table_host.fill(copy, read) + _bind_ngram_table(core) + return set(keys) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 220f46095266..d2a1834ea5e2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -431,6 +431,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo +from .. import ple_host_table as _ple_host from . import pool as _pool from . import vision as _vision from . import weights as _weights @@ -905,6 +906,18 @@ def _moe_prefill_copy(one_rank: bool, trunk_moe_backend: str, max_num_tokens: in ) +def _ple_window(core): + """The n-gram gather's `row_window` for this rank's table (None: the whole table).""" + host = core._ple_table_host + return None if host is None else host.window + + +def _ple_needs_sum(core) -> bool: + """Whether the gathered n-gram rows are this rank's share, summed over attention TP.""" + host = core._ple_table_host + return host is not None and host.needs_sum + + def _ple_host_offload_enabled() -> bool: """Whether the n-gram table is declared in host memory (the default).""" return _switch(_PLE_HOST_OFFLOAD_ENV, default=True) @@ -1194,6 +1207,14 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) self.ple_table_dtype = _weights.PLE_TABLE_DTYPE self.ple_host_offload = _ple_host_offload_enabled() + # How the host-resident table is stored (`ple_host_table`). + self.ple_host_mode = _ple_host.host_table_mode() + if self.ple_host_mode != _ple_host.RANK and not self.ple_host_offload: + logger.warning( + f"{_ple_host.PLE_HOST_TABLE_ENV}={self.ple_host_mode} applies to the host-resident " + f"n-gram table; {_PLE_HOST_OFFLOAD_ENV}=0 keeps one device copy per rank" + ) + self.checkpoint_dir = getattr(composite_config, "_name_or_path", None) _check_switches(self.one_rank) # CPU allocation behind the offloaded table; the `ple_table` parameter is a # device-address view that owns none of those pages. @@ -1354,7 +1375,9 @@ def post_load_weights(self) -> None: } # Reconcile the table height against the checkpoint's id-space tensors: an index past # the end is a device-side assert that poisons the process. - table_rows = w["ple_table"].shape[0] + # Rows of the whole table: under `server` storage `ple_table` holds only this rank's. + host = self._ple_table_host + table_rows = host.rows if host is not None else w["ple_table"].shape[0] used_rows = int(w["ple_offs"][-1]) + int(w["ple_vocab"][-1]) assert used_rows <= table_rows, ( f"the n-gram table has {table_rows} rows but its per-head offsets and vocabulary " @@ -1576,23 +1599,24 @@ def _report_ple_residency(self) -> None: output-identical, so the pointer's memory kind read back from the driver is the signal that cannot be faked.""" table = self.w["ple_table"] - kind = _weights.memory_kind(table.data_ptr()) - expected = ( - _weights.CUDA_MEMORY_TYPE_HOST - if self.ple_host_offload - else _weights.CUDA_MEMORY_TYPE_DEVICE - ) - assert kind == expected, ( - f"the n-gram table was declared with {_PLE_HOST_OFFLOAD_ENV}=" - f"{'1' if self.ple_host_offload else '0'}, but the driver reports cudaMemoryType " - f"{kind} for the pointer the forward gathers from" - ) + host = self._ple_table_host + if host is not None: + assert host.resident_on_host(), ( + f"the n-gram table ({host.describe()}) was declared in host memory, but the " + "driver reports device memory for the pointer the forward gathers from" + ) + else: + kind = _weights.memory_kind(table.data_ptr()) + assert kind == _weights.CUDA_MEMORY_TYPE_DEVICE, ( + f"the n-gram table was declared with {_PLE_HOST_OFFLOAD_ENV}=0, but the driver " + f"reports cudaMemoryType {kind} for the pointer the forward gathers from" + ) mib = 1024.0 * 1024.0 free, total = torch.cuda.mem_get_info() print( f"ple n-gram table: {tuple(table.shape)} {table.dtype}, " f"{table.numel() * table.element_size() / mib:.1f} MiB, " - f"{'HOST-resident (device-addressable)' if self.ple_host_offload else 'DEVICE-resident'}" + f"{f'HOST-resident ({host.describe()})' if host is not None else 'DEVICE-resident'}" f"; after load: allocated {torch.cuda.memory_allocated() / mib:.1f} MiB, " f"driver-used {(total - free) / mib:.1f} MiB of {total / mib:.1f} MiB", flush=True, @@ -4179,7 +4203,10 @@ def _ple_block(self, hc, hc_out, normed_out) -> None: S.ple_offs, core.eos_token_id, num_tokens, + row_window=_ple_window(core), ) + if _ple_needs_sum(core): + ngram = core.tp_sum(ngram) ple_gate_conv_inject_norm( hc[:num_tokens], cublas_mm(ngram, S.ple_key), @@ -5863,7 +5890,10 @@ def _ple_block(self, hc, input_ids, ple_state, next_norm, stage) -> tuple: S.ple_offs, core.eos_token_id, rows, + row_window=_ple_window(core), ) + if _ple_needs_sum(core): + ngram = core.tp_sum(ngram) conv_input = torch.empty_like(hc) if stage else None out = ple_gate_conv_inject_norm( hc, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index f94e9a0abe81..e27e3ab0805a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -70,8 +70,8 @@ checkpoint row order. Under trtllm-gen a second copy of the window in the CuteDSL layout (`fc*_cd`) is declared when the core keeps it (`moe_prefill_copy`); * n-gram table: 128 row chunks concatenated into one `[rows, 160]` buffer (the hash - emits global row indices). It is replicated, and host-resident by default - (`_host_resident_empty`): one copy per rank. + emits global row indices). It is host-resident by default, one copy per rank, per server + or per node (`qwen4_exp/ple_host_table.py`). The n-gram table is not in the table: its height is a checkpoint fact, so `load` registers it with `torch.empty` (`_declare_ngram_table`) and adds its rows to the @@ -103,6 +103,7 @@ from tensorrt_llm.llmapi.utils import get_numa_aware_cpu_affinity from tensorrt_llm.logger import logger +from .. import ple_host_table as _ple_host from .pool import attention_tp_size #: `layers=ALL_LAYERS`, the default: the weight is on every layer. @@ -1591,6 +1592,9 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: set(manifest.keys()) ^ set(core.w.keys()), ) for param_key, sources in manifest.items(): + if param_key == "ple_table" and core._ple_table_host is not None: + consumed |= _fill_ngram_table(core, d, weights, rows_per_shard) + continue for ckpt_key, index, cut, transform in sources: fill(core.w[param_key], ckpt_key, index, cut, transform) if param_key.endswith(_RELEASE_AFTER): @@ -1845,15 +1849,53 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: device = core.w["embed"].device rows = d.ple_shards * rows_per_shard if core.ple_host_offload: - table, core._ple_table_host = _host_resident_empty(rows, head_dim, dtype, device) + mode = getattr(core, "ple_host_mode", _ple_host.RANK) + identity = None + if mode == _ple_host.NODE: + identity = _ple_host.checkpoint_identity( + getattr(core, "checkpoint_dir", None), rows, head_dim, dtype, *keys + ) + mapping = getattr(core, "mapping", None) + core._ple_table_host = _ple_host.PleHostTable( + mode, + rows, + head_dim, + dtype, + device, + rank=mapping.tp_rank if mapping is not None else 0, + world=mapping.tp_size if mapping is not None else 1, + dp=getattr(core, "dp_size", 1), + private=_host_resident_empty, + identity=identity, + ) + _bind_ngram_table(core) else: table = torch.empty(rows, head_dim, dtype=dtype, device=device) core._ple_table_host = None + core.w["ple_table"] = nn.Parameter(table, requires_grad=False) + core.ple_rows_per_shard = rows_per_shard + return rows_per_shard + + +def _bind_ngram_table(core) -> None: + """`core.w["ple_table"]` as the host table's current view (a placeholder until a `node` + table is mapped). `nn.Parameter` (`_make_subclass`) shares storage rather than copying; + host offload depends on that, so it is asserted.""" + host = core._ple_table_host + table = host.table + if table is None: + table = torch.empty(0, host.cols, dtype=host.dtype, device="meta") core.w["ple_table"] = nn.Parameter(table, requires_grad=False) - # `nn.Parameter` (`_make_subclass`) shares storage rather than copying; host offload - # depends on that, so it is asserted. assert core.w["ple_table"].data_ptr() == table.data_ptr(), ( "wrapping the n-gram table in nn.Parameter moved its storage" ) - core.ple_rows_per_shard = rows_per_shard - return rows_per_shard + + +def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: + """Fill this rank's rows of the host-resident n-gram table (`ple_host_table`); return the + shard keys.""" + keys = ngram_shard_keys(d) + copy, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + core._ple_table_host.fill(copy, read) + _bind_ngram_table(core) + return set(keys) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py index 79e4d25e4c7b..7a2c44d4fd62 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py @@ -242,3 +242,30 @@ def test_per_sequence_rows_compile_at_first_call() -> None: for flag in (True, False): counts = {seen[(flag, rows)] for rows in (1, 2, 3, 4)} assert len(counts) == 1, f"carry={not flag}: variants per row count {seen}" + + +@pytest.mark.parametrize("world", (2, 4)) +@pytest.mark.parametrize("lengths", ((3, 1, 4), (300, 7)), ids=("per-sequence", "rows")) +def test_row_windows_sum_to_the_whole_table(world, lengths) -> None: + """Each rank's `row_window` share matches the reference, from a slice of the table (its own + rows, `base = lo`) or from the whole table (`base = 0`); the shares sum to the whole output.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table + + args = _build(dict(lengths=lengths, pool_slots=8), seed=31 + world) + table = args[5] + rows, cols = table.shape + whole = op(*([a.clone() if isinstance(a, torch.Tensor) else a for a in args])) + total = torch.zeros_like(whole, dtype=torch.float32) + for rank in range(world): + lo, hi = ple_host_table.owned_rows(rank, world, rows, cols, align=cols) + for view, base in ((table[lo:hi], lo), (table, 0)): + # Fresh arguments for each call: the op advances the context pool in place. + call, ref_call = ( + [a.clone() if isinstance(a, torch.Tensor) else a for a in args] for _ in range(2) + ) + call[5] = ref_call[5] = view + share = op(*call, row_window=(lo, hi, base)) + ref, _ = op.reference(*ref_call, row_window=(lo, hi, base)) + assert torch.equal(share, ref), (rank, base) + total += share.float() + assert torch.equal(total.to(whole.dtype), whole) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py index 0208bcc279b2..3969c1173f6c 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -514,3 +514,200 @@ def u8(name): def test_load_operands(case, request): """What `load` fills the routed-expert operands from: codes and scales.""" _expert_operands(request.getfixturevalue("configs"), case.split("-")[1]) + + +# ------------------------------------------------------------ the host n-gram table -- + + +def _host_table_module(): + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table + + return ple_host_table + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("rows, cols", [(320001536, 160), (1000, 160), (37, 4)]) +def test_host_table_chunks_cover_the_rows(rows, cols): + """Chunks are whole multiples of the alignment, tile the rows in order, and every server size + that divides `CHUNKS` splits them into disjoint contiguous shares.""" + PH = _host_table_module() + align = PH.CHUNK_ALIGN if cols == 160 else cols + per = PH.chunk_rows(rows, cols, align) + assert (per * cols) % align == 0 and per * PH.CHUNKS >= rows + ranges = [PH.chunk_range(k, rows, cols, align) for k in range(PH.CHUNKS)] + assert ranges[0][0] == 0 and ranges[-1][1] == rows + assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:])) + for world in (1, 2, 4, 8): + shares = [PH.owned_rows(r, world, rows, cols, align) for r in range(world)] + assert shares[0][0] == 0 and shares[-1][1] == rows + assert all(a[1] == b[0] for a, b in zip(shares, shares[1:])), shares + + +def _broker_role(path, key, queue, go, done, die_on_go): + """A child process: the broker loaded from its file, without importing tensorrt_llm. An owner + publishes (or exits, `die_on_go`) when the test sets `go` and stays until it sets `done`.""" + import importlib.util + import mmap + import os + + spec = importlib.util.spec_from_file_location("chunk_broker", path) + broker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(broker) + queue.put("ready") + got = broker.acquire(key, 300) + if got[0] == broker.OWNER: + queue.put("owner") + go.wait(300) + if die_on_go: + os._exit(0) + fd = os.memfd_create("t") + os.ftruncate(fd, 4096) + mmap.mmap(fd, 4096)[:5] = b"bytes" + got[1].publish(fd, {"n": 4096}) + done.wait(300) + got[1].close() + else: + queue.put(("attached", bytes(mmap.mmap(got[1], 4096)[:5]), got[2])) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("owner_dies", [False, True], ids=["publish", "owner-exits"]) +def test_chunk_broker_hands_over(owner_dies): + """The first process owns a key; a waiter that is already waiting attaches to what it + publishes (and so does a late one), or owns the key itself when the owner exits before + publishing. No files are left.""" + import multiprocessing + import os + import time + import uuid + + PH = _host_table_module() + path = PH._broker.__file__ + key = uuid.uuid4().hex + ctx = multiprocessing.get_context("spawn") + go, done = ctx.Event(), ctx.Event() + procs = [] + + def start(die_on_go=False): + queue = ctx.Queue() + proc = ctx.Process(target=_broker_role, args=(path, key, queue, go, done, die_on_go)) + proc.start() + procs.append(proc) + assert queue.get(timeout=300) == "ready" + return queue + + try: + assert start(owner_dies).get(timeout=60) == "owner" + waiter = start() + time.sleep(1.0) # the waiter is connected and blocked on the owner + go.set() + if owner_dies: + assert waiter.get(timeout=60) == "owner" + else: + assert waiter.get(timeout=60) == ("attached", b"bytes", {"n": 4096}) + assert start().get(timeout=60) == ("attached", b"bytes", {"n": 4096}) + finally: + go.set() + done.set() + for proc in procs: + proc.join(60) + assert all(proc.exitcode == 0 for proc in procs), [proc.exitcode for proc in procs] + assert not [f for f in os.listdir("/dev/shm") if key in f] + + +def _fake_shards(rows_per_shard: int, shards: int, cols: int, seed: int) -> tuple: + g = torch.Generator().manual_seed(seed) + data = torch.randint(0, 256, (shards * rows_per_shard, cols), generator=g, dtype=torch.uint8) + keys = [f"s{i}" for i in range(shards)] + weights = { + k: data[i * rows_per_shard : (i + 1) * rows_per_shard].view(WT.PLE_TABLE_DTYPE) + for i, k in enumerate(keys) + } + return data, keys, weights + + +def _host_tables(monkeypatch, mode, world, data, keys, weights, rows_per_shard, identity=None): + """`world` ranks' tables in one process: threads, so `node` ranks wait on each other, and a + barrier in place of the server's collective agreement.""" + import threading + + PH = _host_table_module() + barrier = threading.Barrier(world) + votes = [] + + def agree(ok): + votes.append(ok) + barrier.wait() + return all(votes) + + monkeypatch.setattr(PH, "_all_agree", agree) + rows, cols = data.shape + copy, read = PH.shard_rows(weights, keys, rows_per_shard, WT._read) + tables = [ + PH.PleHostTable( + mode, + rows, + cols, + WT.PLE_TABLE_DTYPE, + torch.device("cuda"), + rank=r, + world=world, + dp=1, + private=WT._host_resident_empty, + identity=identity, + ) # fmt: skip + for r in range(world) + ] + threads = [threading.Thread(target=t.fill, args=(copy, read)) for t in tables] + for t in threads: + t.start() + for t in threads: + t.join(300) + return tables + + +def _vmm_host_numa() -> bool: + PH = _host_table_module() + try: + PH._VmmChunks(torch.cuda.current_device(), PH.CHUNK_ALIGN, 1, 0).close() + except (RuntimeError, AttributeError): + return False + return True + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +@pytest.mark.parametrize("mode", ["rank", "server", "node", "node-fallback"]) +def test_host_table_modes(mode, monkeypatch): + """`rank` holds every row, `server` this rank's rows with its window, `node` every row in + chunks one rank creates and the other attaches to; `node` without a checkpoint identity runs + `server`. Each holds exactly the checkpoint's bytes, in host memory.""" + import uuid + + PH = _host_table_module() + if mode == "node" and not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=5) + rows = data.shape[0] + want = {"rank": PH.RANK, "server": PH.SERVER, "node": PH.NODE, "node-fallback": PH.SERVER} + requested = PH.NODE if mode.startswith("node") else mode + identity = uuid.uuid4().hex if mode == "node" else None + tables = _host_tables(monkeypatch, requested, 2, data, keys, weights, rows_per_shard, identity) + expected = data.cuda() + for rank, t in enumerate(tables): + assert t.mode == want[mode], (rank, t.mode, t.describe()) + assert t.resident_on_host(), t.describe() + if t.mode == PH.SERVER: + lo, hi, base = t.window + assert (lo, hi) == PH.owned_rows(rank, 2, rows, cols) + got = t.table[lo - base : hi - base] + assert torch.equal(got.view(torch.uint8), expected[lo:hi]) + assert t.needs_sum + else: + assert t.window is None and not t.needs_sum + assert torch.equal(t.table.view(torch.uint8), expected) + if mode == "node": + created = sorted(len(t._segments) for t in tables) + assert created == [PH.CHUNKS // 2, PH.CHUNKS // 2], [t.describe() for t in tables] + for t in tables: + t.close() From bd3462fa033b8420cede8c04098f3662d3e78e22 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:05:56 -0800 Subject: [PATCH 14/51] [None][fix] modeling_v2 qwen4_exp: precompile the tp2/tep2 HT replay kernels a mixed step launches Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 20 +++++++++-- .../ops/ssm/gdn_cached_replay_cute__verify.py | 35 +++++++++++++------ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index ab1cf3cb2410..061a6d892892 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1542,8 +1542,7 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, # The CuTe replay kernels of the default mapping a verify step launches, compiled # here: a compile costs 5-10 s, and the padded decode graphs skip the request counts # an eager mixed step carries. PDL up to `_GDN_REPLAY_PDL_MAX_HEAD_ROWS` - # (request x value head) pairs; a tp2/tep2 HT batch takes its own mapping, compiled - # on its first launch (`_gdn_replay_launch`). + # (request x value head) pairs. heads = (width, self.num_k_heads, self.num_v_heads) partition_from = CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE requests = md.max_num_requests @@ -1553,6 +1552,23 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, if get_env_enable_pdl() and pdl_requests > 0: gdn_replay_precompile(*heads, pdl_requests, pdl=True, partition_from=partition_from) gdn_replay_precompile(*heads, requests, pdl=False, partition_from=partition_from) + if ( + heads[1:] == _GDN_REPLAY_HT_GEOMETRY + and md.max_num_requests >= _GDN_REPLAY_HT_MIN_REQUESTS + ): + # A tp2/tep2 HT batch takes its own mapping and conv fold (`_gdn_replay_launch`, + # `_gdn_verify_folds_conv`). The decode warmup compiles its kernels with the norm + # folded (`_gdn_verify_folds_norm`); a mixed step's verify rows + # (`PrefillTarget._gdn_core`) launch them without, compiled here. + gdn_replay_precompile( + *heads, + md.max_num_requests, + pdl=False, + partition_from=partition_from, + min_requests=_GDN_REPLAY_HT_MIN_REQUESTS, + launch=lambda n: _gdn_replay_launch(self, n), + conv_fold=lambda n: _gdn_verify_folds_conv(self, n), + ) return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) def _bind_draft_cache_manager(self, md) -> None: diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py index e46225a50fc6..91152cd92a23 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -19,7 +19,7 @@ from __future__ import annotations -from typing import Optional +from typing import Callable, Optional import cutlass import cutlass.cute as cute @@ -2224,30 +2224,43 @@ def _kernel_kw( def precompile( - t: int, h: int, hv: int, max_requests: int, *, pdl: bool, partition_from: int + t: int, + h: int, + hv: int, + max_requests: int, + *, + pdl: bool, + partition_from: int, + min_requests: int = 1, + launch: Optional[Callable[[int], tuple]] = None, + conv_fold: Optional[Callable[[int], bool]] = None, ) -> int: - """Compile every kernel a verify step of 1..`max_requests` requests launches through the - default mapping, the way a target calls this entry: work items, and with them no in-kernel - commit, from `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 - `a` / `b`; no `config`, `conv_fold` or `gated_norm`. A compile costs 5-10 s, so a target - runs this at bind, before any step: the captured decode graphs do not reach every mapping - (padded graphs skip 9 to 15 requests, which an eager mixed step carries). Returns the - number of distinct kernels.""" + """Compile every kernel a verify step of `min_requests`..`max_requests` requests launches, + the way a target calls this entry: work items, and with them no in-kernel commit, from + `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 `a` / `b`; no + `gated_norm`. By default the default mapping at `pdl` without `conv_fold`; `launch(n)` + instead returns `(pdl, config)` for a batch of `n` requests and `conv_fold(n)` whether it + folds the conv. A compile costs 5-10 s, so a target runs this at bind, before any step: the + captured decode graphs do not reach every mapping (padded graphs skip 9 to 15 requests, + which an eager mixed step carries). Returns the number of distinct kernels.""" keys = set() - for n in range(1, max_requests + 1): + for n in range(min_requests, max_requests + 1): partitioned = n >= partition_from + n_pdl, config = launch(n) if launch is not None else (pdl, None) kw = _kernel_kw( n, t, h, hv, commit=not partitioned, - pdl=pdl, + pdl=n_pdl, l2norm=True, gating=True, g_f32=False, b_f32=False, work_items=partitioned, + config=config, + fold=conv_fold is not None and conv_fold(n), ) key = tuple(sorted(kw.items())) if key not in keys: From c581161a19bb549e2727556da532ebf32bba3b1b Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:23:39 -0800 Subject: [PATCH 15/51] [None][test] modeling_v2 qwen4_exp: schedule multi-GPU stages when the FP8 target's collectives change Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 9941d4466c31..30776956bc23 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -2042,6 +2042,8 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "requirements.txt", "security_scanning/pyproject.toml", "tensorrt_llm/_ipc_utils.py", + "tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/", + "tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/", "tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py", "tensorrt_llm/_torch/compilation/patterns/ub_allreduce.py", "tensorrt_llm/_torch/custom_ops/torch_custom_ops.py", @@ -2125,6 +2127,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tests/integration/test_lists/test-db/l0_model_express.yml", "tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml", "tests/unittest/_torch/attention/multi_gpu/", + "tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/", "tests/unittest/_torch/multi_gpu/", "tests/unittest/_torch/moe/multi_gpu/", "tests/unittest/kv_cache_manager_v2_tests/", From 6a4ad9730536381ab10076047dda7ecc64737ce1 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:46:20 -0800 Subject: [PATCH 16/51] [None][perf] modeling_v2 qwen4_exp: order the wide decode selection by a per-step sort instead of an in-kernel rank Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../ops/attention/qsa_topk_expand_decode.py | 136 +++++++++--------- .../modeling.py | 47 +++++- ...test_modeling_v2_qsa_topk_expand_decode.py | 65 +++++++-- 3 files changed, 167 insertions(+), 81 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py index b663fbe2d10e..38fb558a9a43 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py @@ -106,12 +106,13 @@ def __call__( len_ptr: cute.Pointer, topk_ptr: cute.Pointer, sel_ptr: cute.Pointer, + order_ptr: cute.Pointer, rows: cutlass.Int32, stream, ): # The keys arrive as a tensor, as the TVM-FFI entry takes its stream from a GPU # tensor; their layout is rebuilt with the static width and row stride the 16-byte - # loads need. The row count is the call's. + # loads need. The row count is the call's. `order_ptr` is the wide kernel's (unread). mKeys = cute.make_tensor( keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) ) @@ -427,14 +428,14 @@ def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): mSel[row, col] = val -def _cache_slabs(width: int, threads: int, ctas: int, extra_words: int = 0) -> int: +def _cache_slabs(width: int, threads: int, ctas: int) -> int: """Slabs of the wide kernel's window cache: as many of the table's as the shared memory left after the fixed state and the candidate list hold with `ctas` CTAs an SM (11 of 4096 columns at one CTA of 1024 threads: a window of up to 45056 blocks, 180224 tokens, is read from global once).""" slab = KPT * threads per_cta = min(_SMEM_LIMIT, _SMEM_SM // ctas - _SMEM_RESERVED) - room = per_cta - (_WIDE_FIXED_WORDS + 2 * slab + extra_words) * 4 - _WIDE_PAD + room = per_cta - (_WIDE_FIXED_WORDS + 2 * slab) * 4 - _WIDE_PAD return max(0, min(-(-width // slab), room // (slab * 4))) @@ -487,20 +488,15 @@ def __init__( self.warps = threads // 32 self.slab = KPT * threads self.ctas = ctas - # `order`: CTA b takes the row of the b-th longest window (ties to the lower row), so the - # CTAs launched last get the shortest windows. Every row's selection is the same. + # `order`: CTA b takes row `row_order[b]` (longest window first), so the CTAs launched + # last get the shortest windows. Every row's selection is the same. self.order = bool(order) - self.cache_slabs = _cache_slabs( - width, threads, ctas, (_ORDER_MAX_ROWS + 32) if self.order else 0 - ) + self.cache_slabs = _cache_slabs(width, threads, ctas) self.per_warp = -(-block_topk // self.warps) self.prefilter = bool(prefilter) and self.per_warp <= 32 def smem_bytes(self) -> int: - order_words = (_ORDER_MAX_ROWS + 32) if self.order else 0 - return ( - self.cache_slabs * self.slab + 2 * self.slab + _WIDE_FIXED_WORDS + order_words - ) * 4 + _WIDE_PAD + return (self.cache_slabs * self.slab + 2 * self.slab + _WIDE_FIXED_WORDS) * 4 + _WIDE_PAD @cute.jit def __call__( @@ -511,6 +507,7 @@ def __call__( len_ptr: cute.Pointer, topk_ptr: cute.Pointer, sel_ptr: cute.Pointer, + order_ptr: cute.Pointer, rows: cutlass.Int32, stream, ): @@ -525,11 +522,12 @@ def __call__( mSel = cute.make_tensor( sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) ) + mOrder = cute.make_tensor(order_ptr, per_row) ld_keys = cute.make_copy_atom( cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT ) g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.Int32, num_bits_per_copy=32 * KPT) - self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys, g2s).launch( + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, mOrder, ld_keys, g2s).launch( grid=[rows, 1, 1], block=[self.threads, 1, 1], smem=self.smem_bytes(), @@ -885,7 +883,16 @@ def _select_window( @cute.kernel def kernel( - self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom, g2s: cute.CopyAtom + self, + mKeys, + mEnd, + mPos, + mLen, + mTopk, + mSel, + mOrder, + ld_keys: cute.CopyAtom, + g2s: cute.CopyAtom, ): tidx, _, _ = cute.arch.thread_idx() row, _, _ = cute.arch.block_idx() @@ -897,37 +904,6 @@ def kernel( k: cutlass.Constexpr = self.k cr: cutlass.Constexpr = self.cr smem = cutlass.utils.SmemAllocator() - if const_expr(self.order): - # 0a. This CTA's row: the one whose window ranks `block_idx` by length, from the - # host-filled row ends (read before the wait). Four threads rank a row, each over - # every fourth row; the ranks are a permutation, so every row has one CTA. - sEnd = smem.allocate_tensor( - cutlass.Int32, cute.make_layout((_ORDER_MAX_ROWS,), stride=(1,)), byte_alignment=128 - ) - sRow = smem.allocate_tensor( - cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 - ) - nrows = cute.size(mEnd) - for t0 in cutlass.range(0, nrows, nt): - if t0 + tidx < nrows: - sEnd[t0 + tidx] = mEnd[t0 + tidx] - cute.arch.sync_threads() - q4 = tidx % 4 - for r0 in cutlass.range(0, nrows, nt // 4): - tt = r0 + tidx // 4 - cnt = cutlass.Int32(0) - if tt < nrows: - e = sEnd[tt] - for u in cutlass.range(q4, nrows, 4): - eu = sEnd[u] - if (eu > e) | ((eu == e) & (u < tt)): - cnt = cnt + 1 - cnt = cnt + cute.arch.shuffle_sync_bfly(cnt, offset=1) - cnt = cnt + cute.arch.shuffle_sync_bfly(cnt, offset=2) - if (tt < nrows) & (q4 == 0) & (cnt == row): - sRow[0] = tt - cute.arch.sync_threads() - row = sRow[0] sK = None if const_expr(self.cache_slabs > 0): # `KPT` consecutive keys (mode 0) of each thread-slab column (mode 1). @@ -968,16 +944,22 @@ def kernel( cutlass.Int32, cute.make_layout((KPT, nt), stride=(1, KPT)), byte_alignment=128 ) - # 0. Shared state and the row's host-filled metadata, before the wait. + # 0. Shared state and the row's host-filled metadata, before the wait. Under `order` + # the row is `row_order`'s, a device sort earlier in the step: read after the wait. for b in cutlass.range_constexpr(-(-_HIST // nt)): if tidx + b * nt < _HIST: hist[tidx + b * nt] = cutlass.Int32(0) if tidx == 0: state[5] = cutlass.Int32(0) # candidates compacted + if const_expr(self.order): + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + row = mOrder[row] n = mEnd[row] seqlen = mLen[row] visible = cutlass.Int32(mPos[row]) + 1 - if const_expr(self.pdl): + if const_expr(self.pdl and not self.order): cute.arch.griddepcontrol_wait() cute.arch.griddepcontrol_launch_dependents() @@ -1168,24 +1150,31 @@ def kernel( _SMS: dict = {} -#: Rows the ordered launch ranks (its shared row-end table). -_ORDER_MAX_ROWS = 1024 +def _sms(device) -> int: + sms = _SMS.get(device.index) + if sms is None: + sms = _SMS[device.index] = torch.cuda.get_device_properties(device).multi_processor_count + return sms def _wide_config(rows: int, device, ordered: bool = False) -> tuple: """The wide kernel's `(threads, CTAs an SM, prefilter, order)` for a call of `rows` rows: one 1024-thread CTA an SM (the largest window cache) while the rows fit one wave, two once - they do not, so one CTA's walks overlap the other's loads. With `ordered`, past one wave and - up to `_ORDER_MAX_ROWS` rows: one CTA an SM with the rows ranked by window length, so the - second wave takes the shortest windows. The same selection either way.""" - sms = _SMS.get(device.index) - if sms is None: - sms = _SMS[device.index] = torch.cuda.get_device_properties(device).multi_processor_count - if ordered and sms < rows <= _ORDER_MAX_ROWS: - return (_WIDE_THREADS, 1, True, True) + they do not, so one CTA's walks overlap the other's loads. With `ordered` (a `row_order` + given), past one wave the CTAs take the rows in that order, so the last wave takes the + shortest windows: one CTA an SM up to two waves, two past them. The same selection either + way.""" + sms = _sms(device) + if ordered and rows > sms: + return (_WIDE_THREADS, 1 if rows <= 2 * sms else 2, True, True) return (_WIDE_THREADS, 1 if rows <= sms else 2, True, False) +def orders_rows(scores) -> bool: + """Whether a call over `scores` reads `row_order`: the wide kernel past one wave of rows.""" + return scores.shape[1] > MAX_WIDTH and scores.shape[0] > _sms(scores.device) + + _INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} @@ -1199,10 +1188,11 @@ def launch( token_topk, pdl=True, out=None, - ordered_wide=False, + row_order=None, ): """The launch behind `QsaTopkExpandDecode.__call__`: writes `topk_indices` and returns the - token selection `[rows, token_topk + compress_ratio - 1]` int32 (`out` when given).""" + token selection `[rows, token_topk + compress_ratio - 1]` int32 (`out` when given). + `row_order`: int32 `[rows]`, a permutation of the rows, longest window first.""" rows, width = scores.shape block_topk = topk_indices.shape[1] sel = out @@ -1223,7 +1213,7 @@ def launch( query_positions.dtype, sequence_lengths.dtype, pdl, - None if width <= MAX_WIDTH else _wide_config(rows, scores.device, ordered_wide), + None if width <= MAX_WIDTH else _wide_config(rows, scores.device, row_order is not None), ) ptrs = ( row_ends.data_ptr(), @@ -1231,6 +1221,7 @@ def launch( sequence_lengths.data_ptr(), topk_indices.data_ptr(), sel.data_ptr(), + (row_ends if row_order is None else row_order).data_ptr(), ) fn = _COMPILED.get(key) if fn is None: @@ -1268,6 +1259,7 @@ def launch( ), make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[5], gm, assumed_align=4), cutlass.Int32(rows), cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi", @@ -1458,6 +1450,9 @@ class QsaTopkExpandDecode(OpWrapper): kernel, `_TopkExpandWideKernel`); `topk_indices` int32 `[rows, K]` contiguous; `row_ends` int32, the positions and lengths int32 or int64, each a contiguous `[rows]`. Ties: the higher block at the threshold, as `qsa_canonical_selection` rewrites the CUDA top-k's pick. + `row_order` (optional): int32 `[rows]`, a permutation of the rows, longest window first, + read by the wide kernel past one wave of rows (`orders_rows`), after the wait; a caller + sorts once a step for every layer. Compiled once per score-table width and row stride (the block table's, fixed for an engine), k, compress ratio, token budget, index dtypes and PDL: the row count is a @@ -1490,7 +1485,7 @@ class QsaTopkExpandDecode(OpWrapper): The widths are derived from `max_seq_len` (288, 512 and 8192 (+4) pages of 32 tokens), not read off an engine. 4096 is `MAX_WIDTH`: 1024 threads, the largest CTA; wider tables take the wide kernel, compiled per row-count class (one CTA an SM up to the - SM count, two past it). + SM count, two past it; with a `row_order`, one up to twice the SM count, two past it). `is_valid` refuses `token_topk != k * compress_ratio`. It sizes the selection's rows while `k * compress_ratio` places the tokens, so a smaller budget writes past each row. @@ -1518,7 +1513,7 @@ def __call__( compress_ratio, token_topk, out=None, - ordered_wide=False, + row_order=None, ): return launch( scores, @@ -1529,7 +1524,7 @@ def __call__( compress_ratio, token_topk, out=out, - ordered_wide=ordered_wide, + row_order=row_order, ) def reference( @@ -1542,11 +1537,11 @@ def reference( compress_ratio, token_topk, out=None, - ordered_wide=False, + row_order=None, ): """`(topk_indices_after, selection)`: a stable descending sort per row (ties to the higher block), the k first in block order, then the expansion entry's reference. - `ordered_wide` picks a launch, not a result.""" + `row_order` picks a launch, not a result.""" rows, _ = scores.shape k = topk_indices.shape[1] out = torch.full_like(topk_indices, -1) @@ -1578,7 +1573,7 @@ def is_valid( compress_ratio, token_topk, out=None, - ordered_wide=False, + row_order=None, ): assert scores.dtype == torch.float32 and scores.stride(1) == 1, ( "fp32 scores, unit column stride" @@ -1605,6 +1600,15 @@ def is_valid( and out.is_contiguous() and out.shape == (scores.shape[0], token_topk + compress_ratio - 1) ), "out: contiguous int32 [rows, token_topk + compress_ratio - 1]" + assert row_order is None or ( + row_order.dtype == torch.int32 + and row_order.is_contiguous() + and row_order.shape == (rows,) + and torch.equal( + torch.sort(row_order).values, + torch.arange(rows, dtype=torch.int32, device=row_order.device), + ) + ), "row_order: a contiguous int32 permutation of the rows" qsa_topk_expand_decode = QsaTopkExpandDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 061a6d892892..f39d76817580 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -161,6 +161,9 @@ qsa_rollback_gather, qsa_rollback_restore, ) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + orders_rows as _topk_expand_orders_rows, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( qsa_topk_expand_decode, ) @@ -3303,14 +3306,15 @@ def _index_split_plan(split, qsa, num_tokens: int, rows_per_seq: int): def _decode_index_selection( - core, split, q, qsa, num_tokens, p, device, index_cache, plan=None, q_permuted=False + core, split, q, qsa, num_tokens, p, device, index_cache, plan=None, q_permuted=False, order=None ): """A sparse decode layer's int32 `[num_tokens, token_topk + compress_ratio - 1]` selection. Index scores of `q` (`p` rows a request), then `_selection`, over every row or, under `split`, over this rank's rows gathered in rank order. `qsa` is `(req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts, block_table, tokens_per_block)`. With a `plan`, - this rank takes its slice of the permuted order (`q_permuted`: `q` is in it already).""" + this rank takes its slice of the permuted order (`q_permuted`: `q` is in it already). `order`: + the step's `_RowOrder`, shared by its layers.""" req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts = qsa[:6] block_table, tokens_per_block = qsa[6], qsa[7] part = _index_split_rows(split, num_tokens, p) @@ -3377,6 +3381,7 @@ def _decode_index_selection( topk_indices[lo:hi], r_starts, selected, + order=order, ) if part is None: return selected @@ -3420,10 +3425,37 @@ def _draft_continue_rows(core, md, gen_seqs: int, p: int, share: bool): return torch.arange(gen_seqs, dtype=torch.int64, device=last.device) * p + last +class _RowOrder: + """A step's sparse decode rows, longest visible window first, for the ordered launch of + `qsa_topk_expand_decode`: sorted at the first layer that launches it, then read by the step's + other layers, which select over the same rows.""" + + def __init__(self): + self._order = None + + def __call__(self, visible_blocks): + if self._order is None: + self._order = torch.argsort(visible_blocks, descending=True, stable=True).to( + torch.int32 + ) + assert self._order.shape == visible_blocks.shape, "one row order serves one row set" + return self._order + + def _selection( - core, scores, visible_blocks, logical, seq_lens_q, topk_indices, row_starts, out, context=False + core, + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + row_starts, + out, + context=False, + order=None, ): - """The canonical block top-k and expansion of `scores`' rows into `out`, fused if possible.""" + """The canonical block top-k and expansion of `scores`' rows into `out`, fused if possible. + `order`: a `_RowOrder` over these rows, for the ordered wide launch.""" if ( context and scores.shape[0] >= _TOPK_ROWS_MIN_ROWS @@ -3453,6 +3485,9 @@ def _selection( ) return if _topk_expand_supports(scores): + row_order = None + if order is not None and _topk_expand_orders_rows(scores): + row_order = order(visible_blocks) qsa_topk_expand_decode( scores, visible_blocks, @@ -3462,7 +3497,7 @@ def _selection( core.compress_ratio, core.token_topk, out=out, - ordered_wide=True, + row_order=row_order, ) return _select(core, scores, row_starts, visible_blocks, topk_indices) @@ -5707,6 +5742,7 @@ def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, sta core._bound.pools.index_cache[i], plan, q_permuted=prologue, + order=getattr(self, "_row_order", None), ) attn = _sparse_gqa( core, @@ -5996,6 +6032,7 @@ def forward( self._index_plan = None if sparse and split is not None: self._index_plan = _index_split_plan(split, qsa, num_tokens, gen_p) + self._row_order = _RowOrder() if sparse else None # Scratch shared by all attention layers; each layer rewrites what it reads. qkv = torch.empty([num_tokens, core.qkv_width], dtype=dt, device=dev) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py index 355afd2538cd..023a8bb6400f 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py @@ -139,27 +139,33 @@ def test_wide_rows_equal_the_canonical_composition(width: int, distribution: str assert torch.equal(fused, composed), "fused token expansion" -@pytest.mark.parametrize("rows", [149, 192, 288, 576]) +@pytest.mark.parametrize("rows", [149, 192, 288, 576, 1500]) @pytest.mark.parametrize("distribution", ["relu_sum", "relu_sparse"]) -def test_wide_rows_past_one_wave_ordered_by_window(rows: int, distribution: str) -> None: - """`ordered_wide` past one wave of rows: block lists and token selections equal the default - launch's and the reference's, over ragged windows with ties.""" +def test_wide_rows_past_one_wave_in_row_order(rows: int, distribution: str) -> None: + """A `row_order` past one wave of rows (longest window first, and an arbitrary permutation): + block lists and token selections equal the default launch's and the reference's, over + ragged windows with ties.""" g = torch.Generator(device=DEV).manual_seed(rows) pos = torch.randint(1, 262143, (rows,), generator=g, device=DEV) pos[::17] = 99999 # ties of window length pos[3] = 262143 spec = dict(rows=rows, pos=pos.tolist(), width=65568, **{distribution: True}) + ends = _build(spec, seed=rows + 1)[1] + orders = ( + None, + torch.argsort(ends, descending=True, stable=True).to(torch.int32), + torch.randperm(rows, generator=g, device=DEV).to(torch.int32), + ) outs = [] - for order in (True, False): + for row_order in orders: args = _build(spec, seed=rows + 1) - sel = op(*args, ordered_wide=order) + sel = op(*args, row_order=row_order) torch.cuda.synchronize() outs.append((args[4].clone(), sel.clone())) want_topk, want_sel = op.reference(*_build(spec, seed=rows + 1)) - assert torch.equal(outs[0][0], outs[1][0]), "block list: ordered vs two CTAs an SM" - assert torch.equal(outs[0][1], outs[1][1]), "token selection: ordered vs two CTAs an SM" - assert torch.equal(outs[0][0], want_topk), "block list vs reference" - assert torch.equal(outs[0][1], want_sel.to(torch.int32)), "token selection vs reference" + for topk, sel in outs: + assert torch.equal(topk, want_topk), "block list vs reference" + assert torch.equal(sel, want_sel.to(torch.int32)), "token selection vs reference" # The decode-path shapes, captured in the decode CUDA graph: tp1's one row at 2560 columns, @@ -203,6 +209,45 @@ def step(): assert torch.equal(graph_sel, want_sel), f"replay {trial}: token selection" +def test_row_order_sorted_in_the_graph_follows_the_replayed_metadata() -> None: + """The decode step's ordered launch: the row order sorted inside the graph from the row + ends, then read by the selection after the wait. Each replay refills the scores and the + rows' positions, lengths and ends (288 ragged rows at the 65568-wide table).""" + rows = 288 + g = torch.Generator(device=DEV).manual_seed(rows) + + def spec(): + pos = torch.randint(1, 262143, (rows,), generator=g, device=DEV) + return dict(rows=rows, pos=pos.tolist(), width=65568, relu_sum=True) + + args = _build(spec(), seed=21) + src = args[0].clone() + + def step(): + args[0].copy_(src * 1.0) + order = torch.argsort(args[1], descending=True, stable=True).to(torch.int32) + return op(*args, row_order=order) + + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + step() + torch.cuda.current_stream().wait_stream(warmup) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_sel = step() + for trial in range(3): + fresh = _build(spec(), seed=100 + trial) + src.copy_(fresh[0]) + for i in (1, 2, 3): + args[i].copy_(fresh[i]) + graph.replay() + torch.cuda.synchronize() + want_topk, want_sel = op.reference(src, *args[1:]) + assert torch.equal(args[4], want_topk), f"replay {trial}: block list" + assert torch.equal(graph_sel, want_sel), f"replay {trial}: token selection" + + def test_guard_refuses_a_token_budget_that_is_not_k_blocks() -> None: """`token_topk` sizes the selection's rows while `k * compress_ratio` places the selected tokens: a smaller budget writes past each row, into the next row and, for the last one, From 67295bcc761ad121a914b4fd0c5d33b25a6e2514 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:26:54 -0800 Subject: [PATCH 17/51] [None][fix] modeling_v2 qwen4_exp: harden the FP8 MoE and hc all-reduce ops and follow the C++ coding guidelines Use named casts, k-prefixed constants, camelCase hc kernel names and members, #if defined(...) instead of an in-namespace macro, checked argument comments and Doxygen on the public declarations. The TPB kernel reads its grid-barrier target after griddepcontrol.wait and syncs the warp before overwriting staging slots whose copies only the issuing lanes waited on. The ops check the device of every tensor, the hc rank range and workspace dtype under a CUDAGuard, and the MoE front rejects RenormalizeNaive, which it does not compute. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../csrc/fp8BlockMoeTpb.cu | 96 +++++++------ .../csrc/fp8BlockMoeTpb.h | 60 ++++---- .../csrc/fp8BlockMoeTpbOp.cpp | 5 + .../csrc/fp8BlockScaleMoeFront.cu | 104 +++++++------- .../csrc/fp8BlockScaleMoeFront.h | 33 ++--- .../csrc/fp8BlockScaleMoeFrontOp.cpp | 6 +- .../csrc/hcAllreduceInjectNorm.cu | 136 +++++++++--------- .../csrc/hcAllreduceInjectNorm.h | 45 +++--- .../csrc/hcAllreduceInjectNormOp.cpp | 60 +++++--- 9 files changed, 287 insertions(+), 258 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu index d2b7bc988601..83df1e66a134 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu @@ -67,7 +67,7 @@ constexpr float kAmaxFloor = 1.0e-10f; // [kMaxClusters][8][2560] at 4096. constexpr unsigned long long kStride = 1024; constexpr size_t kWsPart = 4096; -constexpr size_t kWsBytes = kWsPart + size_t(kMaxClusters) * kRows * kH * 4; +constexpr size_t kWsBytes = kWsPart + static_cast(kMaxClusters) * kRows * kH * 4; constexpr int kDefaultStages = 4; // as fast as five, and leaves ~40 KB of shared memory an SM to side-stream kernels struct alignas(16) Smem @@ -102,17 +102,13 @@ template using SmemT = std::conditional_t; template -constexpr int kStaticBytes = (int(sizeof(SmemT)) + 127) / 128 * 128; +constexpr int kStaticBytes = (static_cast(sizeof(SmemT)) + 127) / 128 * 128; constexpr int kShMark = 1024; // block-list entries >= this are shared-slice items (kShMark + item) constexpr int kShCols = 32; // intermediate columns of a shared item constexpr int kShRowBytes = kH * 2; // one bf16 weight / activation row #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) -#define FP8_BLOCK_MOE_TPB_DEVICE 1 -#endif - -#ifdef FP8_BLOCK_MOE_TPB_DEVICE __device__ __forceinline__ uint32_t smemAddr(void const* p) { return static_cast(__cvta_generic_to_shared(p)); @@ -323,7 +319,7 @@ __device__ __forceinline__ float siluFast(float x) // Row t's top-10 logits (ties to the lower id) and their softmax rounded to bf16; lane l holds experts 16l..16l+15. __device__ __forceinline__ void routeRow(TpbParams const& p, int t, int lane, Smem& s) { - uint8_t const* row = static_cast(p.logits) + size_t(t) * kE * 2 + lane * 32; + uint8_t const* row = static_cast(p.logits) + static_cast(t) * kE * 2 + lane * 32; uint4 r0 = ldcg128(row); uint4 r1 = ldcg128(row + 16); uint32_t words[8] = {r0.x, r0.y, r0.z, r0.w, r1.x, r1.y, r1.z, r1.w}; @@ -333,7 +329,7 @@ __device__ __forceinline__ void routeRow(TpbParams const& p, int t, int lane, Sm { uint32_t b = (words[i >> 1] >> ((i & 1) * 16)) & 0xffffu; uint32_t ord = (b & 0x8000u) ? (~b & 0xffffu) : (b | 0x8000u); - key[i] = (ord << 16) | uint32_t(kE - 1 - (lane * 16 + i)); + key[i] = (ord << 16) | static_cast(kE - 1 - (lane * 16 + i)); } uint32_t mine = 0; #pragma unroll 1 @@ -362,7 +358,7 @@ __device__ __forceinline__ void routeRow(TpbParams const& p, int t, int lane, Sm uint32_t ord = mine >> 16; uint32_t b = (ord & 0x8000u) ? (ord & 0x7fffu) : (~ord & 0xffffu); v = __uint_as_float(b << 16); - s.topkId[t][lane] = kE - 1 - int(mine & 0xffffu); + s.topkId[t][lane] = kE - 1 - static_cast(mine & 0xffffu); } float const mx = __shfl_sync(0xffffffffu, v, 0); float e = lane < kK ? expf(v - mx) : 0.f; @@ -394,14 +390,14 @@ struct Ring } }; -#endif // FP8_BLOCK_MOE_TPB_DEVICE +#endif // !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) // CS: CTAs a cluster (4 or 8); kSh: also fold in this rank's shared-expert slice. template __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( TpbParams const p, int const numStages, int const xbufOffset, int const ringOffset, int const shXOffset) { -#ifdef FP8_BLOCK_MOE_TPB_DEVICE +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) static_assert(!kSh || CS == 4, "the shared-expert fold takes 4-CTA clusters"); constexpr int kCs = CS; constexpr int kUpRows = 128 / kCs; // a CTA's up rows (and gate rows) of a block @@ -418,7 +414,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( int const G = gridDim.x; int const cta = blockIdx.x; int const nSlots = T * kK; - int const crank = int(clusterCtaRank()); + int const crank = static_cast(clusterCtaRank()); int const cl = cta / kCs; int const NC = G / kCs; @@ -435,8 +431,6 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( mbarInit(&s.xbar[0], 1); // the local arrive.expect_tx; the peers' copies complete the bytes mbarInit(&s.xbar[1], 1); fenceBarrierInit(); - // Read before arriving: the counter is a multiple of kStride plus fewer than all of this call's arrivals. - s.barTarget = (ldAcquireGpu64(counter) / kStride + 1) * kStride; } for (int i = threadIdx.x; i < kMaxSlots * kRows; i += kThreads) { @@ -450,6 +444,12 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( clusterSync(); gridDepWait(); + if (threadIdx.x == 0) + { + // Read past the wait, so the previous call's arrivals have all landed, and before arriving: the counter is a + // multiple of kStride plus fewer than all of this call's arrivals. + s.barTarget = (ldAcquireGpu64(counter) / kStride + 1) * kStride; + } for (int t = warp; t < T; t += kThreads / 32) { @@ -492,7 +492,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( int const bit = myExpert & 31; int const d = s.wordBase[wd] + __popc(s.bitmap[wd] & ((1u << bit) - 1u)); s.expertOf[d] = myExpert; - s.slotOf[d][t] = int8_t(threadIdx.x); + s.slotOf[d][t] = static_cast(threadIdx.x); } __syncthreads(); int const D = s.numDistinct; @@ -508,16 +508,16 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( { int const e = s.expertOf[d]; int const b0 = (p.rank - e) & (kTpbRanks - 1); - s.blkJ[d] = uint16_t(e); - s.blkD[d] = uint8_t(d); + s.blkJ[d] = static_cast(e); + s.blkD[d] = static_cast(d); extra = b0 == 0; } uint32_t const bal = __ballot_sync(0xffffffffu, extra); if (extra) { int const pos = D + base + __popc(bal & ((1u << lane) - 1u)); - s.blkJ[pos] = uint16_t(kE + (s.expertOf[d] >> 2)); - s.blkD[pos] = uint8_t(d); + s.blkJ[pos] = static_cast(kE + (s.expertOf[d] >> 2)); + s.blkD[pos] = static_cast(d); } base += __popc(bal); } @@ -525,7 +525,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( { if (lane < p.shBlocks) { - s.blkJ[D + base + lane] = uint16_t(kShMark + lane); + s.blkJ[D + base + lane] = static_cast(kShMark + lane); s.blkD[D + base + lane] = 0; } } @@ -568,9 +568,10 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( { uint8_t* dst = ring + rs.stage * kStageBytes; mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes); - uint8_t const* src = q < 2 - ? static_cast(p.shGu) + (size_t(sb) * 64 + crank * 16 + q * 8) * kShRowBytes - : static_cast(p.shDn) + (size_t(sb) * kH + crank * kFc2Rows) * kShCols * 2; + uint8_t const* src = q < 2 ? static_cast(p.shGu) + + (static_cast(sb) * 64 + crank * 16 + q * 8) * kShRowBytes + : static_cast(p.shDn) + + (static_cast(sb) * kH + crank * kFc2Rows) * kShCols * 2; bulkG2S(dst, src, kItemRowBytes, &s.fullBar[rs.stage]); } __syncwarp(); @@ -589,16 +590,17 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes + kKb * 4); if (kind < 2) { - size_t const row = size_t(j) * kBlkRows + kind * 128 + crank * kUpRows + u * 16; + size_t const row = static_cast(j) * kBlkRows + kind * 128 + crank * kUpRows + u * 16; bulkG2S(dst, p.w1 + row * kH, kItemRowBytes, &s.fullBar[rs.stage]); - bulkG2S( - dst + kItemRowBytes, p.s1 + (size_t(j) * 2 + kind) * kKb, kKb * 4, &s.fullBar[rs.stage]); + bulkG2S(dst + kItemRowBytes, p.s1 + (static_cast(j) * 2 + kind) * kKb, kKb * 4, + &s.fullBar[rs.stage]); } else { - size_t const row = size_t(j) * kH + crank * kFc2Rows + u * kItemFc2Rows; + size_t const row = static_cast(j) * kH + crank * kFc2Rows + u * kItemFc2Rows; bulkG2S(dst, p.w2 + row * 128, kItemRowBytes, &s.fullBar[rs.stage]); - bulkG2S(dst + kItemRowBytes, p.s2 + size_t(j) * kKb, kKb * 4, &s.fullBar[rs.stage]); + bulkG2S( + dst + kItemRowBytes, p.s2 + static_cast(j) * kKb, kKb * 4, &s.fullBar[rs.stage]); } } __syncwarp(); @@ -659,6 +661,8 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( { bulkWaitRead<2 * kNh>(); } + // Only the issuing lanes waited; the whole warp overwrites the slots below. + __syncwarp(); } float* stg = &s.stage[n & 1][q][0][0]; if (t0 < T) @@ -675,9 +679,10 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( __syncwarp(); if (lane < kCs) { - uint32_t const off = uint32_t((n & 1) * T * kBlkRows + ((crank * nq + q) * T) * 16) * 4u; - bulkS2Cluster(mapaShared(xbase + off, uint32_t(lane)), stg, uint32_t(T * 16 * 4), - mapaShared(xbarBase + uint32_t(n & 1) * 8u, uint32_t(lane))); + uint32_t const off = static_cast((n & 1) * T * kBlkRows + ((crank * nq + q) * T) * 16) * 4u; + bulkS2Cluster(mapaShared(xbase + off, static_cast(lane)), stg, + static_cast(T * 16 * 4), + mapaShared(xbarBase + static_cast(n & 1) * 8u, static_cast(lane))); } if (lane < kCs) { @@ -702,7 +707,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( for (int m = 0; m < 2; ++m) { int const kpos = kb * 128 + m * 64 + tig * 16; - bx[q][m] = g < T ? ldcg128(p.xq + size_t(g) * kH + kpos) : make_uint4(0, 0, 0, 0); + bx[q][m] = g < T ? ldcg128(p.xq + static_cast(g) * kH + kpos) : make_uint4(0, 0, 0, 0); } xs[q][0] = 2 * tig < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig) : 0.f; xs[q][1] = 2 * tig + 1 < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig + 1) : 0.f; @@ -727,7 +732,8 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( float acc = 0.f; for (int c = lane; c < kH / 8; c += 32) { - uint4 const xv = ldcg128(static_cast(p.shX) + size_t(t) * kShRowBytes + c * 16); + uint4 const xv = ldcg128( + static_cast(p.shX) + static_cast(t) * kShRowBytes + c * 16); uint4 const wv = ldcg128(static_cast(p.shGl) + c * 16); __nv_bfloat162 const* xa = reinterpret_cast<__nv_bfloat162 const*>(&xv); __nv_bfloat162 const* wa = reinterpret_cast<__nv_bfloat162 const*>(&wv); @@ -813,7 +819,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( } if (warp == 0 && lane == 0) { - mbarArriveExpectTx(&s.xbar[pb], uint32_t(kCs * 2 * T * 16 * 4)); + mbarArriveExpectTx(&s.xbar[pb], static_cast(kCs * 2 * T * 16 * 4)); } mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); xphase ^= 1u << pb; @@ -912,7 +918,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( if (warp == 0 && lane == 0) { - mbarArriveExpectTx(&s.xbar[pb], uint32_t(kCs * 2 * kNh * T * 16 * 4)); + mbarArriveExpectTx(&s.xbar[pb], static_cast(kCs * 2 * kNh * T * 16 * 4)); } mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); xphase ^= 1u << pb; @@ -1035,7 +1041,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( for (int mi = 0; mi < kMtPerWarp; ++mi) { int const row = crank * kFc2Rows + u * kItemFc2Rows + (warp + kConsumerWarps * mi) * 16 + g; - float* pp = part + size_t(cl) * kRows * kH + row; + float* pp = part + static_cast(cl) * kRows * kH + row; if (t0 < T) { __stcg(pp + t0 * kH, out[u][mi][0]); @@ -1058,7 +1064,7 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( { // Every CTA arrives, inactive ones too, so past the wait every CTA is past its prologue. fenceAcqRelGpu(); - redAddRelease64(counter, cta == 0 ? kStride - (unsigned long long) (G - 1) : 1ull); + redAddRelease64(counter, cta == 0 ? kStride - static_cast(G - 1) : 1ull); unsigned long long const target = s.barTarget; while (ldAcquireGpu64(counter) < target) { @@ -1067,8 +1073,8 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( __syncthreads(); { int const nUnits = T * (kH / 4); - int const u0 = int((long long) cta * nUnits / G); - int const u1 = int((long long) (cta + 1) * nUnits / G); + int const u0 = static_cast(static_cast(cta) * nUnits / G); + int const u1 = static_cast(static_cast(cta + 1) * nUnits / G); int const grp = threadIdx.x >> 4; int const q = threadIdx.x & 15; constexpr int kGroups = kThreads / 16; @@ -1079,13 +1085,13 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( bool const live = u < u1; int const t = live ? u / (kH / 4) : 0; int const row = live ? (u % (kH / 4)) * 4 : 0; - float const* src = part + size_t(t) * kH + row; + float const* src = part + static_cast(t) * kH + row; float4 v = make_float4(0.f, 0.f, 0.f, 0.f); if (live) { for (int k = q; k < NCa; k += 16) { - float4 const x = __ldcg(reinterpret_cast(src + size_t(k) * kRows * kH)); + float4 const x = __ldcg(reinterpret_cast(src + static_cast(k) * kRows * kH)); v.x += x.x; v.y += x.y; v.z += x.z; @@ -1107,11 +1113,11 @@ __global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( uint2 pk; pk.x = *reinterpret_cast(&lo); pk.y = *reinterpret_cast(&hi); - *reinterpret_cast(o + size_t(t) * kH + row) = pk; + *reinterpret_cast(o + static_cast(t) * kH + row) = pk; } } } -#endif // FP8_BLOCK_MOE_TPB_DEVICE +#endif // !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) } int smemOptinTpb(int device) @@ -1141,7 +1147,7 @@ int activeClustersT(int device) cudaLaunchConfig_t cfg{}; cfg.gridDim = dim3(CS * kMaxClusters); cfg.blockDim = dim3(kThreads); - cfg.dynamicSmemBytes = size_t(maxSmemBytes); + cfg.dynamicSmemBytes = static_cast(maxSmemBytes); cudaLaunchAttribute attr[1]; attr[0].id = cudaLaunchAttributeClusterDimension; attr[0].val.clusterDim.x = CS; @@ -1186,7 +1192,7 @@ void launchTpbT(TpbParams const& params, bool pdl, cudaStream_t stream) { throw std::invalid_argument("fp8_block_moe_tpb: three ring stages do not fit in shared memory"); } - size_t const smem = size_t(ringOffset) + size_t(stages) * kStageBytes; + size_t const smem = static_cast(ringOffset) + static_cast(stages) * kStageBytes; cudaLaunchConfig_t cfg{}; cfg.gridDim = dim3(clusters * CS); cfg.blockDim = dim3(kThreads); diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h index 9e1d989b1691..f4ef92ebbdf4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h @@ -28,46 +28,48 @@ TRTLLM_MV2_NAMESPACE_BEGIN namespace kernels::fp8_block_moe_tpb { -// Decode routed experts over 128-row intermediate blocks (e4m3, fp32 128x128 weight and 1x128 activation scales). -// Block j of a rank is one (expert, intermediate block) pair, block-major: -// w1 [blocks, 256, 2560] e4m3 [128 up rows ; 128 gate rows], s1 [blocks, 2, 20] fp32 their block scales; -// w2 [blocks, 2560, 128] e4m3 the block's down_proj columns, s2 [blocks, 20] fp32 their block scales. -// Expert e's block b lives on attention-TP rank (e + b) % 4: rank r stores block b < 4 of e at j = e and block 4 -// (when e % 4 == r) at j = 512 + e / 4, so each rank holds 640 blocks and at least one block of every expert. +//! Decode routed experts over 128-row intermediate blocks (e4m3, fp32 128x128 weight and 1x128 activation scales). +//! Block j of a rank is one (expert, intermediate block) pair, block-major: +//! w1 [blocks, 256, 2560] e4m3 [128 up rows ; 128 gate rows], s1 [blocks, 2, 20] fp32 their block scales; +//! w2 [blocks, 2560, 128] e4m3 the block's down_proj columns, s2 [blocks, 20] fp32 their block scales. +//! Expert e's block b lives on attention-TP rank (e + b) % 4: rank r stores block b < 4 of e at j = e and block 4 +//! (when e % 4 == r) at j = 512 + e / 4, so each rank holds 640 blocks and at least one block of every expert. constexpr int kHidden = 2560; constexpr int kExperts = 512; constexpr int kMaxRows = 8; constexpr int kTpbRanks = 4; -constexpr int kTpbBlocks = 640; // blocks a rank holds (512 * 5 / 4) -constexpr int kShItems = 5; // work items of the folded shared-expert slice -constexpr int kShItemCols = 32; // intermediate columns of one shared item +constexpr int kTpbBlocks = 640; //!< blocks a rank holds (512 * 5 / 4) +constexpr int kShItems = 5; //!< work items of the folded shared-expert slice +constexpr int kShItemCols = 32; //!< intermediate columns of one shared item +//! Arguments of one TPB launch (layouts above; T = numRows). struct TpbParams { - void const* logits; // [T, 512] bf16 - uint8_t const* xq; // [T, 2560] e4m3 - float const* xsf; // [20, xsfStride] fp32: the 1x128 scales of xq, transposed - int xsfStride; - uint8_t const* w1; - float const* s1; - uint8_t const* w2; - float const* s2; - int rank; // this rank of kTpbRanks - int numRows; // 1..8 - // Optional fold of this rank's TP slice of the bf16 shared expert (shBlocks = 5 items of 32 columns; 3..8 rows): - // the partial also accumulates sigmoid(gate logit) * the slice's down projection in fp32 before the bf16 rounding. - void const* shX; // [T, 2560] bf16: the MoE site's rows - void const* shGu; // [5, 64, 2560] bf16: item sb, 8 groups of 4 columns, each [gate rows (4) ; up rows (4)] - void const* shGl; // [2560] bf16: the gate-logit row - void const* shDn; // [5, 2560, 32] bf16: down_proj's columns of item sb - int shBlocks; - void* out; // [T, 2560] bf16: this rank's partial (the TP sum of the ranks' partials is the layer) - void* workspace; // tpbWorkspaceBytes(), zeroed once; its grid counter only grows (one per device) + void const* logits; //!< [T, 512] bf16 + uint8_t const* xq; //!< [T, 2560] e4m3 + float const* xsf; //!< [20, xsfStride] fp32: the 1x128 scales of xq, transposed + int xsfStride; //!< row stride of xsf, in elements + uint8_t const* w1; //!< [640, 256, 2560] e4m3 + float const* s1; //!< [640, 2, 20] fp32 + uint8_t const* w2; //!< [640, 2560, 128] e4m3 + float const* s2; //!< [640, 20] fp32 + int rank; //!< this rank of kTpbRanks + int numRows; //!< 1..8 + //! Optional fold of this rank's TP slice of the bf16 shared expert (shBlocks = 5 items of 32 columns; 3..8 rows): + //! the partial also accumulates sigmoid(gate logit) * the slice's down projection in fp32 before the bf16 rounding. + void const* shX; //!< [T, 2560] bf16: the MoE site's rows + void const* shGu; //!< [5, 64, 2560] bf16: item sb, 8 groups of 4 columns, each [gate rows (4) ; up rows (4)] + void const* shGl; //!< [2560] bf16: the gate-logit row + void const* shDn; //!< [5, 2560, 32] bf16: down_proj's columns of item sb + int shBlocks; //!< kShItems with the shared-expert fold, else 0 + void* out; //!< [T, 2560] bf16: this rank's partial (the TP sum of the ranks' partials is the layer) + void* workspace; //!< tpbWorkspaceBytes(), zeroed once; its grid counter only grows (one per device) }; +//! Bytes of the per-device workspace launchTpb takes. size_t tpbWorkspaceBytes(); -// One launch of 8-CTA clusters (up to 2 rows) or 4-CTA clusters (3..8 rows). +//! One launch of 8-CTA clusters (up to 2 rows) or 4-CTA clusters (3..8 rows). void launchTpb(TpbParams const& params, bool pdl, cudaStream_t stream); } // namespace kernels::fp8_block_moe_tpb diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp index 019944abd398..6caf72083a7a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp @@ -84,6 +84,10 @@ at::Tensor fp8BlockMoeTpb(at::Tensor const& logits, at::Tensor const& xq, at::Te { checkAligned(*t, "operand"); } + for (at::Tensor const* t : std::initializer_list{&xq, &xsf, &w1, &s1, &w2, &s2, &workspace}) + { + TORCH_CHECK(t->device() == logits.device(), "every operand must be on the logits' CUDA device"); + } c10::cuda::CUDAGuard const guard(logits.device()); int const device = logits.get_device(); auto out = at::empty({rows, kd::kHidden}, logits.options()); @@ -116,6 +120,7 @@ at::Tensor fp8BlockMoeTpb(at::Tensor const& logits, at::Tensor const& xq, at::Te for (at::Tensor const* t : std::initializer_list{&*shX, &*shGu, &*shGl, &*shDn}) { checkAligned(*t, "shared operand"); + TORCH_CHECK(t->device() == logits.device(), "every shared operand must be on the logits' CUDA device"); } p.shX = shX->data_ptr(); p.shGu = shGu->data_ptr(); diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu index 9ed76e280cd8..a833a795b913 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu @@ -40,18 +40,18 @@ namespace cg = cooperative_groups; namespace routing = ::moe::dev::routing; using routing::WarpSize; -constexpr int NumThreads = 256; -constexpr int NumWarps = NumThreads / WarpSize; -constexpr int VecSize = MaxNumExperts / WarpSize; -constexpr int ScanItemsPerThread = MaxNumExperts / NumThreads; -static_assert(MaxNumExperts % NumThreads == 0); +constexpr int kNumThreads = 256; +constexpr int kNumWarps = kNumThreads / WarpSize; +constexpr int kVecSize = kMaxNumExperts / WarpSize; +constexpr int kScanItemsPerThread = kMaxNumExperts / kNumThreads; +static_assert(kMaxNumExperts % kNumThreads == 0); // Per-device state, zeroed at allocation: two banks of per-local-expert counters (a launch counts in bank // generation % 2 and zeroes the other), then the grid barrier's arrival count and generation flag. -constexpr int CounterInts = 2 * MaxNumExperts + 64; -constexpr int BarrierCount = 2 * MaxNumExperts; -constexpr int BarrierFlag = 2 * MaxNumExperts + 32; -constexpr int QuantIters = QuantHidden / 256; // two 1x128 groups a warp iteration +constexpr int kCounterInts = 2 * kMaxNumExperts + 64; +constexpr int kBarrierCount = 2 * kMaxNumExperts; +constexpr int kBarrierFlag = 2 * kMaxNumExperts + 32; +constexpr int kQuantIters = kQuantHidden / 256; // two 1x128 groups a warp iteration __device__ __forceinline__ int32_t ldAcquire(int32_t const* p) { @@ -88,14 +88,14 @@ __device__ __forceinline__ void quantizeGroup(uint4 in, __nv_fp8_e4m3* out, floa { laneAmax = __hmax(laneAmax, __habs(x.v[i])); } - float groupAmax = float(laneAmax); + float groupAmax = static_cast(laneAmax); #pragma unroll for (int mask = 8; mask > 0; mask /= 2) { groupAmax = std::max(groupAmax, __shfl_xor_sync(0xFFFFFFFF, groupAmax, mask)); } __nv_bfloat16 amax = ::tensorrt_llm::common::cuda_max(__nv_bfloat16(groupAmax), __nv_bfloat16(1e-10f)); - float quantScale = 448.f / float(amax); + float quantScale = 448.f / static_cast(amax); float dequantScale = 1.f / quantScale; if (laneInGroup == 0) { @@ -111,7 +111,7 @@ __device__ __forceinline__ void quantizeGroup(uint4 in, __nv_fp8_e4m3* out, floa #pragma unroll for (int i = 0; i < 8; i++) { - y.v[i] = __nv_fp8_e4m3(float(x.v[i]) * quantScale); + y.v[i] = __nv_fp8_e4m3(static_cast(x.v[i]) * quantScale); } *reinterpret_cast(out) = y.pack; } @@ -119,51 +119,51 @@ __device__ __forceinline__ void quantizeGroup(uint4 in, __nv_fp8_e4m3* out, floa // One warp a token selects its experts with the trtllm-gen routing device functions and takes each slot's offset in // its local expert from a global atomic counter. After a grid barrier of the routing CTAs, every CTA scans the counts // into expert offsets and writes its own slots' permutation; CTA 0 writes the grouped-GEMM launch tables. -__global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_t* state) +__global__ void __launch_bounds__(kNumThreads) moeFrontKernel(Data params, int32_t* state) { #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) using InputT = __nv_bfloat16; using BaseType = __nv_bfloat16; using TypePacked = routing::PackedScoreIdx; - using Scan = cub::BlockScan; + using Scan = cub::BlockScan; __shared__ typename Scan::TempStorage tempStorage; - __shared__ int32_t smemExpertOffset[MaxNumExperts]; + __shared__ int32_t smemExpertOffset[kMaxNumExperts]; int32_t const warpIdx = __shfl_sync(0xffffffff, threadIdx.x / WarpSize, 0); int32_t const laneIdx = threadIdx.x % WarpSize; - int32_t const tokenIdx = blockIdx.x * NumWarps + warpIdx; + int32_t const tokenIdx = blockIdx.x * kNumWarps + warpIdx; bool const validToken = tokenIdx < params.numTokens; auto block = cg::this_thread_block(); auto warp = cg::tiled_partition(block); - int32_t* barrierCount = state + BarrierCount; - int32_t* barrierFlag = state + BarrierFlag; + int32_t* barrierCount = state + kBarrierCount; + int32_t* barrierFlag = state + kBarrierFlag; if (params.usePdl) { cudaGridDependencySynchronize(); } - int32_t const routeCtas = (params.numTokens + NumWarps - 1) / NumWarps; + int32_t const routeCtas = (params.numTokens + kNumWarps - 1) / kNumWarps; if (static_cast(blockIdx.x) >= routeCtas) { // Quantization CTAs, outside the routing CTAs' barrier: half a hidden row a warp. - int32_t const half = (blockIdx.x - routeCtas) * NumWarps + warpIdx; + int32_t const half = (blockIdx.x - routeCtas) * kNumWarps + warpIdx; int32_t const tokenQ = half / 2; if (tokenQ < params.numTokens) { - int32_t const it0 = (half % 2) * (QuantIters / 2); + int32_t const it0 = (half % 2) * (kQuantIters / 2); auto const* in - = static_cast<__nv_bfloat16 const*>(params.ptrHiddenBf16) + static_cast(tokenQ) * QuantHidden; - uint4 part[QuantIters / 2]; + = static_cast<__nv_bfloat16 const*>(params.ptrHiddenBf16) + static_cast(tokenQ) * kQuantHidden; + uint4 part[kQuantIters / 2]; #pragma unroll - for (int it = 0; it < QuantIters / 2; it++) + for (int it = 0; it < kQuantIters / 2; it++) { part[it] = *reinterpret_cast(in + (it0 + it) * 256 + laneIdx * 8); } - auto* out = static_cast<__nv_fp8_e4m3*>(params.ptrHiddenFp8) + static_cast(tokenQ) * QuantHidden; + auto* out = static_cast<__nv_fp8_e4m3*>(params.ptrHiddenFp8) + static_cast(tokenQ) * kQuantHidden; #pragma unroll - for (int it = 0; it < QuantIters / 2; it++) + for (int it = 0; it < kQuantIters / 2; it++) { int32_t const group = 2 * (it0 + it) + laneIdx / 16; quantizeGroup(part[it], out + (it0 + it) * 256 + laneIdx * 8, @@ -179,13 +179,13 @@ __global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_ } BaseType const minScore = BaseType{-INFINITY}; - BaseType score[VecSize]; - int32_t idx[VecSize]; + BaseType score[kVecSize]; + int32_t idx[kVecSize]; if (validToken) { InputT const* scores = static_cast(params.ptrScores) + tokenIdx * params.numExperts; #pragma unroll - for (int i = 0; i < VecSize; i++) + for (int i = 0; i < kVecSize; i++) { auto expertIdx = i * WarpSize + laneIdx; score[i] = expertIdx < params.numExperts ? static_cast(scores[expertIdx]) : minScore; @@ -196,12 +196,12 @@ __global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_ // No CTA flips the generation before every CTA, this one included, has arrived; the previous launch flipped it // before it completed. int32_t const generation = *reinterpret_cast(barrierFlag); - int32_t* counters = state + (generation & 1) * MaxNumExperts; + int32_t* counters = state + (generation & 1) * kMaxNumExperts; if (blockIdx.x == 0) { // The other bank is the previous launch's, which is complete (this grid depends on it transitively). - int32_t* other = state + ((generation + 1) & 1) * MaxNumExperts; - for (int e = threadIdx.x; e < MaxNumExperts; e += NumThreads) + int32_t* other = state + ((generation + 1) & 1) * kMaxNumExperts; + for (int e = threadIdx.x; e < kMaxNumExperts; e += kNumThreads) { other[e] = 0; } @@ -211,8 +211,8 @@ __global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_ int32_t slotOffset = 0; if (validToken) { - BaseType warpTopKScore[MaxTopK]; - int32_t warpTopKExpertIdx[MaxTopK]; + BaseType warpTopKScore[kMaxTopK]; + int32_t warpTopKExpertIdx[kMaxTopK]; routing::topk::reduceTopK(warp, warpTopKScore, warpTopKExpertIdx, score, idx, minScore, params.topK); BaseType const softmaxScore = routing::calcSoftmax( warp, laneIdx < params.topK ? warpTopKScore[laneIdx] : minScore, laneIdx, params.topK); @@ -251,26 +251,26 @@ __global__ void __launch_bounds__(NumThreads) moeFrontKernel(Data params, int32_ __syncthreads(); int32_t const tileN = params.tileTokensDim; - int32_t count[ScanItemsPerThread]; - int32_t numCta[ScanItemsPerThread]; + int32_t count[kScanItemsPerThread]; + int32_t numCta[kScanItemsPerThread]; #pragma unroll - for (int e = 0; e < ScanItemsPerThread; e++) + for (int e = 0; e < kScanItemsPerThread; e++) { - int32_t const expert = threadIdx.x * ScanItemsPerThread + e; + int32_t const expert = threadIdx.x * kScanItemsPerThread + e; count[e] = expert < params.numLocalExperts ? __ldcg(counters + expert) : 0; } #pragma unroll - for (int e = 0; e < ScanItemsPerThread; e++) + for (int e = 0; e < kScanItemsPerThread; e++) { numCta[e] = (count[e] + tileN - 1) / tileN; } - int32_t ctaOffset[ScanItemsPerThread]; + int32_t ctaOffset[kScanItemsPerThread]; int32_t numNonExitingCtas; Scan(tempStorage).ExclusiveSum(numCta, ctaOffset, numNonExitingCtas); #pragma unroll - for (int e = 0; e < ScanItemsPerThread; e++) + for (int e = 0; e < kScanItemsPerThread; e++) { - int32_t const expert = threadIdx.x * ScanItemsPerThread + e; + int32_t const expert = threadIdx.x * kScanItemsPerThread + e; smemExpertOffset[expert] = ctaOffset[e] * tileN; if (blockIdx.x == 0 && expert < params.numLocalExperts) { @@ -326,10 +326,10 @@ int32_t* deviceCounters(int device) cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); int32_t* ptr = nullptr; - TLLM_CUDA_CHECK(cudaMalloc(&ptr, CounterInts * sizeof(int32_t))); + TLLM_CUDA_CHECK(cudaMalloc(&ptr, kCounterInts * sizeof(int32_t))); cudaStream_t side; TLLM_CUDA_CHECK(cudaStreamCreateWithFlags(&side, cudaStreamNonBlocking)); - TLLM_CUDA_CHECK(cudaMemsetAsync(ptr, 0, CounterInts * sizeof(int32_t), side)); + TLLM_CUDA_CHECK(cudaMemsetAsync(ptr, 0, kCounterInts * sizeof(int32_t), side)); TLLM_CUDA_CHECK(cudaStreamSynchronize(side)); TLLM_CUDA_CHECK(cudaStreamDestroy(side)); TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); @@ -342,8 +342,8 @@ int32_t* deviceCounters(int device) bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim) { - return numTokens > 0 && numTokens <= MaxNumTokens && numExperts > 0 && numExperts <= MaxNumExperts && topK > 0 - && topK <= MaxTopK && numLocalExperts > 0 && numLocalExperts <= MaxNumExperts && hiddenDim % 128 == 0; + return numTokens > 0 && numTokens <= kMaxNumTokens && numExperts > 0 && numExperts <= kMaxNumExperts && topK > 0 + && topK <= kMaxTopK && numLocalExperts > 0 && numLocalExperts <= kMaxNumExperts && hiddenDim % 128 == 0; } void run(Data const& data, cudaStream_t stream) @@ -352,22 +352,22 @@ void run(Data const& data, cudaStream_t stream) "fp8_block_scale_moe_front: unsupported problem (tokens %d, experts %d, topK %d, local %d, hidden %d)", data.numTokens, data.numExperts, data.topK, data.numLocalExperts, data.hiddenDim); TLLM_CHECK_WITH_INFO(data.ptrHiddenBf16 == nullptr - || (data.ptrHiddenFp8 != nullptr && data.ptrHiddenScale != nullptr && data.hiddenDim == QuantHidden), - "fp8_block_scale_moe_front: quantization needs both outputs and hidden %d", QuantHidden); + || (data.ptrHiddenFp8 != nullptr && data.ptrHiddenScale != nullptr && data.hiddenDim == kQuantHidden), + "fp8_block_scale_moe_front: quantization needs both outputs and hidden %d", kQuantHidden); int device = 0; TLLM_CUDA_CHECK(cudaGetDevice(&device)); int32_t* counters = deviceCounters(device); cudaLaunchConfig_t config{}; - int32_t const routeCtas = (data.numTokens + NumWarps - 1) / NumWarps; - int32_t const quantCtas = data.ptrHiddenBf16 != nullptr ? (2 * data.numTokens + NumWarps - 1) / NumWarps : 0; + int32_t const routeCtas = (data.numTokens + kNumWarps - 1) / kNumWarps; + int32_t const quantCtas = data.ptrHiddenBf16 != nullptr ? (2 * data.numTokens + kNumWarps - 1) / kNumWarps : 0; config.gridDim = routeCtas + quantCtas; - config.blockDim = NumThreads; + config.blockDim = kNumThreads; config.dynamicSmemBytes = 0; config.stream = stream; cudaLaunchAttribute attributes[1] = {}; attributes[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attributes[0].val.programmaticStreamSerializationAllowed = int(data.usePdl); + attributes[0].val.programmaticStreamSerializationAllowed = static_cast(data.usePdl); config.attrs = attributes; config.numAttrs = 1; TLLM_CUDA_CHECK(cudaLaunchKernelEx(&config, moeFrontKernel, data, counters)); diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h index 646f5c94a979..945e57ef7fd4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h @@ -27,44 +27,45 @@ TRTLLM_MV2_NAMESPACE_BEGIN namespace kernels::fp8_block_scale_moe_front { -// Renormalize routing (top-k of the bf16 logits, softmax over those k) in front of the trtllm-gen FP8 block-scale MoE -// runner for a decode-sized batch: one launch writes the runner's routing tables, optionally fused with the 1x128 e4m3 -// quantization of the bf16 hidden states. Top-k ids, order and bf16 weights match the runner bitwise; a slot's offset -// within its expert may differ (atomic order), which no consumer depends on. Launches on one device must not overlap. +//! Renormalize routing (top-k of the bf16 logits, softmax over those k) in front of the trtllm-gen FP8 block-scale MoE +//! runner for a decode-sized batch: one launch writes the runner's routing tables, optionally fused with the 1x128 e4m3 +//! quantization of the bf16 hidden states. Top-k ids, order and bf16 weights match the runner bitwise; a slot's offset +//! within its expert may differ (atomic order), which no consumer depends on. Launches on one device must not overlap. struct Data { - void const* ptrScores{nullptr}; // [numTokens, numExperts] bf16 logits - void const* ptrHiddenBf16{nullptr}; // optional [numTokens, hiddenDim] bf16 hidden states to quantize + void const* ptrScores{nullptr}; //!< [numTokens, numExperts] bf16 logits + void const* ptrHiddenBf16{nullptr}; //!< optional [numTokens, hiddenDim] bf16 hidden states to quantize int32_t numTokens{0}; int32_t numExperts{0}; int32_t topK{0}; - int32_t tileTokensDim{0}; + int32_t tileTokensDim{0}; //!< token tile of the grouped GEMMs int32_t localExpertsStartIdx{0}; int32_t numLocalExperts{0}; int32_t hiddenDim{0}; bool usePdl{false}; - // Outputs: the runner's routing tables. - void* ptrTopKWeights{nullptr}; // [numTokens, topK] bf16 + //! Outputs: the runner's routing tables. + void* ptrTopKWeights{nullptr}; //!< [numTokens, topK] bf16 int32_t* ptrExpandedIdxToPermutedIdx{nullptr}; int32_t* ptrPermutedIdxToTokenIdx{nullptr}; int32_t* ptrPermutedIdxSize{nullptr}; int32_t* ptrCtaIdxXyToBatchIdx{nullptr}; int32_t* ptrCtaIdxXyToMnLimit{nullptr}; int32_t* ptrNumNonExitingCtas{nullptr}; - void* ptrHiddenFp8{nullptr}; // [numTokens, hiddenDim] e4m3 (with ptrHiddenBf16) - float* ptrHiddenScale{nullptr}; // [hiddenDim / 128, numTokens] fp32 (with ptrHiddenBf16) + void* ptrHiddenFp8{nullptr}; //!< [numTokens, hiddenDim] e4m3 (with ptrHiddenBf16) + float* ptrHiddenScale{nullptr}; //!< [hiddenDim / 128, numTokens] fp32 (with ptrHiddenBf16) }; -constexpr int MaxNumExperts = 512; -constexpr int MaxTopK = 16; -constexpr int MaxNumTokens = 1024; +constexpr int kMaxNumExperts = 512; +constexpr int kMaxTopK = 16; +constexpr int kMaxNumTokens = 1024; -constexpr int QuantHidden = 2560; // hidden width the fused quantization is compiled for +constexpr int kQuantHidden = 2560; //!< hidden width the fused quantization is compiled for -// Whether `run` takes this problem (bf16 logits, no fused shared experts, stride-1 local window). +//! Whether `run` takes this problem (bf16 logits, no fused shared experts, stride-1 local window). bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim); +//! Writes the routing tables (and, with ptrHiddenBf16, the quantized hidden states) of `data` on `stream`. void run(Data const& data, cudaStream_t stream); } // namespace kernels::fp8_block_scale_moe_front diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp index 6c5936d0f3df..276e181916c8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp @@ -69,7 +69,7 @@ trtllmGenMoe::MoE::Runner& moeRunner(int64_t tileN) if (runner == nullptr) { runner = std::make_unique( - btg::Dtype::E4m3, /* useDeepSeekFp8 */ true, static_cast(tileN)); + /*dtypeElt=*/btg::Dtype::E4m3, /*useDeepSeekFp8=*/true, /*tileTokensDim=*/static_cast(tileN)); } return *runner; } @@ -89,8 +89,8 @@ void checkOperands(at::Tensor const& routing_logits, std::optional c "routing_logits and hidden_states must have the same number of tokens."); TORCH_CHECK(routing_logits.device() == hidden_states.device(), "routing_logits and hidden_states must be on the same device"); - auto const routingMethod = static_cast(routing_method_type); - TORCH_CHECK(routingMethod == RoutingMethodType::Renormalize || routingMethod == RoutingMethodType::RenormalizeNaive, + // The kernel computes Renormalize (softmax over the top-k logits), so RenormalizeNaive is not taken. + TORCH_CHECK(static_cast(routing_method_type) == RoutingMethodType::Renormalize, "fp8_block_scale_moe_front: logits-routed Renormalize without bias or fused shared experts only"); TORCH_CHECK(top_k <= 32 && top_k > 0, "Current routing kernel (no groups, renormalize) only supports top_k<=32 && top_k>0."); diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu index 81e83bdfc98c..7311bc2e5020 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu @@ -50,7 +50,7 @@ using ::tensorrt_llm::kernels::ar_fusion::ld_global_volatile; } // namespace hc_inject // Thread t holds columns [8t, 8t + 8) of a row's block output and of each of its 4 streams. The Triton program runs a -// stream on G = 32 * program_warps threads over a 4096-wide block, its thread u in chunk c holding this CTA's thread +// stream on G = 32 * programWarps threads over a 4096-wide block, its thread u in chunk c holding this CTA's thread // c * G + u. Its sum of squares, reproduced bitwise: per thread and chunk the fp32 pair (fma(x0, x0, x2 * x2) + // fma(x4, x4, x6 * x6), fma(x1, x1, x3 * x3) + fma(x5, x5, x7 * x7)), a pairwise tree over chunks (masked ones are // zero), low + high, the warp's xor butterfly 16..1, then ((w0 + w2) + (w1 + w3)) at 4 warps or (w0 + w1) at 2. @@ -60,36 +60,36 @@ static constexpr int kHidden = 2560; static constexpr int kStreams = 4; static constexpr int kThreads = kHidden / 8; -__device__ __forceinline__ float bf16_lo(uint32_t w) +__device__ __forceinline__ float bf16Lo(uint32_t w) { return __uint_as_float(w << 16); } -__device__ __forceinline__ float bf16_hi(uint32_t w) +__device__ __forceinline__ float bf16Hi(uint32_t w) { return __uint_as_float(w & 0xFFFF0000u); } -__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) +__device__ __forceinline__ uint32_t packBf16x2(float lo, float hi) { uint32_t r; asm("cvt.rn.bf16x2.f32 %0, %1, %2;" : "=r"(r) : "f"(hi), "f"(lo)); return r; } -__device__ __forceinline__ float round_bf16(float x) +__device__ __forceinline__ float roundBf16(float x) { - return bf16_lo(pack_bf16x2(x, 0.f)); + return bf16Lo(packBf16x2(x, 0.f)); } -__device__ __forceinline__ float ld_bf16(void const* base, int64_t elem) +__device__ __forceinline__ float ldBf16(void const* base, int64_t elem) { uint16_t v = reinterpret_cast(base)[elem]; return __uint_as_float(static_cast(v) << 16); } //! `tl.sigmoid` as the Triton kernel compiles it. -__device__ __forceinline__ float sigmoid_triton(float x) +__device__ __forceinline__ float sigmoidTriton(float x) { float y; asm("{ .reg .f32 t; mul.f32 t, %1, 0fBFB8AA3B; ex2.approx.f32 t, t; add.f32 t, t, 0f3F800000; " @@ -100,7 +100,7 @@ __device__ __forceinline__ float sigmoid_triton(float x) } //! `rsqrt(ss / 2560 + eps)` as the Triton kernel compiles it (`div.full`, `add`, `rsqrt.approx.ftz`). -__device__ __forceinline__ float rstd_triton(float ss, float eps) +__device__ __forceinline__ float rstdTriton(float ss, float eps) { float y; asm("{ .reg .f32 t; div.full.f32 t, %1, 0f45200000; add.f32 t, t, %2; rsqrt.approx.ftz.f32 %0, t; }" @@ -116,7 +116,7 @@ __device__ __forceinline__ float2 add2(float2 a, float2 b) //! A program thread's sum of squares: the pairwise tree over its unmasked chunk pairs, then low + high. template -__device__ __forceinline__ float thread_ssq(float2 const* leaves, int u) +__device__ __forceinline__ float threadSsq(float2 const* leaves, int u) { if constexpr (G == 128) { @@ -149,12 +149,12 @@ struct HcVariant template __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) - hc_allreduce_inject_norm_kernel(HcInjectNormParams params) + hcAllreduceInjectNormKernel(HcInjectNormParams params) { static_assert(kHidden % (8 * G) == 0 || G == 128, "the chunk layout above"); - __shared__ float2 s_leaf[kStreams][kThreads]; - __shared__ float s_red[kStreams][G / 32]; - __shared__ uint4 s_w[kStreams][kThreads]; + __shared__ float2 sLeaf[kStreams][kThreads]; + __shared__ float sRed[kStreams][G / 32]; + __shared__ uint4 sW[kStreams][kThreads]; int const tid = threadIdx.x; int const lane = tid & 31; @@ -162,7 +162,7 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #pragma unroll for (int s = 0; s < kStreams; ++s) { - s_w[s][tid] = reinterpret_cast(params.norm_weight)[s * kThreads + tid]; + sW[s][tid] = reinterpret_cast(params.normWeight)[s * kThreads + tid]; } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) @@ -174,8 +174,8 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #endif LamportComm comm(params.workspace, params.rank); int const rows = params.rows; - int const tot_access = rows * kThreads; - int const clear_access = static_cast((comm.clear_size + sizeof(float4) - 1) / sizeof(float4)); + int const totAccess = rows * kThreads; + int const clearAccess = static_cast((comm.clear_size + sizeof(float4) - 1) / sizeof(float4)); // Push this rank's partial rows, -0.0f stored as +0.0f. for (int row = blockIdx.x; row < rows; row += gridDim.x) @@ -195,14 +195,14 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #pragma unroll for (int r = 0; r < NRanks; ++r) { - reinterpret_cast(comm.data_bufs[r])[params.rank * tot_access + idx] + reinterpret_cast(comm.data_bufs[r])[params.rank * totAccess + idx] = *reinterpret_cast(val); } } - float4 const clear_vec = get_neg_zero(); - for (int idx = blockIdx.x * kThreads + tid; idx < clear_access; idx += gridDim.x * kThreads) + float4 const clearVec = get_neg_zero(); + for (int idx = blockIdx.x * kThreads + tid; idx < clearAccess; idx += gridDim.x * kThreads) { - reinterpret_cast(comm.clear_buf)[idx] = clear_vec; + reinterpret_cast(comm.clear_buf)[idx] = clearVec; } for (int row = blockIdx.x; row < rows; row += gridDim.x) @@ -211,20 +211,20 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) // The site's operands (issued before the poll, consumed after it). uint4 hcw[kStreams]; float inj[kStreams]; - auto const* hc_row = reinterpret_cast(params.hc) + row * params.hc_stride * 2; + auto const* hcRow = reinterpret_cast(params.hc) + row * params.hcStride * 2; #pragma unroll for (int s = 0; s < kStreams; ++s) { - hcw[s] = reinterpret_cast(hc_row + s * kHidden * 2)[tid]; - inj[s] = ld_bf16(params.inject, row * params.inject_stride + s); + hcw[s] = reinterpret_cast(hcRow + s * kHidden * 2)[tid]; + inj[s] = ldBf16(params.inject, row * params.injectStride + s); } [[maybe_unused]] uint4 shw = make_uint4(0, 0, 0, 0); [[maybe_unused]] float gate = 0.f; if constexpr (HasShared) { shw = reinterpret_cast( - reinterpret_cast(params.shared) + row * params.shared_stride * 2)[tid]; - gate = ld_bf16(params.gate, row * params.gate_stride); + reinterpret_cast(params.shared) + row * params.sharedStride * 2)[tid]; + gate = ldBf16(params.gate, row * params.gateStride); } float4 vals[NRanks]; @@ -236,7 +236,7 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) for (int r = 0; r < NRanks; ++r) { vals[r] - = ld_global_volatile(&reinterpret_cast(comm.data_bufs[params.rank])[r * tot_access + idx]); + = ld_global_volatile(&reinterpret_cast(comm.data_bufs[params.rank])[r * totAccess + idx]); done &= !is_neg_zero(vals[r]); } } @@ -251,17 +251,17 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) { acc += static_cast(reinterpret_cast<__nv_bfloat16 const*>(&vals[r])[i]); } - b[i] = round_bf16(acc); + b[i] = roundBf16(acc); } if constexpr (HasShared) { - float const sg = sigmoid_triton(gate); + float const sg = sigmoidTriton(gate); uint32_t const* sw = reinterpret_cast(&shw); #pragma unroll for (int p = 0; p < 4; ++p) { - b[2 * p] = round_bf16(__fmaf_rn(sg, bf16_lo(sw[p]), b[2 * p])); - b[2 * p + 1] = round_bf16(__fmaf_rn(sg, bf16_hi(sw[p]), b[2 * p + 1])); + b[2 * p] = roundBf16(__fmaf_rn(sg, bf16Lo(sw[p]), b[2 * p])); + b[2 * p + 1] = roundBf16(__fmaf_rn(sg, bf16Hi(sw[p]), b[2 * p + 1])); } } // Injection (rounded once, stored) and the per-thread sum-of-squares pairs. @@ -274,19 +274,19 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #pragma unroll for (int p = 0; p < 4; ++p) { - ow[s][p] = pack_bf16x2( - __fmaf_rn(inj[s], b[2 * p], bf16_lo(xw[p])), __fmaf_rn(inj[s], b[2 * p + 1], bf16_hi(xw[p]))); - e[2 * p] = bf16_lo(ow[s][p]); - e[2 * p + 1] = bf16_hi(ow[s][p]); + ow[s][p] = packBf16x2( + __fmaf_rn(inj[s], b[2 * p], bf16Lo(xw[p])), __fmaf_rn(inj[s], b[2 * p + 1], bf16Hi(xw[p]))); + e[2 * p] = bf16Lo(ow[s][p]); + e[2 * p + 1] = bf16Hi(ow[s][p]); } reinterpret_cast( - reinterpret_cast(params.hc_out) + (row * params.hc_out_stride + s * kHidden) * 2)[tid] + reinterpret_cast(params.hcOut) + (row * params.hcOutStride + s * kHidden) * 2)[tid] = make_uint4(ow[s][0], ow[s][1], ow[s][2], ow[s][3]); float const lo = __fadd_rn(__fmaf_rn(e[0], e[0], __fmul_rn(e[2], e[2])), __fmaf_rn(e[4], e[4], __fmul_rn(e[6], e[6]))); float const hi = __fadd_rn(__fmaf_rn(e[1], e[1], __fmul_rn(e[3], e[3])), __fmaf_rn(e[5], e[5], __fmul_rn(e[7], e[7]))); - s_leaf[s][tid] = make_float2(lo, hi); + sLeaf[s][tid] = make_float2(lo, hi); } __syncthreads(); if (tid < G) @@ -295,7 +295,7 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #pragma unroll for (int s = 0; s < kStreams; ++s) { - v[s] = thread_ssq(s_leaf[s], tid); + v[s] = threadSsq(sLeaf[s], tid); } #pragma unroll for (int off = 16; off >= 1; off >>= 1) @@ -311,7 +311,7 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) #pragma unroll for (int s = 0; s < kStreams; ++s) { - s_red[s][tid / 32] = v[s]; + sRed[s][tid / 32] = v[s]; } } } @@ -322,25 +322,25 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) float tot; if constexpr (G == 128) { - tot = __fadd_rn(__fadd_rn(s_red[s][0], s_red[s][2]), __fadd_rn(s_red[s][1], s_red[s][3])); + tot = __fadd_rn(__fadd_rn(sRed[s][0], sRed[s][2]), __fadd_rn(sRed[s][1], sRed[s][3])); } else { - tot = __fadd_rn(s_red[s][0], s_red[s][1]); + tot = __fadd_rn(sRed[s][0], sRed[s][1]); } - float const rstd = rstd_triton(tot, params.eps); - uint4 const wq = s_w[s][tid]; + float const rstd = rstdTriton(tot, params.eps); + uint4 const wq = sW[s][tid]; uint32_t const* ww = reinterpret_cast(&wq); uint32_t nw[4]; #pragma unroll for (int p = 0; p < 4; ++p) { - float const ylo = __fmul_rn(__fmul_rn(bf16_lo(ow[s][p]), rstd), __fadd_rn(1.f, bf16_lo(ww[p]))); - float const yhi = __fmul_rn(__fmul_rn(bf16_hi(ow[s][p]), rstd), __fadd_rn(1.f, bf16_hi(ww[p]))); - nw[p] = pack_bf16x2(ylo, yhi); + float const ylo = __fmul_rn(__fmul_rn(bf16Lo(ow[s][p]), rstd), __fadd_rn(1.f, bf16Lo(ww[p]))); + float const yhi = __fmul_rn(__fmul_rn(bf16Hi(ow[s][p]), rstd), __fadd_rn(1.f, bf16Hi(ww[p]))); + nw[p] = packBf16x2(ylo, yhi); } reinterpret_cast( - reinterpret_cast(params.normed) + (row * params.normed_stride + s * kHidden) * 2)[tid] + reinterpret_cast(params.normed) + (row * params.normedStride + s * kHidden) * 2)[tid] = make_uint4(nw[0], nw[1], nw[2], nw[3]); } } @@ -356,26 +356,26 @@ __global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) } template -int max_grid() +int maxGrid() { static int const grid = []() { - int per_sm = 0; + int perSm = 0; TLLM_CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &per_sm, hc_allreduce_inject_norm_kernel, kThreads, 0)); - return std::max(1, per_sm) * get_sm_count(); + &perSm, hcAllreduceInjectNormKernel, kThreads, 0)); + return std::max(1, perSm) * get_sm_count(); }(); return grid; } template -void launch_var(HcInjectNormParams const& params) +void launchVar(HcInjectNormParams const& params) { - auto kernel = hc_allreduce_inject_norm_kernel; - int const max_grid = hc_inject::max_grid(); + auto kernel = hcAllreduceInjectNormKernel; + int const maxGrid = hc_inject::maxGrid(); cudaLaunchConfig_t cfg; cudaLaunchAttribute attribute[1]; - cfg.gridDim = std::min(params.rows, max_grid); + cfg.gridDim = std::min(params.rows, maxGrid); cfg.blockDim = kThreads; cfg.dynamicSmemBytes = 0; cfg.stream = params.stream; @@ -389,23 +389,23 @@ void launch_var(HcInjectNormParams const& params) template void launch(HcInjectNormParams const& params) { - int const g3 = hc_inject::max_grid(); - int const g2 = hc_inject::max_grid(); + int const g3 = hc_inject::maxGrid(); + int const g2 = hc_inject::maxGrid(); if (params.rows > g3 && params.rows <= g2) { - launch_var(params); + launchVar(params); } else { - launch_var(params); + launchVar(params); } } template -void dispatch_flags(HcInjectNormParams const& params) +void dispatchFlags(HcInjectNormParams const& params) { bool const shared = params.shared != nullptr; - if (shared && params.trigger_completion_at_end) + if (shared && params.triggerCompletionAtEnd) { launch(params); } @@ -413,7 +413,7 @@ void dispatch_flags(HcInjectNormParams const& params) { launch(params); } - else if (params.trigger_completion_at_end) + else if (params.triggerCompletionAtEnd) { launch(params); } @@ -424,25 +424,23 @@ void dispatch_flags(HcInjectNormParams const& params) } } // namespace hc_inject -void hc_allreduce_inject_norm_op(HcInjectNormParams const& params) +void launchHcAllreduceInjectNorm(HcInjectNormParams const& params) { - TLLM_CHECK_WITH_INFO(params.hidden_dim == hc_inject::kHidden && params.hc_count == hc_inject::kStreams, + TLLM_CHECK_WITH_INFO(params.hiddenDim == hc_inject::kHidden && params.hcCount == hc_inject::kStreams, "hc_allreduce_inject_norm: hidden 2560 and 4 streams only"); TLLM_CHECK_WITH_INFO(params.rows > 0, "hc_allreduce_inject_norm: no rows"); - TLLM_CHECK_WITH_INFO(params.program_warps == 4 || params.program_warps == 2, + TLLM_CHECK_WITH_INFO(params.programWarps == 4 || params.programWarps == 2, "hc_allreduce_inject_norm: the Triton program's warps must be 4 or 2"); TLLM_CHECK_WITH_INFO((params.shared == nullptr) == (params.gate == nullptr), "hc_allreduce_inject_norm: shared and gate come together"); TLLM_CHECK_WITH_INFO(params.partial != nullptr, "hc_allreduce_inject_norm: no partial"); if (params.nranks == 2) { - params.program_warps == 4 ? hc_inject::dispatch_flags<2, 128>(params) - : hc_inject::dispatch_flags<2, 64>(params); + params.programWarps == 4 ? hc_inject::dispatchFlags<2, 128>(params) : hc_inject::dispatchFlags<2, 64>(params); } else if (params.nranks == 4) { - params.program_warps == 4 ? hc_inject::dispatch_flags<4, 128>(params) - : hc_inject::dispatch_flags<4, 64>(params); + params.programWarps == 4 ? hc_inject::dispatchFlags<4, 128>(params) : hc_inject::dispatchFlags<4, 64>(params); } else { diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h index 8ea3e1fe3326..93014f52cfc7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h @@ -28,38 +28,39 @@ namespace kernels { //! IPC one-shot Lamport all-reduce of a block output (stock `allreduce_fusion_kernel_oneshot_lamport` workspace and -//! protocol) fused with a hyper-connection site's `hc_out = bf16(fma(inject, block, hc))` and grouped RMS norm -//! `normed = bf16((hc_out * rstd) * (1 + w))`; bitwise that all-reduce followed by the Triton `_hc_inject_norm_kernel`. +//! protocol) fused with a hyper-connection site's `hcOut = bf16(fma(inject, block, hc))` and grouped RMS norm +//! `normed = bf16((hcOut * rstd) * (1 + w))`; bitwise that all-reduce followed by the Triton `_hc_inject_norm_kernel`. //! With `shared` and `gate`, the block is first `bf16(fma(sigmoid(gate), shared, routed))`. struct HcInjectNormParams { int nranks; int rank; int rows; - int hidden_dim; // 2560 - int hc_count; // 4 - int program_warps; // 4 or 2: warps of the Triton program whose reduction order is reproduced - void** workspace; // the stock all-reduce fusion IPC workspace - void const* partial; // [rows, hidden] bf16, contiguous - void const* hc; // [rows, hc_count * hidden] bf16 - int64_t hc_stride; - void const* inject; // [rows, >= hc_count] bf16 - int64_t inject_stride; - void const* shared; // optional [rows, hidden] bf16 - int64_t shared_stride; - void const* gate; // optional [rows, 1] bf16 - int64_t gate_stride; - void const* norm_weight; // [hc_count * hidden] bf16 - void* hc_out; // [rows, hc_count * hidden] bf16 - int64_t hc_out_stride; - void* normed; // [rows, hc_count * hidden] bf16 - int64_t normed_stride; + int hiddenDim; //!< 2560 + int hcCount; //!< 4 + int programWarps; //!< 4 or 2: warps of the Triton program whose reduction order is reproduced + void** workspace; //!< the stock all-reduce fusion IPC workspace + void const* partial; //!< [rows, hidden] bf16, contiguous + void const* hc; //!< [rows, hcCount * hidden] bf16 + int64_t hcStride; //!< row stride of hc, in elements + void const* inject; //!< [rows, >= hcCount] bf16 + int64_t injectStride; //!< row stride of inject, in elements + void const* shared; //!< optional [rows, hidden] bf16 + int64_t sharedStride; //!< row stride of shared, in elements + void const* gate; //!< optional [rows, 1] bf16 + int64_t gateStride; //!< row stride of gate, in elements + void const* normWeight; //!< [hcCount * hidden] bf16 + void* hcOut; //!< [rows, hcCount * hidden] bf16 + int64_t hcOutStride; //!< row stride of hcOut, in elements + void* normed; //!< [rows, hcCount * hidden] bf16 + int64_t normedStride; //!< row stride of normed, in elements float eps; - bool trigger_completion_at_end; + bool triggerCompletionAtEnd; //!< trigger the dependent launch at the end instead of after the grid dependency wait cudaStream_t stream; }; -void hc_allreduce_inject_norm_op(HcInjectNormParams const& params); +//! Launches the fused all-reduce + inject + norm of `params` on `params.stream`. +void launchHcAllreduceInjectNorm(HcInjectNormParams const& params); } // namespace kernels diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp index c2f6311a958f..c05c080ba7c5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp @@ -21,8 +21,10 @@ #include #include +#include #include +#include #include #include @@ -31,13 +33,27 @@ TRTLLM_MV2_NAMESPACE_BEGIN namespace { -//! Returns `(hc_out, normed)` for the block output summed over ranks; see kernels::HcInjectNormParams. +//! Returns `(hcOut, normed)` for the block output summed over ranks; see kernels::HcInjectNormParams. std::vector hcAllreduceInjectNorm(at::Tensor const& partial, at::Tensor workspace, int64_t const rank, int64_t const nranks, at::Tensor const& hc, at::Tensor const& inject, std::optional const& shared, - std::optional const& gate, at::Tensor const& norm_weight, double const eps, int64_t const program_warps, - bool const trigger_completion_at_end) + std::optional const& gate, at::Tensor const& normWeight, double const eps, int64_t const programWarps, + bool const triggerCompletionAtEnd) { auto const bf16 = at::ScalarType::BFloat16; + TORCH_CHECK(hc.is_cuda(), "hc_allreduce_inject_norm: hc must be a CUDA tensor"); + c10::cuda::CUDAGuard const guard(hc.device()); + for (at::Tensor const* t : std::initializer_list{&partial, &workspace, &inject, &normWeight}) + { + TORCH_CHECK(t->device() == hc.device(), "hc_allreduce_inject_norm: every tensor must be on hc's device"); + } + for (std::optional const* t : std::initializer_list const*>{&shared, &gate}) + { + TORCH_CHECK(!t->has_value() || (*t)->device() == hc.device(), + "hc_allreduce_inject_norm: every tensor must be on hc's device"); + } + TORCH_CHECK(workspace.scalar_type() == at::ScalarType::Long && workspace.is_contiguous(), + "hc_allreduce_inject_norm: workspace must be the stock all-reduce fusion workspace (contiguous int64)"); + TORCH_CHECK(0 <= rank && rank < nranks, "hc_allreduce_inject_norm: rank must be in [0, nranks)"); int64_t const rows = hc.size(0); TORCH_CHECK( partial.dim() == 2 && partial.is_contiguous() && partial.scalar_type() == bf16 && partial.size(0) == rows, @@ -45,14 +61,14 @@ std::vector hcAllreduceInjectNorm(at::Tensor const& partial, at::Ten int64_t const hidden = partial.size(1); TORCH_CHECK(hc.dim() == 2 && hc.size(1) % hidden == 0 && hc.stride(1) == 1 && hc.scalar_type() == bf16, "hc_allreduce_inject_norm: hc must be [rows, hc_count * hidden] bf16 with unit column stride"); - int64_t const hc_count = hc.size(1) / hidden; + int64_t const hcCount = hc.size(1) / hidden; TORCH_CHECK(hc.stride(0) % 8 == 0 && reinterpret_cast(hc.data_ptr()) % 16 == 0, "hc_allreduce_inject_norm: hc rows must be 16-byte aligned"); - TORCH_CHECK(inject.dim() == 2 && inject.size(0) == rows && inject.size(1) >= hc_count && inject.stride(1) == 1 + TORCH_CHECK(inject.dim() == 2 && inject.size(0) == rows && inject.size(1) >= hcCount && inject.stride(1) == 1 && inject.scalar_type() == bf16, "hc_allreduce_inject_norm: inject must be [rows, >= hc_count] bf16 with unit column stride"); TORCH_CHECK( - norm_weight.numel() == hc_count * hidden && norm_weight.is_contiguous() && norm_weight.scalar_type() == bf16, + normWeight.numel() == hcCount * hidden && normWeight.is_contiguous() && normWeight.scalar_type() == bf16, "hc_allreduce_inject_norm: norm_weight must be [hc_count * hidden] bf16"); TORCH_CHECK(shared.has_value() == gate.has_value(), "hc_allreduce_inject_norm: shared and gate come together"); if (shared.has_value()) @@ -64,36 +80,36 @@ std::vector hcAllreduceInjectNorm(at::Tensor const& partial, at::Ten TORCH_CHECK(gate->dim() == 2 && gate->size(0) == rows && gate->scalar_type() == bf16, "hc_allreduce_inject_norm: gate must be [rows, 1] bf16"); } - auto hc_out = at::empty({rows, hc_count * hidden}, hc.options()); - auto normed = at::empty({rows, hc_count * hidden}, hc.options()); + auto hcOut = at::empty({rows, hcCount * hidden}, hc.options()); + auto normed = at::empty({rows, hcCount * hidden}, hc.options()); kernels::HcInjectNormParams params; params.nranks = static_cast(nranks); params.rank = static_cast(rank); params.rows = static_cast(rows); - params.hidden_dim = static_cast(hidden); - params.hc_count = static_cast(hc_count); - params.program_warps = static_cast(program_warps); + params.hiddenDim = static_cast(hidden); + params.hcCount = static_cast(hcCount); + params.programWarps = static_cast(programWarps); params.workspace = reinterpret_cast(workspace.mutable_data_ptr()); params.partial = partial.data_ptr(); params.hc = hc.data_ptr(); - params.hc_stride = hc.stride(0); + params.hcStride = hc.stride(0); params.inject = inject.data_ptr(); - params.inject_stride = inject.stride(0); + params.injectStride = inject.stride(0); params.shared = shared.has_value() ? shared->data_ptr() : nullptr; - params.shared_stride = shared.has_value() ? shared->stride(0) : 0; + params.sharedStride = shared.has_value() ? shared->stride(0) : 0; params.gate = gate.has_value() ? gate->data_ptr() : nullptr; - params.gate_stride = gate.has_value() ? gate->stride(0) : 0; - params.norm_weight = norm_weight.data_ptr(); - params.hc_out = hc_out.mutable_data_ptr(); - params.hc_out_stride = hc_out.stride(0); + params.gateStride = gate.has_value() ? gate->stride(0) : 0; + params.normWeight = normWeight.data_ptr(); + params.hcOut = hcOut.mutable_data_ptr(); + params.hcOutStride = hcOut.stride(0); params.normed = normed.mutable_data_ptr(); - params.normed_stride = normed.stride(0); + params.normedStride = normed.stride(0); params.eps = static_cast(eps); - params.trigger_completion_at_end = trigger_completion_at_end; + params.triggerCompletionAtEnd = triggerCompletionAtEnd; params.stream = at::cuda::getCurrentCUDAStream(hc.get_device()); - kernels::hc_allreduce_inject_norm_op(params); - return {hc_out, normed}; + kernels::launchHcAllreduceInjectNorm(params); + return {hcOut, normed}; } } // namespace From 7dd2bcfca696157bea8e617a17db7faea9024952 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:37:53 -0800 Subject: [PATCH 18/51] [None][fix] modeling_v2 qwen4_exp: direct-launch through the ops' public launchers Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../activation/fused_sigmoid_mul_inplace.py | 59 ++++--------------- .../ops/norm/rms_norm_gated_token_major.py | 40 +++---------- .../_torch/modules/mamba/layernorm_gated.py | 22 ++++++- 3 files changed, 38 insertions(+), 83 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py index 816cf9c801f6..09de9e186c3b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py @@ -11,54 +11,15 @@ from __future__ import annotations import torch -import triton + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper # Registered by the module that defines it: after `import # tensorrt_llm._torch.custom_ops` alone, torch.ops.trtllm has no # `fused_sigmoid_mul_inplace` (measured). -import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate as _gate_mod -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper - - -def _launch(attention_output: torch.Tensor, gate: torch.Tensor) -> bool: - """The registered op's launch, made here when its preconditions hold: the same kernel, - grid, block, warps and PDL policy, without the custom-op dispatch around it. False - (nothing launched) when they do not, for the op to refuse loudly.""" - if attention_output.dim() != 2 or attention_output.stride(-1) != 1 or gate.stride(-1) != 1: - return False - num_tokens, hidden = attention_output.shape - if gate.dim() == 3: - if gate.shape[0] != num_tokens or gate.shape[1] * gate.shape[2] != hidden: - return False - head_dim, gate_stride_head = gate.shape[2], gate.stride(1) - elif gate.dim() == 2 and gate.shape == attention_output.shape: - head_dim, gate_stride_head = hidden, hidden - else: - return False - if num_tokens == 0: - return True - block = min(triton.next_power_of_2(hidden), 1024 if num_tokens < 1024 else 2048) - pdl = ( - dict(LAUNCH_WITH_PDL=True, launch_pdl=True) - if _gate_mod._sigmoid_mul_pdl(attention_output.device.index) - else {} - ) - _gate_mod._fused_sigmoid_mul_kernel[(num_tokens, triton.cdiv(hidden, block))]( - attention_output, - attention_output, - gate, - attention_output.stride(0), - attention_output.stride(0), - gate.stride(0), - gate_stride_head, - hidden_size=hidden, - head_dim=head_dim, - block_size=block, - num_warps=4, - **pdl, - ) - return True - +from tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate import ( + launch_fused_sigmoid_mul_inplace, +) #: The checkpoint's gated attention: 24 q heads of 256, q projection [T, 24, 512]. _CKPT = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") @@ -176,9 +137,9 @@ class FusedSigmoidMulInplace(OpWrapper): the running device at the first call of each (hidden, head_dim, block) specialization, no autotuner: grid `(T, cdiv(hidden, block))`, `block = min(next_pow2(hidden), 1024 if T < 1024 else 2048)`, 4 warps. The - cells ran on sm_103. With `direct_launch=True` (the FP8 target), `_launch` makes - that same launch itself where the preconditions below hold and skips only the - custom-op dispatch; otherwise the op is called and refuses the call. + cells ran on sm_103. With `direct_launch=True` (the FP8 target) the op's own + launch `launch_fused_sigmoid_mul_inplace` (the op's body) is called, skipping + only the custom-op dispatch: 15 instead of 28 us of host time a call on GB300. Call sites, each the attention output gate between the attention core and o_proj: @@ -257,7 +218,9 @@ class FusedSigmoidMulInplace(OpWrapper): def __call__( self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False ) -> torch.Tensor: - if not (direct_launch and _launch(attention_output, gate)): + if direct_launch: + launch_fused_sigmoid_mul_inplace(attention_output, gate) + else: torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) return attention_output diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py index adc049f7feb2..b7955cff9893 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py @@ -49,25 +49,6 @@ def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> return x.shape[0] >= _PREFILL_MIN_ROWS and _multirow(x, z, gate_activation) -def _direct_launch(x, z, weight, eps, gate_activation): - """The registered op's own multi-row launch (same kernel, 4 rows a program, PDL by the - op's policy), skipping only the custom-op dispatch.""" - rows, n = x.shape - out = torch.empty_like(x) - grid = triton.cdiv(rows, _layernorm_gated._MULTIROW_ROWS) - device = x.device.index - pdl = _layernorm_gated._multirow_pdl(grid, device) - with torch.cuda.device(device): - _layernorm_gated._rms_norm_gated_fwd_multirow_kernel[(grid,)]( - x, out, weight, z, None, None, x.stride(0), out.stride(0), z.stride(0), rows, eps, - N=n, ROWS=_layernorm_gated._MULTIROW_ROWS, HEADS_PER_TOK=z.shape[1], - GATE_SIGMOID=gate_activation == "sigmoid", WEIGHT_IS_DELTA=False, - LAUNCH_WITH_PDL=pdl, num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, - EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], launch_pdl=pdl, - ) # fmt: skip - return out - - def _prefill_launch(x, z, weight, eps, gate_activation): """The multi-row kernel at `_PREFILL_ROWS` rows a program (no PDL: the grid covers every SM).""" rows, n = x.shape @@ -154,8 +135,9 @@ class RmsNormGatedTokenMajor(OpWrapper): SM count and `TRTLLM_ENABLE_PDL != 0`; results are bitwise identical with it off. The 8-row programs reduce a row's mean square in another order than the 4-row ones (one-ulp differences on some outputs). With - `direct_launch=True` (the FP8 target), the 4-row launch is made here, the - registered op's own, skipping only the custom-op dispatch. + `direct_launch=True` (the FP8 target) below the 8-row threshold, the op's own + launch `launch_rms_norm_gated_token_major` (the op's body) is called, skipping + only the custom-op dispatch: 24 instead of 39 us of host time a call on GB300. Loud (measured), so not guarded: 3-D `x` (ValueError), a `z` not covering `x` (AssertionError "does not match x shape"; 2-D `z` ValueError), an unknown @@ -218,18 +200,12 @@ def __call__( ) -> torch.Tensor: if fp8_scale is None and z.dim() == 3 and _prefill_multirow(x, z, gate_activation): return _prefill_launch(x, z, weight, eps, gate_activation) - if ( - direct_launch - and fp8_scale is None - and x.dim() == 2 - and z.dim() == 3 - and weight.is_contiguous() - and _multirow(x, z, gate_activation) - ): - return _direct_launch(x, z, weight, eps, gate_activation) - return torch.ops.trtllm.rms_norm_gated_token_major( - x, z, weight, eps, fp8_scale, gate_activation + launch = ( + _layernorm_gated.launch_rms_norm_gated_token_major + if direct_launch + else torch.ops.trtllm.rms_norm_gated_token_major ) + return launch(x, z, weight, eps, fp8_scale, gate_activation) def reference( self, diff --git a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py index 98754aaf0f76..c0ad516eb838 100644 --- a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py +++ b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py @@ -321,8 +321,7 @@ def _multirow_gated_rmsnorm_eligible(N, ngroups, bias, z, norm_before_gate, and ngroups == 1 and N <= _MULTIROW_MAX_N and (N & (N - 1)) == 0) -@torch.library.custom_op("trtllm::rms_norm_gated_token_major", mutates_args=()) -def rms_norm_gated_token_major( +def launch_rms_norm_gated_token_major( x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor, @@ -330,7 +329,10 @@ def rms_norm_gated_token_major( fp8_scale: torch.Tensor | None = None, gate_activation: str = "silu", ) -> torch.Tensor: - """Gated rmsnorm(x) with z read from a 3D token-major view. + """``trtllm::rms_norm_gated_token_major``'s launch, for eager callers that + skip the custom-op dispatch. + + Gated rmsnorm(x) with z read from a 3D token-major view. x: [num_tokens * heads, N] with contiguous rows. z: [num_tokens, heads, N] whose (heads, N) block is contiguous per token and whose token stride is @@ -407,6 +409,20 @@ def rms_norm_gated_token_major( return out +@torch.library.custom_op("trtllm::rms_norm_gated_token_major", mutates_args=()) +def rms_norm_gated_token_major( + x: torch.Tensor, + z: torch.Tensor, + weight: torch.Tensor, + eps: float, + fp8_scale: torch.Tensor | None = None, + gate_activation: str = "silu", +) -> torch.Tensor: + """``launch_rms_norm_gated_token_major`` as a custom op.""" + return launch_rms_norm_gated_token_major(x, z, weight, eps, fp8_scale, + gate_activation) + + @rms_norm_gated_token_major.register_fake def _( x: torch.Tensor, From 896e3e93dd83e88efd620ef9dfb2829c1f91ca70 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:29:41 -0800 Subject: [PATCH 19/51] [None][test] modeling_v2 qwen4_exp: build the TPB test operands through the loader Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../ops/moe/moe_tpb_decode__decode.py | 37 ++----------- .../weights.py | 2 +- .../test_moe_and_dense_ops.py | 54 +++++++++++++++++-- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py index 664cb25cefc8..acb1faafccaf 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py @@ -9,7 +9,8 @@ runs a block's FC1 -> activation -> FC2, sharing the FC1 outputs through distributed shared memory; partials are summed in cluster order (deterministic). -Operands (`BlockExperts`), block-major, a rank's 640 blocks: +Operands (`BlockExperts`), block-major, a rank's 640 blocks as the loader lays them out +(`weights._block_experts`, blocks dealt by `block_ids`): `w1 [640, 256, 2560]` e4m3 (rows: the block's 128 up rows, then its 128 gate rows), `s1 [640, 2, 20]` fp32, `w2 [640, 2560, 128]` e4m3 (the block's 128 down_proj columns), `s2 [640, 20]` fp32. """ @@ -62,7 +63,7 @@ class SharedBlocks(NamedTuple): """This rank's TP4 slice of the bf16 shared expert, folded into the launch (3..8 rows): `x` the MoE site's `[T, 2560]` bf16 rows, `gu` `[5, 64, 2560]` the slice's gate / up rows (item sb = columns `[160 rank + 32 sb, +32)` as 8 groups of 4 columns, each `[gate 4 ; up 4]`), `gl` the `[2560]` gate-logit row, - `dn` `[5, 2560, 32]` the slice's down_proj columns (`shared_blocks`).""" + `dn` `[5, 2560, 32]` the slice's down_proj columns (the loader's `tpb_sh_gu` / `tpb_sh_dn`).""" x: torch.Tensor gu: torch.Tensor @@ -70,23 +71,6 @@ class SharedBlocks(NamedTuple): dn: torch.Tensor -def shared_blocks(gate, up, dn, rank: int): - """The checkpoint's shared-expert gate / up rows `[640, 2560]` and down `[2560, 640]` (bf16) -> this rank's - `(gu [5, 64, 2560], dn [5, 2560, 32])`.""" - w = SHARED_INTER // RANKS - lo = rank * w - g = gate[lo : lo + w].reshape(SHARED_ITEMS, 8, 4, -1) - u = up[lo : lo + w].reshape(SHARED_ITEMS, 8, 4, -1) - gu = torch.stack([g, u], dim=2).reshape(SHARED_ITEMS, 64, -1).contiguous() - d = ( - dn[:, lo : lo + w] - .reshape(dn.shape[0], SHARED_ITEMS, SHARED_ITEM_COLS) - .permute(1, 0, 2) - .contiguous() - ) - return gu, d - - def block_ids(rank: int) -> tuple[list[int], list[int]]: """(expert, block) of each of a rank's 640 blocks: j < 512 is expert j's block (rank - j) % 4, j >= 512 is expert 4 (j - 512) + rank's block 4.""" @@ -95,21 +79,6 @@ def block_ids(rank: int) -> tuple[list[int], list[int]]: return experts, blocks -def block_major(fc1_w, fc1_s, fc2_w, fc2_s, rank: int) -> BlockExperts: - """Whole-expert operands of all 512 experts indexed by routing id (`[512, 1280, 2560]` rows - `[up ; gate]`, `[512, 10, 20]`, `[512, 2560, 640]`, `[512, 20, 5]`) -> the rank's block-major - operands.""" - experts, blocks = block_ids(rank) - e = torch.tensor(experts, device=fc1_w.device) - b = torch.tensor(blocks, device=fc1_w.device) - n = fc1_w.shape[0] - w1 = fc1_w.view(n, 2, EXPERT_BLOCKS, 128, HIDDEN)[e, :, b].reshape(BLOCKS, 256, HIDDEN) - s1 = fc1_s.view(n, 2, EXPERT_BLOCKS, HIDDEN // 128)[e, :, b].reshape(BLOCKS, 2, HIDDEN // 128) - w2 = fc2_w.view(n, HIDDEN, EXPERT_BLOCKS, 128)[e, :, b] - s2 = fc2_s[e, :, b] - return BlockExperts(w1.contiguous(), s1.contiguous(), w2.contiguous(), s2.contiguous(), rank) - - _WORKSPACE: dict = {} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index e0661fdc50fa..ecf776b0ae88 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -509,7 +509,7 @@ def _tpb_fc2_s(t: torch.Tensor, d) -> torch.Tensor: def _tpb_sh_gu(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: """(gate, up) rows of one 32-column shared-expert item -> `[64, hidden]`. - 8 groups of 4 columns, each `[gate 4 ; up 4]` (`moe_tpb_decode.shared_blocks`). + 8 groups of 4 columns, each `[gate 4 ; up 4]` (`moe_tpb_decode.SharedBlocks`). """ gate, up = t return ( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py index 5c84c8777a1e..b592eadfa885 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py @@ -774,8 +774,56 @@ def composed(x, routed, gate, shared, inject): # ------------------------------------------------------------ moe_tpb_decode__decode --- +_TPB_BLOCKS: dict = {} + + +def _tpb_dims(rank: int) -> SimpleNamespace: + """The loader's dims of tep4 rank `rank` with the block-major copy on.""" + return SimpleNamespace( + moe_tpb=True, tp_rank=rank, num_experts=512, moe_inter=640, shared_inter=640, + hidden=2560, dtype=torch.bfloat16, + ) # fmt: skip + + +def _loaded(d, fake, names) -> list: + """The loader's parameters `names` of trunk layer 0, filled from `fake` as `load` fills them.""" + entries = {e.name: e for e in M._weights.MODEL_WEIGHTS.WEIGHTS} + out = [] + for name in names: + e = entries[name] + param = torch.empty(e.shape(d), dtype=e.dtype or d.dtype, device=DEV) + sources = [(*s, e.transform) for s in e.sources(d, 0, "")] + M._weights.MODEL_WEIGHTS.fill(param, sources, fake, d) + out.append(param) + return out + + def _tpb_blocks(rank: int): - return _tpb.block_major(*_weights(2560, 640, 512, seed=2560 + 640 + 512), rank) + """Tep4 rank `rank`'s block-major routed experts as the loader lays them out from a + checkpoint holding the stack `_weights` draws (rows `[up ; gate]`). Cached per rank.""" + if rank not in _TPB_BLOCKS: + w1, w1s, w2, w2s = _weights(2560, 640, 512, seed=2560 + 640 + 512) + p = f"{M._weights._trunk_mlp(0)}.experts" + fake = {} + for e in range(512): + fake.update({ + f"{p}.{e}.up_proj.weight": w1[e, :640], + f"{p}.{e}.gate_proj.weight": w1[e, 640:], + f"{p}.{e}.down_proj.weight": w2[e], + f"{p}.{e}.up_proj.weight_scale_inv": w1s[e, :5], + f"{p}.{e}.gate_proj.weight_scale_inv": w1s[e, 5:], + f"{p}.{e}.down_proj.weight_scale_inv": w2s[e], + }) # fmt: skip + names = ("tpb_w1", "tpb_s1", "tpb_w2", "tpb_s2") + _TPB_BLOCKS[rank] = _tpb.BlockExperts(*_loaded(_tpb_dims(rank), fake, names), rank) + return _TPB_BLOCKS[rank] + + +def _tpb_shared_blocks(gate, up, dn, rank: int): + """Tep4 rank `rank`'s shared-expert slice `(gu, dn)` as the loader lays it out.""" + p = f"{M._weights._trunk_mlp(0)}.shared_expert" + fake = {f"{p}.gate_proj.weight": gate, f"{p}.up_proj.weight": up, f"{p}.down_proj.weight": dn} + return tuple(_loaded(_tpb_dims(rank), fake, ("tpb_sh_gu", "tpb_sh_dn"))) def _tpb_call(fn, logits, args, blocks, **kw): @@ -863,7 +911,7 @@ def _tpb_shared_slice(rows): x, gate, up, gl, dn = _tpb_shared(rows, _seed(f"tpb-shared-w-{rows}")) total = torch.zeros(rows, 2560, device=DEV) for r in range(4): - gu_r, dn_r = _tpb.shared_blocks(gate, up, dn, r) + gu_r, dn_r = _tpb_shared_blocks(gate, up, dn, r) sh = _tpb.SharedBlocks(x, gu_r, gl, dn_r) blocks = _tpb_blocks(r) out = _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks, shared=sh) @@ -897,7 +945,7 @@ def _tpb_window(rank: int, shared) -> "M._weights.ExpertWindow": """Tep4 rank `rank`'s 128-expert window with its block-major copy and shared-expert slice.""" bl = _tpb_blocks(rank) gate, up, gl, dn = shared - sh_gu, sh_dn = _tpb.shared_blocks(gate, up, dn, rank) + sh_gu, sh_dn = _tpb_shared_blocks(gate, up, dn, rank) bw = M._weights.BlockWindow(bl.w1, bl.s1, bl.w2, bl.s2, rank, sh_gu, gl, sh_dn) return _pad_window(128 * rank, 128)._replace(blocks=bw) From 9f45b45523e13b1ab67228135ea9ebb2b8c3a13c Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:29:40 -0800 Subject: [PATCH 20/51] [None][test] modeling_v2 qwen4_exp: pin the stable-JIT QSA kernels bitwise to the shared launchers Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../test_attention_ops.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index b82cc22ec8f9..5351d3ffc40e 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1479,6 +1479,55 @@ def test_certified_cells(entry, cell, check, certifying, monkeypatch) -> None: check(cell, monkeypatch) +def _stable_kv_store_is_shared(cell) -> None: + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + if not isinstance(pool, _KvEnginePool): + shared = _stable.qsa_paged_kv_store.reference(**args) + _stable.qsa_paged_kv_store(**args) + _kv_store_compare((args["k_cache"], args["v_cache"]), shared) + return + try: + # The shared launcher stores into dense copies of the touched pages; the copy into the + # engine's pool, whose pages straddle 2^32 elements. + dense_k, dense_v, renamed = pool.dense(args["block_table"]) + shared = _stable.qsa_paged_kv_store.reference( + **{**args, "k_cache": dense_k, "v_cache": dense_v, "block_table": renamed} + ) + _stable.qsa_paged_kv_store(**args) + _kv_store_compare(pool.gathered(), shared) + finally: + pool.close() + + +def _stable_sparse_gqa_is_shared(cell) -> None: + args = _sparse_gqa_build(cell.spec, _seed(cell.why)) + got = _stable.qsa_paged_sparse_gqa(**args) + want = _stable.qsa_paged_sparse_gqa.reference(**args) + assert got.dtype == want.dtype and got.shape == want.shape, (got.shape, want.shape) + assert torch.equal(_bits(got), _bits(want)), "the stable-JIT kernels differ from the shared" + + +_STABLE_CASES = [ + pytest.param( + check, cell, + id=f"{name}: {cell.why[:44]}", + marks=[pytest.mark.high_cuda_memory] if cell.spec.get("large") else [], + ) + for name, cells, check in ( + ("kv_store", qsa_paged_kv_store.CELLS, _stable_kv_store_is_shared), + ("sparse_gqa", qsa_paged_sparse_gqa.CELLS, _stable_sparse_gqa_is_shared), + ) + for cell in cells +] # fmt: skip + + +@pytest.mark.parametrize("check, cell", _STABLE_CASES) +def test_stable_kernels_are_the_shared_launchers(check, cell) -> None: + """`qsa_triton_kernels__stable_jit`'s kernel copies write bitwise what the shared QSA + launchers they copy write, at every cell of the entries that launch them.""" + check(cell) + + # `DecodeTarget._attention_block` at the checkpoint's shapes for one rank of a layout. PRL_DEV, PRL_BF, PRL_F8 = "cuda", torch.bfloat16, torch.float8_e4m3fn From fff27132ac015ffab8694fe05c957069d5c4894f Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:26:12 -0800 Subject: [PATCH 21/51] [None][chore] modeling_v2 qwen4_exp: build the window runner's keywords in one helper Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 73 ++++++------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index f39d76817580..5b46a915aa3e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -2353,8 +2353,24 @@ def _window_runner( out = _window_runner(core, experts, xq, xsf, topk_ids, topk_w, output) after_first(out) return out - return fp8_block_scale_moe_runner( + return _window_runner_call( + core, + experts, + xq, + xsf, routing_logits=None, + topk_weights=topk_w, + topk_ids=topk_ids, + output=output, + tune_max_num_tokens=_runner_tune_rows(core), + ) + + +def _window_runner_call(core, experts: _weights.ExpertWindow, xq, xsf, **routing) -> torch.Tensor: + """`fp8_block_scale_moe_runner` over this rank's whole-expert window, renormalized SwiGLU; + `routing` is the call's routing (`routing_logits`, or `topk_ids` / `topk_weights`) and + any further runner keywords.""" + return fp8_block_scale_moe_runner( routing_bias=None, hidden_states=xq, hidden_states_scale=xsf, @@ -2372,12 +2388,9 @@ def _window_runner( local_num_experts=experts.count, routed_scaling_factor=None, routing_method_type=_ROUTING_METHOD_RENORM, - topk_weights=topk_w, - topk_ids=topk_ids, act_type=_ACT_TYPE_SWIGLU, gemm1_clamp_limit=None, - output=output, - tune_max_num_tokens=_runner_tune_rows(core), + **routing, ) @@ -2545,29 +2558,8 @@ def _window_runner_front(core, experts: _weights.ExpertWindow, x, logits, before def _window_runner_logits(core, experts: _weights.ExpertWindow, xq, xsf, logits) -> torch.Tensor: """Run the window with the runner routing `logits` itself (bitwise `renorm_moe_routing_op`).""" - return fp8_block_scale_moe_runner( - routing_logits=logits, - routing_bias=None, - hidden_states=xq, - hidden_states_scale=xsf, - gemm1_weights=experts.fc1_w, - gemm1_weights_scale=experts.fc1_s, - gemm2_weights=experts.fc2_w, - gemm2_weights_scale=experts.fc2_s, - num_experts=core.num_experts, - top_k=core.topk, - num_fused_shared_experts=None, - n_group=None, - topk_group=None, - intermediate_size=core.moe_inter, - local_expert_offset=experts.offset, - local_num_experts=experts.count, - routed_scaling_factor=None, - routing_method_type=_ROUTING_METHOD_RENORM, - topk_weights=None, - topk_ids=None, - act_type=_ACT_TYPE_SWIGLU, - gemm1_clamp_limit=None, + return _window_runner_call( + core, experts, xq, xsf, routing_logits=logits, topk_weights=None, topk_ids=None ) @@ -2590,30 +2582,7 @@ def _routed_experts_front( shared, ) assert shared is None, "the shared-expert fold rides the block launch" - return fp8_block_scale_moe_runner( - routing_logits=logits, - routing_bias=None, - hidden_states=xq, - hidden_states_scale=xsf, - gemm1_weights=experts.fc1_w, - gemm1_weights_scale=experts.fc1_s, - gemm2_weights=experts.fc2_w, - gemm2_weights_scale=experts.fc2_s, - num_experts=core.num_experts, - top_k=core.topk, - num_fused_shared_experts=None, - n_group=None, - topk_group=None, - intermediate_size=core.moe_inter, - local_expert_offset=experts.offset, - local_num_experts=experts.count, - routed_scaling_factor=None, - routing_method_type=_ROUTING_METHOD_RENORM, - topk_weights=None, - topk_ids=None, - act_type=_ACT_TYPE_SWIGLU, - gemm1_clamp_limit=None, - ) + return _window_runner_logits(core, experts, xq, xsf, logits) class _WindowRun(NamedTuple): From a3ae84e2563e663f189311f1e578cbed665aa1d5 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:51:14 -0800 Subject: [PATCH 22/51] [None][fix] modeling_v2 qwen4_exp: decline in-forward snapshots the split cannot serve at bind The FP8 target's in-forward snapshot split needs the manager to stage the PLE prompt-end state and to plan splits at least _INFORWARD_MIN_MAIN tokens before the snapshot point; the stock manager does neither (it plans from its conv width, 3). Check both at bind and raise NotImplementedError naming them, instead of an attribute assert at bind or a FlashInfer assert mid-step, and drop the shell's inforward_snapshot_min_main property, which nothing reads. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 27 +++++++++++-------- .../test_inforward_snapshot.py | 26 ++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 5b46a915aa3e..e1e8ba910b3e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -713,7 +713,8 @@ def _moe_comm_method() -> str: # end L runs through L, leaving the GDN and PLE state at P in the live slot and staging the state # at L (`PrefillTarget._gdn_split`). The fewest chunk tokens before P a split request takes: past # `_GDN_INPLACE_MAX_TOKENS` (the main chunk's FlashInfer path) and covering the conv (3) and PLE -# (9 columns, 2 ids) windows that end at P. +# (9 columns, 2 ids) windows that end at P. The manager plans the splits, so the bind declines one +# whose `inforward_snapshot_min_main` is lower (`_inforward_stage`). _INFORWARD_MIN_MAIN = _GDN_INPLACE_MAX_TOKENS + 1 #: Rows up to which `tp_sum` takes the multicast NVLink one-shot kernel @@ -808,11 +809,20 @@ def _inforward_stage(cm, gdn_layers, ple_layer: int) -> _InforwardStage | None: """The in-forward snapshot staging rows of `cm`, or None when it plans no snapshots.""" if getattr(cm, "inforward_snapshot_rows", 0) <= 0: return None - _pool.require( - cm, - ("inforward_snapshot_stage_views", "inforward_ple_stage_views", "conv_state_shape"), - "stages the in-forward snapshot's prompt-end GDN and PLE state", - ) + missing = [ + name + for name in ("inforward_snapshot_stage_views", "inforward_ple_stage_views") + if not hasattr(cm, name) + ] + min_main = getattr(cm, "inforward_snapshot_min_main", 0) + if missing or min_main < _INFORWARD_MIN_MAIN: + raise NotImplementedError( + f"the KV cache manager {type(cm).__name__} plans in-forward Mamba snapshots; this " + "target serves them only from a manager that stages the GDN and PLE prompt-end " + f"state and splits at least {_INFORWARD_MIN_MAIN} tokens before the snapshot point, " + f"and this one lacks {missing} and splits from {min_main} tokens. " + "kv_cache_config.enable_block_reuse=False turns the snapshots off" + ) ple = cm.inforward_ple_stage_views(ple_layer) assert ple is not None, ( f"the KV cache manager plans in-forward snapshots but stages no PLE state for decoder " @@ -6866,11 +6876,6 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: "(kv_cache_config.mamba_ssm_cache_dtype)" ) - @property - def inforward_snapshot_min_main(self) -> int: - """Fewest chunk tokens before the snapshot point for an in-forward snapshot split.""" - return _INFORWARD_MIN_MAIN - def __init__(self, model_config: ModelConfig): cfg = model_config.pretrained_config assert cfg is not None diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py index 18ef562a97c1..ec202e0bcd40 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py @@ -358,3 +358,29 @@ def _ple_publish_end() -> None: def test_launch_variants_are_bitwise_the_plain_launch(helper, args) -> None: """A padded CUDA-graph GDN call and a partly publishing PLE launch against the plain ones.""" helper(*args) + + +def _manager(rows=ROWS, min_main=_m._INFORWARD_MIN_MAIN, ple=True): + """A manager stub that plans `rows` staging rows and splits from `min_main` tokens.""" + views = { + "inforward_snapshot_stage_views": lambda i: (f"ssm{i}", f"conv{i}"), + "inforward_ple_stage_views": lambda layer: ("ple_conv", "ple_ngram"), + } + if not ple: + del views["inforward_ple_stage_views"] + return types.SimpleNamespace( + inforward_snapshot_rows=rows, inforward_snapshot_min_main=min_main, **views + ) + + +def test_the_bind_takes_only_snapshots_the_split_serves() -> None: + """No staging rows: no stage. A manager that stages no PLE state, or plans splits of fewer + tokens before P than the main chunk's FlashInfer path takes (the stock manager's conv-width + minimum, 3), is declined at bind rather than asserted on mid-step.""" + assert _m._inforward_stage(_manager(rows=0), [0, 2], 1) is None + stage = _m._inforward_stage(_manager(), [0, 2], 1) + assert stage.ssm == {0: "ssm0", 2: "ssm2"} and stage.ple_conv == "ple_conv" + with pytest.raises(NotImplementedError, match=r"lacks \[\] and splits from 3 tokens"): + _m._inforward_stage(_manager(min_main=D_CONV - 1), [0, 2], 1) + with pytest.raises(NotImplementedError, match="lacks \\['inforward_ple_stage_views'\\]"): + _m._inforward_stage(_manager(ple=False), [0, 2], 1) From 13c100b0dd78f0f03f58791e6fd73c162812d5d0 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:51:22 -0800 Subject: [PATCH 23/51] [None][fix] modeling_v2 qwen4_exp: port the FP8 KV range probe to the FP8 target Under TRTLLM_MODELING_V2_VALIDATE over an fp8 KV pool, the prefill and decode step classes report once when the first eager forward stores a K/V magnitude past the e4m3 maximum (the pool is written at the unit scale), as the NVFP4 target does. The probe and its tests are the NVFP4 target's. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 54 +++++++++++ .../test_attention_ops.py | 94 ++++++++++++++++++- 2 files changed, 143 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index e1e8ba910b3e..8e36754664cc 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -3967,6 +3967,45 @@ def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens, paged_ ) +class _KvRangeProbe: + """`TRTLLM_MODELING_V2_VALIDATE` over an fp8 KV pool: reports, once, the largest K/V + magnitude the first eager forward's attention layers store. + + The pool takes K/V at the unit scale, so anything past the e4m3 maximum saturates silently + (a bf16 inf included). One host sync per attention layer of that one forward; a forward + under CUDA graph capture is passed over without a read.""" + + def __init__(self, last_layer: int) -> None: + self._last = last_layer + self._worst = 0.0 + + @classmethod + def for_target(cls, core, enabled: bool) -> "_KvRangeProbe | None": + """A probe over `core`'s attention layers when `enabled` and the pool is fp8.""" + if not enabled or core.quant_mode != QM_FP8_KV_CACHE: + return None + return cls(core.attn_layers[-1]) + + def observe(self, layer: int, k: torch.Tensor, v: torch.Tensor) -> bool: + """Fold one attention layer's K/V in; False once the forward's last layer has reported.""" + if torch.cuda.is_current_stream_capturing(): + return True + peak = float(torch.maximum(k.abs().amax(), v.abs().amax())) + # A NaN stays: `max` would drop it. + self._worst = math.nan if math.isnan(peak + self._worst) else max(self._worst, peak) + if layer != self._last: + return True + limit = torch.finfo(torch.float8_e4m3fn).max + # Written so that NaN fails too. + if not self._worst <= limit: + logger.warning( + f"FP8 KV cache: the first forward stores a K/V magnitude of {self._worst:g}, " + f"past the e4m3 maximum {limit:g}; the pool is written at the unit scale, " + "so those values saturate" + ) + return False + + # ------------------------------------------------------------------ prefill --- @@ -4011,6 +4050,8 @@ class PrefillTarget: def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: self.core = core self._contract_pending = step_contract_enabled() + # Dropped once it has reported. + self._kv_probe = _KvRangeProbe.for_target(core, self._contract_pending) self._layers = _layer_views(core) self._stack = _stack_views(core) self._step: _PrefillStep | None = None @@ -4387,6 +4428,8 @@ def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: num_tokens = step.num_tokens proj = proj[:num_tokens] q_rows, k_dst, v_dst = _attn_in_heads(core, proj) + if self._kv_probe is not None and not self._kv_probe.observe(i, k_dst, v_dst): + self._kv_probe = None qkv = proj_qkv[:num_tokens] if not sparse: # The dense op reads a compact `[q | k | v]`. @@ -5616,6 +5659,8 @@ class DecodeTarget: def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: self.core = core self._contract_pending = step_contract_enabled() + # Dropped once it has reported. + self._kv_probe = _KvRangeProbe.for_target(core, self._contract_pending) self._layers = _layer_views(core) self._stack = _stack_views(core) self._index_split = _index_split_group(core) @@ -5705,6 +5750,15 @@ def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, sta ) # One compact `[q | k | v]` copy: read flat by the dense op, per head by sparse GQA. qkv.copy_(qkv_view) + if self._kv_probe is not None: + # Both branches leave the normalized, rotated K/V in the packed copy. + k_rows, v_rows = torch.split( + torch.reshape(qkv, [qkv.shape[0], -1, core.head_dim])[:, core.heads_q :], + core.heads_kv, + 1, + ) + if not self._kv_probe.observe(i, k_rows, v_rows): + self._kv_probe = None if sparse: kv = pools.kv[i] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 5351d3ffc40e..2fa1ba6699ae 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the FP8 unified target's attention entries other than `thop_attention`: every certified cell against the entry's `reference` over the engine's paged state, plus the TP index -split, captured stores, kernels without cells and the call sites.""" +split, captured stores, kernels without cells, the call sites and the FP8 KV range probe.""" import math import types import zlib from types import SimpleNamespace +from unittest.mock import Mock import cuda.bindings.driver as cuda_drv import pytest @@ -1662,7 +1663,9 @@ def select(rank, q, vectors, n, rpr, gather=lambda part, sizes, g_, pg: part, ** return M._decode_index_selection(core, split, q, vectors, n, rpr, PRL_DEV, cache, **kw) # The selections are taken over the planes the layer left (its rows' index keys stored). - full_out, scratch = layer(types.SimpleNamespace(core=core, _index_split=None), None) + full_out, scratch = layer( + types.SimpleNamespace(core=core, _index_split=None, _kv_probe=None), None + ) q = M._index_query(core, scratch) full = select(None, q, q8, rows, p) halves = [select(rank, q, q8, rows, p) for rank in (0, 1)] @@ -1713,13 +1716,16 @@ def nccl(part, sizes, group, process_group): handed.append(part.clone()) return torch.cat(halves) - out, _ = layer(types.SimpleNamespace(core=core, _index_split=([0, 1], None, rank)), nccl) + out, _ = layer( + types.SimpleNamespace(core=core, _index_split=([0, 1], None, rank), _kv_probe=None), + nccl, + ) restore() assert len(handed) == 1 and torch.equal(handed[0], halves[rank]) assert torch.equal(out, full_out), "the layer over the gathered selection" # The plan: the prologue stores the index queries in the permuted order. target = types.SimpleNamespace( - core=core, _index_split=([0, 1], None, rank), _index_plan=plans[rank] + core=core, _index_split=([0, 1], None, rank), _index_plan=plans[rank], _kv_probe=None ) out, scr = layer(target, lambda part, sizes, g_, pg: gathered) restore() @@ -1958,10 +1964,13 @@ def counted(*args, **kwargs): if site == "trunk": qkv = torch.empty(rows, core.qkv_width, dtype=PRL_BF, device=PRL_DEV) scratch = torch.empty(rows, core.index_scratch_width, dtype=PRL_BF, device=PRL_DEV) + # The FP8 KV range probe reads the packed K/V on both branches; this is its last layer. + target = types.SimpleNamespace(core=core, _kv_probe=M._KvRangeProbe(PRL_LAYER)) out = M.DecodeTarget._attention_block( - types.SimpleNamespace(core=core), x, PRL_LAYER, L, qkv, scratch, None, qsa, + target, x, PRL_LAYER, L, qkv, scratch, None, qsa, {"predicted_tokens_per_seq": p}, False, ) # fmt: skip + assert target._kv_probe is None, "the probe reported on its last layer and was dropped" else: roles, planes, position = pools draft = types.SimpleNamespace( @@ -2112,3 +2121,78 @@ def md(q_lens, kv_lens): def test_call_sites(check, case, monkeypatch) -> None: """What the target's call sites hand the entries beyond a cell.""" check(case, monkeypatch) + + +# ---- the FP8 KV range probe (`TRTLLM_MODELING_V2_VALIDATE` over an fp8 pool) ---------------- +# The pool stores K/V at the unit scale, so a magnitude past the e4m3 maximum saturates. The probe +# reports the largest magnitude of the first eager forward's attention layers, once, and never +# reads a tensor under CUDA graph capture. + +PROBE_LAST = 11 + + +def _probe_kv(peak: float = 1.0): + """K and V of one layer, 5 rows x 2 heads x 8 dims, with `peak` as V's largest value.""" + k = torch.linspace(-1.0, 1.0, 80, device=DEV).reshape(5, 2, 8).to(torch.bfloat16) + v = torch.linspace(-0.5, 0.5, 80, device=DEV).reshape(5, 2, 8).to(torch.bfloat16) + v[3, 1, 4] = peak + return k, v + + +def _probe_forward(probe, peaks): + """One forward's attention layers 3, 7 and PROBE_LAST; returns each `observe` result.""" + return [ + probe.observe(layer, *_probe_kv(peak)) for layer, peak in zip((3, 7, PROBE_LAST), peaks) + ] + + +def test_a_forward_within_the_e4m3_range_is_silent(monkeypatch) -> None: + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + assert _probe_forward(M._KvRangeProbe(PROBE_LAST), (1.0, 300.0, 448.0)) == [True, True, False] + logger.warning.assert_not_called() + + +@pytest.mark.parametrize("peak", [-600.0, float("nan")]) +def test_a_magnitude_past_the_e4m3_maximum_is_reported_once(monkeypatch, peak) -> None: + """A negative value counts by its magnitude; a NaN counts as out of range. The oversized value + is in a middle layer: the report waits for the forward's last one.""" + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + assert _probe_forward(M._KvRangeProbe(PROBE_LAST), (1.0, peak, 1.0)) == [True, True, False] + logger.warning.assert_called_once() + assert "e4m3 maximum 448" in logger.warning.call_args.args[0] + + +def test_the_probe_runs_only_when_asked_over_an_fp8_pool() -> None: + core = SimpleNamespace(quant_mode=M.QM_FP8_KV_CACHE, attn_layers=[3, 7, PROBE_LAST]) + assert M._KvRangeProbe.for_target(core, True)._last == PROBE_LAST + assert M._KvRangeProbe.for_target(core, False) is None + for quant_mode in (0, M.QM_NVFP4_KV_CACHE): + other = SimpleNamespace(quant_mode=quant_mode, attn_layers=core.attn_layers) + assert M._KvRangeProbe.for_target(other, True) is None + + +def test_observing_under_capture_reads_nothing(monkeypatch) -> None: + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + probe = M._KvRangeProbe(PROBE_LAST) + k, v = _probe_kv(1000.0) + scratch = torch.zeros(1, device=DEV) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + scratch.add_(1) # warm the allocator off the capture stream + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + kept = [] + with torch.cuda.graph(graph): + scratch.add_(1) + # The last layer, with an out-of-range value: a read here would fail the capture (a host + # sync) and a report would fire. + kept.append(probe.observe(PROBE_LAST, k, v)) + assert kept == [True], "a captured forward does not end the probe" + logger.warning.assert_not_called() + # The first eager forward after capture is what reports. + assert _probe_forward(probe, (1.0, 1.0, 1000.0)) == [True, True, False] + logger.warning.assert_called_once() From c4617a07822ecd1df8f15316abc0dda34f447b11 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:51:23 -0800 Subject: [PATCH 24/51] [None][fix] modeling_v2 qwen4_exp: run the vision tower on a chunk's frames only Under chunked prefill every chunk overlapping a request's placeholder rows reran the tower over all of the request's media and kept its slice. A frame's rows depend on its own patches alone, so run one tower call over only the frames holding the chunk's rows; the rows are bitwise the one-pass rows. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../vision.py | 142 +++++++++++------- .../test_vision.py | 86 ++++++++--- 2 files changed, 148 insertions(+), 80 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py index f462e4ede4c7..1bec3a9f33b5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -422,15 +422,15 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): `rows` is `[R, hidden]` in packed-batch order, `mm_token_indices` the `[R]` rows they replace. The shell calls it: the inherited `DecoderModelForCausalLM.forward` drops `multimodal_params`, which covers a subset of the context requests (never zip it positionally). - `multimodal_runtime` counts rows over a request's media together, in prompt order.""" + `multimodal_runtime` counts rows over a request's media together, in prompt order; a chunk + runs the tower on only the frames that hold its rows (`_tower_rows`).""" mm_params = kwargs.get("multimodal_params") or [] if not mm_params: return None rows = [] for params in mm_params: data = getattr(params, "multimodal_data", None) or {} - image, video = data.get("image"), data.get("video") - if image is None and video is None: + if data.get("image") is None and data.get("video") is None: continue mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") assert ( @@ -444,26 +444,21 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): # Every placeholder row of this request lies in another chunk. continue assert tower is not None, "post_load_weights builds the vision tower before any step" - towers = {} - if image is not None: - towers["image"] = run_tower(tower, image, *MEDIA_KEYS["image"]) - if video is not None: - towers["video"] = run_tower(tower, video, *MEDIA_KEYS["video"]) - embeds = in_prompt_order(tower.merge, params, towers) + items = _items_in_prompt_order(params, data) + merge_unit = tower.merge * tower.merge + total = sum(t * h * w for _, _, (t, h, w) in items) // merge_unit + start, count = 0, total if runtime is not None: - # Chunked prefill or reused prefix: splice only this chunk's slice. + # Chunked prefill or reused prefix: only this chunk's slice. start = int(runtime.num_cached_mm_tokens) count = int(runtime.num_mm_tokens_in_chunk) - total = embeds.shape[0] assert int(runtime.total_embeds_in_request) == total, ( f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " - f"this request and the tower produced {total} over its {sorted(towers)}; the " - "two are derived from the same grids and a disagreement means the merger's " - "row count is wrong" + f"this request and its grids describe {total}; the two are derived from the " + "same grids and a disagreement means the merger's row count is wrong" ) assert 0 <= start <= start + count <= total, (start, count, total) - embeds = embeds[start : start + count] - rows.append(embeds) + rows.append(_tower_rows(tower, items, start, count)) if not rows: return None vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) @@ -480,49 +475,86 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): return vision, mm_token_indices -def run_tower(tower, media: dict, pixel_key: str, grid_key: str): - """Run the tower on one modality's payload: `(rows, grid_thw)`, items in processor order.""" - pixel_values, grid_thw = media.get(pixel_key), media.get(grid_key) - assert pixel_values is not None and grid_thw is not None, ( - f"the multimodal payload carries {sorted(media)}; this target reads {pixel_key!r} and " - f"{grid_key!r} from it" - ) - assert not grid_thw.is_cuda, f"{grid_key} must stay on the host; see DEVICE_PATHS" - return tower(pixel_values, grid_thw), grid_thw - - -def in_prompt_order(merge: int, params, towers: dict) -> torch.Tensor: - """One request's tower rows (`towers`: modality -> `(rows, grid_thw)`) in placeholder order. +def _items_in_prompt_order(params, data: dict) -> list[tuple[torch.Tensor, int, tuple]]: + """One request's media items in placeholder order: `(pixel_values, first patch row, (t, h, w))`. Two modalities interleave by `item_metadata.item_refs`, the only host-side record of order.""" - if len(towers) == 1: - return next(iter(towers.values()))[0] - metadata = params.item_metadata - order = metadata.item_refs if metadata is not None else None - assert order, ( - f"a request carrying {sorted(towers)} needs item metadata to splice them in prompt " - "order; the payload holds each modality's rows apart" - ) - merge_unit = merge * merge - items = {} - for modality, (embeds, grid_thw) in towers.items(): - counts = [t * h * w // merge_unit for t, h, w in grid_thw.tolist()] - assert sum(counts) == embeds.shape[0], ( - f"the {modality} grid describes {sum(counts)} rows and the tower produced " - f"{embeds.shape[0]}" - ) - items[modality] = torch.split(embeds, counts, 0) - listed = {modality: [] for modality in items} - for modality, index in order: - assert modality in items, ( - f"item_refs lists a {modality!r} item; this request carries {sorted(items)}" + media = {} + for modality, (pixel_key, grid_key) in MEDIA_KEYS.items(): + payload = data.get(modality) + if payload is None: + continue + pixel_values, grid_thw = payload.get(pixel_key), payload.get(grid_key) + assert pixel_values is not None and grid_thw is not None, ( + f"the multimodal payload carries {sorted(payload)}; this target reads {pixel_key!r} " + f"and {grid_key!r} from it" ) - listed[modality].append(int(index)) - for modality, parts in items.items(): - assert sorted(listed[modality]) == list(range(len(parts))), ( - f"item_refs lists {modality} items {listed[modality]}; the payload carries {len(parts)}" + assert not grid_thw.is_cuda, f"{grid_key} must stay on the host; see DEVICE_PATHS" + grid = [tuple(int(v) for v in row) for row in grid_thw.tolist()] + firsts, patches = [], 0 + for frames, height, width in grid: + firsts.append(patches) + patches += frames * height * width + media[modality] = (pixel_values, grid, firsts) + if len(media) == 1: + modality, (_, grid, _) = next(iter(media.items())) + order = [(modality, index) for index in range(len(grid))] + else: + metadata = params.item_metadata + order = metadata.item_refs if metadata is not None else None + assert order, ( + f"a request carrying {sorted(media)} needs item metadata to splice them in prompt " + "order; the payload holds each modality's rows apart" ) - return torch.cat([items[modality][int(index)] for modality, index in order], 0) + listed = {modality: [] for modality in media} + for modality, index in order: + assert modality in media, ( + f"item_refs lists a {modality!r} item; this request carries {sorted(media)}" + ) + listed[modality].append(int(index)) + for modality, (_, grid, _) in media.items(): + assert sorted(listed[modality]) == list(range(len(grid))), ( + f"item_refs lists {modality} items {listed[modality]}; the payload carries " + f"{len(grid)}" + ) + return [ + (media[modality][0], media[modality][2][int(index)], media[modality][1][int(index)]) + for modality, index in order + ] + + +def _tower_rows(tower: VisionTower, items: list, start: int, count: int) -> torch.Tensor: + """Placeholder rows `[start, start + count)` of one request's `items` (prompt order). + + One tower call over only the frames that hold them: a frame's rows depend on its own patches + alone (attention is per frame, every other op per row), so a chunk of a long video does not + rerun the request's other frames. A frame a chunk boundary cuts runs in both chunks. Adjacent + frames of one pixel tensor stay one view; items of two modalities are copied into one tensor.""" + merge_unit = tower.merge * tower.merge + end = start + count + spans: list[list] = [] # [pixel_values, first patch row, end patch row] + grid: list[tuple[int, int, int]] = [] + first_row = None + row = 0 # the item's first placeholder row + for pixel_values, first_patch, (frames, height, width) in items: + frame_rows = height * width // merge_unit + lo = max(start - row, 0) // frame_rows + hi = min(-(-(end - row) // frame_rows), frames) + if lo < hi: + if first_row is None: + first_row = row + lo * frame_rows + patch_lo = first_patch + lo * height * width + patch_hi = first_patch + hi * height * width + if spans and spans[-1][0] is pixel_values and spans[-1][2] == patch_lo: + spans[-1][2] = patch_hi + else: + spans.append([pixel_values, patch_lo, patch_hi]) + grid.append((hi - lo, height, width)) + row += frames * frame_rows + assert first_row is not None, (start, count, row) + parts = [pixel_values[lo:hi] for pixel_values, lo, hi in spans] + out = tower(parts[0] if len(parts) == 1 else torch.cat(parts, 0), torch.tensor(grid)) + return out[start - first_row : end - first_row] def validate_request(tower, request) -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py index aec661fdfbca..efccb55f4210 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -130,30 +130,32 @@ def _angle_table(grid_thw: torch.Tensor, merge: int, theta: float, head_dim: int class _Tower: - """Stub `VisionTower`: every row of item `i` of modality `m` holds `100 * m + i`.""" + """Stub `VisionTower`: a merged row carries its first patch's pixel value (`_media`).""" merge = MERGE def __init__(self): self.calls = 0 + self.patches = 0 def __call__(self, pixel_values, grid_thw): self.calls += 1 - modality = int(pixel_values[0, 0]) - return torch.cat( - [ - torch.full((t * h * w // (MERGE * MERGE), HIDDEN), float(100 * modality + item)) - for item, (t, h, w) in enumerate(grid_thw.tolist()) - ] - ) + self.patches += pixel_values.shape[0] + assert sum(t * h * w for t, h, w in grid_thw.tolist()) == pixel_values.shape[0] + return pixel_values[:: MERGE * MERGE].expand(-1, HIDDEN) def _media(modality, grids): - """One `multimodal_data` entry; `pixel_values` only carries the stub's modality.""" - patches = sum(t * h * w for t, h, w in grids) + """One `multimodal_data` entry; every patch row of item `i` holds `100 * modality + i`.""" key = "pixel_values" if modality == IMAGE else "pixel_values_videos" grid_key = "image_grid_thw" if modality == IMAGE else "video_grid_thw" - return {key: torch.full((patches, 1), float(modality)), grid_key: torch.tensor(grids)} + pixels = torch.cat( + [ + torch.full((t * h * w, 1), float(100 * modality + item)) + for item, (t, h, w) in enumerate(grids) + ] + ) + return {key: pixels, grid_key: torch.tensor(grids)} def _params(image=None, video=None, order=None, runtime=None, mrope=True): @@ -542,6 +544,33 @@ def _hf_video_in_two_calls(tower, media, references, monkeypatch): _check("video in two calls", out, *references["video"]) +def _hf_video_in_chunks(tower, media, references, monkeypatch): + """The video spliced over three chunked-prefill chunks whose boundaries cut its two 96-row + frames: each chunk runs only the frames holding its rows, and together they are the one-pass + rows bitwise.""" + px, grid = media["video"] + total = int(grid.prod(-1).sum()) // tower.merge**2 + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)}, + "video": {"pixel_values_videos": px, "video_grid_thw": grid}, + } + chunks = [] + with torch.inference_mode(): + whole = tower(px, grid) + for start, end in ((0, 50), (50, 150), (150, total)): + runtime = SimpleNamespace( + num_cached_mm_tokens=start, + num_mm_tokens_in_chunk=end - start, + total_embeds_in_request=total, + ) + params = MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(end - start)} + chunks.append(V.multimodal_embeds(tower, kwargs)[0]) + out = torch.cat(chunks) + assert torch.equal(out, whole), _mismatch(out, whole) + _check("video in three chunks", out, *references["video"]) + + _HF_CASES = { "image0": _hf_single("image0"), "image1": _hf_single("image1"), @@ -549,6 +578,7 @@ def _hf_video_in_two_calls(tower, media, references, monkeypatch): "image_pair": _hf_pair(), "image_pair_two_calls": _hf_pair(cap=1176), "video_two_calls": _hf_video_in_two_calls, + "video_chunks": _hf_video_in_chunks, } @@ -748,33 +778,39 @@ def test_admission_serves_what_the_tower_serves(build, refusal, reads_tower): Shell.validate_request(shell, build()) -def _chunk(cached, in_chunk): +def _chunk(cached, in_chunk, total=len(MIXED_ROWS)): return SimpleNamespace( num_cached_mm_tokens=cached, num_mm_tokens_in_chunk=in_chunk, - total_embeds_in_request=len(MIXED_ROWS), + total_embeds_in_request=total, ) -#: `(id, _params kwargs, placeholder rows, rows that land there (None: no media), tower calls)`. -#: The chunked cases take prompt-order rows 3..9 of the mixed request, or none. +#: `(id, _params kwargs, placeholder rows, rows that land there (None: no media), tower calls, +#: patch rows the tower runs)`. The mixed chunk takes prompt-order rows 3..9: the second frame of +#: video 0, image 0 and the first frame of video 1 (40 of the request's 68 patch rows); the video +#: chunk rows 5..7, inside video 1's first frame. _SPLICES = [ - ("text", dict(), 0, None, 0), - ("image", dict(image=IMAGES), 5, [100.0] * 4 + [101.0], 1), - ("video", dict(video=VIDEOS), 12, [200.0] * 4 + [201.0] * 8, 1), - ("mixed", _MIXED, len(MIXED_ROWS), MIXED_ROWS, 2), - ("chunk", dict(_MIXED, runtime=_chunk(3, 6)), 6, MIXED_ROWS[3:9], 2), - ("chunk_without_placeholders", dict(_MIXED, runtime=_chunk(len(MIXED_ROWS), 0)), 0, None, 0), + ("text", dict(), 0, None, 0, 0), + ("image", dict(image=IMAGES), 5, [100.0] * 4 + [101.0], 1, 20), + ("video", dict(video=VIDEOS), 12, [200.0] * 4 + [201.0] * 8, 1, 48), + ("mixed", _MIXED, len(MIXED_ROWS), MIXED_ROWS, 1, 68), + ("chunk", dict(_MIXED, runtime=_chunk(3, 6)), 6, MIXED_ROWS[3:9], 1, 40), + ("video_chunk", dict(video=VIDEOS, runtime=_chunk(5, 2, 12)), 2, [201.0] * 2, 1, 16), + ("chunk_without_placeholders", dict(_MIXED, runtime=_chunk(len(MIXED_ROWS), 0)), 0, None, 0, 0), ] @pytest.mark.cpu_only @pytest.mark.parametrize( - "params,num_rows,want,calls", [s[1:] for s in _SPLICES], ids=[s[0] for s in _SPLICES] + "params,num_rows,want,calls,patches", [s[1:] for s in _SPLICES], ids=[s[0] for s in _SPLICES] ) -def test_the_splice_puts_each_items_rows_on_its_placeholders(params, num_rows, want, calls): +def test_the_splice_puts_each_items_rows_on_its_placeholders( + params, num_rows, want, calls, patches +): """The tower's rows reach `begin_step` with the placeholder indices, and the splice writes them - over the placeholders in prompt order and zeroes the padding rows after the step.""" + over the placeholders in prompt order and zeroes the padding rows after the step. A chunk runs + the tower on only the frames that hold its rows.""" tower = _Tower() indices = torch.arange(1, 1 + num_rows, dtype=torch.int64) kwargs = { @@ -783,7 +819,7 @@ def test_the_splice_puts_each_items_rows_on_its_placeholders(params, num_rows, w "resource_manager": object(), } vision = _forward_to_begin_step(tower, kwargs) - assert tower.calls == calls + assert (tower.calls, tower.patches) == (calls, patches) total = num_rows + 2 h = torch.full((total + 3, HIDDEN), -1.0) M.PrefillTarget._splice( From a70f1ca1f222412d6de06d5b4d33069166674af9 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:59:19 -0800 Subject: [PATCH 25/51] [None][fix] modeling_v2 qwen4_exp: pass the draft-extend step its host fp8 page count Under IndexShare the draft-extend step's sparse attention over context rows called the fp8 row-tile pass without a host page count, so the pass read it back from the device (one sync per step with context rows). The context rows reach the trunk step's pages, so the trunk's host count serves; the pass now declines an fp8 pool without one, as _rowtile_takes does. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 8e36754664cc..8b53ff854b9c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -3738,19 +3738,17 @@ def _sparse_gqa_rowtile( """The row-tile pass over a context launch, or None where it does not serve it. An fp8 pool's reached pages are widened exactly into a bf16 scratch pool; `fp8_pages` is - their host count (read from the device when None).""" + their host count, without which the pass does not serve an fp8 pool (`_rowtile_takes`).""" k_cache, v_cache, table = kv.k, kv.v, block_table if kv.k.dtype == torch.float8_e4m3fn: + if fp8_pages is None or fp8_pages * kv.k[0].numel() * 4 > _FP8_ROWTILE_MAX_SCRATCH: + return None # Each slot's highest position, -1 if absent: a compare-and-max over `[slots, rows]` # (a scatter max would serialize a request's rows on one address). slots = torch.arange(block_table.shape[0], dtype=request_indices.dtype, device=q.device) held = request_indices[None, :] == slots[:, None] last = torch.where(held, query_positions[None, :].long(), -1).amax(1) counts = torch.div(last + tpb, tpb, rounding_mode="floor") - if fp8_pages is None: - fp8_pages = int(counts.sum()) - if fp8_pages * kv.k[0].numel() * 4 > _FP8_ROWTILE_MAX_SCRATCH: - return None k_cache = torch.empty([fp8_pages, *kv.k.shape[1:]], dtype=q.dtype, device=q.device) v_cache = torch.empty_like(k_cache) table = torch.full_like(block_table, -1) @@ -5595,6 +5593,7 @@ def mtp_forward( self._capture_selection(md, topk_indices, visible_blocks) attn = torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device) for lo, hi, context_rows in parts: + # The context rows reach the trunk step's pages: its host count serves here. out = _sparse_gqa( core, kv, @@ -5604,6 +5603,7 @@ def mtp_forward( req_idx[lo:hi], tokens_per_block, logical[lo:hi] if context_rows else None, + self._step.fp8_pages if context_rows else None, ) attn[lo:hi].copy_(torch.reshape(out, [hi - lo, core.q_width])) else: From 1329367f7772216ca3882f1ffee34e0983a647bd Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:43:51 -0800 Subject: [PATCH 26/51] [None][fix] modeling_v2 qwen4_exp: run eager draft steps' projections on cuBLAS The draft steps after the first of a step with context rows run DecodeTarget's mtp_forward eagerly, at the batch's request count, not a captured padded size. Their projections went to _decode_projection's FlashInfer low-M kernels, which compile on the first call at each shape (the direct kernel's rows_per_block is in the compile key): an uncaptured count such as 3 requests compiled a kernel inside the served step, about 0.24 s on GB300. Those steps now take cublas_mm; captured and generation-only steps keep the low-M kernels. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 33 +++++++++++-------- .../test_attention_ops.py | 4 ++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 8b53ff854b9c..5b735850cf57 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1107,6 +1107,9 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self._binds = 0 # Set by `begin_step`, cleared by `forward`. self._step_begun = False + # Whether the step `begin_step` saw carries context rows: its draft steps after the + # first run eagerly on `DecodeTarget`, at any batch size (`DecodeTarget.mtp_forward`). + self._context_step = False self._mtp: "_MtpWeights | None" = None self._mtp_cos_sin: torch.Tensor | None = None # The verify step's side-cache rollback, registered with the spec worker. @@ -1689,6 +1692,7 @@ def begin_step( target.prepare(attn_metadata, input_ids, position_ids, vision, spec_metadata) else: assert vision is None, "a generation-only step carries vision rows" + self._context_step = attn_metadata.num_contexts > 0 self._step_begun = True def forward( @@ -2145,7 +2149,7 @@ def _mtp_front(core, M: _MtpWeights, embed_tokens, input_ids, hidden_states, mm= `fc_embedding` of the normalized next-token embedding is added to every stream of `fc_hidden` as the trunk site's unit-weight injection; returns `(x, mixed, inject)`. `mm` runs dense projections here and in every draft-layer and MoE helper: `cublas_mm` when - None, `_decode_projection` from `DecodeTarget`.""" + None, `_decode_projection` from `DecodeTarget` (except after a step's context rows).""" mm = mm or cublas_mm rows = hidden_states.shape[0] assert rows <= M.unit_inject.shape[0], ( @@ -2195,17 +2199,17 @@ def _mtp_attention_in(core, M: _MtpWeights, mixed, pos3, pos, with_index: bool, def _mtp_attention_planes( - core, M: _MtpWeights, mixed, draft: _DraftBound, qsa, rows_per_request, with_index, sparse + core, M: _MtpWeights, mixed, draft: _DraftBound, qsa, rows_per_request, with_index, sparse, mm ) -> tuple: """Compute a generation-only draft step's attention input and write its planes. - Returns `(qkv, gate, q_index)`. The K/V append happens here only on the sparse path (the - dense op appends itself). The draft-extend step runs everything after the projection as - the trunk's one-launch `qwen4_exp_attn_decode_prologue`.""" + Returns `(qkv, gate, q_index)`; `mm` runs the input projection. The K/V append happens here + only on the sparse path (the dense op appends itself). The draft-extend step runs everything + after the projection as the trunk's one-launch `qwen4_exp_attn_decode_prologue`.""" req_idx, logical, block_table, tokens_per_block, pos3, pos, coords = qsa if with_index and _decode_prologue(draft.kv, sparse): rows = mixed.shape[0] - proj = _decode_projection(mixed, M.attn_in) + proj = mm(mixed, M.attn_in) qkv = torch.empty([rows, core.qkv_width], dtype=mixed.dtype, device=mixed.device) index_scratch = torch.empty( [rows, core.index_scratch_width], dtype=mixed.dtype, device=mixed.device @@ -2246,7 +2250,7 @@ def _mtp_attention_planes( gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] return qkv, gate, q_index proj, qkv, gate, k_dst, v_dst, index_scratch = _mtp_attention_in( - core, M, mixed, pos3, pos, with_index, mm=_decode_projection + core, M, mixed, pos3, pos, with_index, mm=mm ) q_index = None if with_index: @@ -6192,6 +6196,11 @@ def mtp_forward( """One draft step over generation rows only (`p` rows a request on the first step of a batch without context, one otherwise), inside the decode graph when the trunk is. + The steps after the first of a step with context rows (`core._context_step`) run + eagerly at any batch size, so their projections take `cublas_mm`: `_decode_projection`'s + FlashInfer kernels compile on the first call at each shape, and only captured batch + sizes compile before serving. + Under IndexShare the draft-extend step captures each request's selection and later steps skip the index branch, rebuilding it with `qsa_mtp_reuse_topk`; the index keys they skip lie past the committed length and the next draft-extend step rewrites them.""" @@ -6229,10 +6238,9 @@ def mtp_forward( logical = md.qsa_logical_positions[:rows] block_table = md.qsa_block_table tokens_per_block = md.tokens_per_block + mm = cublas_mm if core._context_step else _decode_projection - x, mixed, inject = _mtp_front( - core, M, embed_tokens, input_ids, hidden_states, mm=_decode_projection - ) + x, mixed, inject = _mtp_front(core, M, embed_tokens, input_ids, hidden_states, mm=mm) qkv, gate, q_index = _mtp_attention_planes( core, M, @@ -6242,6 +6250,7 @@ def mtp_forward( p, not reuse, sparse, + mm, ) # Rows whose outputs are read: each request's continuing row on draft-extend, else all. cont, pre = None, {} @@ -6359,9 +6368,7 @@ def sel(t, name=None): ) fused_sigmoid_mul_inplace(attn, gate, direct_launch=True) attn = sel(attn) - out = _mtp_tail( - core, M, sel(x, "x"), attn, sel(inject, "inject"), pad_rows, mm=_decode_projection - ) + out = _mtp_tail(core, M, sel(x, "x"), attn, sel(inject, "inject"), pad_rows, mm=mm) if capture is not None: torch.cuda.current_stream().wait_stream(capture) if cont is None: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 2fa1ba6699ae..673d4d2478a5 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1979,7 +1979,9 @@ def counted(*args, **kwargs): position_cache=position, ) vectors = (qsa[0], qsa[1], qsa[6], qsa[7], qsa[8], qsa[9], qsa[10]) - qkv, gate, iq = M._mtp_attention_planes(core, L, x, draft, vectors, p, True, True) + qkv, gate, iq = M._mtp_attention_planes( + core, L, x, draft, vectors, p, True, True, M._decode_projection + ) torch.cuda.synchronize() if site == "trunk": got = dict(out=out, qkv=qkv, iq=M._index_query(core, scratch).clone()) From 739b8736a5cc7230fd8dff165c13add89967a1a7 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:47:53 -0800 Subject: [PATCH 27/51] [None][fix] modeling_v2 qwen4_exp: agree on the copy-engine exchange's IPC setup before opening Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../ops/comm/ce_exchange__sp_pair.py | 101 ++++++++++++++---- .../comm/_ce_exchange__sp_pair_op_matrix.py | 26 ++++- 2 files changed, 107 insertions(+), 20 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py index 799e4460dad1..1b545b7bf590 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py @@ -21,6 +21,8 @@ from __future__ import annotations +import ctypes +import sys from typing import Sequence import torch @@ -30,11 +32,69 @@ #: Bytes kept for the two flags after the data region (one 128-byte line each). _FLAG_BYTES = 256 +#: The buffer's size is a multiple of this (2 MiB), as for the stock IPC buffers. +_ALIGN = 1 << 21 +#: Bytes of a `cudaIpcMemHandle_t` (CUDA_IPC_HANDLE_SIZE). +_HANDLE_BYTES = 64 HIDDEN = 2560 #: Words of one QSA selection row: the target's `token_topk + compress_ratio - 1`. _SELECTION_WIDTH = 2051 +def _checked(runtime, result): + """The value of a `cuda.bindings.runtime` call's `(err, value)` result; raises RuntimeError + (and clears the error) when `err` is not success.""" + err = result[0] + if err != runtime.cudaError_t.cudaSuccess: + runtime.cudaGetLastError() + raise RuntimeError(f"CUDA runtime error: {err!r}") + return result[1] if len(result) > 1 else None + + +def _release(runtime, local: int, peer: int) -> None: + """Close the peer's mapping and free this rank's buffer (either may be 0).""" + if peer: + runtime.cudaIpcCloseMemHandle(peer) + if local: + runtime.cudaFree(local) + + +def _open_pair(runtime, rank: int, nbytes: int, allgather) -> tuple: + """`(local, peer)` pointers of the pair's buffers, or None on both ranks. Each rank allocates + and exports its zeroed buffer, `allgather((ok, handle))` gives every rank both outcomes, a rank + opens the peer's handle only when both exports succeeded, and `allgather(opened)` makes a + failed open on either rank fail both. `allgather` is the pair's object allgather (rank order); + every rank calls it exactly twice. Whatever a rank allocated or opened is released when it + returns None.""" + size = -(-nbytes // _ALIGN) * _ALIGN + local = peer = 0 + handle = b"" + try: + local = _checked(runtime, runtime.cudaMalloc(size)) + _checked(runtime, runtime.cudaMemset(local, 0, size)) + # The peer writes the flags after the agreement below; the zeroing must land first. + _checked(runtime, runtime.cudaDeviceSynchronize()) + exported = _checked(runtime, runtime.cudaIpcGetMemHandle(local)) + handle = ctypes.string_at(exported.getPtr(), _HANDLE_BYTES) + except RuntimeError as error: + logger.warning(f"copy-engine exchange: no IPC buffer ({error})") + peer_ok, peer_handle = allgather((bool(handle), handle))[1 - rank] + if handle and peer_ok: + try: + imported = runtime.cudaIpcMemHandle_t() + ctypes.memmove(imported.getPtr(), peer_handle, _HANDLE_BYTES) + peer = _checked( + runtime, + runtime.cudaIpcOpenMemHandle(imported, runtime.cudaIpcMemLazyEnablePeerAccess), + ) + except RuntimeError as error: + logger.warning(f"copy-engine exchange: the peer's IPC buffer did not open ({error})") + if all(allgather(bool(peer))): + return local, peer + _release(runtime, local, peer) + return None + + class _CudaArray: """`__cuda_array_interface__` over a raw device allocation (uint8), for `torch.as_tensor`.""" @@ -52,42 +112,45 @@ class CopyEngineExchange: """The pair's buffers and flags: `exchange(x)` -> the peer's `x` (`ce_exchange`, a view of this rank's receive buffer, same shape and dtype). `capacity` bytes a call at most. Built on the attention-TP group of a two-rank `mapping` (IPC memory inside a node); `create` returns - None where that does not hold.""" + None where that does not hold. Owns this rank's buffer and its mapping of the peer's, and + releases both when collected.""" - def __init__(self, mapping, capacity: int): + def __init__(self, local: int, peer: int, capacity: int): from cuda.bindings import driver, runtime - from tensorrt_llm._ipc_utils import IpcMemory - - assert mapping.tp_size == 2 self._driver, self._runtime = driver, runtime self.capacity = capacity - self._ipc = IpcMemory(mapping, capacity + _FLAG_BYTES, open_ipc=True) - rank = mapping.tp_rank - local, peer = self._ipc.peer_ptrs[rank], self._ipc.peer_ptrs[1 - rank] - assert local and peer, "IPC memory did not open" - self._peer = peer + self._local, self._peer = local, peer self._my_ready, self._my_data = local + capacity, local + capacity + 128 self._peer_ready, self._peer_data = peer + capacity, peer + capacity + 128 self._recv = torch.as_tensor(_CudaArray(local, capacity), device=torch.device("cuda")) + def __del__(self): + if not sys.is_finalizing() and self._local: + _release(self._runtime, self._local, self._peer) + self._local = self._peer = 0 + @classmethod def create(cls, mapping, capacity: int): """The exchange over `mapping`'s attention-TP pair, or None (not two ranks, ranks on - different nodes, or IPC memory unavailable on either rank). A collective of the pair: - both ranks build it or neither.""" + different nodes, or IPC memory unavailable on either rank: `_open_pair`). A collective + of the pair: both ranks build it or neither, and both make the same collective calls.""" if mapping.tp_size != 2 or mapping.tp_size > mapping.gpus_per_node: return None + from cuda.bindings import runtime + from tensorrt_llm._torch.distributed.communicator import Distributed - try: - exchange = cls(mapping, capacity) - except (RuntimeError, OSError, MemoryError) as error: - logger.warning(f"copy-engine exchange unavailable ({error}); the pair runs NCCL") - exchange = None - if not all(Distributed.get(mapping).tp_allgather(exchange is not None)): + ptrs = _open_pair( + runtime, + mapping.tp_rank, + capacity + _FLAG_BYTES, + Distributed.get(mapping).tp_allgather, + ) + if ptrs is None: + logger.warning("copy-engine exchange unavailable on the pair; the pair runs NCCL") return None - return exchange + return cls(*ptrs, capacity) def exchange(self, x: torch.Tensor) -> torch.Tensor: """`ce_exchange(self, x)`.""" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py index 459ca994f84f..cd818269f696 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Certification matrix for the FP8 target's `ce_exchange__sp_pair` copy-engine exchange at the -tep2 / tp2 pair under mpirun: eager exchanges and graph replays with new inputs, byte-exact. +tep2 / tp2 pair under mpirun: setup failing on one rank yields None on both, then eager exchanges +and graph replays with new inputs, byte-exact. Run as `CUDA_VISIBLE_DEVICES=0,1 python _ce_exchange__sp_pair_op_matrix.py --world-size 2`.""" import argparse @@ -44,6 +45,29 @@ def fill(t, r, it): else: t.copy_(torch.randn(t.shape, generator=g, device=dev).to(t.dtype)) + # check_failed_setup: one rank's allocation, then one rank's open of the peer's handle, + # fails; both ranks get None (no hang), and each frees what it allocated. + from cuda.bindings import runtime + + def fail(*_): + return (runtime.cudaError_t.cudaErrorMemoryAllocation, 0) + + for name, failing in (("cudaMalloc", 1), ("cudaIpcOpenMemHandle", 0)): + torch.cuda.synchronize() + free_before = torch.cuda.mem_get_info()[0] + real = getattr(runtime, name) + if rank == failing: + setattr(runtime, name, fail) + try: + got = ce_mod.CopyEngineExchange.create(mapping, 512 << 20) + finally: + setattr(runtime, name, real) + assert got is None, f"rank {rank}: built an exchange although {name} failed on {failing}" + torch.cuda.synchronize() + leaked = free_before - torch.cuda.mem_get_info()[0] + assert leaked < 64 << 20, f"rank {rank}: {leaked} bytes kept after {name} failed" + comm.Barrier() + # check_create: the buffer holds the largest cell, 8192 int32 rows of 2051. capacity = 72 << 20 ce = ce_mod.CopyEngineExchange.create(mapping, capacity) From 6e5c42572ef81497f356a842b6b3f21d2ee612de Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:47:54 -0800 Subject: [PATCH 28/51] [None][fix] modeling_v2 qwen4_exp: refuse a foreign-uid PLE broker owner and bound the refused-connect retry Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../models/qwen4_exp/chunk_broker.py | 32 +++++++++----- .../test_weights.py | 43 +++++++++++++++++++ 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py index e081217ac82f..187cbf4d4e42 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py @@ -4,10 +4,13 @@ A segment is named by a key. The first process to bind the key's abstract Unix socket (`\\0trtllm-ple--`) owns it: it creates and fills the allocation, then serves the -allocation's file descriptor to every process that connects (`SCM_RIGHTS`; a peer of another -uid is refused through `SO_PEERCRED`). A process that cannot bind connects and waits. When the -owner exits or aborts before publishing, the kernel closes its sockets and the waiter tries to -own the segment itself. Nothing is written to the filesystem, so a crash leaves nothing behind. +allocation's file descriptor to every process that connects (`SCM_RIGHTS`). Abstract names carry +no permissions, so both sides check the other's uid through `SO_PEERCRED`: the owner refuses a +peer of another uid, and a waiter whose name is held by another uid's socket gets +`PermissionError` at once. A process that cannot bind connects and waits, at most `timeout_s` +(also when the name is bound but nothing listens on it). When the owner exits or aborts before +publishing, the kernel closes its sockets and the waiter tries to own the segment itself. +Nothing is written to the filesystem, so a crash leaves nothing behind. Abstract socket names belong to a network namespace: processes in different namespaces do not meet, and each keeps its own copy. @@ -36,6 +39,12 @@ def socket_name(key: str) -> str: return f"\0trtllm-ple-{os.getuid()}-{key}" +def _peer_uid(conn: socket.socket) -> int: + """The uid of the process at the other end of the connected Unix socket `conn`.""" + _, uid, _ = _CRED.unpack(conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, _CRED.size)) + return uid + + class OwnedSegment: """The owner's side: serves `publish`ed `(fd, meta)` to every same-uid peer, until `close`.""" @@ -83,10 +92,7 @@ def _serve(self) -> None: def _answer(self, conn: socket.socket) -> None: with conn: try: - _, uid, _ = _CRED.unpack( - conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, _CRED.size) - ) - if uid != os.getuid(): + if _peer_uid(conn) != os.getuid(): return while not self._ready.wait(_POLL_SECONDS): if self._closed.is_set(): @@ -117,7 +123,8 @@ def acquire( ) -> tuple: """`(OWNER, OwnedSegment)` when no process owns `key`, else `(ATTACHED, fd, meta)` once the owner publishes; an owner that exits or aborts first hands `key` to its waiters. - Raises `TimeoutError` when an owner has not published within `timeout_s`.""" + Raises `TimeoutError` when no owner has published within `timeout_s`, and `PermissionError` + when a process of another uid holds the name.""" name = socket_name(key) deadline = time.monotonic() + timeout_s next_report = time.monotonic() + report_s @@ -129,12 +136,17 @@ def acquire( try: conn.connect(name) except (ConnectionRefusedError, FileNotFoundError): - # The owner just exited, or is between `bind` and `listen`. + # The owner just exited, or is between `bind` and `listen` (or never listens). conn.close() + if time.monotonic() >= deadline: + raise TimeoutError(f"no owner of {key!r} listens within {timeout_s:.0f} s") time.sleep(0.05) continue got: Optional[tuple] = None try: + owner_uid = _peer_uid(conn) + if owner_uid != os.getuid(): + raise PermissionError(f"the owner of {key!r} runs as uid {owner_uid}") while got is None: remaining = deadline - time.monotonic() if remaining <= 0: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py index 3969c1173f6c..4896503c3b86 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -615,6 +615,49 @@ def start(die_on_go=False): assert not [f for f in os.listdir("/dev/shm") if key in f] +@pytest.mark.cpu_only +def test_chunk_broker_refuses_a_foreign_owner(monkeypatch): + """A waiter whose key's name is held by a socket of another uid raises `PermissionError` (an + `OSError`, so `node` falls back) at once instead of waiting for it to publish.""" + import os + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + owner = broker.try_own(key) + assert owner is not None + monkeypatch.setattr(broker, "_peer_uid", lambda conn: os.getuid() + 1) + try: + start = time.monotonic() + with pytest.raises(PermissionError): + broker.acquire(key, 60) + assert time.monotonic() - start < 5 + finally: + owner.close() + + +@pytest.mark.cpu_only +def test_chunk_broker_times_out_on_a_name_nobody_listens_on(): + """A key's name bound by a socket that never listens refuses every connection; the waiter + raises `TimeoutError` at its deadline.""" + import socket + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + squatter = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + squatter.bind(broker.socket_name(key)) + try: + start = time.monotonic() + with pytest.raises(TimeoutError): + broker.acquire(key, 2.0) + assert 2.0 <= time.monotonic() - start < 10 + finally: + squatter.close() + + def _fake_shards(rows_per_shard: int, shards: int, cols: int, seed: int) -> tuple: g = torch.Generator().manual_seed(seed) data = torch.randint(0, 256, (shards * rows_per_shard, cols), generator=g, dtype=torch.uint8) From 066001d715e5307c114f37d16325abb87e267af2 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:49:08 -0800 Subject: [PATCH 29/51] [None][fix] modeling_v2 qwen4_exp: decline a sliding window at an inferred max_seq_len With max_seq_len unset, check_deployment takes the text config's max_position_embeddings as the full window (what the engine infers after the build), so a window below it is declined at routing instead of failing the bind assert. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 8 ++++---- .../test_routing_and_layout.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 5b735850cf57..ad719da74c38 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -6892,11 +6892,11 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: "pads every rank to one prefill bucket, so a rank holding only generation " "requests is handed more rows than its batch, which the decode step refuses" ) + # Unset, max_seq_len is inferred after the build from the text tower's + # max_position_embeddings (the pinned rope declares no scaling factor). windows = llm_args.kv_cache_config.max_attention_window - full = llm_args.max_seq_len or model_config.max_seq_len - if windows is not None and ( - any(w < full for w in windows) if full else len(set(windows)) > 1 - ): + full = llm_args.max_seq_len or model_config.max_seq_len or text.max_position_embeddings + if windows is not None and any(w < full for w in windows): raise NoTarget( f"kv_cache_config.max_attention_window={windows}; the target implements no " "sliding window: every attention layer's window is max_seq_len" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index 83a6d5f38989..6d4c825ae9ed 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -144,6 +144,7 @@ def _qwen4_exp_config(text_overrides=None, composite_overrides=None, **vision_ov rope_theta=10000000, partial_rotary_factor=0.25, rope_scaling=dict(type="mrope", mrope_section=[11, 11, 10], mrope_interleaved=True), + max_position_embeddings=262144, mtp=dict(_MTP_BLOCK), mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=False, @@ -240,6 +241,8 @@ def test_each_member_is_admitted_as_the_llm_api_spells_it(on_sm103, member, para dict(kv_cache_config=KvCacheConfig(dtype="nvfp4")), ), ("tp4", None, dict(kv_cache_config=KvCacheConfig(mamba_ssm_cache_dtype="bfloat16"))), + # A window of the full length, max_seq_len unset (inferred from max_position_embeddings). + ("tep4", None, dict(kv_cache_config=KvCacheConfig(max_attention_window=[262144]))), ], ids=[ "tp1-mtp1", @@ -247,6 +250,7 @@ def test_each_member_is_admitted_as_the_llm_api_spells_it(on_sm103, member, para "tep4-mtp3-fp8-kv", "dep4-mtp3-nvfp4-kv", "tp4-bf16-state", + "tep4-full-window", ], ) def test_the_engine_choices_the_target_runs_are_admitted( @@ -336,6 +340,11 @@ def test_the_engine_choices_the_target_runs_are_admitted( dict(max_seq_len=8192, kv_cache_config=KvCacheConfig(max_attention_window=[4096])), "sliding window", ), + ( + dict(member="tep4"), + dict(kv_cache_config=KvCacheConfig(max_attention_window=[32768])), + "sliding window", + ), ( dict(member="tep4"), dict(moe_config=MoeConfig(load_balancer={"num_slots": 512})), @@ -367,6 +376,7 @@ def test_the_engine_choices_the_target_runs_are_admitted( "other-rope-theta", "other-mtp-layer", "sliding-window", + "sliding-window-inferred-max-seq-len", "eplb", "vision-gpu-normalize", ], From 055596b83d388ca036dc6cb0a6dc87a68a07dcf8 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:49:13 -0800 Subject: [PATCH 30/51] [None][fix] modeling_v2 qwen4_exp: score IndexShare draft context rows in chunks PrefillTarget.mtp_forward with IndexShare scored every context row in one call; at max_seq_len 262144 past 32752 rows that passes the int32 score-table range. It now runs the trunk prefill's chunked _score_and_select, whose chunks are equal so the last one stays past the index scores' decode-row limit. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 136 ++++++++++++------ .../test_attention_ops.py | 64 +++++++++ 2 files changed, 155 insertions(+), 45 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index ad719da74c38..a67d692d0b4d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -3495,6 +3495,68 @@ def _selection( ) +def _score_chunk_rows(core, block_table, tokens_per_block: int) -> int: + """Rows one `_score_and_select` call scores: fewer than `_SCORE_TABLE_ELEMENTS / columns`, + in whole query tiles.""" + columns = block_table.shape[1] * tokens_per_block // core.compress_ratio + chunk = max(1, (_SCORE_TABLE_ELEMENTS - 1) // columns) + if chunk >= _SELECTION_CUT_ALIGN: + chunk -= chunk % _SELECTION_CUT_ALIGN + return chunk + + +def _score_and_select( + core, + index_cache, + q_index, + block_table, + logical, + req_idx, + tokens_per_block: int, + visible_blocks, + seq_lens_q, + topk_indices, + row_starts, + lo: int, + hi: int, + context_rows: bool, + out, +) -> None: + """Score rows `[lo, hi)` against `index_cache` and select them into `out` (`_selection`), + in equal chunks of at most `_score_chunk_rows` (whole query tiles): a row's scores and + selection read only its query, position and request, so the chunks select what one call + would; below the bound the rows are one call. Equal chunks keep the last one past the + index scores' decode-row limit, whose launch rounds the scores differently.""" + bound = _score_chunk_rows(core, block_table, tokens_per_block) + pieces = max(1, -(-(hi - lo) // bound)) + chunk = -(-(hi - lo) // pieces) + if bound >= _SELECTION_CUT_ALIGN: + chunk = -(-chunk // _SELECTION_CUT_ALIGN) * _SELECTION_CUT_ALIGN + for a in range(lo, hi, max(1, chunk)): + b = min(a + chunk, hi) + scores = _context_index_scores( + q_index[a:b], + index_cache, + block_table, + logical[a:b], + req_idx[a:b], + tokens_per_block, + core.compress_ratio, + context_rows, + ) + _selection( + core, + scores, + visible_blocks[a:b], + logical[a:b], + seq_lens_q[a:b], + topk_indices[a:b], + row_starts[a:b], + out[a - lo : b - lo], + context=context_rows, + ) + + def _prefill_split_enabled(attn_tp_size: int, dp_size: int) -> bool: """Whether a prefill step splits its selection and row-wise work over the attention-TP group.""" return dp_size == 1 and attn_tp_size in _PREFILL_SPLIT_DEFAULT_TP @@ -4456,38 +4518,24 @@ def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: device=proj.device, ) - # A call scores fewer than `_SCORE_TABLE_ELEMENTS / columns` rows (whole query - # tiles): a row's scores and selection read only its query, position and request, - # so the chunks select what one call would; below the bound a part is one call. - columns = block_table.shape[1] * tokens_per_block // core.compress_ratio - chunk = max(1, (_SCORE_TABLE_ELEMENTS - 1) // columns) - if chunk >= _SELECTION_CUT_ALIGN: - chunk -= chunk % _SELECTION_CUT_ALIGN - def score_and_select(lo, hi, context_rows, out): - for a in range(lo, hi, chunk): - b = min(a + chunk, hi) - scores = _context_index_scores( - q_index[a:b], - core._bound.pools.index_cache[i], - block_table, - logical[a:b], - req_idx[a:b], - tokens_per_block, - core.compress_ratio, - context_rows, - ) - _selection( - core, - scores, - visible_blocks[a:b], - logical[a:b], - seq_lens_q[a:b], - topk_indices[a:b], - topk_row_starts[a:b], - out[a - lo : b - lo], - context=context_rows, - ) + _score_and_select( + core, + core._bound.pools.index_cache[i], + q_index, + block_table, + logical, + req_idx, + tokens_per_block, + visible_blocks, + seq_lens_q, + topk_indices, + topk_row_starts, + lo, + hi, + context_rows, + out, + ) for lo, hi, context_rows in parts: if context_rows and self._split_context_rows(hi - lo): @@ -5572,25 +5620,23 @@ def mtp_forward( dtype=torch.int32, device=qkv.device, ) + # Every context row is scored, so they go in the trunk prefill's chunks. for lo, hi, context_rows in parts: - scores = _context_index_scores( - q_index[lo:hi], + _score_and_select( + core, draft.index_cache, + q_index, block_table, - logical[lo:hi], - req_idx[lo:hi], + logical, + req_idx, tokens_per_block, - core.compress_ratio, + visible_blocks, + seq_lens_q, + topk_indices, + row_starts, + lo, + hi, context_rows, - ) - _selection( - core, - scores, - visible_blocks[lo:hi], - logical[lo:hi], - seq_lens_q[lo:hi], - topk_indices[lo:hi], - row_starts[lo:hi], selected[lo:hi], ) if share: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 673d4d2478a5..140af3726a88 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -2105,6 +2105,60 @@ def md(q_lens, kv_lens): assert all(prev[1] == nxt[0] for prev, nxt in zip(groups, groups[1:])) +def _site_draft_share_chunks(case, monkeypatch) -> None: + """The IndexShare draft-extend step's context selection (`_score_and_select`, as + `mtp_forward` calls it) in equal chunks of a lowered score-table bound, `case` = + (((prefix, rows), ...), bound, chunk rows): bitwise the one-call scores and `_selection` + the step ran unchunked, block lists and token selection alike.""" + requests, bound, want = case + tpb, cr, token_topk = 32, 4, TKC_TOKEN_TOPK + need = [-(-(p + r) // tpb) for p, r in requests] + rows = sum(r for _, r in requests) + gen, cache, table, slot_of = _index_pool(len(requests), need, rows, 12000, 64) + req = torch.cat( + [torch.full((r,), int(slot_of[i]), dtype=torch.int32) for i, (_, r) in enumerate(requests)] + ).to(DEV) + logical = torch.cat([torch.arange(p, p + r, dtype=torch.int64) for p, r in requests]).to(DEV) + seq_lens = torch.cat([torch.full((r,), p + r, dtype=torch.int32) for p, r in requests]).to(DEV) + visible = ((logical + 1) // cr).to(torch.int32) + row_starts = torch.zeros(rows, dtype=torch.int32, device=DEV) + q = torch.randn(rows, 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + core = SimpleNamespace(compress_ratio=cr, token_topk=token_topk) + columns = table.shape[1] * tpb // cr + # At the engine's bound, `max_seq_len` 262144 (8196 pages of 32) scores 32704 rows a call. + assert M._score_chunk_rows(core, table, tpb) == 32704 + + def topk(): + return torch.full((rows, token_topk // cr), -7, dtype=torch.int32, device=DEV) + + def selected(): + return torch.full((rows, token_topk + cr - 1), -7, dtype=torch.int32, device=DEV) + + one_topk, one = topk(), selected() + scores = M._context_index_scores(q, cache, table, logical, req, tpb, cr, True) + M._selection(core, scores, visible, logical, seq_lens, one_topk, row_starts, one) + del scores + + monkeypatch.setattr(M, "_SCORE_TABLE_ELEMENTS", bound * columns + 1) + assert M._score_chunk_rows(core, table, tpb) == bound + calls = [] + scorer = M._context_index_scores + + def counted(q_rows, *args): + calls.append(q_rows.shape[0]) + return scorer(q_rows, *args) + + monkeypatch.setattr(M, "_context_index_scores", counted) + chunked_topk, chunked = topk(), selected() + M._score_and_select( + core, cache, q, table, logical, req, tpb, visible, seq_lens, chunked_topk, row_starts, + 0, rows, True, chunked, + ) # fmt: skip + assert calls == list(want) + assert torch.equal(chunked_topk, one_topk), "block lists" + assert torch.equal(chunked, one), "token selection" + + _SITE_CASES = [ *( pytest.param(_site_prologue, (site, layout), id=f"{site} prologue {layout}") @@ -2116,6 +2170,16 @@ def md(q_lens, kv_lens): _site_rowtile_q_in_place, [(3000, 1040), (7000, 1043)], id="rowtile q in place, lone group" ), pytest.param(_site_rowtile_scratch_groups, None, id="rowtile fp8 scratch groups"), + pytest.param( + _site_draft_share_chunks, + (((88064, 900), (40000, 600), (3, 300)), 512, (512, 512, 512, 264)), + id="draft index share chunks, three requests", + ), + pytest.param( + _site_draft_share_chunks, + (((250000, 1100),), 512, (384, 384, 332)), + id="draft index share chunks, 250k", + ), ] From 3936b67d7455000d5c3d3c44a5b62dc03a4a1cd5 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:07:12 -0800 Subject: [PATCH 31/51] [None][perf] modeling_v2 qwen4_exp: order a mixed step's generation rows for the wide decode selection Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 14 +++++-- .../test_attention_ops.py | 37 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index a67d692d0b4d..efa5638053be 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -3521,12 +3521,14 @@ def _score_and_select( hi: int, context_rows: bool, out, + order=None, ) -> None: """Score rows `[lo, hi)` against `index_cache` and select them into `out` (`_selection`), in equal chunks of at most `_score_chunk_rows` (whole query tiles): a row's scores and selection read only its query, position and request, so the chunks select what one call would; below the bound the rows are one call. Equal chunks keep the last one past the - index scores' decode-row limit, whose launch rounds the scores differently.""" + index scores' decode-row limit, whose launch rounds the scores differently. `order`: the + step's `_RowOrder` over rows `[lo, hi)`, read when they are one call.""" bound = _score_chunk_rows(core, block_table, tokens_per_block) pieces = max(1, -(-(hi - lo) // bound)) chunk = -(-(hi - lo) // pieces) @@ -3554,6 +3556,7 @@ def _score_and_select( row_starts[a:b], out[a - lo : b - lo], context=context_rows, + order=order if pieces == 1 else None, ) @@ -4106,6 +4109,8 @@ class _PrefillStep(NamedTuple): qsa_rows: tuple | None = None # `_selection_cuts`, or None for equal row counts. sel_cuts: tuple | None = None + # The generation rows' `_RowOrder`, shared by the step's sparse layers. + gen_order: _RowOrder | None = None class PrefillTarget: @@ -4518,7 +4523,7 @@ def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: device=proj.device, ) - def score_and_select(lo, hi, context_rows, out): + def score_and_select(lo, hi, context_rows, out, order=None): _score_and_select( core, core._bound.pools.index_cache[i], @@ -4535,6 +4540,7 @@ def score_and_select(lo, hi, context_rows, out): hi, context_rows, out, + order, ) for lo, hi, context_rows in parts: @@ -4549,7 +4555,8 @@ def score_and_select(lo, hi, context_rows, out): step.sel_cuts if lo == 0 and hi == num_ctx else None, ) else: - score_and_select(lo, hi, context_rows, selected[lo:hi]) + order = None if context_rows else step.gen_order + score_and_select(lo, hi, context_rows, selected[lo:hi], order) # The row-tile pass reads q in place; the other kernels need a compact copy. launches = [] for lo, hi, context_rows in parts: @@ -5221,6 +5228,7 @@ def prepare( split=self._split_of(mm, num_prefills, num_decodes, hi_p, num_tokens), qsa_rows=qsa_rows, sel_cuts=sel_cuts, + gen_order=_RowOrder(), ) def _qsa_padded_rows(self, md, coords, num_tokens: int, physical_tokens: int) -> tuple: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 140af3726a88..b49079d4e128 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -2159,6 +2159,42 @@ def counted(q_rows, *args): assert torch.equal(chunked, one), "token selection" +def _site_gen_rows_order(rows, monkeypatch) -> None: + """A mixed step's generation rows through `_score_and_select` with the step's `_RowOrder`: + past one wave of rows on the wide (65568-column) table the selection launches in that + order, bitwise the unordered launch, block lists and token selection alike.""" + tpb, cr, token_topk = 32, 4, TKC_TOKEN_TOPK + gen, cache, table, slot_of = _index_pool(1, [2900], rows, 3000, rows) + table[:] = table[int(slot_of[0])] # every request reads the same pages + req = torch.arange(rows, dtype=torch.int32, device=DEV) + logical = torch.randint(1, 2900 * tpb - 1, (rows,), generator=gen, device=DEV) + seq_lens = (logical + 1).to(torch.int32) + visible = ((logical + 1) // cr).to(torch.int32) + row_starts = torch.zeros(rows, dtype=torch.int32, device=DEV) + q = torch.randn(rows, 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + core = SimpleNamespace(compress_ratio=cr, token_topk=token_topk) + seen = [] + op = M.qsa_topk_expand_decode + + def spy(*args, row_order=None, **kwargs): + seen.append(row_order) + return op(*args, row_order=row_order, **kwargs) + + monkeypatch.setattr(M, "qsa_topk_expand_decode", spy) + outs = [] + for order in (None, M._RowOrder()): + topk = torch.full((rows, token_topk // cr), -7, dtype=torch.int32, device=DEV) + sel = torch.full((rows, token_topk + cr - 1), -7, dtype=torch.int32, device=DEV) + M._score_and_select( + core, cache, q, table, logical, req, tpb, visible, seq_lens, topk, row_starts, + 0, rows, False, sel, order, + ) # fmt: skip + outs.append((topk, sel)) + assert seen[0] is None and seen[1] is not None, "only the ordered call passes a row order" + assert torch.equal(outs[0][0], outs[1][0]), "block lists" + assert torch.equal(outs[0][1], outs[1][1]), "token selection" + + _SITE_CASES = [ *( pytest.param(_site_prologue, (site, layout), id=f"{site} prologue {layout}") @@ -2180,6 +2216,7 @@ def counted(q_rows, *args): (((250000, 1100),), 512, (384, 384, 332)), id="draft index share chunks, 250k", ), + pytest.param(_site_gen_rows_order, 300, id="generation rows in the step's row order"), ] From 35d3bac07407763aec7a2b9a8c0e9cd601b7a190 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:13:51 -0800 Subject: [PATCH 32/51] [None][fix] modeling_v2 qwen4_exp: run the vision tower once per frame across a request's chunks A chunk reran the tower over the whole image (one frame) whose rows it held, so an 8192-row image under 2048-token chunked prefill ran the 27-block tower in each of its chunks. Keep the request's last tower call on its multimodal_data until the chunk with the call's last row reads it; later chunks run the tower only on frames no earlier call produced. Every tower block attends over the whole frame, so a frame cannot be split by rows. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../vision.py | 79 ++++++++++++----- .../test_vision.py | 87 ++++++++++++++++--- 2 files changed, 132 insertions(+), 34 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py index 1bec3a9f33b5..855eb6d54fbb 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -149,6 +149,12 @@ #: The position-ids axes of the mRoPE layout a media request carries. _POSITION_AXES = 3 +#: The `multimodal_data` key of a chunked request's last tower call that later chunks still read: +#: `(first placeholder row, rows)`. The executor hands every chunk the request's own +#: `py_multimodal_data` dict, so the entry lives on the request; the chunk that consumes its last +#: row drops it, and the executor's post-prefill and termination release clears the dict. +_TOWER_ROWS_KEY = "qwen4_exp_tower_rows" + class _VisionBlockWeights(NamedTuple): """One vision block's weights (`.t()` GEMM views); its LayerNorms have biases.""" @@ -423,7 +429,8 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): The shell calls it: the inherited `DecoderModelForCausalLM.forward` drops `multimodal_params`, which covers a subset of the context requests (never zip it positionally). `multimodal_runtime` counts rows over a request's media together, in prompt order; a chunk - runs the tower on only the frames that hold its rows (`_tower_rows`).""" + runs the tower on only the frames that hold its rows (`_tower_rows`), once per frame across + the request's chunks (`_chunk_rows`).""" mm_params = kwargs.get("multimodal_params") or [] if not mm_params: return None @@ -447,18 +454,19 @@ def multimodal_embeds(tower: VisionTower | None, kwargs: dict): items = _items_in_prompt_order(params, data) merge_unit = tower.merge * tower.merge total = sum(t * h * w for _, _, (t, h, w) in items) // merge_unit - start, count = 0, total - if runtime is not None: - # Chunked prefill or reused prefix: only this chunk's slice. - start = int(runtime.num_cached_mm_tokens) - count = int(runtime.num_mm_tokens_in_chunk) - assert int(runtime.total_embeds_in_request) == total, ( - f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " - f"this request and its grids describe {total}; the two are derived from the " - "same grids and a disagreement means the merger's row count is wrong" - ) - assert 0 <= start <= start + count <= total, (start, count, total) - rows.append(_tower_rows(tower, items, start, count)) + if runtime is None: + rows.append(_tower_rows(tower, items, 0, total)[1]) + continue + # Chunked prefill or reused prefix: only this chunk's slice. + start = int(runtime.num_cached_mm_tokens) + count = int(runtime.num_mm_tokens_in_chunk) + assert int(runtime.total_embeds_in_request) == total, ( + f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " + f"this request and its grids describe {total}; the two are derived from the " + "same grids and a disagreement means the merger's row count is wrong" + ) + assert 0 <= start <= start + count <= total, (start, count, total) + rows.append(_chunk_rows(tower, data, items, start, count)) if not rows: return None vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) @@ -523,13 +531,16 @@ def _items_in_prompt_order(params, data: dict) -> list[tuple[torch.Tensor, int, ] -def _tower_rows(tower: VisionTower, items: list, start: int, count: int) -> torch.Tensor: - """Placeholder rows `[start, start + count)` of one request's `items` (prompt order). +def _tower_rows( + tower: VisionTower, items: list, start: int, count: int +) -> tuple[int, torch.Tensor]: + """`(first row, rows)`: one tower call over the frames of one request's `items` (prompt + order) that hold its placeholder rows `[start, start + count)`, from the first of those + frames' first row on. - One tower call over only the frames that hold them: a frame's rows depend on its own patches - alone (attention is per frame, every other op per row), so a chunk of a long video does not - rerun the request's other frames. A frame a chunk boundary cuts runs in both chunks. Adjacent - frames of one pixel tensor stay one view; items of two modalities are copied into one tensor.""" + A frame's rows depend on its own patches alone (attention is per frame, every other op per + row), so a chunk of a long video does not run the request's other frames. Adjacent frames of + one pixel tensor stay one view; items of two modalities are copied into one tensor.""" merge_unit = tower.merge * tower.merge end = start + count spans: list[list] = [] # [pixel_values, first patch row, end patch row] @@ -553,8 +564,34 @@ def _tower_rows(tower: VisionTower, items: list, start: int, count: int) -> torc row += frames * frame_rows assert first_row is not None, (start, count, row) parts = [pixel_values[lo:hi] for pixel_values, lo, hi in spans] - out = tower(parts[0] if len(parts) == 1 else torch.cat(parts, 0), torch.tensor(grid)) - return out[start - first_row : end - first_row] + pixels = parts[0] if len(parts) == 1 else torch.cat(parts, 0) + return first_row, tower(pixels, torch.tensor(grid)) + + +def _chunk_rows(tower: VisionTower, data: dict, items: list, start: int, count: int): + """Placeholder rows `[start, start + count)` of one chunk of a request's prefill. + + Rows the request's previous tower call already produced (`_TOWER_ROWS_KEY` in `data`, the + request's `multimodal_data`) are read from it; the tower runs only on the frames holding the + rest. So an image or frame that several chunks cut runs once: an 8192-row image under + 2048-token chunks runs in its first chunk and keeps its `[8192, out_hidden_size]` rows (40 MiB + at 2560 bf16) until the chunk with its last row. The rows are bitwise the one-pass rows.""" + end = start + count + kept = data.pop(_TOWER_ROWS_KEY, None) + parts = [] + if kept is not None: + first, out = kept + if first <= start < first + out.shape[0]: + take = min(end, first + out.shape[0]) + parts.append(out[start - first : take - first]) + start = take + if start < end: + kept = _tower_rows(tower, items, start, end - start) + first, out = kept + parts.append(out[start - first : end - first]) + if kept is not None and end < kept[0] + kept[1].shape[0]: + data[_TOWER_ROWS_KEY] = kept + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) def validate_request(tower, request) -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py index efccb55f4210..dbaf67474efc 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -544,29 +544,71 @@ def _hf_video_in_two_calls(tower, media, references, monkeypatch): _check("video in two calls", out, *references["video"]) +def _counted_runs(tower, monkeypatch) -> list: + """The patch rows of each `_run` call `tower` makes from here on.""" + runs = [] + run = tower._run + + def counted(pixel_values, grid): + runs.append(pixel_values.shape[0]) + return run(pixel_values, grid) + + monkeypatch.setattr(tower, "_run", counted) + return runs + + +def _spliced_in_chunks(tower, data, bounds): + """The rows `multimodal_embeds` splices for each `(start, end)` chunk of one request whose + `multimodal_data` is `data`, the one dict the executor hands all of a request's chunks.""" + total = bounds[-1][1] + chunks = [] + for start, end in bounds: + runtime = SimpleNamespace( + num_cached_mm_tokens=start, + num_mm_tokens_in_chunk=end - start, + total_embeds_in_request=total, + ) + params = MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(end - start)} + chunks.append(V.multimodal_embeds(tower, kwargs)[0]) + assert V._TOWER_ROWS_KEY not in data, "the chunk with the last row drops the kept rows" + return torch.cat(chunks) + + +def _hf_image_in_chunks(tower, media, references, monkeypatch): + """One image spliced over three chunked-prefill chunks: the tower runs once, in the first + chunk, and the later chunks read its kept rows, bitwise the one-pass rows.""" + px, grid = media["image0"] + total = int(grid.prod(-1).sum()) // tower.merge**2 + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)}, + "image": {"pixel_values": px, "image_grid_thw": grid}, + } + with torch.inference_mode(): + whole = tower(px, grid) + runs = _counted_runs(tower, monkeypatch) + out = _spliced_in_chunks(tower, data, ((0, 100), (100, 200), (200, total))) + assert runs == [px.shape[0]], runs + assert torch.equal(out, whole), _mismatch(out, whole) + _check("image in three chunks", out, *references["image0"]) + + def _hf_video_in_chunks(tower, media, references, monkeypatch): """The video spliced over three chunked-prefill chunks whose boundaries cut its two 96-row - frames: each chunk runs only the frames holding its rows, and together they are the one-pass - rows bitwise.""" + frames: each frame runs once, in the first chunk holding its rows, and together the chunks + are the one-pass rows bitwise.""" px, grid = media["video"] total = int(grid.prod(-1).sum()) // tower.merge**2 data = { "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)}, "video": {"pixel_values_videos": px, "video_grid_thw": grid}, } - chunks = [] with torch.inference_mode(): whole = tower(px, grid) - for start, end in ((0, 50), (50, 150), (150, total)): - runtime = SimpleNamespace( - num_cached_mm_tokens=start, - num_mm_tokens_in_chunk=end - start, - total_embeds_in_request=total, - ) - params = MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) - kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(end - start)} - chunks.append(V.multimodal_embeds(tower, kwargs)[0]) - out = torch.cat(chunks) + runs = _counted_runs(tower, monkeypatch) + out = _spliced_in_chunks(tower, data, ((0, 50), (50, 150), (150, total))) + frame = px.shape[0] // 2 + assert runs == [frame, frame], runs assert torch.equal(out, whole), _mismatch(out, whole) _check("video in three chunks", out, *references["video"]) @@ -578,6 +620,7 @@ def _hf_video_in_chunks(tower, media, references, monkeypatch): "image_pair": _hf_pair(), "image_pair_two_calls": _hf_pair(cap=1176), "video_two_calls": _hf_video_in_two_calls, + "image_chunks": _hf_image_in_chunks, "video_chunks": _hf_video_in_chunks, } @@ -832,3 +875,21 @@ def test_the_splice_puts_each_items_rows_on_its_placeholders( assert vision[1] is indices assert h[1 : 1 + num_rows, 0].tolist() == want assert torch.all(h[0] == -1.0) and torch.all(h[total - 1] == -1.0) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("chunk", range(1, len(MIXED_ROWS) + 1)) +def test_a_chunked_request_runs_each_frame_once(chunk): + """Over all of one request's chunks the tower runs each frame once (the one-pass 68 patch + rows) and the chunks splice the one-pass rows; the last chunk drops the kept rows.""" + tower = _Tower() + params = _params(**_MIXED) + spliced = [] + for start in range(0, len(MIXED_ROWS), chunk): + count = min(chunk, len(MIXED_ROWS) - start) + params.multimodal_runtime = _chunk(start, count) + kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(count)} + spliced += V.multimodal_embeds(tower, kwargs)[0][:, 0].tolist() + assert spliced == MIXED_ROWS + assert tower.patches == 68, (tower.calls, tower.patches) + assert V._TOWER_ROWS_KEY not in params.multimodal_data From 19ae5827c37a4e9ab7a4229ca27acce187ac8764 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:17:00 -0800 Subject: [PATCH 33/51] [None][fix] modeling_v2 qwen4_exp: honor the engine's all-reduce strategy and decline idle switches Explicit MNNVL takes the MNNVL TP sum and reaches the raw all-reduce as AUTO (allreduce_group_args maps MNNVL and SYMM_MEM to AUTO, as the AllReduce module does); UB and LOWPRECISION decline under attention TP. The split index gather's Lamport buffers and the copy-engine exchange need AUTO or MNNVL and peer access, like the fused paths. The fused decode TP sum + inject runs only with fp32 all-reduce accumulation. TRTLLM_FORCE_COMM_METHOD is read only under attention DP with dp_size > 1. TRTLLM_MODELING_V2_DECODE_MOE_TPB=rr outside tp4/tep4 and TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS without an FP8 KV cache decline. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling_v2/catalog/comm/allreduce.py | 15 +- .../modeling.py | 110 ++++++++--- .../comm/hc_allreduce_inject_norm__tp_sum.py | 4 + .../weights.py | 7 +- .../test_routing_and_layout.py | 186 +++++++++++++++++- 5 files changed, 280 insertions(+), 42 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py b/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py index 205b11e693ef..8ad58fe4bae2 100644 --- a/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py +++ b/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py @@ -32,6 +32,16 @@ } ) +#: Strategies of the `AllReduce` module's own kernels, which this op has no branch for: the +#: module runs the MNNVL or symmetric-memory kernel where it can and calls this op at AUTO +#: otherwise (`_torch/distributed/ops.py`), and so does a call built by `allreduce_group_args`. +_MODULE_ONLY = frozenset( + { + int(AllReduceStrategy.MNNVL), + int(AllReduceStrategy.SYMM_MEM), + } +) + def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: """The communicator side of the call for this rank's TP group. @@ -41,7 +51,8 @@ def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: The IPC workspace is this op's own fixture: one per mapping, shared with every other user of that mapping in the process, allocated on first use. - It exists only for strategies that read it. + It exists only for strategies that read it. MNNVL and SYMM_MEM reach the + op as AUTO (`_MODULE_ONLY`); running the MNNVL kernel is the caller's part. Mirrors the router in `_torch/distributed/ops.py`: under MPI the plain symbol is called and `process_group` stays None; with MPI disabled the @@ -50,6 +61,8 @@ def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: from tensorrt_llm._torch.distributed.ops import get_allreduce_workspace from tensorrt_llm._utils import mpi_disabled + if strategy in _MODULE_ONLY: + strategy = AUTO workspace = None if strategy in _NO_WORKSPACE else get_allreduce_workspace(mapping) process_group, rank = None, None if mpi_disabled(): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index efa5638053be..7d413fb7b31e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -685,14 +685,16 @@ def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, #: Opt-in switch (`weights.switch`): a context step over an fp8 pool whose pages would widen #: past `_FP8_ROWTILE_MAX_SCRATCH` runs one row-tile launch per group of requests that fits #: (`_rowtile_scratch_groups`) instead of the per-row kernel. Off by default because such steps' -#: outputs then move by rounding. Read once, when the target is built. +#: outputs then move by rounding. Read once, when the target is built; `check_deployment` +#: declines it without an FP8 KV cache, where it does nothing. ROWTILE_SCRATCH_GROUPS_ENV = "TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS" #: The stock MoE communication override (`communication_factory`), compared upper-cased like the -#: stock factory. Under attention DP, `NVLINK_ONE_SIDED` runs the MoE round trip as an all-to-all -#: dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster one; it -#: needs MPI, attention DP and MNNVL. Unset or `ALLGATHER` keeps the all-gather / reduce-scatter -#: round trip. `check_deployment` declines any other value, and the all-to-all where it cannot -#: run. Read once, when the target is built. +#: stock factory and, like it, read only under attention DP with `dp_size > 1`: other layouts +#: have no MoE round trip and ignore it. There `NVLINK_ONE_SIDED` runs the round trip as an +#: all-to-all dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster +#: one; it needs MPI and MNNVL. Unset or `ALLGATHER` keeps the all-gather / reduce-scatter round +#: trip. `check_deployment` declines any other value, and the all-to-all where it cannot run. +#: Read once, when the target is built. MOE_COMM_METHOD_ENV = "TRTLLM_FORCE_COMM_METHOD" @@ -721,6 +723,22 @@ def _moe_comm_method() -> str: #: (`ops/comm/mnnvl_allreduce__tp_sum`) instead of the IPC one, where the platform supports MNNVL #: (`MNNVLAllReduce.is_mnnvl`); the two return the same bits. _TP_SUM_MNNVL_MAX_ROWS = 32 +#: The engine `allreduce_strategy` values under which the target runs its own communication: the +#: MNNVL `tp_sum` and the IPC buffers `_own_ipc` admits. Under another strategy every TP sum is +#: the raw all-reduce at that strategy and the gathers run NCCL; `check_deployment` declines UB +#: and LOWPRECISION, whose setup only the stock `AllReduce` module does. +_OWN_COMM_STRATEGIES = frozenset({int(AllReduceStrategy.AUTO), int(AllReduceStrategy.MNNVL)}) +#: The stock IPC fusion all-reduce's accumulation switch (`use_fp32_acc` in +#: `allReduceFusionKernels.cu`): fp32 unless the value starts with `0`. `hc_allreduce_inject_norm` +#: always sums in fp32, so with the opt-out the decode sites sum apart (`_hc_ar_inject`). +AR_FP32_ACC_ENV = "ALL_REDUCE_FUSION_KERNEL_ACC_FP32" + + +def _own_ipc(core) -> bool: + """Whether the target opens its own IPC buffers over the attention-TP group: the split index + gather's Lamport buffers, the two-rank copy-engine exchange and (at AUTO only) the fused TP sum + + inject. Needs an `_OWN_COMM_STRATEGIES` strategy and peer access between the group's ranks.""" + return core.allreduce_strategy in _OWN_COMM_STRATEGIES and can_access_peer(core.mapping) def _runtime_hooks() -> RuntimeHooks: @@ -906,7 +924,8 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self.mapping = mapping # Ranks the attention and recurrent heads are split over (1 under attention DP). self.attn_tp_size = _pool.attention_tp_size(mapping) - # The engine's all-reduce strategy for the TP sums; the fused IPC paths need AUTO. + # The engine's all-reduce strategy for the TP sums; `_OWN_COMM_STRATEGIES` gates the + # target's own MNNVL and IPC paths. self.allreduce_strategy = int(model_config.allreduce_strategy) # Under attention DP each rank serves its own requests; MoE runs over the TP group's rows. self.dp_size = mapping.tp_size if mapping.enable_attention_dp else 1 @@ -1281,7 +1300,11 @@ def post_load_weights(self) -> None: self._decode = DecodeTarget(self) def _tp_sum_op(self): - """Return the attention-TP sum of block partials, bit-identical on every rank.""" + """Return the attention-TP sum of block partials, bit-identical on every rank. + + At AUTO or MNNVL, where the platform supports MNNVL, rows up to `_TP_SUM_MNNVL_MAX_ROWS` + take the MNNVL kernel; every other sum is the raw all-reduce, which `allreduce_group_args` + hands MNNVL as AUTO.""" if self.attn_tp_size == 1: return lambda x: x group = allreduce_group_args(self.mapping, self.allreduce_strategy) @@ -1299,7 +1322,7 @@ def tp_sum(x: torch.Tensor) -> torch.Tensor: **group, ) - if self.allreduce_strategy != int(AllReduceStrategy.AUTO) or not MNNVLAllReduce.is_mnnvl( + if self.allreduce_strategy not in _OWN_COMM_STRATEGIES or not MNNVLAllReduce.is_mnnvl( self.mapping, torch.bfloat16, explicitly_requested=True ): return tp_sum @@ -1317,7 +1340,10 @@ def tp_sum(x: torch.Tensor) -> torch.Tensor: # noqa: F811 return tp_sum def _hc_ar_inject(self): - """Return the decode sites' fused TP-sum + injection state, or None where unsupported.""" + """Return the decode sites' fused TP-sum + injection state, or None where unsupported. + + The kernel sums in fp32, the stock IPC all-reduce's default; under that all-reduce's + opt-out (`AR_FP32_ACC_ENV`) the sites sum through `tp_sum`, which follows it.""" if not ( self.attn_tp_size in (2, 4) and self.dp_size == 1 @@ -1325,7 +1351,8 @@ def _hc_ar_inject(self): and self.hidden == 2560 and self.hc_count == 4 and self.allreduce_strategy == int(AllReduceStrategy.AUTO) - and can_access_peer(self.mapping) + and not os.environ.get(AR_FP32_ACC_ENV, "").startswith("0") + and _own_ipc(self) ): return None group = allreduce_group_args(self.mapping, self.allreduce_strategy) @@ -3232,7 +3259,8 @@ def _index_split_group(core) -> tuple | None: def _index_split_lamport(core): """The split gather's Lamport IPC buffers, or None where the Lamport exchange does not apply. - Needs 2-4 attention-TP ranks with peer access, PDL and captured batches that reach the split. + Needs 2-4 attention-TP ranks, `_own_ipc`, PDL and captured batches that reach the split; + without it the split gathers over NCCL. A collective: every rank builds the buffers here, before any capture.""" graph_batches = tuple( getattr(getattr(core, "model_config", None), "cuda_graph_batch_sizes", ()) or () @@ -3242,7 +3270,7 @@ def _index_split_lamport(core): and get_env_enable_pdl() and _INDEX_SPLIT_LAMPORT_ROWS > 0 and (not graph_batches or max(graph_batches) >= _INDEX_SPLIT_MIN_REQUESTS) - and can_access_peer(core.mapping) + and _own_ipc(core) ): return None width = core.token_topk + core.compress_ratio - 1 @@ -4167,11 +4195,13 @@ def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: self._comm_fork = torch.cuda.Event() self._comm_join = torch.cuda.Event() # Two-rank sequence parallelism only: sized for half a max_num_tokens step's bf16 rows. + # Without `_own_ipc` the pair exchanges over NCCL. self._ce_exchange = None if ( self._comm_stream is not None and self._sequence_parallel is not None and self._sequence_parallel[1] == 2 + and _own_ipc(core) ): from .ops.comm.ce_exchange__sp_pair import CopyEngineExchange @@ -6871,30 +6901,42 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"the checkpoint has Qwen3.8-Flash-Next's shape but declares {differs}; the " f"target implements { ({k: _TEXT_SETTINGS[k] for k in differs}) }" ) - # The opt-in switches, read here so a misspelled value fails before the build. + # The opt-in switches, read here so a misspelled value fails before the build; one that + # would do nothing at this deployment declines rather than being ignored. mapping = model_config.mapping - _weights.moe_tpb_requested() - _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) - comm_method = _moe_comm_method() - if comm_method not in ("ALLGATHER", "NVLINK_ONE_SIDED"): + if _weights.moe_tpb_requested() and not _weights.moe_tpb_enabled(mapping): raise NoTarget( - f"{MOE_COMM_METHOD_ENV}={comm_method} at {layout}: the target runs the MoE round " - "trip as ALLGATHER (the default) or NVLINK_ONE_SIDED" + f"{_weights.MOE_TPB_ENV}=rr at {layout}: the block-major MoE decode copy runs at " + "tp4 and tep4 only; unset it" ) - if comm_method == "NVLINK_ONE_SIDED": - from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( - NVLinkOneSided, + rowtile_groups = _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) + strategy = AllReduceStrategy(int(model_config.allreduce_strategy)) + if _pool.attention_tp_size(mapping) > 1 and strategy in ( + AllReduceStrategy.UB, + AllReduceStrategy.LOWPRECISION, + ): + raise NoTarget( + f"allreduce_strategy={strategy.name} at {layout}: the target calls the all-reduce " + "op directly, without the buffers the stock AllReduce module sets up for it" ) - - if ( - mpi_disabled() - or not mapping.enable_attention_dp - or not NVLinkOneSided.is_platform_supported() - ): + # As the stock factory: only an attention-DP group has a MoE round trip to override. + if mapping.enable_attention_dp and mapping.dp_size > 1: + comm_method = _moe_comm_method() + if comm_method not in ("ALLGATHER", "NVLINK_ONE_SIDED"): raise NoTarget( - f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at {layout}: the all-to-all MoE " - "round trip needs MPI, attention DP and MNNVL on this node" + f"{MOE_COMM_METHOD_ENV}={comm_method} at {layout}: the target runs the " + "attention-DP MoE round trip as ALLGATHER (the default) or NVLINK_ONE_SIDED" ) + if comm_method == "NVLINK_ONE_SIDED": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + NVLinkOneSided, + ) + + if mpi_disabled() or not NVLinkOneSided.is_platform_supported(): + raise NoTarget( + f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at {layout}: the all-to-all MoE " + "round trip needs MPI and MNNVL on this node" + ) spec = model_config.spec_config if spec is not None: # One linear MTP chain, verified at `max_draft_len + 1` rows on the GDN cached replay. @@ -6966,6 +7008,12 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"a {kv_cache_quant_algo} KV cache; the target stores K/V as the checkpoint's " "bf16, as FP8 or as NVFP4" ) + if rowtile_groups and kv_cache_quant_algo != QuantAlgo.FP8: + stored = "bf16" if kv_cache_quant_algo is None else kv_cache_quant_algo + raise NoTarget( + f"{ROWTILE_SCRATCH_GROUPS_ENV} with a {stored} KV cache: the grouped row-tile " + "context launches run over an FP8 KV cache only; unset it" + ) if model_config.torch_dtype != torch.bfloat16: raise NoTarget( f"the engine runs the model in {model_config.torch_dtype}; the target is bf16" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py index 980dbd033db9..17a1abc423f0 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py @@ -8,6 +8,10 @@ norm from the summed row, its sum of squares in the Triton program's reduction tree at `program_warps` warps. Both outputs are bitwise the all-reduce followed by the Triton kernel. +The sum accumulates in fp32 only, the all-reduce's default. Under +`ALL_REDUCE_FUSION_KERNEL_ACC_FP32=0` the all-reduce sums in bf16 and this kernel does not +match it, so the target does not call it then (`modeling._hc_ar_inject`). + Only the norm weights are read before the grid-dependency wait; the residual, injection weights and shared-expert operands are read after it. """ diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index ecf776b0ae88..7d79ac5fac83 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -85,9 +85,10 @@ def switch(env: str, on: frozenset = SWITCH_ON) -> bool: #: Opt-in (`rr`; default off) block-major copy of every MoE layer's routed experts for -#: `ops/moe/moe_tpb_decode` (attention TP4 only): expert e's 128-row intermediate block b on rank -#: (e + b) % 4, plus this rank's slice of the shared expert. The whole-expert window stays for -#: prefill and the runner, so the copy costs one more window per layer (~31 GB / rank at tep4). +#: `ops/moe/moe_tpb_decode` (tp4 and tep4 only, `moe_tpb_enabled`; `check_deployment` declines +#: it at another layout): expert e's 128-row intermediate block b on rank (e + b) % 4, plus this +#: rank's slice of the shared expert. The whole-expert window stays for prefill and the runner, +#: so the copy costs one more window per layer (~31 GB / rank at tep4). MOE_TPB_ENV = "TRTLLM_MODELING_V2_DECODE_MOE_TPB" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index 6d4c825ae9ed..f850e05bb062 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -27,6 +27,7 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache +from tensorrt_llm.functional import AllReduceStrategy from tensorrt_llm.llmapi import QSASparseAttentionConfig from tensorrt_llm.llmapi.llm_args import ( KvCacheConfig, @@ -407,20 +408,26 @@ def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypa with pytest.raises(ValueError, match="TRTLLM_USE_GDN_FLASHINFER_REPLAY"): modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) monkeypatch.delenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY") - # An all-to-all MoE the deployment cannot run: no attention DP, or no MNNVL. + # Attention TP has no MoE round trip: the stock override is ignored there, as the stock + # factory ignores it. + for method in ("nvlink_one_sided", "NVLINK_TWO_SIDED"): + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, method) + for member in ("tp2", "tep4"): + _admitted(_routed_config(member)) + # An attention-DP all-to-all MoE the deployment cannot run: no MNNVL. monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "nvlink_one_sided") - with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED"): - modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) with mock.patch( "tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided.NVLinkOneSided" ".is_platform_supported", return_value=False, ): - with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED"): - modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) + config = _routed_config("dep4") + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at dep4"): + modeling_v2_resolve(config, _llm_args("require")) # A stock method the target does not run declines rather than routing as the default. monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "NVLINK_TWO_SIDED") - with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_TWO_SIDED"): + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_TWO_SIDED at dep4"): modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) # `ALLGATHER`, in any case, is the default round trip. monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "allgather") @@ -432,6 +439,54 @@ def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypa monkeypatch.delenv(W.MOE_TPB_ENV) +@pytest.mark.cpu_only +@pytest.mark.parametrize("strategy", ["UB", "LOWPRECISION"]) +def test_a_strategy_only_the_stock_module_sets_up_declines(on_sm103, strategy): + """The target calls the all-reduce op directly: under attention TP a strategy whose buffers + the stock `AllReduce` module sets up declines; with no TP sum (one rank, attention DP) it is + admitted.""" + for member in ("tp2", "tep2", "tp4", "tep4"): + config = replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy]) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"allreduce_strategy={strategy} at {member}"): + modeling_v2_resolve(config, _llm_args("require")) + for member in ("tp1", "dep2", "dep4"): + _admitted(replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy])) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy", ["NCCL", "NCCL_SYMMETRIC", "MIN_LATENCY", "ONESHOT", "TWOSHOT", "MNNVL"] +) +def test_a_strategy_the_target_runs_is_admitted(on_sm103, strategy): + """The raw all-reduce op runs these; MNNVL takes the target's MNNVL sum and the raw op at + AUTO.""" + for member in ("tp2", "tep4"): + _admitted(replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy])) + + +@pytest.mark.cpu_only +def test_a_switch_that_would_do_nothing_declines(on_sm103, monkeypatch): + """`MOE_TPB_ENV=rr` outside tp4 / tep4 and `ROWTILE_SCRATCH_GROUPS_ENV` without an FP8 KV + cache decline, so a run that sets one never measures the target without it.""" + monkeypatch.setenv(W.MOE_TPB_ENV, "rr") + for member in ("tp1", "tp2", "tep2", "dep2", "dep4"): + config = _routed_config(member) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}=rr at {member}"): + modeling_v2_resolve(config, _llm_args("require")) + for member in ("tp4", "tep4"): + _admitted(_routed_config(member)) + monkeypatch.delenv(W.MOE_TPB_ENV) + monkeypatch.setenv(M.ROWTILE_SCRATCH_GROUPS_ENV, "1") + for kv, stored in ((KvCacheConfig(), "bf16"), (KvCacheConfig(dtype="nvfp4"), "NVFP4")): + config = _routed_config("tep4") + assert modeling_v2_resolve(config, _llm_args("auto", kv_cache_config=kv)) is None + with pytest.raises(ValueError, match=f"{M.ROWTILE_SCRATCH_GROUPS_ENV} with a {stored} KV"): + modeling_v2_resolve(config, _llm_args("require", kv_cache_config=kv)) + _admitted(_routed_config("tep4"), kv_cache_config=KvCacheConfig(dtype="fp8")) + + @pytest.mark.cpu_only def test_the_quantization_picks_the_checkpoint(on_sm103): """MIXED_PRECISION routes to the NVFP4 target (its own checks mocked), FP8_BLOCK_SCALES here.""" @@ -503,7 +558,7 @@ def _mtp(max_draft_len: int = 3) -> MTPDecodingConfig: return MTPDecodingConfig(max_draft_len=max_draft_len) -def _core(member: str, rank: int, spec_config=None): +def _core(member: str, rank: int, spec_config=None, **config): model_config = ModelConfig( pretrained_config=Qwen4ExpTextConfig(**_TEXT), mapping=_mapping(member, rank), @@ -512,6 +567,7 @@ def _core(member: str, rank: int, spec_config=None): moe_backend_requested="TRTLLM", sparse_attention_config=QSASparseAttentionConfig(), spec_config=spec_config, + **config, ) composite = SimpleNamespace( vision_config=Qwen4ExpVisionConfig(**_VISION), image_token_id=248056, video_token_id=248057 @@ -524,6 +580,122 @@ def _core(member: str, rank: int, spec_config=None): return M.Qwen4ExpUnifiedModelingV2(model_config, composite) +@pytest.mark.cpu_only +@pytest.mark.parametrize("strategy", ["AUTO", "MNNVL", "NCCL"]) +def test_the_tp_sum_runs_at_the_engines_allreduce_strategy(strategy, monkeypatch) -> None: + """At AUTO or MNNVL, sums up to `_TP_SUM_MNNVL_MAX_ROWS` rows take the MNNVL kernel (asked + for by name, so one node qualifies) and wider ones the raw op at AUTO, which has no MNNVL + branch. Another strategy reaches the raw op as it is; NCCL reads no IPC workspace.""" + import tensorrt_llm._utils as utils + from tensorrt_llm._torch.distributed import ops as dist_ops + + core = _core("tep4", 0, allreduce_strategy=AllReduceStrategy[strategy]) + asked = [] + + def is_mnnvl(mapping, dtype, explicitly_requested=False): + asked.append(explicitly_requested) + return True + + monkeypatch.setattr(M.MNNVLAllReduce, "is_mnnvl", staticmethod(is_mnnvl)) + monkeypatch.setattr(dist_ops, "get_allreduce_workspace", lambda mapping: "ipc workspace") + monkeypatch.setattr(utils, "mpi_disabled", lambda: False) + monkeypatch.setattr(M, "mnnvl_workspace_args", lambda *args: dict(buffers="mnnvl workspace")) + monkeypatch.setattr(M, "mnnvl_tp_sum", lambda x, buffers: ("mnnvl", buffers)) + monkeypatch.setattr( + M, "allreduce", lambda x, *args, **kwargs: ("raw", kwargs["strategy"], kwargs["workspace"]) + ) + tp_sum = core._tp_sum_op() + narrow = torch.zeros(M._TP_SUM_MNNVL_MAX_ROWS, 8) + wide = torch.zeros(M._TP_SUM_MNNVL_MAX_ROWS + 1, 8) + if strategy == "NCCL": + assert asked == [] + assert tp_sum(narrow) == tp_sum(wide) == ("raw", int(AllReduceStrategy.NCCL), None) + return + assert asked == [True] + assert tp_sum(narrow) == ("mnnvl", "mnnvl workspace") + assert tp_sum(wide) == ("raw", int(AllReduceStrategy.AUTO), "ipc workspace") + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy, own", [("AUTO", True), ("MNNVL", True), ("NCCL", False), ("ONESHOT", False)] +) +def test_the_target_opens_its_own_ipc_buffers_at_auto_or_mnnvl_only( + strategy, own, monkeypatch +) -> None: + """`_own_ipc` gates the split index gather's Lamport buffers and the two-rank copy-engine + exchange; under another strategy neither is built and both gather over NCCL.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm.ce_exchange__sp_pair import ( # noqa: E501 + CopyEngineExchange, + ) + + built = [] + monkeypatch.setattr(M, "can_access_peer", lambda mapping: True) + monkeypatch.setattr(M, "get_env_enable_pdl", lambda: True) + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + monkeypatch.setattr( + M, "_selection_lamport_build", lambda mapping, size: built.append("lamport") or "lamport" + ) + monkeypatch.setattr( + CopyEngineExchange, + "create", + classmethod(lambda cls, mapping, capacity: built.append("ce") or "ce"), + ) + core = SimpleNamespace( + attn_tp_size=2, + dp_size=1, + mapping=_mapping("tep2", 0), + allreduce_strategy=int(AllReduceStrategy[strategy]), + token_topk=2048, + compress_ratio=4, + max_num_tokens=8192, + hidden=HIDDEN, + model_config=SimpleNamespace(cuda_graph_batch_sizes=()), + _side_stream=SimpleNamespace(device="cpu"), + tp_sum=None, + ) + assert M._own_ipc(core) is own + assert (M._index_split_lamport(core) is not None) is own + # The prefill target's construction, its views and graphs stubbed out. + monkeypatch.setattr(M, "_layer_views", lambda core: None) + monkeypatch.setattr(M, "_stack_views", lambda core: None) + monkeypatch.setattr(M, "_keyed_eager_on_graph", lambda *args, **kwargs: None) + monkeypatch.setattr(M._KvRangeProbe, "for_target", classmethod(lambda cls, core, on: None)) + monkeypatch.setattr(torch.cuda, "Stream", lambda **kwargs: SimpleNamespace(**kwargs)) + monkeypatch.setattr(torch.cuda, "Event", lambda: None) + assert (M.PrefillTarget(core)._ce_exchange is not None) is own + assert built == (["lamport", "ce"] if own else []) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy, fp32_acc, fused", + [ + ("AUTO", None, True), + ("AUTO", "1", True), + ("AUTO", "0", False), + ("MNNVL", None, False), + ("NCCL", None, False), + ], +) +def test_the_fused_decode_sum_runs_at_auto_with_fp32_accumulation( + strategy, fp32_acc, fused, monkeypatch +) -> None: + """`hc_allreduce_inject_norm` sums in fp32 only: under the stock all-reduce's opt-out + (`AR_FP32_ACC_ENV=0`) and at a strategy other than AUTO the decode sites sum through + `tp_sum`.""" + monkeypatch.setattr(M, "can_access_peer", lambda mapping: True) + monkeypatch.setattr(M, "allreduce_group_args", lambda mapping, s: dict(workspace="ipc")) + if fp32_acc is None: + monkeypatch.delenv(M.AR_FP32_ACC_ENV, raising=False) + else: + monkeypatch.setenv(M.AR_FP32_ACC_ENV, fp32_acc) + state = _core("tep4", 0, allreduce_strategy=AllReduceStrategy[strategy])._hc_ar_inject() + assert (state is not None) is fused + if fused: + assert (state.workspace, state.rank, state.nranks) == ("ipc", 0, 4) + + #: Every rank; MTP with 7 drafts at a member's last rank, 1 elsewhere, and none at dep2. _CORE_CASES = [(m, r, None if m == "dep2" else 7 if r == _world(m) - 1 else 1) for m, r in _RANKS] From 1c0db6b26fe60f2d96e85413f0a55011790f1a5f Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:21:51 -0800 Subject: [PATCH 34/51] [None][fix] modeling_v2 qwen4_exp: harden the node PLE host table - An owner records a chunk's broker segment before it creates the chunk, so a failed allocation closes the name and the waiting processes take the chunk over; `_VmmChunks._map` unmaps when `cuMemSetAccess` fails; both targets' `load` release the n-gram table on any error after declaring it. - The broker closes its listener when `accept` fails, and a waiter whose connection is reset retries, so no name stays bound without a server. - `SO_PEERCRED` is unpacked as `struct ucred` (pid_t, uid_t, gid_t), so a uid of 2**31 or more reads back unsigned. - The `node` key is a digest of a strided sample of every chunk's rows, not the config's `_name_or_path`; an attaching rank compares another strided sample with its checkpoint and raises on a difference. - The FP8 target's load checks the n-gram table's height against its id space with explicit raises, including the per-shard padding bound. - A requested PLE host-table mode that does not run is a warning, once per process; both targets read the mode in `check_deployment`. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../models/qwen4_exp/chunk_broker.py | 15 +- .../models/qwen4_exp/ple_host_table.py | 123 ++++--- .../modeling.py | 13 +- .../weights.py | 53 ++- .../modeling.py | 3 +- .../weights.py | 24 +- .../test_routing_and_layout.py | 7 +- .../test_weights.py | 315 +++++++++++++++++- .../test_routing.py | 15 +- 9 files changed, 490 insertions(+), 78 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py index 187cbf4d4e42..865cbe724c61 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py @@ -30,7 +30,7 @@ OWNER = "owner" ATTACHED = "attached" -_CRED = struct.Struct("3i") # struct ucred: pid, uid, gid +_CRED = struct.Struct("iII") # struct ucred: pid_t pid, uid_t uid, gid_t gid _POLL_SECONDS = 1.0 @@ -84,6 +84,10 @@ def _serve(self) -> None: try: conn, _ = self._server.accept() except OSError: + # `close`, or `accept` failing (EMFILE, ENFILE, ENOMEM, ...). Either way the name + # is freed with the thread: connected peers already answered keep their answer, + # and every other peer sees its connection reset and owns the key itself. + self._server.close() return answer = threading.Thread(target=self._answer, args=(conn,), daemon=True) self._answers = [t for t in self._answers if t.is_alive()] + [answer] @@ -122,9 +126,9 @@ def acquire( key: str, timeout_s: float, log: Callable[[str], None] = print, report_s: float = 60.0 ) -> tuple: """`(OWNER, OwnedSegment)` when no process owns `key`, else `(ATTACHED, fd, meta)` once - the owner publishes; an owner that exits or aborts first hands `key` to its waiters. - Raises `TimeoutError` when no owner has published within `timeout_s`, and `PermissionError` - when a process of another uid holds the name.""" + the owner publishes; an owner that exits, aborts or stops accepting first hands `key` to its + waiters. Raises `TimeoutError` when no owner has published within `timeout_s`, and + `PermissionError` when a process of another uid holds the name.""" name = socket_name(key) deadline = time.monotonic() + timeout_s next_report = time.monotonic() + report_s @@ -158,6 +162,9 @@ def acquire( if time.monotonic() >= next_report: log(f"ple host table: waiting for the owner of {key!r} to publish it") next_report = time.monotonic() + report_s + except ConnectionResetError: + # The owner closed its listener before accepting this connection. + got = (b"", [], 0, None) finally: conn.close() msg, fds, _, _ = got diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py index 7cf949aa89a6..bd7b896f98eb 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py @@ -14,15 +14,18 @@ * `node`: every chunk is one CUDA VMM allocation on host NUMA memory, shared by every process of the host through `chunk_broker`; every rank maps all chunks back to back and gathers locally. A rank fills the chunks it is responsible for and attaches to the others. One copy per node. + Processes share a chunk when their tables have one `table_identity`: the bytes of a strided + sample of every chunk's rows, whatever directory the checkpoint was read from. A rank that + attaches compares the chunk with its own checkpoint at other sampled rows. `server` needs attention TP with a size that divides `CHUNKS`, else it runs as `rank`. When a -rank cannot take part in `node` (no VMM host NUMA allocation, no checkpoint identity, no owner -within the timeout), every rank of the server runs `server` instead, or `rank`. +rank cannot take part in `node` (no VMM host NUMA allocation, no table identity, no owner +within the timeout), every rank of the server runs `server` instead, or `rank`. A requested +mode that does not run is a warning, once per process. """ from __future__ import annotations -import glob import hashlib import math import os @@ -52,7 +55,12 @@ CHUNK_ALIGN = 2 << 20 #: How long a rank waits for another process to publish a chunk before giving up on `node`. _ATTACH_TIMEOUT_S = 3600.0 +#: A chunk's sample (`sample_blocks`): this many blocks of `_SAMPLE_ROWS` rows at an even stride. +_SAMPLE_BLOCKS = 256 +_SAMPLE_ROWS = 8 _CUDA_MEMORY_TYPE_HOST = 1 +#: `(requested, running)` fallbacks this process has warned about. +_WARNED: set = set() Copy = Callable[[torch.Tensor, int, int, int], None] Read = Callable[[int, int], torch.Tensor] @@ -67,6 +75,17 @@ def host_table_mode() -> str: return value +def _warn_fallback(requested: str, running: str, why: str) -> None: + """Warn, once per process and pair, that the requested mode runs as `running`, and why: the + table then takes more host memory than the deployment asked for.""" + if (requested, running) in _WARNED: + return + _WARNED.add((requested, running)) + logger.warning( + f"ple host table: {PLE_HOST_TABLE_ENV}={requested} requested, running {running}: {why}" + ) + + def chunk_rows(rows: int, cols: int, align: int = CHUNK_ALIGN) -> int: """Rows of one chunk: an even split of `rows` over `CHUNKS`, rounded up so a chunk's bytes (`cols` one-byte columns a row) are a multiple of `align`.""" @@ -96,24 +115,37 @@ def owned_rows(rank: int, world: int, rows: int, cols: int, align: int = CHUNK_A ] -def checkpoint_identity(checkpoint_dir: Optional[str], *parts) -> Optional[str]: - """A key for the table's bytes: the checkpoint directory, its safetensors files' names, sizes - and modification times, and `parts`; None when the directory or its files cannot be read.""" - if not checkpoint_dir: - return None - root = os.path.realpath(checkpoint_dir) - try: - files = sorted(glob.glob(os.path.join(root, "*.safetensors"))) - stats = [(os.path.basename(f), os.stat(f)) for f in files] - except OSError: - return None - if not stats: - return None - digest = hashlib.sha256(root.encode()) - for name, st in stats: - digest.update(f"\0{name}:{st.st_size}:{st.st_mtime_ns}".encode()) +def sample_blocks(k: int, rows: int, cols: int, align: int = CHUNK_ALIGN, phase: int = 0) -> list: + """Row blocks `[lo, hi)` of chunk `k` that `node` compares: `_SAMPLE_BLOCKS` blocks of + `_SAMPLE_ROWS` rows at an even stride, `phase` 1 halfway between phase 0's; the whole chunk + when it is too short to stride.""" + lo, hi = chunk_range(k, rows, cols, align) + if hi - lo <= 2 * _SAMPLE_BLOCKS * _SAMPLE_ROWS: + return [(lo, hi)] if hi > lo else [] + stride = (hi - lo) // _SAMPLE_BLOCKS + first = lo + phase * (stride // 2) + return [ + (start, min(start + _SAMPLE_ROWS, hi)) + for start in range(first, hi, stride)[:_SAMPLE_BLOCKS] + ] + + +def _sample(read: Read, blocks) -> torch.Tensor: + """The checkpoint's rows at `blocks`, in order, as bytes on the CPU.""" + return torch.cat([read(lo, hi).contiguous().view(torch.uint8).cpu() for lo, hi in blocks]) + + +def table_identity(read: Read, rows: int, cols: int, *parts, align: int = CHUNK_ALIGN) -> str: + """The `node` key of a `rows` x `cols` table that `read` returns rows of: a digest of + `parts` and of every chunk's phase-0 sample (`sample_blocks`). Two checkpoints share it only + if their tables agree at those rows, wherever each was read from.""" + digest = hashlib.sha256(f"{rows}x{cols}".encode()) for part in parts: digest.update(f"\0{part}".encode()) + for k in range(CHUNKS): + blocks = sample_blocks(k, rows, cols, align) + if blocks: + digest.update(_sample(read, blocks).numpy()) return digest.hexdigest()[:32] @@ -213,9 +245,18 @@ def _prop(node: int): return prop def _map(self, k: int, handle, read_only: bool) -> None: - D = cuda_driver address = self.va + k * self.chunk_bytes - _cu(D.cuMemMap(address, self.chunk_bytes, 0, handle, 0)) + _cu(cuda_driver.cuMemMap(address, self.chunk_bytes, 0, handle, 0)) + try: + self._set_access(address, read_only) + except BaseException: + # The caller releases `handle`; nothing stays mapped to it. + cuda_driver.cuMemUnmap(address, self.chunk_bytes) + raise + self._handles[k] = handle + + def _set_access(self, address: int, read_only: bool) -> None: + D = cuda_driver desc = D.CUmemAccessDesc() desc.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE desc.location.id = self.device_index @@ -231,7 +272,6 @@ def _map(self, k: int, handle, read_only: bool) -> None: raise desc.flags = D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE _cu(D.cuMemSetAccess(address, self.chunk_bytes, [desc], 1)) - self._handles[k] = handle def create(self, k: int) -> None: handle = _cu(cuda_driver.cuMemCreate(self.chunk_bytes, self._prop(self.node), 0)) @@ -358,9 +398,11 @@ def __init__( else: self.mode = SERVER if mode == SERVER and self._shards() else RANK if mode == SERVER and self.mode == RANK and world > 1: - self._log( - f"ple host table: {SERVER} needs attention TP over a size that divides " - f"{CHUNKS} (tp {world}, attention DP {dp}); keeping one copy per rank" + _warn_fallback( + mode, + self.mode, + f"{SERVER} needs attention TP over a size that divides {CHUNKS} (tp {world}, " + f"attention DP {dp})", ) self._declare_private() @@ -391,7 +433,7 @@ def _declare_node(self) -> None: """Map nothing yet; own (create) the chunks this rank is responsible for if no process does. A failure is kept for `fill`, where every rank agrees on the mode.""" if self.identity is None: - self._node_error = "no checkpoint identity" + self._node_error = "no table identity" return try: chunk_bytes = chunk_rows(self.rows, self.cols, self.align) * self.cols @@ -404,8 +446,10 @@ def _declare_node(self) -> None: for k in owned_chunks(self.rank, self.world): segment = _broker.try_own(self._key(k)) if segment is not None: - self._vmm.create(k) + # Recorded first: if `create` fails, `_abort_node` closes it, and the + # processes waiting on it take the chunk over. self._segments[k] = segment + self._vmm.create(k) except (RuntimeError, OSError) as error: self._node_error = str(error) self._abort_node() @@ -429,7 +473,7 @@ def fill(self, copy: Copy, read: Read) -> None: return reason = self._node_error or "another rank of this server could not share the table" self.mode = SERVER if self._shards() else RANK - self._log(f"ple host table: {NODE} unavailable ({reason}); running {self.mode}") + _warn_fallback(NODE, self.mode, reason) if ok: # Keep the full shared view; `server` reads only this rank's rows from it. if self.mode == SERVER: @@ -453,8 +497,8 @@ def _fill_node(self, copy: Copy, read: Read) -> bool: continue got = _broker.acquire(self._key(k), _ATTACH_TIMEOUT_S, self._log) if got[0] == _broker.OWNER: - vmm.create(k) self._segments[k] = got[1] + vmm.create(k) self._fill_and_publish(k, copy) continue _, fd, meta = got @@ -466,7 +510,7 @@ def _fill_node(self, copy: Copy, read: Read) -> bool: os.close(fd) self._attached += 1 self._check_chunk(k, read) - except (RuntimeError, OSError, TimeoutError, AssertionError) as error: + except (RuntimeError, OSError, TimeoutError) as error: self._node_error = str(error) self._abort_node() return False @@ -482,14 +526,17 @@ def _fill_and_publish(self, k: int, copy: Copy) -> None: self._segments[k].publish(fd, {"chunk": k, "chunk_bytes": self._vmm.chunk_bytes}) def _check_chunk(self, k: int, read: Read) -> None: - """An attached chunk's first and last rows are the checkpoint's.""" - lo, hi = chunk_range(k, self.rows, self.cols, self.align) - for row in {lo, hi - 1} if hi > lo else (): - got = self.table[row : row + 1].view(torch.uint8).cpu() - want = read(row, row + 1).contiguous().view(torch.uint8).cpu() - assert torch.equal(got, want), ( - f"shared chunk {k} differs from the checkpoint at row {row}" - ) + """An attached chunk holds the checkpoint's bytes at its phase-1 sample, rows the key does + not hash; raises `RuntimeError` otherwise.""" + blocks = sample_blocks(k, self.rows, self.cols, self.align, phase=1) + if not blocks: + return + rows = torch.cat([torch.arange(lo, hi) for lo, hi in blocks]) + got = self.table.view(torch.uint8).index_select(0, rows.to(self.device)).cpu() + want = _sample(read, blocks) + if not torch.equal(got, want): + first = int(rows[(got != want).any(1)][0]) + raise RuntimeError(f"shared chunk {k} differs from the checkpoint at row {first}") def _abort_node(self) -> None: for segment in self._segments.values(): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 7d413fb7b31e..0a045ff290d2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1042,13 +1042,11 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self.ple_layer = list(cfg.ple_layer_ids)[0] - 1 self.ngram_size = cfg.ngram_size self.ple_dilation = self.ngram_size - self.ngram_vocab_divisor = cfg.make_ngram_vocab_size_divisible_by eos = cfg.eos_token_id self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) self.ple_table_dtype = _weights.PLE_TABLE_DTYPE # How the table is stored (`ple_host_table`); the storage is `_ple_table_host` after load. self.ple_host_mode = _ple_host.host_table_mode() - self.checkpoint_dir = getattr(composite_config, "_name_or_path", None) self._ple_table_host = None vcfg = getattr(composite_config, "vision_config", None) @@ -1252,16 +1250,6 @@ def post_load_weights(self) -> None: # Zero the GDN in-proj alignment rows so the decode GEMM's unread pad columns are finite. for i in self.gdn_layers: w[f"l{i}_in_proj_w"][sum(self.in_proj_split) :].zero_() - # An index past the table's end is a device-side assert that poisons the process. - # Rows of the whole table: under `server` storage `ple_table` holds only this rank's. - host = self._ple_table_host - table_rows = host.rows if host is not None else w["ple_table"].shape[0] - used_rows = int(w["ple_offs"][-1]) + int(w["ple_vocab"][-1]) - assert used_rows <= table_rows, ( - f"the n-gram table has {table_rows} rows but its per-head offsets and vocabulary " - f"sizes address {used_rows}" - ) - assert table_rows % self.ngram_vocab_divisor == 0, (table_rows, self.ngram_vocab_divisor) self._cos_sin = self._index_rope_table(self.theta, device) # Before the draft layer's views, which read its fused first projections. self._fuse_hc_first_projections() @@ -6904,6 +6892,7 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: # The opt-in switches, read here so a misspelled value fails before the build; one that # would do nothing at this deployment declines rather than being ignored. mapping = model_config.mapping + _ple_host.host_table_mode() if _weights.moe_tpb_requested() and not _weights.moe_tpb_enabled(mapping): raise NoTarget( f"{_weights.MOE_TPB_ENV}=rr at {layout}: the block-major MoE decode copy runs at " diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py index 7d79ac5fac83..de3990bc40d6 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -1265,6 +1265,14 @@ def load(self, model, weights) -> None: d = self.dims(core) # Declared before the manifest: its height comes from the shards. rows_per_shard = _declare_ngram_table(core, d, weights) + try: + self._fill_declared(model, d, weights, rows_per_shard) + except BaseException: + _close_ngram_table(core) + raise + + def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int) -> None: + core = model.model manifest = self.manifest(d) manifest["ple_table"] = ngram_table_sources(d, rows_per_shard) consumed: set = set() @@ -1468,6 +1476,7 @@ def rank_dims(cfg, vcfg, mapping, *, mtp_enabled: bool, dtype: torch.dtype) -> S ngram_heads=ngram_heads, ple_head_dim=cfg.ple_embed_dim // ngram_heads, ple_shards=cfg.split_ngram_parts, + ngram_vocab_divisor=cfg.make_ngram_vocab_size_divisible_by, ple_table_dtype=PLE_TABLE_DTYPE, v_depth=vcfg.depth, v_hidden=v_hidden, @@ -1545,12 +1554,12 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: ) device = core.w["embed"].device rows = d.ple_shards * rows_per_shard + _check_ngram_id_space(d, weights, rows) mode = getattr(core, "ple_host_mode", _ple_host.RANK) identity = None if mode == _ple_host.NODE: - identity = _ple_host.checkpoint_identity( - getattr(core, "checkpoint_dir", None), rows, head_dim, dtype, *keys - ) + _, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + identity = _ple_host.table_identity(read, rows, head_dim, dtype, *keys) mapping = getattr(core, "mapping", None) core._ple_table_host = _ple_host.PleHostTable( mode, @@ -1568,6 +1577,33 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: return rows_per_shard +def _check_ngram_id_space(d: SimpleNamespace, weights, rows: int) -> None: + """Raise `ValueError` unless a table of `rows` rows holds the id space the checkpoint's + per-head offsets and vocabulary sizes address, padded to a multiple of + `ngram_vocab_divisor` by less than one divisor. + + An id past the end is a device-side assert that poisons the process. Shards padded one by + one carry more padding than that, and do not concatenate to the padded id space: every id + past the first shard would read a shifted row.""" + prefix = f"{_LM}.layers.{d.ple_layer}.ple.ple_embedding" + offsets = _read(weights[f"{prefix}.ngram_heads_offsets"], None) + vocab = _read(weights[f"{prefix}.ngram_heads_vocab_sizes"], None) + used = int(offsets[-1]) + int(vocab[-1]) + divisor = d.ngram_vocab_divisor + if used > rows: + raise ValueError( + f"the n-gram table has {rows} rows but its per-head offsets and vocabulary sizes " + f"address {used}" + ) + if rows % divisor or rows - used >= divisor: + raise ValueError( + f"the n-gram table has {rows} rows for an id space of {used}: not padded to a " + f"multiple of make_ngram_vocab_size_divisible_by {divisor} by less than one " + "divisor. Shards padded one by one do not concatenate to the padded id space, and " + "every id past the first shard would read a shifted row" + ) + + def _bind_ngram_table(core) -> None: """`core.w["ple_table"]` as the host table's current view (a placeholder until a `node` table is mapped).""" @@ -1582,6 +1618,17 @@ def _bind_ngram_table(core) -> None: ) +def _close_ngram_table(core) -> None: + """Release the n-gram table (in `node`, its chunks and the broker names other processes of + the host may wait on); `ple_table` becomes the placeholder.""" + host = core._ple_table_host + if host is None: + return + host.close() + _bind_ngram_table(core) + core._ple_table_host = None + + def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: """Fill this rank's rows of the n-gram table (`ple_host_table`); return the shard keys.""" keys = ngram_shard_keys(d) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index d2a1834ea5e2..e6437bb637c1 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -1214,7 +1214,6 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig f"{_ple_host.PLE_HOST_TABLE_ENV}={self.ple_host_mode} applies to the host-resident " f"n-gram table; {_PLE_HOST_OFFLOAD_ENV}=0 keeps one device copy per rank" ) - self.checkpoint_dir = getattr(composite_config, "_name_or_path", None) _check_switches(self.one_rank) # CPU allocation behind the offloaded table; the `ple_table` parameter is a # device-address view that owns none of those pages. @@ -6858,6 +6857,8 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"target implements { ({k: _TEXT_SETTINGS[k] for k in differs}) }" ) + # Read here so a misspelled value fails before the build. + _ple_host.host_table_mode() mapping = model_config.mapping one_rank = mapping.world_size == 1 spec = model_config.spec_config diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index e27e3ab0805a..6b0a48b44ab8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -1573,6 +1573,14 @@ def load(self, model, weights) -> None: # The one load-time declaration, before the manifest is built: the table's # height comes from the shards, and the manifest addresses it by row range. rows_per_shard = _declare_ngram_table(core, d, weights) + try: + self._fill_declared(model, d, weights, rows_per_shard) + except BaseException: + _close_ngram_table(core) + raise + + def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int) -> None: + core = model.model manifest = self.manifest(d) manifest["ple_table"] = ngram_table_sources(d, rows_per_shard) consumed: set = set() @@ -1852,9 +1860,8 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: mode = getattr(core, "ple_host_mode", _ple_host.RANK) identity = None if mode == _ple_host.NODE: - identity = _ple_host.checkpoint_identity( - getattr(core, "checkpoint_dir", None), rows, head_dim, dtype, *keys - ) + _, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + identity = _ple_host.table_identity(read, rows, head_dim, dtype, *keys) mapping = getattr(core, "mapping", None) core._ple_table_host = _ple_host.PleHostTable( mode, @@ -1891,6 +1898,17 @@ def _bind_ngram_table(core) -> None: ) +def _close_ngram_table(core) -> None: + """Release the host-resident n-gram table (in `node`, its chunks and the broker names other + processes of the host may wait on); `ple_table` becomes the placeholder.""" + host = core._ple_table_host + if host is None: + return + host.close() + _bind_ngram_table(core) + core._ple_table_host = None + + def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: """Fill this rank's rows of the host-resident n-gram table (`ple_host_table`); return the shard keys.""" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index f850e05bb062..c5f59d83b4fb 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -13,7 +13,7 @@ from transformers import PretrainedConfig from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import routing +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 modeling as M, ) @@ -105,6 +105,7 @@ def on_sm103(monkeypatch): M.MOE_COMM_METHOD_ENV, M.ROWTILE_SCRATCH_GROUPS_ENV, W.MOE_TPB_ENV, + ple_host_table.PLE_HOST_TABLE_ENV, ): monkeypatch.delenv(env, raising=False) @@ -437,6 +438,10 @@ def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypa with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}='1': expected rr"): modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) monkeypatch.delenv(W.MOE_TPB_ENV) + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "nodes") + with pytest.raises(ValueError, match=f"{ple_host_table.PLE_HOST_TABLE_ENV}='nodes': expected"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) + monkeypatch.delenv(ple_host_table.PLE_HOST_TABLE_ENV) @pytest.mark.cpu_only diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py index 4896503c3b86..24767f759759 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -75,6 +75,19 @@ def stored(checkpoint, index) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: return specs +@pytest.fixture(scope="module") +def id_space(checkpoint, index) -> dict[str, torch.Tensor]: + """The n-gram id space's tensors (per-head offsets and vocabulary sizes), read whole.""" + from safetensors import safe_open + + tensors = {} + for key, file in index.items(): + if key.endswith((".ngram_heads_offsets", ".ngram_heads_vocab_sizes")): + with safe_open(str(checkpoint / file), framework="pt") as fh: + tensors[key] = fh.get_tensor(key) + return tensors + + @pytest.fixture(scope="module") def configs(checkpoint): raw = json.loads((checkpoint / "config.json").read_text()) @@ -406,7 +419,7 @@ def named_parameters(): [(member, True) for member in MEMBERS] + [("tep4", False)], ids=[f"{member}-mtp" for member in MEMBERS] + ["tep4-no-mtp"], ) -def test_member_covers_the_checkpoint(stored, configs, member, mtp, monkeypatch): +def test_member_covers_the_checkpoint(stored, configs, id_space, member, mtp, monkeypatch): """Each tensor is read or predicted unread, never both; each expert's tensors by one rank and the rest by every rank; the windows tile the 512 experts; and `load` fills every parameter.""" everything = set(stored) @@ -449,12 +462,41 @@ def test_member_covers_the_checkpoint(stored, configs, member, mtp, monkeypatch) monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None) monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None) model = _stub_model(configs, member, _LOAD_RANK[member, mtp], mtp) - WT.MODEL_WEIGHTS.load(model, {key: _HeaderEntry(*spec) for key, spec in stored.items()}) + WT.MODEL_WEIGHTS.load(model, _header_entries(stored, id_space)) d = WT.MODEL_WEIGHTS.dims(model.model) assert tuple(model.model.w["ple_table"].shape) == (128 * _rows_per_shard(stored, d), 160) assert model.lm_head.weight.shape[0] == _lm_head_rows(member, 248320) +def _header_entries(stored, id_space) -> dict: + """Every checkpoint tensor by its header, but the n-gram id space's, which `load` reads.""" + return { + key: id_space[key] if key in id_space else _HeaderEntry(*spec) + for key, spec in stored.items() + } + + +@pytest.mark.cpu_only +def test_load_rejects_a_per_shard_padded_ngram_table(stored, configs, id_space, monkeypatch): + """Shards padded one by one hold more padding than one divisor past the id space the + per-head offsets and sizes address; `load` raises before it declares the table.""" + d = _dims(configs, "tp1", 0) + rows_per_shard = _rows_per_shard(stored, d) + prefix = f"{WT._LM}.layers.{d.ple_layer}.ple.ple_embedding" + offsets = id_space[f"{prefix}.ngram_heads_offsets"] + vocab = id_space[f"{prefix}.ngram_heads_vocab_sizes"].clone() + # The ids of each shard end `pad` rows before it does. + pad = -(-d.ngram_vocab_divisor // d.ple_shards) + vocab[-1] = d.ple_shards * (rows_per_shard - pad) - int(offsets[-1]) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None) + monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None) + model = _stub_model(configs, "tp1", 0, True) + entries = _header_entries(stored, {**id_space, f"{prefix}.ngram_heads_vocab_sizes": vocab}) + with pytest.raises(ValueError, match="Shards padded one by one"): + WT.MODEL_WEIGHTS.load(model, entries) + assert "ple_table" not in model.model.w + + def _random_bytes(gen, *shape) -> torch.Tensor: return torch.randint(0, 256, shape, generator=gen, dtype=torch.uint8) @@ -658,6 +700,61 @@ def test_chunk_broker_times_out_on_a_name_nobody_listens_on(): squatter.close() +@pytest.mark.cpu_only +def test_chunk_broker_frees_the_name_when_accept_fails(monkeypatch): + """An owner whose `accept` fails (EMFILE here) closes its listener: a waiter in its backlog + sees the connection reset and owns the key itself at once, instead of waiting out its + timeout on a name nobody accepts on.""" + import errno + import socket + import threading + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + gate = threading.Event() + + def failing_accept(self): + gate.wait(30) + raise OSError(errno.EMFILE, "Too many open files") + + monkeypatch.setattr(socket.socket, "accept", failing_accept) + owner = broker.try_own(key) + got = [] + waiter = threading.Thread(target=lambda: got.append(broker.acquire(key, 60))) + waiter.start() + time.sleep(0.5) # the waiter is connected, in the owner's backlog + start = time.monotonic() + gate.set() + waiter.join(30) + try: + assert got and got[0][0] == broker.OWNER, got + assert time.monotonic() - start < 10 + assert not owner._thread.is_alive() + finally: + owner.close() + for result in got: + result[1].close() + + +@pytest.mark.cpu_only +def test_chunk_broker_reads_a_uid_above_2_31(): + """`SO_PEERCRED` is `struct ucred` (pid_t, uid_t, gid_t): an LDAP-mapped uid of 2**31 or + more reads back unsigned.""" + import struct + + broker = _host_table_module()._broker + uid = 2**31 + 5 + + class _Conn: + def getsockopt(self, level, option, size): + assert size == broker._CRED.size == 12 + return struct.pack("iII", 4242, uid, 2**31 + 7) + + assert broker._peer_uid(_Conn()) == uid + + def _fake_shards(rows_per_shard: int, shards: int, cols: int, seed: int) -> tuple: g = torch.Generator().manual_seed(seed) data = torch.randint(0, 256, (shards * rows_per_shard, cols), generator=g, dtype=torch.uint8) @@ -671,7 +768,8 @@ def _fake_shards(rows_per_shard: int, shards: int, cols: int, seed: int) -> tupl def _host_tables(monkeypatch, mode, world, data, keys, weights, rows_per_shard, identity=None): """`world` ranks' tables in one process: threads, so `node` ranks wait on each other, and a - barrier in place of the server's collective agreement.""" + barrier in place of the server's collective agreement. `weights` is every rank's checkpoint, + or a list of one per rank.""" import threading PH = _host_table_module() @@ -685,7 +783,8 @@ def agree(ok): monkeypatch.setattr(PH, "_all_agree", agree) rows, cols = data.shape - copy, read = PH.shard_rows(weights, keys, rows_per_shard, WT._read) + checkpoints = weights if isinstance(weights, list) else [weights] * world + sources = [PH.shard_rows(w, keys, rows_per_shard, WT._read) for w in checkpoints] tables = [ PH.PleHostTable( mode, @@ -701,7 +800,7 @@ def agree(ok): ) # fmt: skip for r in range(world) ] - threads = [threading.Thread(target=t.fill, args=(copy, read)) for t in tables] + threads = [threading.Thread(target=t.fill, args=sources[r]) for r, t in enumerate(tables)] for t in threads: t.start() for t in threads: @@ -722,7 +821,7 @@ def _vmm_host_numa() -> bool: @pytest.mark.parametrize("mode", ["rank", "server", "node", "node-fallback"]) def test_host_table_modes(mode, monkeypatch): """`rank` holds every row, `server` this rank's rows with its window, `node` every row in - chunks one rank creates and the other attaches to; `node` without a checkpoint identity runs + chunks one rank creates and the other attaches to; `node` without a table identity runs `server`. Each holds exactly the checkpoint's bytes, in host memory.""" import uuid @@ -731,26 +830,214 @@ def test_host_table_modes(mode, monkeypatch): pytest.skip("no CUDA VMM host NUMA allocation on this platform") rows_per_shard, shards, cols = 40960, 8, 160 data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=5) - rows = data.shape[0] want = {"rank": PH.RANK, "server": PH.SERVER, "node": PH.NODE, "node-fallback": PH.SERVER} requested = PH.NODE if mode.startswith("node") else mode identity = uuid.uuid4().hex if mode == "node" else None tables = _host_tables(monkeypatch, requested, 2, data, keys, weights, rows_per_shard, identity) + _check_tables(tables, data, want[mode]) + if mode == "node": + created = sorted(len(t._segments) for t in tables) + assert created == [PH.CHUNKS // 2, PH.CHUNKS // 2], [t.describe() for t in tables] + for t in tables: + t.close() + + +def _check_tables(tables, data, mode) -> None: + """Every rank runs `mode` and holds exactly `data`'s bytes (its rows under `server`), in + host memory.""" + PH = _host_table_module() + rows, cols = data.shape expected = data.cuda() for rank, t in enumerate(tables): - assert t.mode == want[mode], (rank, t.mode, t.describe()) + assert t.mode == mode, (rank, t.mode, t.describe()) assert t.resident_on_host(), t.describe() if t.mode == PH.SERVER: lo, hi, base = t.window - assert (lo, hi) == PH.owned_rows(rank, 2, rows, cols) + assert (lo, hi) == PH.owned_rows(rank, len(tables), rows, cols) got = t.table[lo - base : hi - base] assert torch.equal(got.view(torch.uint8), expected[lo:hi]) assert t.needs_sum else: assert t.window is None and not t.needs_sum assert torch.equal(t.table.view(torch.uint8), expected) - if mode == "node": - created = sorted(len(t._segments) for t in tables) - assert created == [PH.CHUNKS // 2, PH.CHUNKS // 2], [t.describe() for t in tables] - for t in tables: - t.close() + + +def _fallback_warnings(monkeypatch) -> list: + """The warnings `ple_host_table` emits from here on, with none emitted yet this process.""" + PH = _host_table_module() + warnings = [] + monkeypatch.setattr(PH, "_WARNED", set()) + monkeypatch.setattr(PH.logger, "warning", warnings.append) + return warnings + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +def test_host_table_owner_that_cannot_create_a_chunk_hands_it_over(monkeypatch): + """A rank whose chunk allocation fails closes the chunk it just owned: the other rank owns + it at once instead of waiting out the attach timeout, and both run `server`, with one + warning naming the requested mode, the mode that runs and why.""" + import time + import uuid + + PH = _host_table_module() + if not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + monkeypatch.setattr(PH, "_ATTACH_TIMEOUT_S", 60.0) + warnings = _fallback_warnings(monkeypatch) + create, failed = PH._VmmChunks.create, [] + + def create_once(self, k): + if k == 0 and not failed: + failed.append(k) + raise RuntimeError("CUDA driver call failed: out of host memory") + create(self, k) + + monkeypatch.setattr(PH._VmmChunks, "create", create_once) + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=7) + start = time.monotonic() + tables = _host_tables( + monkeypatch, PH.NODE, 2, data, keys, weights, rows_per_shard, uuid.uuid4().hex + ) + elapsed = time.monotonic() - start + try: + assert failed == [0] and "out of host memory" in tables[0]._node_error + assert elapsed < 20, f"the other rank waited {elapsed:.0f} s for the failed chunk" + _check_tables(tables, data, PH.SERVER) + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=node requested, running server" in warnings[0] + finally: + for t in tables: + t.close() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +def test_host_table_refuses_a_chunk_that_differs_from_its_checkpoint(monkeypatch): + """Two ranks whose keys agree but whose checkpoints differ at one row of an attached chunk's + sample: the attaching rank raises on it, and both run `server` from their own bytes.""" + import uuid + + PH = _host_table_module() + if not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + warnings = _fallback_warnings(monkeypatch) + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=9) + other = data.clone() + row = PH.sample_blocks(0, *data.shape, phase=1)[3][0] + other[row] ^= 1 + other_weights = { + k: other[i * rows_per_shard : (i + 1) * rows_per_shard].view(WT.PLE_TABLE_DTYPE) + for i, k in enumerate(keys) + } + tables = _host_tables( + monkeypatch, + PH.NODE, + 2, + data, + keys, + [weights, other_weights], + rows_per_shard, + uuid.uuid4().hex, + ) + try: + # Row `row` is rank 0's (chunk 0): each rank's own rows are its checkpoint's. + _check_tables(tables, data, PH.SERVER) + assert tables[1]._node_error == f"shared chunk 0 differs from the checkpoint at row {row}" + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=node requested, running server" in warnings[0] + finally: + for t in tables: + t.close() + + +@pytest.mark.cpu_only +def test_table_identity_is_the_sampled_rows(): + """The `node` key is the table's bytes at every chunk's phase-0 sample and nothing about + where they were read from; phase 1, the rows an attaching rank compares, is disjoint.""" + PH = _host_table_module() + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, _ = _fake_shards(rows_per_shard, shards, cols, seed=11) + rows = data.shape[0] + + def identity(table) -> str: + parts = { + k: table[i * rows_per_shard : (i + 1) * rows_per_shard] for i, k in enumerate(keys) + } + _, read = PH.shard_rows(parts, keys, rows_per_shard, WT._read) + return PH.table_identity(read, rows, cols, "layer 3") + + def sample(phase: int) -> set: + return { + r + for k in range(PH.CHUNKS) + for lo, hi in PH.sample_blocks(k, rows, cols, phase=phase) + for r in range(lo, hi) + } + + key = identity(data) + assert identity(data.clone()) == key + sampled, checked = sample(0), sample(1) + assert sampled and checked and not sampled & checked + assert all(0 <= r < rows for r in sampled | checked) + changed = data.clone() + changed[max(sampled)] ^= 1 + assert identity(changed) != key + # The key hashes its sample only; the attaching rank's check covers this row. + changed = data.clone() + changed[min(checked)] ^= 1 + assert identity(changed) == key + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "ids, rows_per_shard, error", + [ + (800, 208, None), + (800, 256, "Shards padded one by one"), + (800, 200, "not padded to a multiple"), + (900, 208, "address 900"), + ], + ids=["padded-once", "padded-per-shard", "unpadded", "short"], +) +def test_ngram_table_height_matches_the_id_space(ids, rows_per_shard, error): + """The table's height is the id space its offsets and sizes address, padded to the divisor by + less than one divisor; otherwise the declaration raises.""" + d = SimpleNamespace( + ple_layer=3, ple_shards=4, ple_head_dim=4, ngram_heads=2, ple_embed_dim=8, + ple_table_dtype=WT.PLE_TABLE_DTYPE, ngram_vocab_divisor=64, + ) # fmt: skip + prefix = f"{WT._LM}.layers.3.ple.ple_embedding" + weights = { + key: torch.zeros(rows_per_shard, 4, dtype=torch.uint8).view(WT.PLE_TABLE_DTYPE) + for key in WT.ngram_shard_keys(d) + } + weights[f"{prefix}.ngram_heads_offsets"] = torch.tensor([0, ids // 2]) + weights[f"{prefix}.ngram_heads_vocab_sizes"] = torch.tensor([ids // 2, ids - ids // 2]) + core = SimpleNamespace(w={"embed": torch.empty(0)}) + if error is None: + assert WT._declare_ngram_table(core, d, weights) == rows_per_shard + assert tuple(core.w["ple_table"].shape) == (4 * rows_per_shard, 4) + else: + with pytest.raises(ValueError, match=error): + WT._declare_ngram_table(core, d, weights) + + +@pytest.mark.cpu_only +def test_host_table_warns_once_when_server_runs_as_rank(monkeypatch): + """`server` under attention DP keeps a copy per rank: one warning per process says so.""" + PH = _host_table_module() + warnings = _fallback_warnings(monkeypatch) + + def private(rows, cols, dtype, device): + return torch.empty(rows, cols, dtype=dtype), None + + for _ in range(2): + t = PH.PleHostTable( + PH.SERVER, 64, 4, WT.PLE_TABLE_DTYPE, torch.device("cuda"), + rank=0, world=2, dp=2, private=private, + ) # fmt: skip + assert t.mode == PH.RANK + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=server requested, running rank" in warnings[0] + assert "attention DP 2" in warnings[0] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py index 48b65e311d00..eae8d4159190 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py @@ -15,12 +15,12 @@ import torch from transformers import PretrainedConfig -from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import routing +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 modeling as M, ) +from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm.llmapi.llm_args import ( KvCacheConfig, MoeConfig, @@ -74,6 +74,7 @@ def _on_sm103(monkeypatch): monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + monkeypatch.delenv(ple_host_table.PLE_HOST_TABLE_ENV, raising=False) def _llm_args(mode="auto", **kwargs) -> TorchLlmArgs: @@ -535,3 +536,13 @@ def test_mtp_needs_the_cached_replay(monkeypatch, max_draft_len): _claimed(_model_config(**_LAYOUTS["tep4"])) monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY") _claimed(config) + + +def test_a_misspelled_ple_host_table_mode_fails_at_routing(monkeypatch): + """The n-gram table's storage mode is read at routing, so a misspelled value fails there, + under `auto` too, rather than at the build.""" + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "nodes") + with pytest.raises(ValueError, match=f"{ple_host_table.PLE_HOST_TABLE_ENV}='nodes': expected"): + modeling_v2_resolve(_model_config(**_LAYOUTS["tep4"]), _llm_args("auto")) + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "Node") + _claimed(_model_config(**_LAYOUTS["tep4"])) From 5dd2270283a5d39290ef2453a978497298f46a1e Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:07:31 -0800 Subject: [PATCH 35/51] [None][fix] modeling_v2 qwen4_exp: acquire at cluster scope on the FP8 hc decode rows mailbox wait Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../hc/qwen4_exp_hc_decode_rows_tc__decode.py | 18 ++- ...ing_v2_qwen4_exp_hc_decode_rows_tc_sync.py | 114 +++++++++++------- 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py index 11717c851d94..7e8f07f0ba72 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py @@ -29,6 +29,7 @@ from cutlass.cute.nvgpu import cpasync, warp from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.experimental import primitives as prims from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( @@ -94,6 +95,7 @@ def _mbar_init_tx(mbar, nbytes, *, loc=None, ip=None): @dsl_user_op def _mbar_wait0(mbar, *, loc=None, ip=None): + """Wait (CTA scope) for phase 0 of an mbarrier that this CTA's own bulk copies complete.""" _asm( T.i32(), [cutlass.Int32(mbar)], @@ -254,8 +256,9 @@ def kernel( sBar = smem.allocate_tensor(cutlass.Int64, cute.make_layout((2,)), byte_alignment=8) rbar = cutlass.Int32(sBar.iterator.toint()) wbar = rbar + 8 # the weight rows' bulk copies + rbar_ptr = sBar.iterator # pointer form for the cluster-scope wait below # This CTA receives rows [me * RPO, me * RPO + RPO) of the cluster's partials from all CL - # CTAs, by st.async on `rbar` (no cluster barrier, no fence, after the pushes). + # CTAs, by st.async on `rbar` (no cluster barrier after the pushes). rows_me = cutlass.max(cutlass.min(cutlass.Int32(RPO), rows - me * RPO), cutlass.Int32(0)) if tidx == 0: _mbar_init_tx(rbar, rows_me * (CL * (NC + 1) * 4)) @@ -457,7 +460,18 @@ def kernel( v = sSsr[rr] dst = red_addr + ((me * RPO + rl) * rld + nn) * 4 _st_async_f32(_mapa(dst, o), v, _mapa(rbar, o)) - _mbar_wait0(rbar) + # The cluster's partials for this rank's rows have landed: the peers pushed them by + # st.async, whose complete-tx releases at cluster scope, so a relaxed cluster-scope wait + # followed by an acquire fence restricted to shared::cluster is the ISA's acquire pattern + # for it (and, unlike .acquire.cluster on the wait, costs no L1 invalidate). + while not prims.mbarrier_try_wait_parity( + rbar_ptr, + 0, + scope=prims.MBarrierScope.CLUSTER, + order=prims.MemOrder.RELAXED, + ): + pass + prims.fence_sync_restrict(prims.MemOrder.ACQUIRE) # 5. Rank `me` finishes rows (2 me, 2 me + 1): rstd per stream, the logits, low / injection. for it in cutlass.range_constexpr(-(-(RPO * NC) // THREADS)): diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py index 3ea63f29bd8e..9136af1e7868 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py @@ -1,17 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Source-level check of the decode hyper-connection kernel's two mailbox waits. - -``_First2`` receives two kinds of data from the peer CTAs of its cluster through shared-memory -mailboxes credited with ``st.async``: the K half's block rows (``bbar``, folded sites only) and -the cluster's row partials (``rbar``). ``st.async``'s complete-tx releases at cluster scope, so -the receiver's mailbox reads must be ordered by a cluster-scope acquire; ``cute.arch.mbarrier_wait`` -acquires at CTA scope only. The kernel uses a relaxed cluster-scope try-wait followed by -``fence.acquire.sync_restrict::shared::cluster.cluster`` at both waits, which orders the reads -without the L1 invalidation an acquire-qualified try-wait incurs. An accuracy test cannot pin -this down (the payload and its completion are not reordered on current hardware), so this test -scans the kernel's source and fails if either ordering is changed back. No GPU, no CUDA and no -cutlass import are needed. +"""Source-level check of the decode hyper-connection kernels' mailbox waits. + +``_First2`` receives data from the peer CTAs of its cluster through shared-memory mailboxes +credited with ``st.async``: the family kernel receives the K half's block rows (``bbar``, folded +sites only) and the cluster's row partials (``rbar``); the FP8 target's copy receives the row +partials (``rbar``). ``st.async``'s complete-tx releases at cluster scope, so the receiver's +mailbox reads must be ordered by a cluster-scope acquire; ``cute.arch.mbarrier_wait`` and an +inline ``mbarrier.try_wait`` without a scope acquire at CTA scope only. Both kernels use a relaxed +cluster-scope try-wait followed by ``fence.acquire.sync_restrict::shared::cluster.cluster`` at +every mailbox wait, which orders the reads without the L1 invalidation an acquire-qualified +try-wait incurs. An accuracy test cannot pin this down (the payload and its completion are not +reordered on current hardware), so this test scans the kernels' sources and fails if either +ordering is changed back. No GPU, no CUDA and no cutlass import are needed. """ import ast @@ -21,30 +22,45 @@ import tensorrt_llm -_KERNEL = ( - Path(tensorrt_llm.__file__).parent - / "_torch" - / "modeling_v2" - / "models" - / "qwen4_exp" +_QWEN4_EXP = Path(tensorrt_llm.__file__).parent.joinpath( + "_torch", "_experimental", "modeling_v2", "models", "qwen4_exp" +) +_FAMILY = _QWEN4_EXP / "ops" / "hc" / "qwen4_exp_hc_decode_rows_tc.py" +_FP8_TARGET = ( + _QWEN4_EXP + / "qwen3_8_flash_next_fp8__sm_103__unified" / "ops" / "hc" - / "qwen4_exp_hc_decode_rows_tc.py" + / "qwen4_exp_hc_decode_rows_tc__decode.py" ) _WAIT = "prims.mbarrier_try_wait_parity" _FENCE = "prims.fence_sync_restrict" # (barrier pointer, barrier address, scatter helper, mailbox tensor read after the wait) -_MAILBOXES = [ - ("bbar_ptr", "bbar", "_st_async_v4", "sblk_vec"), - ("rbar_ptr", "rbar", "_st_async_f32", "sRed"), +_RBAR = ("rbar_ptr", "rbar", "_st_async_f32", "sRed") +_BBAR = ("bbar_ptr", "bbar", "_st_async_v4", "sblk_vec") + +# kernel -> (source file, its mailboxes, the barriers an inline CTA-scope wait may take). The FP8 +# target's weight-row barrier (``wbar``) completes on this CTA's own bulk copies from global +# memory, for which a CTA-scope acquire is enough. +_KERNELS = { + "family": (_FAMILY, [_BBAR, _RBAR], set()), + "fp8_target": (_FP8_TARGET, [_RBAR], {"wbar"}), +} +_CASES = [ + pytest.param(name, mailbox, id=f"{name}-{mailbox[1]}") + for name, (_, mailboxes, _) in _KERNELS.items() + for mailbox in mailboxes ] @pytest.fixture(scope="module") -def module() -> ast.Module: - """The kernel module's AST, parsed once from the installed source file.""" - return ast.parse(_KERNEL.read_text(), filename=str(_KERNEL)) +def modules() -> dict[str, ast.Module]: + """Each kernel module's AST, parsed once from the installed source file.""" + return { + name: ast.parse(path.read_text(), filename=str(path)) + for name, (path, _, _) in _KERNELS.items() + } def _method(module: ast.Module, cls: str, name: str) -> ast.FunctionDef: @@ -54,7 +70,7 @@ def _method(module: ast.Module, cls: str, name: str) -> ast.FunctionDef: for item in node.body: if isinstance(item, ast.FunctionDef) and item.name == name: return item - raise AssertionError(f"{cls}.{name} not found in {_KERNEL.name}") + raise AssertionError(f"{cls}.{name} not found") def _calls(node: ast.AST) -> list[ast.Call]: @@ -79,10 +95,28 @@ def _only(calls: list[ast.Call], what: str) -> ast.Call: return calls[0] -def test_no_cta_scope_wait_on_the_mailbox_barriers(module) -> None: +def _inline_waits(module: ast.Module) -> set[str]: + """The module-level helpers whose inline PTX is an ``mbarrier.try_wait``.""" + return { + node.name + for node in module.body + if isinstance(node, ast.FunctionDef) + and any( + isinstance(s, ast.Constant) + and isinstance(s.value, str) + and "mbarrier.try_wait" in s.value + for s in ast.walk(node) + ) + } + + +@pytest.mark.parametrize("name", list(_KERNELS)) +def test_no_cta_scope_wait_on_the_mailbox_barriers(modules, name: str) -> None: """Neither the DSL's CTA-scope waits nor an inline CTA-scope try-wait touch the mailbox barriers.""" + _, mailboxes, inline_ok = _KERNELS[name] + module = modules[name] kernel = _method(module, "_First2", "kernel") - barriers = {name for mailbox in _MAILBOXES for name in mailbox[:2]} + barriers = {n for mailbox in mailboxes for n in mailbox[:2]} cta_scope_waits = [ c for c in _calls(kernel) @@ -92,22 +126,20 @@ def test_no_cta_scope_wait_on_the_mailbox_barriers(module) -> None: assert not cta_scope_waits, "a mailbox wait must not acquire at CTA scope:\n" + "\n".join( f" line {c.lineno}: {ast.unparse(c)}" for c in cta_scope_waits ) - inline_waits = [ - s.value - for s in ast.walk(module) - if isinstance(s, ast.Constant) - and isinstance(s.value, str) - and "mbarrier.try_wait" in s.value - ] - assert not inline_waits, f"no inline-PTX mbarrier wait is expected any more: {inline_waits}" + helpers = _inline_waits(module) + inline_calls = [c for c in _calls(module) if _callee(c) in helpers] + unexpected = [c for c in inline_calls if _first_arg(c) not in inline_ok] + assert not unexpected, ( + f"an inline CTA-scope mbarrier wait may take only {sorted(inline_ok)}:\n" + + "\n".join(f" line {c.lineno}: {ast.unparse(c)}" for c in unexpected) + ) -@pytest.mark.parametrize(("barrier", "address", "scatter", "mailbox"), _MAILBOXES) -def test_mailbox_wait_acquires_at_cluster_scope( - module, barrier: str, address: str, scatter: str, mailbox: str -) -> None: +@pytest.mark.parametrize(("name", "mailbox_case"), _CASES) +def test_mailbox_wait_acquires_at_cluster_scope(modules, name: str, mailbox_case: tuple) -> None: """A relaxed cluster-scope wait plus a cluster acquire fence, after the pushes and before the reads.""" - kernel = _method(module, "_First2", "kernel") + barrier, address, scatter, mailbox = mailbox_case + kernel = _method(modules[name], "_First2", "kernel") calls = _calls(kernel) wait = _only( From 48e73391f9af1e88cd354889d49ca486f3a5386a Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:27:55 -0800 Subject: [PATCH 36/51] [None][fix] modeling_v2 qwen4_exp: run the FP8 low-M projections and the attention-DP decode router in graphed steps only Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 63 ++++++++++++------- .../test_attention_ops.py | 1 + .../test_routing_and_layout.py | 50 +++++++++++++++ 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 0a045ff290d2..fb961fe30bc2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1124,9 +1124,12 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self._binds = 0 # Set by `begin_step`, cleared by `forward`. self._step_begun = False - # Whether the step `begin_step` saw carries context rows: its draft steps after the - # first run eagerly on `DecodeTarget`, at any batch size (`DecodeTarget.mtp_forward`). - self._context_step = False + # Whether the step `begin_step` saw is a generation-only step inside a decode CUDA graph, + # which attention-DP ranks agree on (the engine captures only when every rank can). Only + # such a step runs `_decode_projection` (`_step_projection`) and, under attention DP, the + # decode router (`_routed_experts`); an eager step can run at any batch size and meet + # another rank's prefill step. + self._decode_graph_step = False self._mtp: "_MtpWeights | None" = None self._mtp_cos_sin: torch.Tensor | None = None # The verify step's side-cache rollback, registered with the spec worker. @@ -1707,7 +1710,7 @@ def begin_step( target.prepare(attn_metadata, input_ids, position_ids, vision, spec_metadata) else: assert vision is None, "a generation-only step carries vision rows" - self._context_step = attn_metadata.num_contexts > 0 + self._decode_graph_step = attn_metadata.is_cuda_graph and attn_metadata.num_contexts == 0 self._step_begun = True def forward( @@ -2164,7 +2167,7 @@ def _mtp_front(core, M: _MtpWeights, embed_tokens, input_ids, hidden_states, mm= `fc_embedding` of the normalized next-token embedding is added to every stream of `fc_hidden` as the trunk site's unit-weight injection; returns `(x, mixed, inject)`. `mm` runs dense projections here and in every draft-layer and MoE helper: `cublas_mm` when - None, `_decode_projection` from `DecodeTarget` (except after a step's context rows).""" + None, `_step_projection` from `DecodeTarget`.""" mm = mm or cublas_mm rows = hidden_states.shape[0] assert rows <= M.unit_inject.shape[0], ( @@ -2451,10 +2454,12 @@ def _routed_experts( `pad_mask` (`[T, 1]` bool) routes a prefill graph bucket's padding rows to expert -1, so they come back zero instead of all landing on the first window's ten lowest experts.""" mm = mm or cublas_mm - if getattr(core, "dp_size", 1) > 1: - # Attention DP routes the same gathered rows on every rank, whose steps may pick another - # `mm`: `_decode_projection`'s low-M kernels and cuBLAS differ in the last bits, and a - # reordered top-k is a different MoE sum. One router kernel on every rank. + if mm is _decode_projection and getattr(core, "dp_size", 1) > 1 and not core._decode_graph_step: + # Attention DP routes the same gathered rows on every rank, so every rank must compute + # their logits with one kernel: `_decode_projection`'s low-M kernels and cuBLAS differ in + # the last bits, and a reordered top-k is a different MoE sum. A graphed step is a decode + # step on every rank and keeps the decode router; an eager one can meet another rank's + # prefill step and takes the prefill router, `cublas_mm`. mm = cublas_mm rows = x.shape[0] if pad_mask is None and mm is _decode_projection and 0 < rows <= _MOE_FRONT_MAX_ROWS: @@ -2733,13 +2738,22 @@ def _draft_pad_rows(core, all_rank_num_tokens, rows: int) -> _DpPadding | None: def _decode_projection(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor: - """Dense bf16 projection for generation-only steps and their draft layer (graph-captured). + """Dense bf16 projection for graphed generation-only steps and their draft layer. `low_m_bf16_mm` picks FlashInfer's low-M PDL kernels (`_DECODE_PROJ_PDL`) where its table names the shape, else cuBLAS; they compile on the engine's warmup forward before capture.""" return low_m_bf16_mm(mat_a, mat_b, _DECODE_PROJ_PDL) +def _step_projection(core): + """The dense projection of a `DecodeTarget` step: `_decode_projection` in a graphed step + (`core._decode_graph_step`), else `cublas_mm`. An eager generation-only step runs at any + batch size (under attention DP whenever another rank has context rows), and the FlashInfer + kernels compile on the first call at each shape: only captured batch sizes compile before + serving.""" + return _decode_projection if core._decode_graph_step else cublas_mm + + def _hc_first(core, normed, first, with_inject): """Run a hyper-connection mixer's first half: first projection, then SiLU and injection. @@ -5797,7 +5811,8 @@ def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, sta sparse, ) = qsa num_tokens = x.shape[0] - proj = _decode_projection(x, L.attn_in) + mm = _step_projection(core) + proj = mm(x, L.attn_in) qkv_view, gate, _ = torch.split( proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1 ) @@ -5881,7 +5896,7 @@ def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, sta **_CALL_CONSTANTS, ) fused_sigmoid_mul_inplace(out, gate, direct_launch=True) - return _tp_partial(core, _decode_projection(out, L.o)) + return _tp_partial(core, mm(out, L.o)) def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: """One Gated-DeltaNet layer over a generation-only batch: one GEMM for the four input @@ -5890,8 +5905,12 @@ def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: step (`verify`, `gen_p` rows a sequence) takes `_gdn_verify` instead.""" core = self.core pools = core._bound.pools + mm = _step_projection(core) + # The fused launch is FlashInfer's warp split-K kernel: graphed steps only, as `mm`. tactic = ( - _gdn_in_proj_conv_tactic(core, x, L, si_all, verify) if verify is not None else None + _gdn_in_proj_conv_tactic(core, x, L, si_all, verify) + if verify is not None and mm is _decode_projection + else None ) mq_out = None if tactic is not None: @@ -5912,7 +5931,7 @@ def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: proj = full[:, : sum(core.in_proj_split)] else: # Its alignment columns, past `sum(in_proj_split)`, are dropped by the view. - proj = _decode_projection(x, L.in_proj_dec)[:, : sum(core.in_proj_split)] + proj = mm(x, L.in_proj_dec)[:, : sum(core.in_proj_split)] if verify is not None: mixed_qkv, z, a, b = torch.split(proj, core.in_proj_split, 1) num_tokens = x.shape[0] @@ -5934,7 +5953,7 @@ def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: if fold_norm: # The replay returned the normed, gated rows. o = torch.reshape(out, [num_tokens, core.value_dim]) - return _tp_partial(core, _decode_projection(o, L.out)) + return _tp_partial(core, mm(o, L.out)) return self._gdn_epilogue(out, z, L, num_tokens) mixed_qkv = torch.split(proj, core.in_proj_split, 1)[0] causal_conv1d_update( @@ -5959,7 +5978,7 @@ def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: core.num_k_heads, core.head_k_dim, ) - return _tp_partial(core, _decode_projection(o, L.out)) + return _tp_partial(core, mm(o, L.out)) def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: """A verify step's gated RMS norm (sigmoid gate, plain weight) and output projection.""" @@ -5974,7 +5993,7 @@ def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: direct_launch=True, ) o = torch.reshape(o, [num_tokens, core.value_dim]) - return _tp_partial(core, _decode_projection(o, L.out)) + return _tp_partial(core, _step_projection(core)(o, L.out)) def _ple_front(self, input_ids, ple_state, stage) -> tuple: """The PLE part that reads no residual: verify staging of both pools (before the n-gram @@ -6212,7 +6231,7 @@ def forward( core, hc, block_out, inject, L.mhc_norm, L.mhc_first, L.mhc_up, True ) routed, gate_logits, shared = _moe_block( - core, mixed, L, pad_rows, mm=_decode_projection, defer=True + core, mixed, L, pad_rows, mm=_step_projection(core), defer=True ) moe = (routed, inject, shared, gate_logits) del routed, gate_logits, shared @@ -6268,10 +6287,8 @@ def mtp_forward( """One draft step over generation rows only (`p` rows a request on the first step of a batch without context, one otherwise), inside the decode graph when the trunk is. - The steps after the first of a step with context rows (`core._context_step`) run - eagerly at any batch size, so their projections take `cublas_mm`: `_decode_projection`'s - FlashInfer kernels compile on the first call at each shape, and only captured batch - sizes compile before serving. + Its projections take `_step_projection`: the steps after the first of a step with + context rows, and every step of an eager trunk step, run eagerly at any batch size. Under IndexShare the draft-extend step captures each request's selection and later steps skip the index branch, rebuilding it with `qsa_mtp_reuse_topk`; the index keys they skip @@ -6310,7 +6327,7 @@ def mtp_forward( logical = md.qsa_logical_positions[:rows] block_table = md.qsa_block_table tokens_per_block = md.tokens_per_block - mm = cublas_mm if core._context_step else _decode_projection + mm = _step_projection(core) x, mixed, inject = _mtp_front(core, M, embed_tokens, input_ids, hidden_states, mm=mm) qkv, gate, q_index = _mtp_attention_planes( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index b49079d4e128..4894363619e6 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1563,6 +1563,7 @@ def _prl_core(hq, hk): c.index_col = c.qkv_width + c.q_width c._cos_sin = M.Qwen4ExpUnifiedModelingV2._index_rope_table(c, c.theta, PRL_DEV) c.tp_sum = lambda x: x + c._decode_graph_step = True # the trunk sites run as in a captured decode step return c diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index c5f59d83b4fb..f03fdda54ccd 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -835,6 +835,56 @@ def grouped(xq, xsf, ids, w, *weights): lo += n +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "dp_size, graphed, want", + [(1, True, "front"), (1, False, "front"), (2, True, "front"), (4, False, "cublas")], + ids=["tp-graphed", "tp-eager", "adp-graphed", "adp-eager"], +) +def test_a_decode_router_under_attention_dp_follows_the_graphed_step( + dp_size, graphed, want, monkeypatch +) -> None: + """Under attention DP a graphed decode step keeps the single-launch decode router (every rank + runs the same graph) and an eager one takes cuBLAS, the router of another rank's prefill.""" + core = SimpleNamespace( + topk=TOPK, + num_experts=NUM_EXPERTS, + moe_inter=640, + dp_size=dp_size, + moe_max_rows=8192, + _decode_graph_step=graphed, + ) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384, 128) + x = torch.zeros(4, 8) + seen = [] + + def front(core_, rows, router, experts_): + seen.append("front") + return rows + + def cublas(chunk, mat_b): + seen.append("cublas") + return chunk[:, :1] + + monkeypatch.setattr(M, "_routed_experts_front", front) + monkeypatch.setattr(M, "cublas_mm", cublas) + monkeypatch.setattr( + M, + "renorm_moe_routing_op", + lambda logits, topk, output_dtype: ( + logits.int().expand(-1, topk).contiguous(), + logits.expand(-1, topk).contiguous(), + ), + ) + monkeypatch.setattr( + M, "fp8_quantize_1x128", lambda chunk, use_ue8m0: (chunk, chunk[:, :2].t().contiguous()) + ) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", lambda **kw: kw["hidden_states"]) + M._routed_experts(core, x, "router", experts, mm=M._decode_projection) + assert seen == [want] + assert M._step_projection(core) is (M._decode_projection if graphed else cublas) + + def _pad(x: torch.Tensor, rows: int) -> torch.Tensor: return torch.cat([x, x.new_zeros(rows - x.shape[0], x.shape[1])], 0) From fd37f37e50cafeebf6bdcf56f6f46325fc178655 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:28:05 -0800 Subject: [PATCH 37/51] [None][fix] modeling_v2 qwen4_exp: compile every split-K decode plan on a geometry's first call Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 4 +- .../qsa_sparse_gqa_splitk_cute__decode.py | 164 +++++++++++------- .../test_attention_ops.py | 57 ++++++ 3 files changed, 161 insertions(+), 64 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index fb961fe30bc2..4c5ac0e96f45 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -5559,10 +5559,12 @@ def _draft_extend_query_rows(self, md, draft, qkv, q_index, num_ctx: int, rows: md.qsa_topk_row_starts[:rows].index_select(0, query), selected, ) + # q keeps the packed rows' strides (gathered whole), so the call shares the draft decode + # step's split-K geometry and the kernels its warmup compiled. out = _sparse_gqa( core, draft.kv, - _packed_q(core, qkv).index_select(0, query), + _packed_q(core, qkv.index_select(0, query)), md.qsa_block_table, selected, req_idx, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py index 887f2bb0d45a..296c9785387d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py @@ -1110,6 +1110,8 @@ def kernel( # --------------------------------------------------------------------------------------- _COMPILED: dict = {} +#: (geometry, gated, gate tensor rank) forms whose every reachable split plan is compiled. +_PLANNED: set = set() #: Splits by (rows x KV heads), per pool element size, measured at the routed geometries #: (`MAX_ROWS`): one tile a CTA while the grid is under about a wave, then fewer, longer splits; @@ -1149,6 +1151,39 @@ def plan_splits(rows: int, kv_heads: int, width: int, elem: int) -> tuple[int, i return -(-n_tiles // tps), tps +def reachable_plans(max_rows: int, kv_heads: int, width: int, elem: int) -> list: + """Every `(splits, tiles per split)` `plan_splits` returns for 1..max_rows rows.""" + return sorted({plan_splits(r, kv_heads, width, elem) for r in range(1, max_rows + 1)}) + + +def _compile_split(geometry: tuple, splits: int, tps: int, gated: bool, kern_kw: dict, args): + """The split pass of `geometry` at plan `(splits, tps)` (`gated`: the one-split pass stores + the gated output), compiled once; `args()` builds the compile arguments.""" + key = geometry + (splits, tps, gated) + fn = _COMPILED.get(key) + if fn is None: + kern = _SplitKernel(splits=splits, tps=tps, gated=gated, **kern_kw) + fn = cute.compile(kern, *args(), options="--opt-level 2 --enable-tvm-ffi") + _COMPILED[key] = fn + return fn + + +def _compile_merge(heads: int, splits: int, gated: bool, pdl: bool, gdim: int, args): + """The merge of `splits` partials, compiled once per gate tensor rank `gdim` (the trunk's + `[rows, Hq * D]` gate, the draft layer's `[rows, Hq, D]` view, q when ungated); `args()` + builds the compile arguments.""" + key = ("merge", heads, splits, gated, gdim, pdl) + fn = _COMPILED.get(key) + if fn is None: + fn = cute.compile( + _MergeKernel(heads=heads, splits=splits, gated=gated, pdl=pdl), + *args(), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + def _ptr(t: torch.Tensor, dtype, align=16): return make_ptr(dtype, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) @@ -1193,54 +1228,48 @@ def sparse_gqa_decode( else: assert output_gate.shape == (rows, num_q_heads * head_dim) req64 = request_indices.dtype == torch.int64 - key = ( + if n_splits > 1: + part = torch.empty( + (n_splits, rows, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_splits, rows, num_q_heads), dtype=torch.float32, device=q.device) + o_t, l_t = part, lse + else: + o_t, l_t = out, out + gate_t = output_gate if gated else q + gate_stride = output_gate.stride(0) if gated else 0 + # The compile key without the split plan (rows pick it) and the one-split gate. + geometry = ( elem, group, num_kv_heads, width, tpb, - n_splits, - tps, float(softmax_scale), num_pages, tuple(block_table.shape), tuple(k_cache.stride()[:3]), tuple(q.stride()[:2]), req64, - gated and n_splits == 1, use_pdl, ) - if n_splits > 1: - part = torch.empty( - (n_splits, rows, num_q_heads, head_dim), dtype=torch.float32, device=q.device - ) - lse = torch.empty((n_splits, rows, num_q_heads), dtype=torch.float32, device=q.device) - o_t, l_t = part, lse - else: - o_t, l_t = out, out - gate_t = output_gate if gated else q - gate_stride = output_gate.stride(0) if gated else 0 - fn = _COMPILED.get(key) - if fn is None: - kern = _SplitKernel( - elem=elem, - group=group, - kv_heads=num_kv_heads, - width=width, - tpb=tpb, - splits=n_splits, - tps=tps, - scale=float(softmax_scale), - num_pages=num_pages, - bt_width=block_table.shape[1], - num_requests=block_table.shape[0], - kv_strides=tuple(k_cache.stride()[:3]), - q_strides=tuple(q.stride()[:2]), - gated=gated and n_splits == 1, - pdl=use_pdl, - ) - fn = cute.compile( - kern, + kern_kw = dict( + elem=elem, + group=group, + kv_heads=num_kv_heads, + width=width, + tpb=tpb, + scale=float(softmax_scale), + num_pages=num_pages, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + kv_strides=tuple(k_cache.stride()[:3]), + q_strides=tuple(q.stride()[:2]), + pdl=use_pdl, + ) + + def split_args(): + return ( from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), _ptr(q, cutlass.BFloat16), _ptr(k_cache, cutlass.Uint8), @@ -1255,9 +1284,41 @@ def sparse_gqa_decode( cutlass.Int32(gate_stride), cutlass.Int32(gate_hs), cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), - options="--opt-level 2 --enable-tvm-ffi", ) - _COMPILED[key] = fn + + def merge_args(gate, stride): + # The partials' pointers only type the compile (a one-split call has none: `out`'s). + return lambda: ( + from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=gate.dim() - 1), + _ptr(o_t, cutlass.Float32), + _ptr(l_t, cutlass.Float32, 4), + _ptr(out, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + + # The plan depends on the call's row count: the first call of a geometry in a gate form + # compiles the split pass and merge of every plan its rows can reach (`MAX_ROWS`), so no + # serving step compiles one; a gated call also compiles the ungated flavours, which a mixed + # step's generation rows (and a draft-extend step's) call at the geometry a decode step + # calls gated. The engine's warmup runs each captured decode batch eagerly first. + form = geometry + (gated, gate_t.dim()) + if form not in _PLANNED: + _PLANNED.add(form) + cap = max(rows, MAX_ROWS.get((num_q_heads, num_kv_heads, k_cache.dtype), 0)) + for splits, per in reachable_plans(cap, num_kv_heads, width, elem): + _compile_split(geometry, splits, per, gated and splits == 1, kern_kw, split_args) + if splits == 1: + if gated: + _compile_split(geometry, splits, per, False, kern_kw, split_args) + continue + gm = merge_args(gate_t, gate_stride) + _compile_merge(num_q_heads, splits, gated, use_pdl, gate_t.dim(), gm) + if gated: + _compile_merge(num_q_heads, splits, False, use_pdl, q.dim(), merge_args(q, 0)) + fn = _compile_split(geometry, n_splits, tps, gated and n_splits == 1, kern_kw, split_args) fn( selected_tokens, q.data_ptr(), @@ -1275,32 +1336,9 @@ def sparse_gqa_decode( ) if n_splits == 1: return out - - def _compile_merge(merge_gated, gate, stride): - return cute.compile( - _MergeKernel(heads=num_q_heads, splits=n_splits, gated=merge_gated, pdl=use_pdl), - from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=gate.dim() - 1), - _ptr(part, cutlass.Float32), - _ptr(lse, cutlass.Float32, 4), - _ptr(out, cutlass.BFloat16), - cutlass.Int32(rows), - cutlass.Int32(stride), - cutlass.Int32(gate_hs), - cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), - options="--opt-level 2 --enable-tvm-ffi", - ) - - mkey = ("merge", num_q_heads, n_splits, gated, gate_t.dim(), use_pdl) - mfn = _COMPILED.get(mkey) - if mfn is None: - mfn = _compile_merge(gated, gate_t, gate_stride) - _COMPILED[mkey] = mfn - # A mixed step's generation rows (and a draft-extend step's) call the merge ungated at - # the geometry a decode step calls it gated: compile that flavour now as well, so it - # never compiles inside a served step. Ungated, the gate operand is `q` (3-D). - ukey = ("merge", num_q_heads, n_splits, False, q.dim(), use_pdl) - if gated and ukey not in _COMPILED: - _COMPILED[ukey] = _compile_merge(False, q, 0) + mfn = _compile_merge( + num_q_heads, n_splits, gated, use_pdl, gate_t.dim(), merge_args(gate_t, gate_stride) + ) mfn(gate_t, part.data_ptr(), lse.data_ptr(), out.data_ptr(), rows, gate_stride, gate_hs) return out diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 4894363619e6..3dea0e43d24b 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -1157,6 +1157,63 @@ def _check_sparse_gqa_splitk(cell, monkeypatch) -> None: _replay_in_a_graph(_sgc_op, args, lambda: _refill(args, seed + 1, _rolled_selection), out) +@pytest.mark.parametrize( + "spec", + [ + pytest.param( + mod._verify(dict(mod._TEP2, slots=128), 128, last=30000, step=150, **mod._TEP2_ROW), + id="tep2", + ), + pytest.param(mod._verify(mod._DEP4, 32, last=60000, step=1700, **mod._DEP4_ROW), id="dep4"), + pytest.param(mod._verify(mod._TEP4, 4, **mod._TEP4_ROW), id="tep4"), + ], +) +def test_sparse_gqa_splitk_first_decode_call_compiles_every_plan(spec, monkeypatch) -> None: + """A geometry's first gated decode call compiles the split pass and both merge flavours of + every plan its row counts reach (`MAX_ROWS`). A mixed step's ungated generation rows then + compile nothing at any plan (at tp2/tep2, 164..256 and 324..512 rows take three splits), + nor do gated calls, and the results stay within the reference band.""" + monkeypatch.setattr(mod, "_COMPILED", {}) + monkeypatch.setattr(mod, "_PLANNED", set()) + args = _sgc_build(spec, seed=4242) + q, k = args["q"], args["k_cache"] + cap = mod.MAX_ROWS[(q.shape[1], k.shape[1], k.dtype)] + assert q.shape[0] == cap, "the build covers every routed row count" + width = args["selected_tokens"].shape[1] + elem = 1 if k.dtype == torch.float8_e4m3fn else 2 + sliced = ("q", "selected_tokens", "request_indices", "output_gate") + + def call(rows, gated): + a = {n: (v[:rows] if n in sliced else v) for n, v in args.items()} + if not gated: + del a["output_gate"] + return a, _sgc_op(**a) + + call(4, True) # the decode warmup's first call of the geometry + compiled = set(mod._COMPILED) + by_plan = {} + for r in range(1, cap + 1): + by_plan.setdefault(mod.plan_splits(r, k.shape[1], width, elem), []).append(r) + assert sorted(by_plan) == mod.reachable_plans(cap, k.shape[1], width, elem) + pdl = bool(mod.get_env_enable_pdl()) + for splits, _ in by_plan: + if splits > 1: + assert ("merge", q.shape[1], splits, False, 3, pdl) in compiled, splits + assert ("merge", q.shape[1], splits, True, 2, pdl) in compiled, splits + # Each plan's first and last row count; at tp2/tep2 also the mixed steps' three-split ranges. + picks = {r for rows in by_plan.values() for r in (rows[0], rows[-1])} + if cap == 512: + extra = (164, 256, 324, 512) + assert {mod.plan_splits(r, 1, width, elem)[0] for r in extra} == {3} + picks |= set(extra) + for r in sorted(picks): + for gated in (False, True): + a, out = call(r, gated) + assert set(mod._COMPILED) == compiled, f"{r} rows, gated={gated} compiled" + if r == by_plan[mod.plan_splits(r, k.shape[1], width, elem)][-1]: + _gqa_compare(out, _sgc_op.reference(**a)) # each plan's widest call + + def _sgg_selection(spec, seed): """Rows of a request select from its block scores plus `overlap` x noise (None: independent).""" cr, token_topk, spread = spec["compress_ratio"], spec["token_topk"], spec.get("overlap") From c37741438801f5eaaf82ce1b7242180fb040554f Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:28:07 -0800 Subject: [PATCH 38/51] [None][fix] modeling_v2 qwen4_exp: precompile the norm-folded GDN replay kernels and only launched keys Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 53 +++++------- .../ops/ssm/gdn_cached_replay_cute__verify.py | 76 ++++++++++------- .../test_ssm_ops.py | 85 +++++++++++++++++++ 3 files changed, 151 insertions(+), 63 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 4c5ac0e96f45..70616f436bec 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1573,36 +1573,16 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, _pool.check_replay_history(g, i, history, ssm0.shape[0], *old[i][:3]) replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) if ssm0.dtype == torch.bfloat16 and ssm0.is_cuda: - # The CuTe replay kernels of the default mapping a verify step launches, compiled - # here: a compile costs 5-10 s, and the padded decode graphs skip the request counts - # an eager mixed step carries. PDL up to `_GDN_REPLAY_PDL_MAX_HEAD_ROWS` - # (request x value head) pairs. - heads = (width, self.num_k_heads, self.num_v_heads) - partition_from = CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE - requests = md.max_num_requests - if heads[1:] == _GDN_REPLAY_HT_GEOMETRY: - requests = min(requests, _GDN_REPLAY_HT_MIN_REQUESTS - 1) - pdl_requests = min(requests, _GDN_REPLAY_PDL_MAX_HEAD_ROWS // self.num_v_heads) - if get_env_enable_pdl() and pdl_requests > 0: - gdn_replay_precompile(*heads, pdl_requests, pdl=True, partition_from=partition_from) - gdn_replay_precompile(*heads, requests, pdl=False, partition_from=partition_from) - if ( - heads[1:] == _GDN_REPLAY_HT_GEOMETRY - and md.max_num_requests >= _GDN_REPLAY_HT_MIN_REQUESTS - ): - # A tp2/tep2 HT batch takes its own mapping and conv fold (`_gdn_replay_launch`, - # `_gdn_verify_folds_conv`). The decode warmup compiles its kernels with the norm - # folded (`_gdn_verify_folds_norm`); a mixed step's verify rows - # (`PrefillTarget._gdn_core`) launch them without, compiled here. - gdn_replay_precompile( - *heads, - md.max_num_requests, - pdl=False, - partition_from=partition_from, - min_requests=_GDN_REPLAY_HT_MIN_REQUESTS, - launch=lambda n: _gdn_replay_launch(self, n), - conv_fold=lambda n: _gdn_verify_folds_conv(self, n), - ) + # Every CuTe replay kernel a verify step launches, compiled here: a compile costs + # 5-10 s, the padded decode graphs skip the request counts an eager step carries, + # and the generic warmup skips this cache manager. + gdn_replay_precompile( + width, + self.num_k_heads, + self.num_v_heads, + md.max_num_requests, + **_gdn_replay_launches(self, width), + ) return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) def _bind_draft_cache_manager(self, md) -> None: @@ -2134,6 +2114,19 @@ def _gdn_replay_launch(core, n: int) -> tuple: return get_env_enable_pdl() and n * core.num_v_heads <= _GDN_REPLAY_PDL_MAX_HEAD_ROWS, None +def _gdn_replay_launches(core, width: int) -> dict: + """The launch rules `gdn_replay_precompile` takes: those `_gdn_verify` and its callers apply + to a verify step of `width` rows a request (`_gdn_replay_launch`, `_gdn_verify_folds_conv`, + `_gdn_verify_folds_norm`).""" + return dict( + partition_from=CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + launch=lambda n: _gdn_replay_launch(core, n), + conv_fold=lambda n: _gdn_verify_folds_conv(core, n), + fold_norm=lambda n: _gdn_verify_folds_norm(core, n, width), + norm_eps=float(core.eps), + ) + + def _draft_positions(position_ids: torch.Tensor) -> tuple: """Return `(pos3, pos, coords)` from `[3, 1, N]` or `[3, N]` draft-step mRoPE positions. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py index 91152cd92a23..cc5a43889f77 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -2223,49 +2223,59 @@ def _kernel_kw( ) -def precompile( +def precompile_keys( t: int, h: int, hv: int, max_requests: int, *, - pdl: bool, partition_from: int, - min_requests: int = 1, - launch: Optional[Callable[[int], tuple]] = None, + launch: Callable[[int], tuple], conv_fold: Optional[Callable[[int], bool]] = None, -) -> int: - """Compile every kernel a verify step of `min_requests`..`max_requests` requests launches, + fold_norm: Optional[Callable[[int], bool]] = None, + norm_eps: Optional[float] = None, +) -> set: + """The `_COMPILED` key of every kernel a verify step of 1..`max_requests` requests launches, the way a target calls this entry: work items, and with them no in-kernel commit, from - `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 `a` / `b`; no - `gated_norm`. By default the default mapping at `pdl` without `conv_fold`; `launch(n)` - instead returns `(pdl, config)` for a batch of `n` requests and `conv_fold(n)` whether it - folds the conv. A compile costs 5-10 s, so a target runs this at bind, before any step: the - captured decode graphs do not reach every mapping (padded graphs skip 9 to 15 requests, - which an eager mixed step carries). Returns the number of distinct kernels.""" + `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 `a` / `b`. + `launch(n)` returns `(pdl, config)` for a batch of `n` requests (config None: the default + mapping), `conv_fold(n)` whether it folds the conv, and `fold_norm(n)` whether a + generation-only batch folds the gated norm (epsilon `norm_eps`); a mixed step's verify rows + never fold it, so such a batch reaches both kernels.""" keys = set() - for n in range(min_requests, max_requests + 1): + for n in range(1, max_requests + 1): partitioned = n >= partition_from - n_pdl, config = launch(n) if launch is not None else (pdl, None) - kw = _kernel_kw( - n, - t, - h, - hv, - commit=not partitioned, - pdl=n_pdl, - l2norm=True, - gating=True, - g_f32=False, - b_f32=False, - work_items=partitioned, - config=config, - fold=conv_fold is not None and conv_fold(n), - ) - key = tuple(sorted(kw.items())) - if key not in keys: - keys.add(key) - _compile(key, _Replay(**kw)) + n_pdl, config = launch(n) + norms = [None] + ([norm_eps] if fold_norm is not None and fold_norm(n) else []) + for eps in norms: + kw = _kernel_kw( + n, + t, + h, + hv, + commit=not partitioned, + pdl=bool(n_pdl), + l2norm=True, + gating=True, + g_f32=False, + b_f32=False, + work_items=partitioned, + config=config, + fold=conv_fold is not None and conv_fold(n), + norm_eps=eps, + ) + keys.add(tuple(sorted(kw.items()))) + return keys + + +def precompile(t: int, h: int, hv: int, max_requests: int, **launches) -> int: + """Compile every kernel of `precompile_keys(t, h, hv, max_requests, **launches)`. A compile + costs 5-10 s, so a target runs this at bind, before any step: the captured decode graphs do + not reach every mapping (padded graphs skip 9 to 15 requests, which an eager mixed step + carries). Returns the number of distinct kernels.""" + keys = precompile_keys(t, h, hv, max_requests, **launches) + for key in keys: + _compile(key, _Replay(**dict(key))) return len(keys) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py index 1474dbea4830..dfe2b2108c33 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py @@ -1121,6 +1121,91 @@ def test_gdn_verify_agrees_with_the_engines_stock_replay(member, n, monkeypatch) assert torch.equal(x[0], y[0]) +#: Requests the precompile test walks: past every `_GDN_REPLAY_HT_LAUNCH` bucket and the partition. +GVR_MAX_REQUESTS = 96 + + +@pytest.mark.parametrize("pdl", [True, False], ids=["pdl", "no_pdl"]) +@pytest.mark.parametrize("member", list(GVR_GEOM)) +def test_gdn_replay_precompile_is_every_key_a_verify_step_launches(member, pdl, monkeypatch): + """The CuTe replay kernels the bind precompiles (`_gdn_replay_launches`) are exactly those + `_gdn_verify` launches over 1..`GVR_MAX_REQUESTS` requests, from a generation-only step + (the gated norm folded where `_gdn_verify_folds_norm` says) and from a mixed step's verify + rows (never folded). Kernels are recorded, not compiled or run.""" + monkeypatch.setattr(_m, "get_env_enable_pdl", lambda: pdl) + monkeypatch.setattr( + _m, + "_GDN_REPLAY_HT_LAUNCH", + tuple((bound, p and pdl, mapping) for bound, p, mapping in _m._GDN_REPLAY_HT_LAUNCH), + ) + launched = [] + # An empty cache: every launch reaches `_compile`, which records the key instead. + monkeypatch.setattr(cr, "_COMPILED", {}) + monkeypatch.setattr(cr, "_compile", lambda key, kern: launched.append(key) or (lambda *a: None)) + # The conv runs as its own kernel unless folded; the fold's operands are the engine's layout. + monkeypatch.setattr(_m, "causal_conv1d_token_major", lambda mq, *a, **k: mq.contiguous()) + monkeypatch.setattr(_m, "gdn_conv_fold_supported", lambda *a: True) + h, hv = GVR_GEOM[member] + key_dim, value_dim = h * GVR_K, hv * GVR_V + conv_dim = 2 * key_dim + value_dim + bf, nmax, t = torch.bfloat16, GVR_MAX_REQUESTS, GVR_T + slots = nmax + 1 + old = { + 0: ( + torch.zeros(slots, 2, GVR_HIST, hv, GVR_V, dtype=bf, device=DEV), + torch.zeros(slots, 2, GVR_HIST, h, GVR_K, dtype=bf, device=DEV), + torch.zeros(slots, 2, hv, GVR_HIST, device=DEV), + torch.zeros(slots, 2, hv, GVR_HIST, device=DEV), + ) + } + zeros_i32 = torch.zeros(slots, dtype=torch.int32, device=DEV) + pools = types.SimpleNamespace( + ssm={0: torch.zeros(slots, hv, GVR_V, GVR_K, dtype=bf, device=DEV)}, + conv={0: torch.zeros(slots, conv_dim, GVR_CONV_W - 1, dtype=bf, device=DEV)}, + ) + core = types.SimpleNamespace( + conv_dim=conv_dim, + key_dim=key_dim, + value_dim=value_dim, + num_k_heads=h, + head_k_dim=GVR_K, + num_v_heads=hv, + head_v_dim=GVR_V, + eps=1e-6, + _bound=types.SimpleNamespace( + pools=pools, + int_conv={0: torch.zeros(nmax, t, conv_dim, GVR_CONV_W - 1, dtype=bf, device=DEV)}, + replay=_m._Replay(old, zeros_i32, zeros_i32, GVR_HIST, t), + ), + ) + layer = types.SimpleNamespace( + conv_w=torch.zeros(conv_dim, GVR_CONV_W, dtype=bf, device=DEV), + a_log=torch.zeros(hv, device=DEV), + dt_bias=torch.zeros(hv, device=DEV), + gnorm=torch.ones(GVR_V, dtype=bf, device=DEV), + ) + proj = torch.zeros(nmax * t, conv_dim + value_dim + 2 * hv, dtype=bf, device=DEV) + for n in range(1, nmax + 1): + mq, z, a, b = torch.split(proj[: n * t], [conv_dim, value_dim, hv, hv], 1) + rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) + partitioned = n >= _m.CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + work = zeros_i32[:n] if partitioned else None + verify = (torch.arange(n, dtype=torch.int32, device=DEV), work, work, partitioned) + _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify) # a mixed step's rows + fold_norm = _m._gdn_verify_folds_norm(core, n, t) + _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify, z=z if fold_norm else None) + want = set(launched) + got = cr.precompile_keys(t, h, hv, nmax, **_m._gdn_replay_launches(core, t)) + assert got == want, ( + f"precompiled but never launched: {len(got - want)}, launched but not precompiled: " + f"{len(want - got)}" + ) + if pdl and hv * 2 <= _m._GDN_REPLAY_PDL_MAX_HEAD_ROWS: + assert not any(not dict(key)["pdl"] and dict(key)["tpc"] == 2 for key in got), ( + "with PDL on, a batch of up to two requests never launches without it" + ) + + # ------------------------------------------- causal_conv1d_token_major__verify --- From 20945fe4f30ed7fd46131f7df253e053ead0afaa Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:17:33 -0800 Subject: [PATCH 39/51] [None][chore] modeling_v2 qwen4_exp: state the measured GDN replay compile cost Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../qwen3_8_flash_next_fp8__sm_103__unified/modeling.py | 4 ++-- .../ops/ssm/gdn_cached_replay_cute__verify.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 70616f436bec..1a56a60e1387 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1574,8 +1574,8 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) if ssm0.dtype == torch.bfloat16 and ssm0.is_cuda: # Every CuTe replay kernel a verify step launches, compiled here: a compile costs - # 5-10 s, the padded decode graphs skip the request counts an eager step carries, - # and the generic warmup skips this cache manager. + # about 0.5 s, the padded decode graphs skip the request counts an eager step + # carries, and the generic warmup skips this cache manager. gdn_replay_precompile( width, self.num_k_heads, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py index cc5a43889f77..62a3da5f29da 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -2270,9 +2270,9 @@ def precompile_keys( def precompile(t: int, h: int, hv: int, max_requests: int, **launches) -> int: """Compile every kernel of `precompile_keys(t, h, hv, max_requests, **launches)`. A compile - costs 5-10 s, so a target runs this at bind, before any step: the captured decode graphs do - not reach every mapping (padded graphs skip 9 to 15 requests, which an eager mixed step - carries). Returns the number of distinct kernels.""" + costs about 0.5 s on GB300, so a target runs this at bind, before any step: the captured + decode graphs do not reach every mapping (padded graphs skip 9 to 15 requests, which an + eager mixed step carries). Returns the number of distinct kernels.""" keys = precompile_keys(t, h, hv, max_requests, **launches) for key in keys: _compile(key, _Replay(**dict(key))) @@ -2600,7 +2600,7 @@ class GdnCachedReplayCute(OpWrapper): note = """ Source: this file's CuTe DSL kernel (`_Replay`), one compile per (geometry, T, launch - mapping, commit, PDL, folds) on first use (5-10 s): the warm-up before CUDA-graph capture + mapping, commit, PDL, folds) on first use (about 0.5 s): the warm-up before CUDA-graph capture must cover it. Called by `_gdn_verify` over the bf16 recurrent pool (slot stride 36 layers' states) when the rank's GDN heads are a `GEOMETRIES` pair, with work items + `use_all_layer_commit` exactly when the Triton entry gets them (N >= 16). PDL only at up to From ffeeb689aad796b46d47c69702042868e0c56657 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:55:05 -0800 Subject: [PATCH 40/51] [None][fix] modeling_v2 qwen4_exp: address the FP8 target's GDN replay history by its replay slots The stock hybrid cache manager can keep the GDN replay history and its bookkeeping in replay slots of their own (the mamba metadata's replay_state_indices) and hands out five-field work items. The FP8 target's bind checks the bookkeeping and history against the replay slots, its verify steps (decode and mixed) pass the generation rows' replay slots to the CuTe replay, and the replay reads the history, its bookkeeping and the work items at their own slot and element strides. The precompile keys carry the work-item width and whether the replay slots are separate, so the bind compiles exactly the kernels a step launches. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling.py | 32 +- .../ops/ssm/gdn_cached_replay_cute__verify.py | 205 ++++++++-- .../pool.py | 26 +- .../test_routing_and_layout.py | 5 +- .../test_ssm_ops.py | 376 +++++++++++++++++- 5 files changed, 575 insertions(+), 69 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index 1a56a60e1387..c928040f5109 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -1564,13 +1564,16 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, ) ssm0 = pools.ssm[g0] history, width = meta.replay_history_size, meta.replay_step_width + # The bookkeeping has one row per replay slot; a step indexes it, and the history, + # with the metadata's `replay_state_indices`. + replay_slots = meta.prev_num_accepted_tokens.shape[0] _pool.check_replay_state( - history, width, meta.prev_num_accepted_tokens, meta.cache_buf_idx, ssm0 + history, width, meta.prev_num_accepted_tokens, meta.cache_buf_idx, ssm0, replay_slots ) old = {} for i, cache in layer_caches.items(): old[i] = (cache.old_x, cache.old_B, cache.old_dt, cache.old_dA_cumsum) - _pool.check_replay_history(g, i, history, ssm0.shape[0], *old[i][:3]) + _pool.check_replay_history(g, i, history, replay_slots, *old[i][:3]) replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) if ssm0.dtype == torch.bfloat16 and ssm0.is_cuda: # Every CuTe replay kernel a verify step launches, compiled here: a compile costs @@ -1582,6 +1585,8 @@ def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, self.num_v_heads, md.max_num_requests, **_gdn_replay_launches(self, width), + work_item_width=md.mamba_metadata.replay_work_items.shape[1], + separate_replay_slots=_has_replay_slots(cm), ) return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) @@ -1956,8 +1961,15 @@ def _verify_rows(core, spec_metadata, num_seqs: int, num_rows: int) -> tuple[int return p, spec_metadata is not None and num_seqs > 0 and p > 1 -def _verify_state(core, mm, num_decodes: int) -> tuple: - """Return what `_gdn_verify` reads besides the layer: promotion rows and replay work items. +def _has_replay_slots(cm) -> bool: + """Whether the manager keeps the replay history in slots of its own, which the metadata's + `replay_state_indices` then address (otherwise it aliases the recurrent-state slots).""" + return getattr(cm, "replay_layer_group_id", None) is not None + + +def _verify_state(core, mm, num_decodes: int, num_prefills: int = 0) -> tuple: + """Return what `_gdn_verify` reads besides the layer: promotion rows, replay work items, + and the generation rows' replay slots (behind `num_prefills` context rows). Work items and the all-layer commit follow the stock Gated-DeltaNet layer's conditions.""" assert core._bound.int_idx is not None, "the cache-manager bind has not run" @@ -1968,7 +1980,12 @@ def _verify_state(core, mm, num_decodes: int) -> tuple: ) work_items = mm.replay_work_items[:num_decodes] if partitioned else None n_writes = mm.replay_n_writes if partitioned else None - return int_idx, work_items, n_writes, all_layer + replay_indices = ( + mm.replay_state_indices[num_prefills : num_prefills + num_decodes] + if _has_replay_slots(core._bound.cm) + else None + ) + return int_idx, work_items, n_writes, all_layer, replay_indices def _gdn_in_proj_conv_tactic(core, x, L, si_d, verify): @@ -1999,7 +2016,7 @@ def _gdn_verify(core, mq, a, b, i, L, si_d, n, p, verify, mq_out=None, z=None) - history, runs the `p` rows and appends them, so rejected rows cost nothing to undo. `mq` is the `[n * p, conv_dim]` q|k|v view, `a`/`b` the raw gate columns (the kernel applies `A_log`, `dt_bias`, q/k L2 norm). Returns `[1, n * p, num_v_heads, head_v_dim]`.""" - int_idx, work_items, n_writes, all_layer = verify + int_idx, work_items, n_writes, all_layer, replay_indices = verify pools = core._bound.pools replay = core._bound.replay ssm = pools.ssm[i] @@ -2078,6 +2095,7 @@ def _gdn_verify(core, mq, a, b, i, L, si_d, n, p, verify, mq_out=None, z=None) - packed_qkv=None if fold is not None else mq_out, use_all_layer_commit=all_layer, output=None, + replay_indices=replay_indices, **({"config": mapping} if mapping is not None else {}), **({"conv_fold": fold} if fold is not None else {}), **({"gated_norm": norm} if norm is not None else {}), @@ -5159,7 +5177,7 @@ def prepare( qsl_long_p, num_decodes, gen_p, - _verify_state(core, mm, num_decodes) if is_verify else None, + _verify_state(core, mm, num_decodes, num_prefills) if is_verify else None, ) # `has_initial_states` covers the context prefix only; every generation row carries. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py index 62a3da5f29da..720e2c8bbc16 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -14,7 +14,10 @@ the tile it just used (u split into bf16 hi and lo, as the Triton commit does). With PDL the state and history (not written by the previous kernel) are read before -`griddepcontrol.wait`, the step's own inputs after it. +`griddepcontrol.wait`, the step's own inputs after it. The history and its bookkeeping are +addressed by a replay slot of their own (`replay_indices`, the checkpoint slot when not given) at +the slot strides of their buffers, so a manager may keep them in pool pages apart from the +checkpoint. """ from __future__ import annotations @@ -42,6 +45,7 @@ TILES = GRP * (V // 16) # m-tiles of one (request, q/k head) PU = 272 # shared row pitch (bytes) of 128-element bf16 rows read by ldmatrix (conflict-free) PK = 320 # shared row pitch of the history keys read as permuted 16-byte fragments +WI_REPLAY_SLOT = 4 # a work item's replay-slot field, when it has one (`REPLAY_WORK_REPLAY_SLOT`) LOG2E = 1.4426950408889634 LN2 = 0.6931471805599453 @@ -966,6 +970,8 @@ def __init__( spec_hist=False, qk_first=True, use_wi=False, + wi_w=4, + rsep=False, evict_first=True, early_acc=False, h=16, @@ -1009,6 +1015,10 @@ def __init__( self.qk_first = qk_first # (row, slot, pending count, half) per item from the engine's replay work items self.use_wi = use_wi + # int32 fields per work item: 4 is one 16-byte load, anything else scalar loads + self.wi_w = wi_w + # the history and its bookkeeping have replay slots of their own (`replay_indices`) + self.rsep = rsep self.evict_first = evict_first nset = 2 if self.spec_hist else 1 self.nset = nset @@ -1084,6 +1094,12 @@ def __call__( ok_ptr: cutlass.Int64, og_ptr: cutlass.Int64, wi_ptr: cutlass.Int64, + ridx_ptr: cutlass.Int64, + u_ss: cutlass.Int64, + k_ss: cutlass.Int64, + g_ss: cutlass.Int64, + pnat_ss: cutlass.Int64, + buf_ss: cutlass.Int64, cx_ptr: cutlass.Int64, cx_st: cutlass.Int64, cw_ptr: cutlass.Int64, @@ -1127,6 +1143,12 @@ def __call__( ok_ptr, og_ptr, wi_ptr, + ridx_ptr, + u_ss, + k_ss, + g_ss, + pnat_ss, + buf_ss, cx_ptr, cx_st, cw_ptr, @@ -1312,6 +1334,15 @@ def _g_f32(self, x): def _b_f32(self, x): return x if self.b_f32 else _b16_f32(x) + def _work_item(self, wi_ptr, item): + """Work item `item`'s int32 fields: (row, slot, pending count, half), then the replay + slot when the history has slots of its own and the items carry them.""" + base = wi_ptr + cutlass.Int64(item) * (4 * self.wi_w) + if self.wi_w == 4: + return _LDG4["plain"](base) + fields = WI_REPLAY_SLOT + 1 if self.rsep and self.wi_w > WI_REPLAY_SLOT else 4 + return [_ldg_i32(base + 4 * f) for f in range(fields)] + def _load_tile(self, addr, pol=None): regs = [] for jj in range(4): @@ -1350,6 +1381,12 @@ def kernel( ok_ptr: cutlass.Int64, og_ptr: cutlass.Int64, wi_ptr: cutlass.Int64, + ridx_ptr: cutlass.Int64, + u_ss: cutlass.Int64, + k_ss: cutlass.Int64, + g_ss: cutlass.Int64, + pnat_ss: cutlass.Int64, + buf_ss: cutlass.Int64, cx_ptr: cutlass.Int64, cx_st: cutlass.Int64, cw_ptr: cutlass.Int64, @@ -1388,7 +1425,7 @@ def kernel( nh = bidx // S h = nh % H if const_expr(self.use_wi): - wi = _LDG4["plain"](wi_ptr + cutlass.Int64(nh // H) * 16) + wi = self._work_item(wi_ptr, nh // H) n = wi[0] else: n = nh // H @@ -1481,8 +1518,16 @@ def kernel( slot = wi[1] else: slot = _ldg_i32(sidx_ptr + n64 * 4) + # the history and bookkeeping row: the checkpoint's slot unless it has its own + rslot = slot + if const_expr(self.rsep): + if const_expr(self.use_wi and self.wi_w > WI_REPLAY_SLOT): + rslot = wi[WI_REPLAY_SLOT] + else: + rslot = _ldg_i32(ridx_ptr + n64 * 4) if slot >= 0: slot64 = cutlass.Int64(slot) + rs64 = cutlass.Int64(rslot) # ---- 1. the state tiles: bulk copies first, then the register tile ----------------- pol = _policy_evict_first() if const_expr(self.evict_first) else None if const_expr(NSM > 0): @@ -1518,28 +1563,29 @@ def kernel( p = wi[2] rb = wi[3] else: - p = _ldg_i32(pnat_ptr + slot64 * 4) - rb = _ldg_i32(buf_ptr + slot64 * 4) + p = _ldg_i32(pnat_ptr + rs64 * pnat_ss) + rb = _ldg_i32(buf_ptr + rs64 * buf_ss) wrap = (p + T) > HIST wb = cutlass.Int32(cutlass.select_(wrap, 1 - rb, rb)) woff = cutlass.Int32(cutlass.select_(wrap, zi, p)) - rslot = slot64 * 2 + cutlass.Int64(rb) - g_rd = og_ptr + rslot * (HV * HIST * 4) - wslot = slot64 * 2 + cutlass.Int64(wb) - u_rd = ou_ptr + rslot * (HIST * HV * V * 2) - k_rd = ok_ptr + rslot * (HIST * H * K * 2) - u_wr = ou_ptr + wslot * (HIST * HV * V * 2) - k_wr = ok_ptr + wslot * (HIST * H * K * 2) - g_wr = og_ptr + wslot * (HV * HIST * 4) + # a replay slot's two history halves are packed behind its slot stride + u_sl = ou_ptr + rs64 * u_ss + k_sl = ok_ptr + rs64 * k_ss + g_sl = og_ptr + rs64 * g_ss + g_rd = g_sl + cutlass.Int64(rb) * (HV * HIST * 4) + u_rd = u_sl + cutlass.Int64(rb) * (HIST * HV * V * 2) + k_rd = k_sl + cutlass.Int64(rb) * (HIST * H * K * 2) + u_wr = u_sl + cutlass.Int64(wb) * (HIST * HV * V * 2) + k_wr = k_sl + cutlass.Int64(wb) * (HIST * H * K * 2) + g_wr = g_sl + cutlass.Int64(wb) * (HV * HIST * 4) n_u: cutlass.Constexpr = NHV * 256 if const_expr(self.spec_hist): # both halves, every row, before p / half are known; rows j >= p masked at use for hb in cutlass.range_constexpr(2): - hslot = slot64 * 2 + hb - u_h = ou_ptr + hslot * (HIST * HV * V * 2) - k_h = ok_ptr + hslot * (HIST * H * K * 2) - g_h = og_ptr + hslot * (HV * HIST * 4) + u_h = u_sl + hb * (HIST * HV * V * 2) + k_h = k_sl + hb * (HIST * H * K * 2) + g_h = g_sl + hb * (HV * HIST * 4) for it in cutlass.range_constexpr(-(-n_u // NTHR)): e = tidx + it * NTHR if e < n_u: @@ -2156,7 +2202,7 @@ def _compile(key, kern): fn = cute.compile( kern, from_dlpack(dummy, assumed_align=4).mark_layout_dynamic(leading_dim=0), - *([i64] * 37), + *([i64] * 43), cutlass.Int32(1), cutlass.Float32(1.0), cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), @@ -2182,9 +2228,13 @@ def _kernel_kw( config: Optional[dict] = None, fold: bool = False, norm_eps: Optional[float] = None, + work_item_width: int = 4, + separate_replay_slots: bool = False, ) -> dict: """The kernel's compile-time parameters for a call of `n` requests: its key in `_COMPILED`. - `fold` is a `conv_fold` call, `norm_eps` a `gated_norm` call's epsilon.""" + `fold` is a `conv_fold` call, `norm_eps` a `gated_norm` call's epsilon, `work_item_width` + the int32 fields per work item, and `separate_replay_slots` says the history is addressed by + replay slots of its own (`replay_indices`).""" cfg = dict(default_config(n, t, commit)) if config: cfg.update(config) @@ -2209,6 +2259,8 @@ def _kernel_kw( # PDL: state / history before the wait; without it, the step's own inputs first cfg.setdefault("qk_first", not pdl) cfg["use_wi"] = bool(cfg.pop("use_wi", True)) and work_items + cfg["wi_w"] = work_item_width if cfg["use_wi"] else 4 + cfg["rsep"] = separate_replay_slots return dict( T=t, h=h, @@ -2234,6 +2286,8 @@ def precompile_keys( conv_fold: Optional[Callable[[int], bool]] = None, fold_norm: Optional[Callable[[int], bool]] = None, norm_eps: Optional[float] = None, + work_item_width: int = 4, + separate_replay_slots: bool = False, ) -> set: """The `_COMPILED` key of every kernel a verify step of 1..`max_requests` requests launches, the way a target calls this entry: work items, and with them no in-kernel commit, from @@ -2241,7 +2295,9 @@ def precompile_keys( `launch(n)` returns `(pdl, config)` for a batch of `n` requests (config None: the default mapping), `conv_fold(n)` whether it folds the conv, and `fold_norm(n)` whether a generation-only batch folds the gated norm (epsilon `norm_eps`); a mixed step's verify rows - never fold it, so such a batch reaches both kernels.""" + never fold it, so such a batch reaches both kernels. `work_item_width` is the int32 fields + of the manager's work items and `separate_replay_slots` whether it keeps the history at + replay slots of its own (every step then passes `replay_indices`).""" keys = set() for n in range(1, max_requests + 1): partitioned = n >= partition_from @@ -2263,6 +2319,8 @@ def precompile_keys( config=config, fold=conv_fold is not None and conv_fold(n), norm_eps=eps, + work_item_width=work_item_width, + separate_replay_slots=separate_replay_slots, ) keys.add(tuple(sorted(kw.items()))) return keys @@ -2323,13 +2381,21 @@ def cached_replay_update( config: Optional[dict] = None, conv_fold: Optional[dict] = None, gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, **ignored, ) -> torch.Tensor: """Same contract as `fused_recurrent_gated_delta_rule_cached_replay_update` (bf16 state). A wrapping row commits its checkpoint in-kernel unless work items + `use_all_layer_commit` are given. Work items must list all N rows (`config={"use_wi": False}` reads - `state_indices` instead). The Triton launch knobs are ignored. + `state_indices` instead); with separate replay slots their fifth field + (`REPLAY_WORK_REPLAY_SLOT`) gives each row's replay slot, else `replay_indices` does. The + Triton launch knobs are ignored. + + `replay_indices` (`state_indices` when None) address the history (`old_u` / `old_k` / + `old_G`) and its bookkeeping (`prev_num_accepted_tokens` / `cache_buf_idx`); each replay + slot's rows are packed behind a slot stride of 16-byte multiples, and the bookkeeping may + have any element stride (a manager keeps both in one row per slot). `conv_fold` folds the verify conv in, bitwise the conv kernel: `x` (the q|k|v projection columns), `weight` `[C, 4]`, `pool` `[slots, C, 3]` (read, not advanced: the engine's @@ -2351,8 +2417,32 @@ def cached_replay_update( assert ssm_states.stride()[1:] == (V * K, K, 1) assert old_u.dtype == torch.bfloat16 and old_k.dtype == torch.bfloat16 assert old_G.dtype == torch.float32 - for x in (old_u, old_k, old_G): - assert x.is_contiguous() + for name, x, rows in ( + ("old_u", old_u, (2, HIST, HV, V)), + ("old_k", old_k, (2, HIST, H, K)), + ("old_G", old_G, (2, HV, HIST)), + ): + assert tuple(x.shape[1:]) == rows and x[0].is_contiguous(), ( + f"{name} is {tuple(x.shape)} with strides {x.stride()}; the kernel reads packed " + f"[slots, {', '.join(map(str, rows))}] rows" + ) + assert x.data_ptr() % 16 == 0 and (x.stride(0) * x.element_size()) % 16 == 0, ( + f"{name}: the kernel copies 16-byte chunks, so its base and slot stride must be " + "16-byte aligned" + ) + for name, x in ( + ("prev_num_accepted_tokens", prev_num_accepted_tokens), + ("cache_buf_idx", cache_buf_idx), + ): + assert x.dim() == 1 and x.dtype == torch.int32, f"{name}: one int32 per replay slot" + rsep = replay_indices is not None and replay_indices.data_ptr() != state_indices.data_ptr() + if rsep: + assert ( + replay_indices.dim() == 1 + and replay_indices.shape[0] >= N + and replay_indices.dtype == torch.int32 + and replay_indices.is_contiguous() + ), "replay_indices: one contiguous int32 replay slot per row" assert g.shape == (N, T, HV) and beta.shape == (N, T, HV) assert g.stride(2) == 1 and beta.stride(2) == 1 assert g.stride(0) == T * g.stride(1) and beta.stride(0) == T * beta.stride(1) @@ -2393,6 +2483,13 @@ def cached_replay_update( ap, dp = A_log.data_ptr(), dt_bias.data_ptr() else: ap = dp = 0 + if replay_work_items is not None: + assert ( + replay_work_items.dim() == 2 + and replay_work_items.shape[1] >= 4 + and replay_work_items.dtype == torch.int32 + and replay_work_items.is_contiguous() + ), "replay_work_items: contiguous int32 rows of (row, slot, pending count, half[, ...])" fold_args = (0,) * 9 if conv_fold is not None: cx, cw, cs = conv_fold["x"], conv_fold["weight"], conv_fold["pool"] @@ -2438,6 +2535,8 @@ def cached_replay_update( config=config, fold=conv_fold is not None, norm_eps=norm_eps, + work_item_width=replay_work_items.shape[1] if replay_work_items is not None else 4, + separate_replay_slots=rsep, ) use_wi = kw["use_wi"] key = tuple(sorted(kw.items())) @@ -2471,6 +2570,12 @@ def cached_replay_update( old_k.data_ptr(), old_G.data_ptr(), replay_work_items.data_ptr() if use_wi else 0, + replay_indices.data_ptr() if rsep else 0, + old_u.stride(0) * old_u.element_size(), + old_k.stride(0) * old_k.element_size(), + old_G.stride(0) * old_G.element_size(), + prev_num_accepted_tokens.stride(0) * prev_num_accepted_tokens.element_size(), + cache_buf_idx.stride(0) * cache_buf_idx.element_size(), *fold_args, *norm_args, N, @@ -2603,8 +2708,10 @@ class GdnCachedReplayCute(OpWrapper): mapping, commit, PDL, folds) on first use (about 0.5 s): the warm-up before CUDA-graph capture must cover it. Called by `_gdn_verify` over the bf16 recurrent pool (slot stride 36 layers' states) when the rank's GDN heads are a `GEOMETRIES` pair, with work items + - `use_all_layer_commit` exactly when the Triton entry gets them (N >= 16). PDL only at up to - 24 (request x value head) pairs: the conv before it triggers its dependents at its start. + `use_all_layer_commit` exactly when the Triton entry gets them (N >= 16). The history is + addressed by the manager's replay slots (`replay_indices`) when it keeps the history in + slots of its own; the work items then carry the replay slot as a fifth field. PDL only at up + to 24 (request x value head) pairs: the conv before it triggers its dependents at its start. Numerics: the Triton kernel's rounding points (bf16 L2-normalized q / k and history coefficients, fp32 recurrence, fp32 commit rounded once to the bf16 pool, bf16 history u / k, @@ -2652,6 +2759,7 @@ def __call__( config: Optional[dict] = None, conv_fold: Optional[dict] = None, gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: return cached_replay_update( q=q, @@ -2681,6 +2789,7 @@ def __call__( config=config, conv_fold=conv_fold, gated_norm=gated_norm, + replay_indices=replay_indices, ) def reference( @@ -2717,13 +2826,15 @@ def reference( output: Optional[torch.Tensor] = None, config: Optional[dict] = None, conv_fold: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """`(out, ssm_states, old_u, old_k, old_G)` as the call leaves them; mutates nothing. The state is rebuilt from the checkpoint and history in fp32 and the new tokens stepped one at a time in fp32 (no matmul, so no TF32), every store rounded once to its buffer's dtype. Rows with a negative slot keep `output`'s prior content (NaN without `output`). - `replay_work_items` only decides whether a wrap commits here; the launch knobs, `config`, - `old_beta` and `n_writes` are ignored.""" + The history and its bookkeeping are read and written at `replay_indices` (the checkpoint + slots when None). `replay_work_items` only decides whether a wrap commits here; the + launch knobs, `config`, `old_beta` and `n_writes` are ignored.""" N, T, H, K = k.shape HV, V = v.shape[2], v.shape[3] rep = HV // H @@ -2757,15 +2868,16 @@ def reference( if live.numel() == 0: return out.to(out_dtype), pool, u_buf, k_buf, G_buf slot = state_indices[:N][live].long() - p = prev_num_accepted_tokens[slot].long() - b = cache_buf_idx[slot].long() + rslot = slot if replay_indices is None else replay_indices[:N][live].long() + p = prev_num_accepted_tokens[rslot].long() + b = cache_buf_idx[rslot].long() M, depth = live.numel(), old_u.shape[2] # 1. the pending state: checkpoint plus the p history entries of half b pending = torch.arange(depth, device=dev)[None, :] < p[:, None] # [M, depth] - Gh = torch.where(pending[:, None, :], old_G[slot, b].float(), 0.0) # [M, HV, depth] - Uh = torch.where(pending[:, :, None, None], old_u[slot, b].float(), 0.0) - Kh = torch.where(pending[:, :, None, None], old_k[slot, b].float(), 0.0) + Gh = torch.where(pending[:, None, :], old_G[rslot, b].float(), 0.0) # [M, HV, depth] + Uh = torch.where(pending[:, :, None, None], old_u[rslot, b].float(), 0.0) + Kh = torch.where(pending[:, :, None, None], old_k[rslot, b].float(), 0.0) Kh = Kh.repeat_interleave(rep, dim=2) # [M, depth, HV, K] last = (p - 1).clamp(min=0)[:, None, None].expand(M, HV, 1) g_start = torch.where(p[:, None] > 0, Gh.gather(2, last).squeeze(2), 0.0) # [M, HV] @@ -2795,10 +2907,10 @@ def reference( w_off = torch.where(wrap, 0, p) for t in range(T): pos = w_off + t - u_buf[slot, w_buf, pos] = u_new[t].to(u_buf.dtype) - k_buf[slot, w_buf, pos] = kf[live, t].to(k_buf.dtype) + u_buf[rslot, w_buf, pos] = u_new[t].to(u_buf.dtype) + k_buf[rslot, w_buf, pos] = kf[live, t].to(k_buf.dtype) G_store = torch.where(wrap[:, None], G_new[t], g_start + G_new[t]) - G_buf[slot, w_buf, :, pos] = G_store.to(G_buf.dtype) + G_buf[rslot, w_buf, :, pos] = G_store.to(G_buf.dtype) commit = replay_work_items is None if commit and bool(wrap.any()): pool[slot[wrap]] = checkpoint[wrap].to(pool.dtype) @@ -2839,6 +2951,7 @@ def is_valid( config: Optional[dict] = None, conv_fold: Optional[dict] = None, gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, ) -> None: N, _, H, K = k.shape HV, V = v.shape[2], v.shape[3] @@ -2847,18 +2960,28 @@ def is_valid( "kernel reads one slot per sequence with no bound, so a shorter tensor hands the " "trailing rows whatever follows it in memory, and they write that slot's history" ) - pool_slots, slots = ssm_states.shape[0], old_u.shape[0] - assert slots >= pool_slots, ( - f"the history buffers have {slots} slots for a {pool_slots}-slot pool -- a slot " - "past the history's end is read and written out of bounds" + # The history is addressed by the replay slots, the state slots when none are given. + if replay_indices is None: + history_slots, slot_space = ssm_states.shape[0], "the recurrent pool" + else: + assert replay_indices.dim() == 1 and replay_indices.shape[0] >= N, ( + f"replay_indices has shape {tuple(replay_indices.shape)} for {N} sequences -- " + "the kernel reads one replay slot per sequence with no bound" + ) + history_slots, slot_space = int(replay_indices[:N].max()) + 1, "the replay slots" + slots = old_u.shape[0] + assert slots >= history_slots, ( + f"the history buffers have {slots} slots for {history_slots} of {slot_space} -- a " + "slot past the history's end is read and written out of bounds" ) for name, tensor in ( ("prev_num_accepted_tokens", prev_num_accepted_tokens), ("cache_buf_idx", cache_buf_idx), ): - assert tensor.dim() == 1 and tensor.is_contiguous() and tensor.shape[0] >= pool_slots, ( - f"{name} has shape {tuple(tensor.shape)} and stride {tensor.stride()}; the kernel " - f"reads one element per slot of the {pool_slots}-slot pool, at the slot's index" + assert tensor.dim() == 1 and tensor.shape[0] >= history_slots, ( + f"{name} has shape {tuple(tensor.shape)}; the kernel reads one element per " + f"history slot ({history_slots} of {slot_space}), at the slot's index times " + "the element stride" ) expected = { "old_u": (old_u, (slots, 2, history_size, HV, V)), diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py index 6fd62cd8dfc6..0d2e18814609 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py @@ -349,8 +349,10 @@ def check_step_buffers(g: PoolGeometry, int_conv, conv, int_idx) -> None: ) -def check_replay_state(history, width, prev_accepted, cache_buf_idx, ssm) -> None: - """Cached replay, manager-wide: the certified geometry and the per-slot bookkeeping.""" +def check_replay_state(history, width, prev_accepted, cache_buf_idx, ssm, replay_slots) -> None: + """Cached replay, manager-wide: the certified geometry and the per-slot bookkeeping, which + must cover every replay slot. The manager may address the history by replay slots of its + own, which then differ from the recurrent pool's.""" assert ssm.dtype in SSM_STATE_DTYPES, ( f"the recurrent pool is {ssm.dtype}; this target runs the replay over a bf16 checkpoint " "state (kv_cache_config.mamba_ssm_cache_dtype: bfloat16)" @@ -363,22 +365,24 @@ def check_replay_state(history, width, prev_accepted, cache_buf_idx, ssm) -> Non f"verify rows and never above the history ({history})" ) for name, t in (("prev_num_accepted_tokens", prev_accepted), ("cache_buf_idx", cache_buf_idx)): - assert t.dim() == 1 and t.shape[0] >= ssm.shape[0] and t.dtype == torch.int32, ( - f"{name} is {tuple(t.shape)} {t.dtype}; the kernel reads one int32 per pool slot " - f"({ssm.shape[0]})" + assert t.dim() == 1 and t.shape[0] >= replay_slots and t.dtype == torch.int32, ( + f"{name} is {tuple(t.shape)} {t.dtype}; the kernel reads one int32 per replay slot " + f"({replay_slots})" ) def check_replay_history( - g: PoolGeometry, layer: int, history, pool_slots, old_x, old_B, old_dt + g: PoolGeometry, layer: int, history, replay_slots, old_x, old_B, old_dt ) -> None: - """One GDN layer's replay history: `[slots, 2 halves, ...]` over this rank's heads, addressed - by `history` and the head geometry, never by the buffer's own shape.""" + """One GDN layer's replay history: `[replay slots, 2 halves, ...]` over this rank's heads, + addressed by `history` and the head geometry, never by the buffer's own shape.""" for name, t in (("old_x", old_x), ("old_B", old_B), ("old_dt", old_dt)): assert t is not None, f"layer {layer}: the manager's layer cache carries no {name}" - assert old_x.shape[0] >= pool_slots, ( - f"layer {layer}: the replay history has {old_x.shape[0]} slots for a {pool_slots}-slot pool" - ) + for name, t in (("old_x", old_x), ("old_B", old_B), ("old_dt", old_dt)): + assert t.shape[0] >= replay_slots, ( + f"layer {layer}: the replay history {name} has {t.shape[0]} slots for " + f"{replay_slots} replay slots" + ) bf16, fp32 = torch.bfloat16, torch.float32 x_tail, b_tail = (history, g.num_v_heads, g.head_v_dim), (history, g.num_k_heads, g.head_k_dim) _check_pool(old_x, f"layer {layer}: old_x", 2, x_tail, bf16) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index f03fdda54ccd..b3a11907148e 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -1814,12 +1814,15 @@ def _stock_mtp(member, rank, max_draft_len): assert cache.intermediate_ssm is None, "the cached replay keeps no per-step state" assert not cm.use_gdn_flashinfer_ring_replay meta = cm.get_replay_state_update_metadata() + # The history and its bookkeeping have replay slots of their own. + replay_slots = meta.prev_num_accepted_tokens.shape[0] P.check_replay_state( meta.replay_history_size, meta.replay_step_width, meta.prev_num_accepted_tokens, meta.cache_buf_idx, ssm0, + replay_slots, ) assert meta.replay_step_width == max_draft_len + 1 P.check_verify_width(meta.replay_step_width) @@ -1828,7 +1831,7 @@ def _stock_mtp(member, rank, max_draft_len): g, i, meta.replay_history_size, - ssm0.shape[0], + replay_slots, cache.old_x, cache.old_B, cache.old_dt, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py index dfe2b2108c33..3947f9d64df6 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py @@ -6,6 +6,7 @@ from __future__ import annotations +import math import types import warnings import zlib @@ -57,10 +58,13 @@ from tensorrt_llm._torch.modules.fla.cached_replay import ( fused_recurrent_gated_delta_rule_cached_replay_update as stock_replay, ) -from tensorrt_llm._torch.modules.mamba.mamba2_metadata import _build_replay_work_items_triton +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import ( + REPLAY_WORK_ITEM_WIDTH, + _build_replay_work_items_triton, +) __extra_import_path__ = ["../../.."] -from _validating import ULP, certifying # noqa: E402,F401 — path above; fixture by name +from _validating import ULP, certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "the Gated-DeltaNet entries require a CUDA device" @@ -578,7 +582,7 @@ def __init__(self, spec, seed, exact=True): if self.path == "all_layer": # The engine hands out work items exactly from the all-layer commit's minimum batch. assert n >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, n - self.work_items = torch.zeros(n, 4, dtype=torch.int32, device=DEV) + self.work_items = torch.zeros(n, REPLAY_WORK_ITEM_WIDTH, dtype=torch.int32, device=DEV) self.n_writes = torch.zeros(1, dtype=torch.int32, device=DEV) self.exact = exact @@ -1066,7 +1070,7 @@ def _gvr_build(member: str, n: int, seed: int): _bound=types.SimpleNamespace(pools=pools, int_conv=int_conv, replay=replay), ) rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) # slot 0 untouched - verify = (torch.arange(n, dtype=torch.int32, device=DEV), None, None, False) + verify = (torch.arange(n, dtype=torch.int32, device=DEV), None, None, False, None) def run(): outs = [] @@ -1125,13 +1129,19 @@ def test_gdn_verify_agrees_with_the_engines_stock_replay(member, n, monkeypatch) GVR_MAX_REQUESTS = 96 +@pytest.mark.parametrize("separate", [False, True], ids=["coupled", "replay_slots"]) @pytest.mark.parametrize("pdl", [True, False], ids=["pdl", "no_pdl"]) @pytest.mark.parametrize("member", list(GVR_GEOM)) -def test_gdn_replay_precompile_is_every_key_a_verify_step_launches(member, pdl, monkeypatch): +def test_gdn_replay_precompile_is_every_key_a_verify_step_launches( + member, pdl, separate, monkeypatch +): """The CuTe replay kernels the bind precompiles (`_gdn_replay_launches`) are exactly those `_gdn_verify` launches over 1..`GVR_MAX_REQUESTS` requests, from a generation-only step (the gated norm folded where `_gdn_verify_folds_norm` says) and from a mixed step's verify - rows (never folded). Kernels are recorded, not compiled or run.""" + rows (never folded), the partitioned ones with work items of the metadata's width. With + `separate` the manager keeps the history at replay slots of its own: precompiled with + `separate_replay_slots`, every step passes `replay_indices`. Kernels are recorded, not + compiled or run.""" monkeypatch.setattr(_m, "get_env_enable_pdl", lambda: pdl) monkeypatch.setattr( _m, @@ -1159,6 +1169,8 @@ def test_gdn_replay_precompile_is_every_key_a_verify_step_launches(member, pdl, ) } zeros_i32 = torch.zeros(slots, dtype=torch.int32, device=DEV) + work_items = torch.zeros(slots, REPLAY_WORK_ITEM_WIDTH, dtype=torch.int32, device=DEV) + replay_slots = torch.arange(slots, 2 * slots, dtype=torch.int32, device=DEV) pools = types.SimpleNamespace( ssm={0: torch.zeros(slots, hv, GVR_V, GVR_K, dtype=bf, device=DEV)}, conv={0: torch.zeros(slots, conv_dim, GVR_CONV_W - 1, dtype=bf, device=DEV)}, @@ -1189,13 +1201,26 @@ def test_gdn_replay_precompile_is_every_key_a_verify_step_launches(member, pdl, mq, z, a, b = torch.split(proj[: n * t], [conv_dim, value_dim, hv, hv], 1) rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) partitioned = n >= _m.CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE - work = zeros_i32[:n] if partitioned else None - verify = (torch.arange(n, dtype=torch.int32, device=DEV), work, work, partitioned) + work = work_items[:n] if partitioned else None + n_writes = zeros_i32[:1] if partitioned else None + replay_indices = replay_slots[:n] if separate else None + int_idx = torch.arange(n, dtype=torch.int32, device=DEV) + verify = (int_idx, work, n_writes, partitioned, replay_indices) _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify) # a mixed step's rows fold_norm = _m._gdn_verify_folds_norm(core, n, t) _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify, z=z if fold_norm else None) want = set(launched) - got = cr.precompile_keys(t, h, hv, nmax, **_m._gdn_replay_launches(core, t)) + got = cr.precompile_keys( + t, + h, + hv, + nmax, + **_m._gdn_replay_launches(core, t), + work_item_width=REPLAY_WORK_ITEM_WIDTH, + separate_replay_slots=separate, + ) + assert all(dict(key)["rsep"] == separate for key in got) + assert {dict(key)["wi_w"] for key in got if dict(key)["use_wi"]} == {REPLAY_WORK_ITEM_WIDTH} assert got == want, ( f"precompiled but never launched: {len(got - want)}, launched but not precompiled: " f"{len(want - got)}" @@ -1206,6 +1231,339 @@ def test_gdn_replay_precompile_is_every_key_a_verify_step_launches(member, pdl, ) +# ------------------------------------------------ the replay history at slots of its own --- + + +def _replay_pages(slots, layers, h, hv): + """The history and its bookkeeping in pages of their own, as a manager keeps them in its + KVCM pool: one page per replay slot holding every layer's `old_u`, then every layer's + `old_k`, then every layer's `old_G`, then the pending count and the half (int32), and 8 + bytes nobody writes; every byte 0xFF (NaN history, counts of -1) until written. The + `[layers, slots, ...]` history views have the page as their slot stride, the bookkeeping + columns the page as their element stride; `data` is the `[slots, page]` bytes, `tail` + the offset of the unwritten bytes.""" + regions = ( + (torch.bfloat16, (2, HIST, hv, V)), + (torch.bfloat16, (2, HIST, h, K)), + (torch.float32, (2, hv, HIST)), + ) + sizes = [math.prod(shape) * dtype.itemsize for dtype, shape in regions] + page = layers * sum(sizes) + 16 + data = torch.full((slots, page), 0xFF, dtype=torch.uint8, device=DEV) + flat = data.view(-1) + views, offset = [], 0 + for (dtype, shape), size in zip(regions, sizes): + es = dtype.itemsize + rows = torch.empty(shape, device="meta").stride() + views.append( + torch.as_strided( + flat.view(dtype), + (layers, slots, *shape), + (size // es, page // es, *rows), + offset // es, + ) + ) + offset += layers * size + pnat, buf = ( + torch.as_strided(flat.view(torch.int32), (slots,), (page // 4,), offset // 4 + field) + for field in (0, 1) + ) + return types.SimpleNamespace( + data=data, u=views[0], k=views[1], G=views[2], pnat=pnat, buf=buf, tail=offset + 8 + ) + + +class _ReplaySlotBench(_ReplayBench): + """`_ReplayBench` (the same draws for the same seed) with the history and its bookkeeping + moved to replay slots of their own (`_replay_pages`): row `i`'s at `replay_indices[i]`, a + permutation of slots past the checkpoint pool's, so no row's replay slot is a checkpoint + slot. The work items carry the replay slot as their fifth field; the kernel is handed their + first `width` fields (4: it reads the replay slot from `replay_indices`), the manager's + all-layer commit the full items. `writes` is the coupled bench's alone (it reads the + bookkeeping at the checkpoint slots).""" + + def __init__(self, spec, seed, width): + super().__init__(spec, seed, exact=False) + assert "dup" not in spec and "page_layers" not in spec, spec + n, slots = self.n, self.state_indices.long() + cpu = torch.Generator().manual_seed(seed + 1) + replay = torch.randperm(2 * n, generator=cpu)[:n] + self.slots + self.replay_indices = replay.to(DEV, torch.int32) + rows = self.replay_indices.long() + self.pages = _replay_pages(self.slots + 2 * n, self.layers, self.h, self.hv) + pages = self.pages + for paged, coupled in zip( + (pages.u, pages.k, pages.G), (self.old_x, self.old_B, self.old_dt) + ): + paged[:, rows] = coupled[:, slots] + pages.pnat[rows] = self.pnat[slots] + pages.buf[rows] = self.buf[slots] + self.old_x, self.old_B, self.old_dt = pages.u, pages.k, pages.G + self.pnat, self.buf = pages.pnat, pages.buf + self.meta = ReplayStateUpdateMetadata(self.pnat, self.buf, self.t, HIST) + self.item_width = width + self.kernel_items = None + + def prepare(self): + if self.path == "all_layer": + self.work_items.zero_() + _build_replay_work_items_triton( + self.live, + self.pnat, + self.buf, + self.work_items, + self.n_writes, + self.t, + HIST, + self.replay_indices, + ) + full = self.work_items.shape[1] + self.kernel_items = ( + self.work_items + if self.item_width == full + else self.work_items[:, : self.item_width].contiguous() + ) + + def kwargs(self, layer, act): + kw = super().kwargs(layer, act) + kw["replay_indices"] = self.replay_indices + if self.kernel_items is not None: + kw["replay_work_items"] = self.kernel_items + return kw + + def accept(self, accepted): + _advance_replay_state(self.meta, self.replay_indices, accepted) + + +_SLOT_STATE = ("checkpoint", "old_u", "old_k", "old_G", "pending count", "half") + + +def _slot_state(bench): + """A bench's checkpoints, and its rows' history and bookkeeping at their history slots + (the replay slots, when it has them).""" + rows = getattr(bench, "replay_indices", bench.state_indices).long() + return ( + bench.backing, + bench.old_x[:, rows], + bench.old_B[:, rows], + bench.old_dt[:, rows], + bench.pnat[rows], + bench.buf[rows], + ) + + +def _rs_spec(heads, path, n, t): + h, hv = heads + return dict( + state_dtype=torch.bfloat16, + H=h, + HV=hv, + path=path, + n=n, + t=t, + layers=2, + steps=HIST // t + 3, + ) + + +#: Replay-slot runs: requests and the work-item width the kernel is handed (None: no work +#: items, the small path, the kernel commits wrapping rows itself). +_REPLAY_SLOT_RUNS = { + "small_n1": (1, None), + "small_n4": (4, None), + "w4_all_layer": (CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, 4), + "w5_all_layer": (32, REPLAY_WORK_ITEM_WIDTH), +} + + +@pytest.mark.parametrize("heads", cr.GEOMETRIES, ids=[f"{h}x{hv}" for h, hv in cr.GEOMETRIES]) +@pytest.mark.parametrize("n,width", list(_REPLAY_SLOT_RUNS.values()), ids=list(_REPLAY_SLOT_RUNS)) +def test_separate_replay_slots_match_coupled(n, width, heads, certifying) -> None: # noqa: F811 + """Replay slots that differ from the checkpoint slots give, bit for bit, the coupled run. + + The coupled side is `_ReplayBench`: history and bookkeeping at the checkpoint slots. The + other (`_ReplaySlotBench`) keeps the same history and bookkeeping at a permuted set of + replay slots past the checkpoint pool's, in pages holding every layer's rows (a slot stride + of the whole page, the bookkeeping as strided int32 fields of it), 0xFF elsewhere. Both run + verify steps through this entry -- strided a / b and PDL, as the target hands them over -- + with checkpoint wraps committed in-kernel (no work items) or by the manager's all-layer + commit, the kernel handed the work items at `width` fields (4: the replay slot from + `replay_indices`; 5: from the items' fifth field), then advance their bookkeeping with the + same acceptance. Each replay-slot call is also held to this entry's `reference` given the + same `replay_indices`, and no byte of the pages outside the rows' replay slots is written. + """ + certifying(_grc_op) + t = 4 + path = "small" if width is None else "all_layer" + spec = _rs_spec(heads, path, n, t) + seed = 31 + n + coupled = _ReplayBench(spec, seed, exact=False) + paged = _ReplaySlotBench(spec, seed, width or REPLAY_WORK_ITEM_WIDTH) + sides = (coupled, paged) + rows = paged.replay_indices.long() + assert not torch.isin(rows, coupled.state_indices.long()).any() + wraps = appends = 0 + for step in range(coupled.steps): + wrapped = coupled.writes()[1].numel() + wraps += wrapped + appends += n - wrapped + for side in sides: + side.prepare() + if width is not None: + assert torch.equal(coupled.n_writes, paged.n_writes) + for layer in range(coupled.layers): + act = coupled.inputs() + a, b = _grc_gates(coupled, act, True) + outs = [] + for side in sides: + kw = dict(side.kwargs(layer, act), g=a, beta=b, launch_with_pdl=True) + if side is paged: + ref = _grc_op.reference(**kw) + outs.append(_grc_op(**kw)) + pool, u, k, G = paged.state(layer) + _replay_compare( + (outs[1], pool, u[rows], k[rows], G[rows]), + (ref[0], ref[1], ref[2][rows], ref[3][rows], ref[4][rows]), + ) + assert torch.equal(outs[0], outs[1]), f"step {step} layer {layer}: outputs differ" + for side in sides: + side.commit() + accepted = coupled.draw_acceptance() + for side in sides: + side.accept(accepted) + for name, x, y in zip(_SLOT_STATE, _slot_state(coupled), _slot_state(paged)): + assert torch.equal(x, y), f"step {step}: {name} differs" + assert wraps > 0 and appends > 0, f"the run never wrapped or appended ({wraps}, {appends})" + pages = paged.pages + elsewhere = torch.ones(pages.data.shape[0], dtype=torch.bool, device=DEV) + elsewhere[rows] = False + assert (pages.data[elsewhere] == 0xFF).all(), "a page outside the rows' replay slots moved" + assert (pages.data[:, pages.tail :] == 0xFF).all(), "a page's bytes past its rows moved" + + +#: Context rows ahead of the generation rows in the dispatch test's mixed-step metadata. +_DISPATCH_PREFILLS = 3 + + +def _dispatch_core(bench, separate): + """What `_verify_state` and `_gdn_verify` read, over `bench`'s pool, history and + bookkeeping: the GDN core's geometry and bind (the pools, the replay buffers, a manager + with the all-layer commit on and, with `separate`, replay slots of its own), and the mamba + metadata of a mixed step whose generation rows follow `_DISPATCH_PREFILLS` context rows + (`bench.prepare` fills the work items).""" + replay = _m._Replay( + { + layer: ( + bench.old_x[layer], + bench.old_B[layer], + bench.old_dt[layer], + bench.old_dA[layer], + ) + for layer in range(bench.layers) + }, + bench.buf, + bench.pnat, + HIST, + bench.t, + ) + cm = types.SimpleNamespace( + use_gdn_cached_replay_all_layer_commit=True, + replay_layer_group_id=1 if separate else None, + ) + core = types.SimpleNamespace( + conv_dim=bench.width, + key_dim=bench.h * K, + value_dim=bench.hv * V, + num_k_heads=bench.h, + head_k_dim=K, + num_v_heads=bench.hv, + head_v_dim=V, + _bound=types.SimpleNamespace( + pools=types.SimpleNamespace(ssm=dict(enumerate(bench.pool))), + replay=replay, + cm=cm, + int_idx=torch.arange(bench.n, dtype=torch.int32, device=DEV), + ), + ) + context = torch.full((_DISPATCH_PREFILLS,), -1, dtype=torch.int32, device=DEV) + rows = bench.replay_indices if separate else bench.state_indices + mm = types.SimpleNamespace( + replay_work_items=bench.work_items, + replay_n_writes=bench.n_writes, + replay_state_indices=torch.cat([context, rows]), + ) + return core, mm + + +@pytest.mark.parametrize("heads", cr.GEOMETRIES, ids=[f"{h}x{hv}" for h, hv in cr.GEOMETRIES]) +@pytest.mark.parametrize("separate", [False, True], ids=["coupled", "replay_slots"]) +@pytest.mark.parametrize("n,t", [(1, 4), (4, 4), (32, 4), (5, 3)]) +def test_target_dispatch_matches_the_plain_call(n, t, separate, heads) -> None: + """The target's verify dispatch over a bf16 pool (`_verify_state` for a mixed step's + generation rows, then `_gdn_verify` per layer: `a` / `b` column views of the input + projection, the conv output as `packed_qkv`, the launch `_gdn_replay_launch` picks) against + the call it replaces (contiguous `a` / `b`, no PDL, the default mapping), each on its own + copy of the pool, the history and the bookkeeping, over consecutive verify steps with + random partial acceptance in between: outputs, checkpoints, history and bookkeeping bitwise + equal at every step, with the entry's guard on. With `separate` the dispatched side's + manager keeps the history and its bookkeeping at replay slots of its own + (`_ReplaySlotBench`), which `_verify_state` takes from the metadata's `replay_state_indices` + behind the context rows and `_gdn_verify` hands this entry as `replay_indices`. From + `CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE` requests both sides take the work items and the + manager's all-layer commit; below it the kernel commits wraps.""" + path = "all_layer" if n >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE else "small" + spec = _rs_spec(heads, path, n, t) + seed = 77 + n + plain = _ReplayBench(spec, seed, exact=False) + disp = ( + _ReplaySlotBench(spec, seed, REPLAY_WORK_ITEM_WIDTH) + if separate + else _ReplayBench(spec, seed, exact=False) + ) + core, mm = _dispatch_core(disp, separate) + wraps = 0 + with validating(_grc_op): + for step in range(plain.steps): + wraps += plain.writes()[1].numel() + plain.prepare() + disp.prepare() + verify = _m._verify_state(core, mm, n, _DISPATCH_PREFILLS) + assert verify[3] == (path == "all_layer"), ( + "the all-layer commit exactly when partitioned" + ) + if separate: + assert torch.equal(verify[4], disp.replay_indices), "the rows' replay slots" + else: + assert verify[4] is None, "no replay slots of its own: the checkpoint slots" + for layer in range(plain.layers): + act = plain.inputs() + a, b = _grc_gates(plain, act, True) + want = _grc_op(**plain.kwargs(layer, act)) + gating = types.SimpleNamespace(a_log=disp.a_log[layer], dt_bias=disp.dt_bias[layer]) + got = _m._gdn_verify( + core, + None, + a.reshape(n * t, -1), + b.reshape(n * t, -1), + layer, + gating, + disp.state_indices, + n, + t, + verify, + mq_out=act["packed"], + ) + assert torch.equal(got.view_as(want), want), f"step {step} layer {layer}" + plain.commit() + disp.commit() + accepted = plain.draw_acceptance() + plain.accept(accepted) + disp.accept(accepted) + for name, x, y in zip(_SLOT_STATE, _slot_state(disp), _slot_state(plain)): + assert torch.equal(x, y), f"step {step}: {name} differs" + assert wraps > 0, "no history wrapped: no checkpoint commit was exercised" + + # ------------------------------------------- causal_conv1d_token_major__verify --- From dc7f172586c04043759fcaab5f967bf2f42f4b58 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:26:49 -0800 Subject: [PATCH 41/51] [None][fix] modeling_v2 qwen4_exp: make the direct launches local copies pinned to the registered ops Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../activation/fused_sigmoid_mul_inplace.py | 62 +++++++++++++++---- .../ops/norm/rms_norm_gated_token_major.py | 44 ++++++++++--- .../_torch/modules/mamba/layernorm_gated.py | 22 +------ ...t_modeling_v2_fused_sigmoid_mul_inplace.py | 41 +++++++++++- ...v2_qwen4_exp_rms_norm_gated_token_major.py | 45 +++++++++++++- 5 files changed, 173 insertions(+), 41 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py index 09de9e186c3b..63fd5a69d27d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py @@ -11,15 +11,54 @@ from __future__ import annotations import torch - -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +import triton # Registered by the module that defines it: after `import # tensorrt_llm._torch.custom_ops` alone, torch.ops.trtllm has no # `fused_sigmoid_mul_inplace` (measured). -from tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate import ( - launch_fused_sigmoid_mul_inplace, -) +import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate as _gate_mod +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + + +def _launch(attention_output: torch.Tensor, gate: torch.Tensor) -> bool: + """The registered op's launch, made here when its preconditions hold: the same kernel, + grid, block, warps and PDL policy, without the custom-op dispatch around it. False + (nothing launched) when they do not, for the op to refuse loudly.""" + if attention_output.dim() != 2 or attention_output.stride(-1) != 1 or gate.stride(-1) != 1: + return False + num_tokens, hidden = attention_output.shape + if gate.dim() == 3: + if gate.shape[0] != num_tokens or gate.shape[1] * gate.shape[2] != hidden: + return False + head_dim, gate_stride_head = gate.shape[2], gate.stride(1) + elif gate.dim() == 2 and gate.shape == attention_output.shape: + head_dim, gate_stride_head = hidden, hidden + else: + return False + if num_tokens == 0: + return True + block = min(triton.next_power_of_2(hidden), 1024 if num_tokens < 1024 else 2048) + pdl = ( + dict(LAUNCH_WITH_PDL=True, launch_pdl=True) + if _gate_mod._sigmoid_mul_pdl(attention_output.device.index) + else {} + ) + _gate_mod._fused_sigmoid_mul_kernel[(num_tokens, triton.cdiv(hidden, block))]( + attention_output, + attention_output, + gate, + attention_output.stride(0), + attention_output.stride(0), + gate.stride(0), + gate_stride_head, + hidden_size=hidden, + head_dim=head_dim, + block_size=block, + num_warps=4, + **pdl, + ) + return True + #: The checkpoint's gated attention: 24 q heads of 256, q projection [T, 24, 512]. _CKPT = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") @@ -137,9 +176,12 @@ class FusedSigmoidMulInplace(OpWrapper): the running device at the first call of each (hidden, head_dim, block) specialization, no autotuner: grid `(T, cdiv(hidden, block))`, `block = min(next_pow2(hidden), 1024 if T < 1024 else 2048)`, 4 warps. The - cells ran on sm_103. With `direct_launch=True` (the FP8 target) the op's own - launch `launch_fused_sigmoid_mul_inplace` (the op's body) is called, skipping - only the custom-op dispatch: 15 instead of 28 us of host time a call on GB300. + cells ran on sm_103. With `direct_launch=True` (the FP8 target), `_launch` makes + that same launch itself where the preconditions below hold and skips only the + custom-op dispatch (15 instead of 28 us of host time a call on GB300); + otherwise the op is called and refuses the call. `_launch` is a copy of the + op's launch: the test holds it bitwise equal to the op at the direct-launch + cells and more shapes, so a change to the op's launch fails there. Call sites, each the attention output gate between the attention core and o_proj: @@ -218,9 +260,7 @@ class FusedSigmoidMulInplace(OpWrapper): def __call__( self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False ) -> torch.Tensor: - if direct_launch: - launch_fused_sigmoid_mul_inplace(attention_output, gate) - else: + if not (direct_launch and _launch(attention_output, gate)): torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) return attention_output diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py index b7955cff9893..2ae4a233728b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py @@ -49,6 +49,25 @@ def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> return x.shape[0] >= _PREFILL_MIN_ROWS and _multirow(x, z, gate_activation) +def _direct_launch(x, z, weight, eps, gate_activation): + """The registered op's own multi-row launch (same kernel, 4 rows a program, PDL by the + op's policy), skipping only the custom-op dispatch.""" + rows, n = x.shape + out = torch.empty_like(x) + grid = triton.cdiv(rows, _layernorm_gated._MULTIROW_ROWS) + device = x.device.index + pdl = _layernorm_gated._multirow_pdl(grid, device) + with torch.cuda.device(device): + _layernorm_gated._rms_norm_gated_fwd_multirow_kernel[(grid,)]( + x, out, weight, z, None, None, x.stride(0), out.stride(0), z.stride(0), rows, eps, + N=n, ROWS=_layernorm_gated._MULTIROW_ROWS, HEADS_PER_TOK=z.shape[1], + GATE_SIGMOID=gate_activation == "sigmoid", WEIGHT_IS_DELTA=False, + LAUNCH_WITH_PDL=pdl, num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, + EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], launch_pdl=pdl, + ) # fmt: skip + return out + + def _prefill_launch(x, z, weight, eps, gate_activation): """The multi-row kernel at `_PREFILL_ROWS` rows a program (no PDL: the grid covers every SM).""" rows, n = x.shape @@ -135,9 +154,12 @@ class RmsNormGatedTokenMajor(OpWrapper): SM count and `TRTLLM_ENABLE_PDL != 0`; results are bitwise identical with it off. The 8-row programs reduce a row's mean square in another order than the 4-row ones (one-ulp differences on some outputs). With - `direct_launch=True` (the FP8 target) below the 8-row threshold, the op's own - launch `launch_rms_norm_gated_token_major` (the op's body) is called, skipping - only the custom-op dispatch: 24 instead of 39 us of host time a call on GB300. + `direct_launch=True` (the FP8 target) below the 8-row threshold, the 4-row + launch is made here (`_direct_launch`, a copy of the registered op's own), + skipping only the custom-op dispatch: 24 instead of 39 us of host time a + call on GB300. The test holds it bitwise equal to the op at the + direct-launch cells and more shapes, so a change to the op's launch fails + there. Loud (measured), so not guarded: 3-D `x` (ValueError), a `z` not covering `x` (AssertionError "does not match x shape"; 2-D `z` ValueError), an unknown @@ -200,12 +222,18 @@ def __call__( ) -> torch.Tensor: if fp8_scale is None and z.dim() == 3 and _prefill_multirow(x, z, gate_activation): return _prefill_launch(x, z, weight, eps, gate_activation) - launch = ( - _layernorm_gated.launch_rms_norm_gated_token_major - if direct_launch - else torch.ops.trtllm.rms_norm_gated_token_major + if ( + direct_launch + and fp8_scale is None + and x.dim() == 2 + and z.dim() == 3 + and weight.is_contiguous() + and _multirow(x, z, gate_activation) + ): + return _direct_launch(x, z, weight, eps, gate_activation) + return torch.ops.trtllm.rms_norm_gated_token_major( + x, z, weight, eps, fp8_scale, gate_activation ) - return launch(x, z, weight, eps, fp8_scale, gate_activation) def reference( self, diff --git a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py index c0ad516eb838..98754aaf0f76 100644 --- a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py +++ b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py @@ -321,7 +321,8 @@ def _multirow_gated_rmsnorm_eligible(N, ngroups, bias, z, norm_before_gate, and ngroups == 1 and N <= _MULTIROW_MAX_N and (N & (N - 1)) == 0) -def launch_rms_norm_gated_token_major( +@torch.library.custom_op("trtllm::rms_norm_gated_token_major", mutates_args=()) +def rms_norm_gated_token_major( x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor, @@ -329,10 +330,7 @@ def launch_rms_norm_gated_token_major( fp8_scale: torch.Tensor | None = None, gate_activation: str = "silu", ) -> torch.Tensor: - """``trtllm::rms_norm_gated_token_major``'s launch, for eager callers that - skip the custom-op dispatch. - - Gated rmsnorm(x) with z read from a 3D token-major view. + """Gated rmsnorm(x) with z read from a 3D token-major view. x: [num_tokens * heads, N] with contiguous rows. z: [num_tokens, heads, N] whose (heads, N) block is contiguous per token and whose token stride is @@ -409,20 +407,6 @@ def launch_rms_norm_gated_token_major( return out -@torch.library.custom_op("trtllm::rms_norm_gated_token_major", mutates_args=()) -def rms_norm_gated_token_major( - x: torch.Tensor, - z: torch.Tensor, - weight: torch.Tensor, - eps: float, - fp8_scale: torch.Tensor | None = None, - gate_activation: str = "silu", -) -> torch.Tensor: - """``launch_rms_norm_gated_token_major`` as a custom op.""" - return launch_rms_norm_gated_token_major(x, z, weight, eps, fp8_scale, - gate_activation) - - @rms_norm_gated_token_major.register_fake def _( x: torch.Tensor, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py index f92723950d4f..41f06f21925a 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py @@ -2,8 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 """GPU test for the fused_sigmoid_mul_inplace catalog entry: every cell against the entry's reference with nothing written past `attention_output`'s last row, -graph capture of the default target's fused-row and MTP forms, the band refusing -a wrong gate read, and the loud layout refusal.""" +the direct launch bitwise equal to the registered op, graph capture of the +default target's fused-row and MTP forms, the band refusing a wrong gate read, +and the loud layout refusal.""" import zlib @@ -11,6 +12,9 @@ import torch from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation import ( + fused_sigmoid_mul_inplace as op_module, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( # noqa: E501 fused_sigmoid_mul_inplace as op, ) @@ -119,6 +123,39 @@ def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves th assert torch.equal(out_buffer[out.numel() :], tail_before), "a masked lane stored" +#: The direct-launch cells and more shapes for the pin test: 0 rows, both block widths +#: either side of 1024 rows, a compact 2-D gate, the default target's fused-row and +#: 6-head forms. +_PIN_SPECS = tuple(c.spec for c in CELLS if c.spec.get("direct_launch")) + ( + dict(tokens=0, **_FP8_TP2_ROW), + dict(tokens=1023, **_FP8_TP2_ROW), + dict(tokens=1024, **_FP8_TP2_ROW), + dict(tokens=1, **_FP8_MTP), + dict(_FP8_TP2_ROW, tokens=64, row_stride=3072, gate_col=0), + dict(next(c.spec for c in op.CELLS if c.spec.get("row_stride") == 4352), tokens=8), + _mtp_spec(512), +) + + +@pytest.mark.parametrize( + "spec", + _PIN_SPECS, + ids=[f"{s['tokens']}x{s['heads']}h-{s['gate']}-{s.get('row_stride', '')}" for s in _PIN_SPECS], +) +def test_direct_launch_is_bitwise_the_registered_op(spec) -> None: + """`_launch` is a copy of `torch.ops.trtllm.fused_sigmoid_mul_inplace`'s launch: it + launches at every shape here and writes the op's bytes, so a change to the op's + launch (kernel, grid, block) fails here.""" + out, _, _, gate = _build(spec, seed=_seed(f"pin {spec}")) + expected = out.clone() + torch.ops.trtllm.fused_sigmoid_mul_inplace(expected, gate) + direct = out.clone() + + assert op_module._launch(direct, gate), "the direct launch declined" + assert torch.equal(direct, expected) + assert torch.equal(op(out, gate, True), expected) + + @pytest.mark.parametrize("direct_launch", [False, True], ids=["op", "direct"]) def test_cuda_graph_capture_and_replay(direct_launch) -> None: """The decode call sites run captured: the tp4 / tep4 form (6 heads off the diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py index e2830fe8f739..12da4ea0ae9d 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""GPU test for the rms_norm_gated_token_major entry.""" +"""GPU test for the rms_norm_gated_token_major entry: every cell against the +entry's reference, the direct launch bitwise equal to the registered op, and the +weight-length guard.""" import zlib @@ -8,6 +10,9 @@ import torch from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm import ( + rms_norm_gated_token_major as op_module, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( # noqa: E501 rms_norm_gated_token_major as op, ) @@ -126,6 +131,44 @@ def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest res torch.testing.assert_close(out, op.reference(*args)) +#: The direct-launch cells below the 8-row threshold and more shapes for the pin test: the +#: last row count before it, a PDL grid at each width, the silu gate over a packed `z`. +_PIN_SPECS = tuple( + c.spec + for c in CELLS + if c.spec.get("direct_launch") + and c.spec["tokens"] * c.spec["heads"] < op_module._PREFILL_MIN_ROWS +) + ( + dict(_FP8_TP2, tokens=4), + dict(_FP8_TP2, tokens=128), + dict(_FP8_TP2, tokens=1365), + dict(op.CELLS[3].spec, tokens=4, direct_launch=True), + dict(op.CELLS[3].spec, tokens=2730, direct_launch=True), + dict(op.CELLS[-1].spec, direct_launch=True), + dict(_PACKED, tokens=16, gate_activation="silu", direct_launch=True), +) + + +@pytest.mark.parametrize( + "spec", + _PIN_SPECS, + ids=[ + f"{s['tokens']}x{s['heads']}h-{s.get('z_row_stride', s['z'])}-{s['gate_activation']}" + for s in _PIN_SPECS + ], +) +def test_direct_launch_is_bitwise_the_registered_op(spec) -> None: + """`_direct_launch` is a copy of `torch.ops.trtllm.rms_norm_gated_token_major`'s + multi-row launch: it is taken at every shape here and returns the op's bytes, so a + change to the op's launch (kernel, rows a program, warps) fails here.""" + x, z, weight, eps, _, act, _ = _build(spec, seed=_seed(f"pin {spec}")) + assert x.shape[0] < op_module._PREFILL_MIN_ROWS and op_module._multirow(x, z, act) + expected = torch.ops.trtllm.rms_norm_gated_token_major(x, z, weight, eps, None, act) + + assert torch.equal(op_module._direct_launch(x, z, weight, eps, act), expected) + assert torch.equal(op(x, z, weight, eps, None, act, True), expected) + + def test_guard_refuses_a_weight_of_the_wrong_length() -> None: """Unguarded, a length-192 weight is silently truncated to weight[:128].""" x, z, weight, eps, _, act, _ = _build(_PACKED, seed=12) From 583d1163e1c776f4a0d809b0eb02b8ec419f3f50 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:26:50 -0800 Subject: [PATCH 42/51] [None][fix] modeling_v2 qwen4_exp: read the AutoTuner capture state without a stock accessor Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- tensorrt_llm/_torch/autotuner.py | 5 ----- .../ops/moe/fp8_block_scale_moe_runner.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/autotuner.py b/tensorrt_llm/_torch/autotuner.py index 7ffcaec389f0..794a8ebede80 100644 --- a/tensorrt_llm/_torch/autotuner.py +++ b/tensorrt_llm/_torch/autotuner.py @@ -1150,11 +1150,6 @@ def get(cls): cls._instance = AutoTuner() return cls._instance - @property - def active_capture(self) -> Optional['AutoTuner.TacticsCapture']: - """The `capture()` / `replay()` context currently open, or None.""" - return self._active_capture - class TacticsCapture: """Object returned by capture() that can be iterated to get all tactic combinations. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py index 6f9915531d0a..2bf7d75d6e1c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py @@ -156,7 +156,7 @@ def _dispatch(geometry: tuple, inputs: list) -> Optional[tuple]: """`(runner, tactic)` as `torch.ops.trtllm.fp8_block_scale_moe_runner` would choose them, remembered; None while the AutoTuner tunes or captures (the custom op then runs as is).""" tuner = AutoTuner.get() - if tuner.is_tuning_mode or tuner.active_capture is not None: + if tuner.is_tuning_mode or tuner._active_capture is not None: return None shapes = tuple(None if t is None else tuple(t.shape) for t in inputs) key = (geometry, shapes, len(tuner.profiling_cache)) From 80aea321a7dfbbf43dad807467c24b10810ff110 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:26:51 -0800 Subject: [PATCH 43/51] [None][fix] modeling_v2 qwen4_exp: call the raw all-reduce at AUTO for MNNVL and SYMM_MEM in the FP8 target Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling_v2/catalog/comm/allreduce.py | 15 +-------------- .../modeling.py | 18 +++++++++++++----- .../test_routing_and_layout.py | 15 +++++++++++---- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py b/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py index 8ad58fe4bae2..205b11e693ef 100644 --- a/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py +++ b/tensorrt_llm/_torch/modeling_v2/catalog/comm/allreduce.py @@ -32,16 +32,6 @@ } ) -#: Strategies of the `AllReduce` module's own kernels, which this op has no branch for: the -#: module runs the MNNVL or symmetric-memory kernel where it can and calls this op at AUTO -#: otherwise (`_torch/distributed/ops.py`), and so does a call built by `allreduce_group_args`. -_MODULE_ONLY = frozenset( - { - int(AllReduceStrategy.MNNVL), - int(AllReduceStrategy.SYMM_MEM), - } -) - def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: """The communicator side of the call for this rank's TP group. @@ -51,8 +41,7 @@ def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: The IPC workspace is this op's own fixture: one per mapping, shared with every other user of that mapping in the process, allocated on first use. - It exists only for strategies that read it. MNNVL and SYMM_MEM reach the - op as AUTO (`_MODULE_ONLY`); running the MNNVL kernel is the caller's part. + It exists only for strategies that read it. Mirrors the router in `_torch/distributed/ops.py`: under MPI the plain symbol is called and `process_group` stays None; with MPI disabled the @@ -61,8 +50,6 @@ def allreduce_group_args(mapping, strategy: int = AUTO) -> dict: from tensorrt_llm._torch.distributed.ops import get_allreduce_workspace from tensorrt_llm._utils import mpi_disabled - if strategy in _MODULE_ONLY: - strategy = AUTO workspace = None if strategy in _NO_WORKSPACE else get_allreduce_workspace(mapping) process_group, rank = None, None if mpi_disabled(): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index c928040f5109..fdde49544360 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -725,9 +725,14 @@ def _moe_comm_method() -> str: _TP_SUM_MNNVL_MAX_ROWS = 32 #: The engine `allreduce_strategy` values under which the target runs its own communication: the #: MNNVL `tp_sum` and the IPC buffers `_own_ipc` admits. Under another strategy every TP sum is -#: the raw all-reduce at that strategy and the gathers run NCCL; `check_deployment` declines UB -#: and LOWPRECISION, whose setup only the stock `AllReduce` module does. +#: the raw all-reduce at that strategy (AUTO for SYMM_MEM) and the gathers run NCCL; +#: `check_deployment` declines UB and LOWPRECISION, whose setup only the stock `AllReduce` +#: module does. _OWN_COMM_STRATEGIES = frozenset({int(AllReduceStrategy.AUTO), int(AllReduceStrategy.MNNVL)}) +#: Strategies of the stock `AllReduce` module's own kernels, which the raw all-reduce op has no +#: branch for: the module runs the MNNVL or symmetric-memory kernel where it can and calls the op +#: at AUTO otherwise (`_torch/distributed/ops.py`), and so does `tp_sum`. +_MODULE_ONLY_STRATEGIES = frozenset({int(AllReduceStrategy.MNNVL), int(AllReduceStrategy.SYMM_MEM)}) #: The stock IPC fusion all-reduce's accumulation switch (`use_fp32_acc` in #: `allReduceFusionKernels.cu`): fp32 unless the value starts with `0`. `hc_allreduce_inject_norm` #: always sums in fp32, so with the opt-out the decode sites sum apart (`_hc_ar_inject`). @@ -1294,11 +1299,14 @@ def _tp_sum_op(self): """Return the attention-TP sum of block partials, bit-identical on every rank. At AUTO or MNNVL, where the platform supports MNNVL, rows up to `_TP_SUM_MNNVL_MAX_ROWS` - take the MNNVL kernel; every other sum is the raw all-reduce, which `allreduce_group_args` - hands MNNVL as AUTO.""" + take the MNNVL kernel; every other sum is the raw all-reduce, called at AUTO for MNNVL + and SYMM_MEM (`_MODULE_ONLY_STRATEGIES`).""" if self.attn_tp_size == 1: return lambda x: x - group = allreduce_group_args(self.mapping, self.allreduce_strategy) + strategy = self.allreduce_strategy + if strategy in _MODULE_ONLY_STRATEGIES: + strategy = int(AllReduceStrategy.AUTO) + group = allreduce_group_args(self.mapping, strategy) def tp_sum(x: torch.Tensor) -> torch.Tensor: return allreduce( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index b3a11907148e..5ca89d5b65a2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -461,11 +461,11 @@ def test_a_strategy_only_the_stock_module_sets_up_declines(on_sm103, strategy): @pytest.mark.cpu_only @pytest.mark.parametrize( - "strategy", ["NCCL", "NCCL_SYMMETRIC", "MIN_LATENCY", "ONESHOT", "TWOSHOT", "MNNVL"] + "strategy", ["NCCL", "NCCL_SYMMETRIC", "MIN_LATENCY", "ONESHOT", "TWOSHOT", "MNNVL", "SYMM_MEM"] ) def test_a_strategy_the_target_runs_is_admitted(on_sm103, strategy): """The raw all-reduce op runs these; MNNVL takes the target's MNNVL sum and the raw op at - AUTO.""" + AUTO, SYMM_MEM the raw op at AUTO.""" for member in ("tp2", "tep4"): _admitted(replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy])) @@ -586,11 +586,12 @@ def _core(member: str, rank: int, spec_config=None, **config): @pytest.mark.cpu_only -@pytest.mark.parametrize("strategy", ["AUTO", "MNNVL", "NCCL"]) +@pytest.mark.parametrize("strategy", ["AUTO", "MNNVL", "SYMM_MEM", "NCCL"]) def test_the_tp_sum_runs_at_the_engines_allreduce_strategy(strategy, monkeypatch) -> None: """At AUTO or MNNVL, sums up to `_TP_SUM_MNNVL_MAX_ROWS` rows take the MNNVL kernel (asked for by name, so one node qualifies) and wider ones the raw op at AUTO, which has no MNNVL - branch. Another strategy reaches the raw op as it is; NCCL reads no IPC workspace.""" + branch. SYMM_MEM, which the raw op has no branch for either, reaches it at AUTO at every + width. Another strategy reaches the raw op as it is; NCCL reads no IPC workspace.""" import tensorrt_llm._utils as utils from tensorrt_llm._torch.distributed import ops as dist_ops @@ -616,6 +617,12 @@ def is_mnnvl(mapping, dtype, explicitly_requested=False): assert asked == [] assert tp_sum(narrow) == tp_sum(wide) == ("raw", int(AllReduceStrategy.NCCL), None) return + if strategy == "SYMM_MEM": + assert asked == [] + assert ( + tp_sum(narrow) == tp_sum(wide) == ("raw", int(AllReduceStrategy.AUTO), "ipc workspace") + ) + return assert asked == [True] assert tp_sum(narrow) == ("mnnvl", "mnnvl workspace") assert tp_sum(wide) == ("raw", int(AllReduceStrategy.AUTO), "ipc workspace") From c4cf317a752a8e1a35b47646ad15aee12d211426 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:26:52 -0800 Subject: [PATCH 44/51] [None][fix] modeling_v2 qwen4_exp: document the multi-GPU trigger for the FP8 target's collectives Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 3 --- .../qwen3_8_flash_next_fp8__sm_103__unified/__init__.py | 9 ++++++++- .../comm/test_fp8_unified_op_matrices.py | 8 +++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 30776956bc23..9941d4466c31 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -2042,8 +2042,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "requirements.txt", "security_scanning/pyproject.toml", "tensorrt_llm/_ipc_utils.py", - "tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/", - "tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/", "tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py", "tensorrt_llm/_torch/compilation/patterns/ub_allreduce.py", "tensorrt_llm/_torch/custom_ops/torch_custom_ops.py", @@ -2127,7 +2125,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tests/integration/test_lists/test-db/l0_model_express.yml", "tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml", "tests/unittest/_torch/attention/multi_gpu/", - "tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/", "tests/unittest/_torch/multi_gpu/", "tests/unittest/_torch/moe/multi_gpu/", "tests/unittest/kv_cache_manager_v2_tests/", diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py index 4343209006aa..366a4aa96e8b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -1,6 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Qwen3.8-Flash-Next-FP8 / sm_103 / unified: tp1, tp2, tep2, dep2, tp4, tep4 and dep4.""" +"""Qwen3.8-Flash-Next-FP8 / sm_103 / unified: tp1, tp2, tep2, dep2, tp4, tep4 and dep4. + +The collective matrices (`tests/unittest/.../qwen3_8_flash_next_fp8__sm_103__unified/comm`) run in +the pre-merge GB300 multi-GPU stage (`l0_gb300.yml`), which CI schedules only when a file it lists +as multi-GPU changes (`getMultiGpuFileChanged` in `jenkins/L0_MergeRequest.groovy`); this +directory is not among them. A change to `csrc/` or `ops/comm/` is therefore run with +`/bot run --add-multi-gpu-test` (`docs/source/developer-guide/ci-overview.md`, Multi-GPU Tests). +""" from tensorrt_llm._torch.modeling_v2._native import load_target_ops diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py index fdee67b97583..a8256b92fef8 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -2,7 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 """Collected entry point for the FP8 target's collective matrices: each `__op_matrix.py` beside this file is its own launcher, which `_rank_job` runs in a fresh interpreter on `world` -sm_103 GPUs of one node.""" +sm_103 GPUs of one node. + +Pre-merge CI runs this directory in the GB300 multi-GPU stage (`l0_gb300.yml`), which it +schedules only when a file listed in `getMultiGpuFileChanged` (`jenkins/L0_MergeRequest.groovy`) +changes; neither the target's `csrc/` and `ops/comm/` nor this directory is listed. A change to +them is run with `/bot run --add-multi-gpu-test` +(`docs/source/developer-guide/ci-overview.md`, Multi-GPU Tests).""" from pathlib import Path From 4b31a2166747ddeedf081d461f1a4726a1a5935d Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:23:00 -0800 Subject: [PATCH 45/51] [None][fix] modeling_v2 qwen4_exp: keep the HC inject-norm Lamport protocol in the FP8 target Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../allReduceFusionKernels.cu | 95 ++++++++++++- .../communicationKernels/lamportComm.cuh | 129 ------------------ .../csrc/hcAllreduceInjectNorm.cu | 110 +++++++++++++-- 3 files changed, 195 insertions(+), 139 deletions(-) delete mode 100644 cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu index 69b84226c935..a70e1b604ec2 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/reduceKernelUtils.cuh" #include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" -#include "tensorrt_llm/kernels/communicationKernels/lamportComm.cuh" #include "tensorrt_llm/kernels/quantization.cuh" #include @@ -65,6 +64,56 @@ struct SyncComm int flag_value; }; +template +struct LamportComm +{ + __device__ __forceinline__ LamportComm(void** workspace, int rank) + { + counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; + flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; + // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and + // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar + // MiniMax kernel writes 4-byte words. + clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; + flag_value = *flag_ptr; + auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; + clear_size = *clear_ptr; + int data_offset = flag_value % 3; + int clear_offset = (flag_value + 2) % 3; + for (int r = 0; r < NRanks; ++r) + { + data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; + } + clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; + __syncthreads(); + if (threadIdx.x == 0) + { + atomicAdd(counter_ptr, 1); + } + } + + __device__ __forceinline__ void update(int64_t new_clear_size) + { + if (blockIdx.x == 0 && threadIdx.x == 0) + { + while (*reinterpret_cast(counter_ptr) != gridDim.x) + { + } + *flag_ptr = (flag_value + 1) % 3; + *clear_ptr = new_clear_size; + *counter_ptr = 0; + } + } + + int* counter_ptr; + int* flag_ptr; + int64_t* clear_ptr; + uint8_t* data_bufs[NRanks]; + uint8_t* clear_buf; + int64_t clear_size; + int flag_value; +}; + template class Barrier { @@ -296,6 +345,36 @@ private: float4 m_gamma_val; }; +__device__ __forceinline__ bool is_neg_zero(float v) +{ + return *reinterpret_cast(&v) == 0x80000000; +} + +__device__ __forceinline__ bool is_neg_zero(float4 v) +{ + return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); +} + +__device__ __forceinline__ float4 get_neg_zero() +{ + float4 vec; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + reinterpret_cast(&vec)[i] = 0x80000000; + } + return vec; +} + +__device__ __forceinline__ float4 ld_global_volatile(float4* addr) +{ + float4 val; + asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" + : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) + : "l"(addr)); + return val; +} + template __device__ __forceinline__ float4 allreduce_sum(float4* vals) { @@ -516,6 +595,20 @@ __global__ void __launch_bounds__(1024) allreduce_fusion_kernel_twoshot_sync( #endif } +int get_sm_count() +{ + static int sm_count = 0; + if (sm_count == 0) + { + int device_id; + TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); + cudaDeviceProp device_prop; + cudaGetDeviceProperties(&device_prop, device_id); + sm_count = device_prop.multiProcessorCount; + } + return sm_count; +} + template void launch_oneshot_lamport(AllReduceFusionParams const& params, cudaLaunchConfig_t& cfg) { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh b/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh deleted file mode 100644 index adf9a943cfa9..000000000000 --- a/cpp/tensorrt_llm/kernels/communicationKernels/lamportComm.cuh +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/cudaUtils.h" - -#include -#include - -TRTLLM_NAMESPACE_BEGIN - -namespace kernels::ar_fusion -{ -//! The one-shot Lamport protocol on the IPC all-reduce workspace (`allreduce_fusion_kernel_oneshot_lamport`), for -//! every kernel that runs on that workspace between its calls: workspace[2 * NRanks + r] is rank r's buffer of three -//! sub-buffers of workspace[3 * NRanks + 1][1] bytes; with flag = workspace[3 * NRanks][2], a call writes and polls -//! sub-buffer flag % 3 and clears its own (flag + 2) % 3 back to -0.0f. -template -struct LamportComm -{ - __device__ __forceinline__ LamportComm(void** workspace, int rank) - { - counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; - flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; - // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and - // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar - // MiniMax kernel writes 4-byte words. - clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; - flag_value = *flag_ptr; - auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; - clear_size = *clear_ptr; - int data_offset = flag_value % 3; - int clear_offset = (flag_value + 2) % 3; - for (int r = 0; r < NRanks; ++r) - { - data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; - } - clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; - __syncthreads(); - if (threadIdx.x == 0) - { - atomicAdd(counter_ptr, 1); - } - } - - __device__ __forceinline__ void update(int64_t new_clear_size) - { - if (blockIdx.x == 0 && threadIdx.x == 0) - { - while (*reinterpret_cast(counter_ptr) != gridDim.x) - { - } - *flag_ptr = (flag_value + 1) % 3; - *clear_ptr = new_clear_size; - *counter_ptr = 0; - } - } - - int* counter_ptr; - int* flag_ptr; - int64_t* clear_ptr; - uint8_t* data_bufs[NRanks]; - uint8_t* clear_buf; - int64_t clear_size; - int flag_value; -}; - -//! -0.0f is the not-arrived marker: a 16-byte slot has arrived once none of its words is -0.0f. -__device__ __forceinline__ bool is_neg_zero(float v) -{ - return *reinterpret_cast(&v) == 0x80000000; -} - -__device__ __forceinline__ bool is_neg_zero(float4 v) -{ - return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); -} - -__device__ __forceinline__ float4 get_neg_zero() -{ - float4 vec; -#pragma unroll - for (int i = 0; i < 4; ++i) - { - reinterpret_cast(&vec)[i] = 0x80000000; - } - return vec; -} - -__device__ __forceinline__ float4 ld_global_volatile(float4* addr) -{ - float4 val; - asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" - : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) - : "l"(addr)); - return val; -} - -//! The current device's SM count, read once per process (per linked library). -inline int get_sm_count() -{ - static int sm_count = 0; - if (sm_count == 0) - { - int device_id; - TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); - cudaDeviceProp device_prop; - cudaGetDeviceProperties(&device_prop, device_id); - sm_count = device_prop.multiProcessorCount; - } - return sm_count; -} -} // namespace kernels::ar_fusion - -TRTLLM_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu index 7311bc2e5020..d32cd7d252fd 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/kernels/communicationKernels/lamportComm.cuh" #include "tensorrt_llm/modeling_v2/common/namespace.h" #include @@ -33,20 +32,113 @@ namespace kernels { namespace hc_inject { -// This kernel runs on the workspace of the stock one-shot all-reduce `allreduce_fusion_kernel_oneshot_lamport` -// (communicationKernels/allReduceFusionKernels.cu) between its calls, so it takes that file's LamportComm and helpers -// (communicationKernels/lamportComm.cuh) and its push / clear / poll / update steps match that kernel's: +// This kernel shares the stock IPC all-reduce workspace with the Lamport kernels of +// communicationKernels/allReduceFusionKernels.cu and runs between their calls, so LamportComm, the helpers below and +// the push / clear / poll / update steps must follow that file's Lamport protocol: // - workspace[2 * NRanks + r] is rank r's buffer of three sub-buffers of workspace[3 * NRanks + 1][1] bytes; with // flag = workspace[3 * NRanks][2], a call writes and polls sub-buffer flag % 3 and clears its own (flag + 2) % 3. // - workspace[3 * NRanks + 1][0] holds the bytes the previous call wrote, cleared as 16-byte stores of -0.0f. // - A 16-byte slot has arrived once none of its 32-bit words is -0.0f, so senders store -0.0f as +0.0f. // - Every CTA increments workspace[3 * NRanks][0] after reading the flag; once all have, CTA 0 advances the flag to // (flag + 1) % 3, stores the bytes this call wrote and resets the counter. -using ::tensorrt_llm::kernels::ar_fusion::get_neg_zero; -using ::tensorrt_llm::kernels::ar_fusion::get_sm_count; -using ::tensorrt_llm::kernels::ar_fusion::is_neg_zero; -using ::tensorrt_llm::kernels::ar_fusion::LamportComm; -using ::tensorrt_llm::kernels::ar_fusion::ld_global_volatile; +// The comm op matrix (tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/ +// _hc_ar_inject_norm_op_matrix.py) pins this: it interleaves this kernel with stock all-reduces on one workspace and +// checks both bitwise at 2 and 4 ranks. +template +struct LamportComm +{ + __device__ __forceinline__ LamportComm(void** workspace, int rank) + { + counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; + flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; + // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and + // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar + // MiniMax kernel writes 4-byte words. + clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; + flag_value = *flag_ptr; + auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; + clear_size = *clear_ptr; + int data_offset = flag_value % 3; + int clear_offset = (flag_value + 2) % 3; + for (int r = 0; r < NRanks; ++r) + { + data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; + } + clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; + __syncthreads(); + if (threadIdx.x == 0) + { + atomicAdd(counter_ptr, 1); + } + } + + __device__ __forceinline__ void update(int64_t new_clear_size) + { + if (blockIdx.x == 0 && threadIdx.x == 0) + { + while (*reinterpret_cast(counter_ptr) != gridDim.x) + { + } + *flag_ptr = (flag_value + 1) % 3; + *clear_ptr = new_clear_size; + *counter_ptr = 0; + } + } + + int* counter_ptr; + int* flag_ptr; + int64_t* clear_ptr; + uint8_t* data_bufs[NRanks]; + uint8_t* clear_buf; + int64_t clear_size; + int flag_value; +}; + +//! -0.0f is the not-arrived marker: a 16-byte slot has arrived once none of its words is -0.0f. +__device__ __forceinline__ bool is_neg_zero(float v) +{ + return *reinterpret_cast(&v) == 0x80000000; +} + +__device__ __forceinline__ bool is_neg_zero(float4 v) +{ + return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); +} + +__device__ __forceinline__ float4 get_neg_zero() +{ + float4 vec; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + reinterpret_cast(&vec)[i] = 0x80000000; + } + return vec; +} + +__device__ __forceinline__ float4 ld_global_volatile(float4* addr) +{ + float4 val; + asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" + : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) + : "l"(addr)); + return val; +} + +//! The current device's SM count, read once per process. +inline int get_sm_count() +{ + static int sm_count = 0; + if (sm_count == 0) + { + int device_id; + TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); + cudaDeviceProp device_prop; + TLLM_CUDA_CHECK(cudaGetDeviceProperties(&device_prop, device_id)); + sm_count = device_prop.multiProcessorCount; + } + return sm_count; +} } // namespace hc_inject // Thread t holds columns [8t, 8t + 8) of a row's block output and of each of its 4 streams. The Triton program runs a From 532864b665101612663bee5e9c3b30a6024d1120 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:00:17 -0800 Subject: [PATCH 46/51] [None][test] modeling_v2 qwen4_exp: fit the FP8 target's post-merge tests in the GB300 shards Record measured durations for test_mtp3_gsm8k[dep4] and test_disaggregated_dep2 so the least-duration split places them, and keep test_mtp3_gsm8k[dep4_alltoall] (the opt-in all-to-all path) in the QA list only. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- tests/integration/defs/.test_durations | 2 + .../test-db/l0_gb300_multi_gpus.yml | 3 +- .../_mnnvl_allreduce__tp_sum_op_matrix.py | 178 --------------- ..._moe_a2a__dp_dispatch_combine_op_matrix.py | 208 ------------------ 4 files changed, 3 insertions(+), 388 deletions(-) delete mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py delete mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index e15d9838903d..1f1665113d82 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -25,6 +25,8 @@ "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp4::test_gsm8k": 931.6819444444445, "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp8::test_eagle3_gsm8k_and_acceptance": 791.973, "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp8::test_gsm8k": 659.1681428571428, + "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2": 360.976, + "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4]": 666.068, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2": 314.12083333333334, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_and_long_context": 585.9465129533679, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[dep4_fp8_kv]": 600.46915, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml index 887e69b38423..6f6ef09a4304 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml @@ -43,9 +43,8 @@ l0_gb300_multi_gpus: - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_fp8_kv_flashinfer_ring] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_nvfp4_kv_no_reuse] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2 TIMEOUT (90) - # qwen4_exp FP8 target, post-merge: dep4 MTP3 with each MoE exchange, dep2 disaggregated. + # qwen4_exp FP8 target, post-merge: dep4 MTP3, dep2 disaggregated. - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) - - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4_alltoall] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) # qwen4_exp target at tp1, post-merge: its AgentX engines (NVFP4 KV, MTP2 with rejection # sampling, the breakable prefill graph, every opt-in projection) on one context and one diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py deleted file mode 100644 index fef3add8edde..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Certification matrix for the FP8 target's `mnnvl_allreduce__tp_sum` under mpirun: each sum is -bitwise the IPC op's and within `_compare`'s band of the fp32 sum, eagerly and graph-replayed. -The MPI-disabled launch (`TLLM_DISABLE_MPI=1`) is not certified here.""" - -import argparse -import functools -import os -import sys -from pathlib import Path -from typing import Any - -import torch - -_MPI_WORKER_FLAG = "--mpi-rank-worker" -DEADLINE_S = 900 - -#: tests/unittest/_torch/modeling_v2: `_validating` and the shared `comm/` helpers. -_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] - -# Bound inside the rank body: module scope imports only torch and the standard library, since -# `_rank_job` executes it to read `DEADLINE_S`. -entry: Any = None -ipc: Any = None -mnnvl: Any = None -modeling: Any = None -validating: Any = None -WORKSPACE: dict = {} -MPI_RANK: Any = None -RANK = 0 -WORLD = 1 - -#: Row counts a captured `tp_sum` meets inside the budget: decode batches 1..8 and the MTP-3 -#: verify steps of 1..8 requests (4..32 rows). -GRAPH_ROWS = (1, 2, 3, 4, 8, 12, 16, 24, 32) - - -def _compare(out, ref) -> None: - """`catalog/comm/allreduce`'s band: 5 ulp per element against the row scale, 1.25 ulp RMS.""" - from _validating import assert_within_ulp - - assert_within_ulp(out, ref, element_ulp=5.0, rms_ulp=1.25) - - -def _payload(rank: int, rows: int, seed: int) -> torch.Tensor: - """One rank's partial: every rank can regenerate every other rank's from the seed.""" - gen = torch.Generator(device="cuda").manual_seed(seed * 977 + rank + 1) - return torch.randn(rows, entry.HIDDEN, generator=gen, device="cuda").to(torch.bfloat16) - - -def _reference(rows: int, seed: int) -> torch.Tensor: - return entry.mnnvl_tp_sum.reference([_payload(r, rows, seed) for r in range(WORLD)]) - - -def _exact(out, ref) -> float: - return (out == ref).float().mean().item() - - -def check_certified_cells() -> None: - cells = [c for c in entry.mnnvl_tp_sum.CELLS if c.spec["world"] == WORLD] - assert cells, f"no cell is certified at world size {WORLD}" - for i, cell in enumerate(cells): - rows = cell.spec["rows"] - for s in range(3): - seed = 4000 + 31 * i + 1000 * s - x = _payload(RANK, rows, seed) - with validating(entry.mnnvl_tp_sum): - out = mnnvl(x) - twin = ipc(x) - ref = _reference(rows, seed) - torch.cuda.synchronize() - assert out.shape == x.shape and out.dtype == x.dtype, (out.shape, out.dtype) - assert torch.equal(out, twin), f"{cell.why}: not the IPC op's bits (seed {seed})" - _compare(out, ref) - if RANK == 0 and s == 0: - print( - f"[world {WORLD}] {cell.why}: bitwise the IPC op; " - f"{100 * _exact(out, ref):.4f}% bit-exact to the fp32 sum rounded once", - flush=True, - ) - MPI_RANK.barrier() - - -def check_cuda_graph_replay_matches_eager() -> None: - """Captured once per row count, refilled with fresh partials and replayed: bitwise the - eager call and the IPC op on those partials (the Lamport buffers rotate per call).""" - for rows in GRAPH_ROWS: - static = _payload(RANK, rows, 900 + rows) - side = torch.cuda.Stream() - side.wait_stream(torch.cuda.current_stream()) - with torch.cuda.stream(side): - for _ in range(3): - mnnvl(static) - torch.cuda.current_stream().wait_stream(side) - MPI_RANK.barrier() - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - captured = mnnvl(static) - MPI_RANK.barrier() - for step in range(3): - fresh = _payload(RANK, rows, 1900 + 10 * rows + step) - static.copy_(fresh) - graph.replay() - eager = mnnvl(fresh) - twin = ipc(fresh) - torch.cuda.synchronize() - assert torch.equal(captured, eager), f"{rows} rows step {step}: replay != eager" - assert torch.equal(captured, twin), f"{rows} rows step {step}: replay != IPC op" - MPI_RANK.barrier() - if RANK == 0: - print(f"[world {WORLD}] captured {rows} rows: replay == eager == IPC op", flush=True) - del graph - - -CHECKS = (check_certified_cells, check_cuda_graph_replay_matches_eager) - - -def _run_one_mpi_rank() -> int: - global entry, ipc, mnnvl, modeling, validating, WORKSPACE, MPI_RANK, RANK, WORLD - os.environ.pop("TLLM_DISABLE_MPI", None) - sys.path.insert(0, str(_MODELING_V2_TESTS)) - sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) - import _lockstep - from _validating import validating as _validating_cm - - MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) - from tensorrt_llm._torch.modeling_v2.catalog.comm import ( - allreduce as allreduce_mod, - ) - from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 - modeling as _modeling, - ) - from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 - mnnvl_allreduce__tp_sum as _entry, - ) - from tensorrt_llm.functional import AllReduceFusionOp - - entry, modeling, validating = _entry, _modeling, _validating_cm - RANK, WORLD = MPI_RANK.rank, MPI_RANK.world - group = allreduce_mod.allreduce_group_args(MPI_RANK.mapping) - ipc = functools.partial( - allreduce_mod.allreduce, - residual=None, - norm_weight=None, - scale=None, - bias=None, - op=int(AllReduceFusionOp.NONE), - eps=1e-6, - trigger_completion_at_end=False, - **group, - ) - WORKSPACE = entry.mnnvl_workspace_args( - MPI_RANK.mapping, modeling._TP_SUM_MNNVL_MAX_ROWS, entry.HIDDEN, torch.bfloat16 - ) - mnnvl = functools.partial(entry.mnnvl_tp_sum, **WORKSPACE) - - code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) - torch.cuda.synchronize() - sys.stdout.flush() - sys.stderr.flush() - MPI_RANK.MPI.Finalize() - os._exit(code) - - -if __name__ == "__main__": - if _MPI_WORKER_FLAG in sys.argv: - sys.exit(_run_one_mpi_rank()) - import _launch # beside this file - - parser = argparse.ArgumentParser() - parser.add_argument("--world-size", type=int, default=None) - world_size = parser.parse_args().world_size or len(_launch.visible_devices()) - _launch.mpirun_ranks( - __file__, world_size, [_MPI_WORKER_FLAG, "--world-size", str(world_size)], DEADLINE_S - ) - print("OK: MPI session (mnnvl_fusion_allreduce, fusion NONE)", flush=True) - print("OK") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py deleted file mode 100644 index 819bf0d36b7c..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Certification matrix for the FP8 target's MoE all-to-all round trip (`moe_a2a_dispatch` and -`moe_a2a_combine` on the stock NVLink one-sided all-to-all) at dep2 and dep4 under mpirun, eagerly -and graph-replayed. Fails where the platform has no MNNVL (the collected test skips there first).""" - -import os -import sys -from pathlib import Path -from typing import Any, List, Tuple - -import torch - -assert torch.cuda.is_available(), "the MoE all-to-all requires CUDA devices" - -_MPI_WORKER_FLAG = "--mpi-rank-worker" -DEADLINE_S = 900 - -#: The attention-DP group sizes this target serves (dep2, dep4). -CERTIFIED_WORLDS: Tuple[int, ...] = (2, 4) - -HIDDEN = 2560 -NUM_EXPERTS = 512 -TOPK = 10 - -_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] -_TARGET = ( - "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." - "qwen3_8_flash_next_fp8__sm_103__unified" -) - -# Bound inside the rank body: importing the entry pulls in tensorrt_llm, which calls MPI_Init at -# import, and the launcher must stay free of MPI and CUDA state. -entry: Any = None -validating: Any = None -MPI_RANK: Any = None -STRATEGY: Any = None -RANK = 0 -WORLD = 1 - - -def _gen(*key: int) -> torch.Generator: - """Seeded alike on every rank for a key, so each rank can rebuild what any other rank holds.""" - seed = 0 - for k in key: - seed = seed * 1000003 + int(k) - return torch.Generator(device="cuda").manual_seed(seed % (2**62)) - - -def _inputs(rows: int, src: int, salt: int) -> tuple: - """Rank `src`'s rows and routing: bf16 rows, ten distinct experts a row, softmax weights.""" - g = _gen(salt, src) - x = torch.randn(rows, HIDDEN, generator=g, device="cuda").to(torch.bfloat16) - ids = torch.rand(rows, NUM_EXPERTS, generator=g, device="cuda").topk(TOPK, dim=1).indices - w = torch.softmax(torch.randn(rows, TOPK, generator=g, device="cuda"), dim=1) - return x, ids.to(torch.int32), w.to(torch.bfloat16) - - -def _window(rows, ids, w, rank: int) -> torch.Tensor: - """Rank `rank`'s stand-in expert window: each row scaled by its in-window experts' weights times - their ids, in fp32, rounded once; a row with no expert there (an empty slot too) is zero.""" - epr = NUM_EXPERTS // WORLD - keep = (ids >= rank * epr) & (ids < (rank + 1) * epr) - scale = (w.float() * keep * (ids.float() + 1) / NUM_EXPERTS).sum(-1, keepdim=True) - out = scale * rows.float() - return torch.where(keep.any(-1, keepdim=True), out, torch.zeros_like(out)).to(torch.bfloat16) - - -def _cells() -> list: - return [c for c in entry.moe_a2a_dispatch.CELLS if c.spec["world"] == WORLD] - - -def _check_received(got, everyone, counts, what: str) -> None: - """Received token slots equal the reference rows as a set: the kernel orders a block itself.""" - want = entry.moe_a2a_dispatch.reference(everyone, counts, NUM_EXPERTS // WORLD, rank=RANK) - by_key = {} - for x, ids, w in want: - for i in range(ids.shape[0]): - by_key[tuple(ids[i].tolist())] = (x[i], w[i]) - valid = (got.topk_ids != entry.NO_EXPERT).any(1) - assert got.rows.shape[0] == WORLD * max(counts), (what, got.rows.shape, counts) - assert int(valid.sum()) == len(by_key), (what, int(valid.sum()), len(by_key)) - for i in torch.nonzero(valid).flatten().tolist(): - key = tuple(got.topk_ids[i].tolist()) - assert key in by_key, f"{what}: slot {i} holds a row no source routed here" - x, w = by_key[key] - assert torch.equal(got.rows[i], x) and torch.equal(got.topk_weights[i], w), (what, i) - - -def _check_combined(out, mine, what: str) -> None: - x, ids, w = mine - want = entry.moe_a2a_combine.reference([_window(x, ids, w, r) for r in range(WORLD)]) - assert out.shape == want.shape, (what, out.shape, want.shape) - torch.testing.assert_close(out.float(), want.float(), rtol=2 * 2**-8, atol=1e-3, msg=what) - - -def _round_trip(x, ids, w, counts): - got = entry.moe_a2a_dispatch(STRATEGY, x, ids, w, counts) - partials = _window(got.rows, got.topk_ids, got.topk_weights, RANK) - return got, entry.moe_a2a_combine(STRATEGY, partials) - - -def check_cells() -> None: - """Every cell of this world size: dispatch as a set against its reference, then the round - trip through the stand-in window against the combine reference.""" - cells = _cells() - assert cells, f"no cell at world {WORLD}" - for i, cell in enumerate(cells): - counts = list(cell.spec["counts"]) - everyone = [_inputs(n, r, 10 + i) for r, n in enumerate(counts)] - with validating(entry.moe_a2a_dispatch): - got, out = _round_trip(*everyone[RANK], counts) - torch.cuda.synchronize() - _check_received(got, everyone, counts, cell.why) - _check_combined(out, everyone[RANK], cell.why) - MPI_RANK.barrier() - - -def check_graph() -> None: - """The round trip captured once at a uniform cell and replayed with fresh rows and routing.""" - cell = max( - (c for c in _cells() if len(set(c.spec["counts"])) == 1), key=lambda c: c.spec["counts"][0] - ) - counts = list(cell.spec["counts"]) - static = [t.clone() for t in _inputs(counts[RANK], RANK, 50)] - _round_trip(*static, counts) # eager first: the graph replays what the warm call set up - torch.cuda.synchronize() - MPI_RANK.barrier() - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - got, out = _round_trip(*static, counts) - for salt in (51, 52): - everyone = [_inputs(n, r, salt) for r, n in enumerate(counts)] - for dst, src in zip(static, everyone[RANK]): - dst.copy_(src) - MPI_RANK.barrier() - graph.replay() - torch.cuda.synchronize() - _check_received(got, everyone, counts, f"replay {salt}") - _check_combined(out, everyone[RANK], f"replay {salt}") - MPI_RANK.barrier() - del graph - torch.cuda.synchronize() - MPI_RANK.barrier() - - -CHECKS = (check_cells, check_graph) - - -def _run_one_mpi_rank() -> int: - """One MPI rank's checks; a failing check aborts the MPI job, which ends its siblings.""" - global entry, validating, MPI_RANK, STRATEGY, RANK, WORLD - os.environ.pop("TLLM_DISABLE_MPI", None) - sys.path.insert(0, str(_MODELING_V2_TESTS)) - sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) - import importlib - - import _lockstep - from _validating import validating as _validating_cm - - MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) - from tensorrt_llm.mapping import Mapping - - entry = importlib.import_module(f"{_TARGET}.ops.comm.moe_a2a__dp_dispatch_combine") - validating = _validating_cm - RANK, WORLD = MPI_RANK.rank, MPI_RANK.world - mapping = Mapping( - world_size=WORLD, - rank=RANK, - gpus_per_node=WORLD, - tp_size=WORLD, - moe_tp_size=1, - moe_ep_size=WORLD, - enable_attention_dp=True, - ) - max_rows = max(max(c.spec["counts"]) for c in entry.moe_a2a_dispatch.CELLS) - STRATEGY = entry.build_strategy(mapping, max_rows) - if STRATEGY is None: - print(f"[rank {RANK}] no MNNVL on this platform: nothing certified", flush=True) - MPI_RANK.MPI.Finalize() - os._exit(1) - _lockstep.run_checks(MPI_RANK, list(CHECKS)) - torch.cuda.synchronize() - print(f"[rank {RANK}] {len(CHECKS)} checks passed over the MPI session", flush=True) - sys.stdout.flush() - sys.stderr.flush() - MPI_RANK.MPI.Finalize() - # Skip static destructors: the symmetric workspace's teardown adds nothing to the verdict. - os._exit(0) - - -def _world_size(argv: List[str]) -> int: - if "--world-size" in argv: - return int(argv[argv.index("--world-size") + 1]) - return len([d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()]) - - -if __name__ == "__main__": - if _MPI_WORKER_FLAG in sys.argv: - sys.exit(_run_one_mpi_rank()) - import _launch # beside this file - - WORLD_SIZE = _world_size(sys.argv) - assert WORLD_SIZE in CERTIFIED_WORLDS, (WORLD_SIZE, CERTIFIED_WORLDS) - _launch.mpirun_ranks( - __file__, WORLD_SIZE, [_MPI_WORKER_FLAG, "--world-size", str(WORLD_SIZE)], DEADLINE_S - ) - print(f"OK: {WORLD_SIZE} ranks over the MPI session", flush=True) From 29aca83c82281c74e96bdb5b5eebf03f4323822f Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:00:19 -0800 Subject: [PATCH 47/51] [None][test] modeling_v2 qwen4_exp: drop the FP8 target's MNNVL tp_sum and MoE all-to-all collective matrices Both ops are thin calls of stock kernels (the MNNVL all-reduce without fusion, NVLinkOneSided dispatch / combine); the matrices kept are the target's own protocols: the fused HC TP sum, the Lamport index-split gather and the copy-engine exchange. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../comm/test_fp8_unified_op_matrices.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py index a8256b92fef8..b3c5539e5576 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -23,11 +23,9 @@ #: The group sizes the target runs each entry at (each launcher's `CERTIFIED_WORLDS`, restated #: so that collecting this module on a machine without GPUs loads no launcher). _WORLDS = { - "mnnvl_allreduce__tp_sum": (2, 4), "ce_exchange__sp_pair": (2,), "lamport_allgather__index_split": (2, 4), "hc_ar_inject_norm": (2, 4), - "moe_a2a__dp_dispatch_combine": (2, 4), } _CASES = [(entry, world) for entry, worlds in _WORLDS.items() for world in worlds] @@ -46,9 +44,4 @@ def _sm103_devices(world: int) -> int: def test_op_matrix(entry: str, world: int) -> None: if _sm103_devices(world) < world: pytest.skip(f"the {entry} matrix at world {world} runs on {world} sm_103 GPUs in one node") - if entry == "moe_a2a__dp_dispatch_combine": - from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided - - if not NVLinkOneSided.is_platform_supported(): - pytest.skip(f"the {entry} matrix needs MNNVL, which this platform lacks") _rank_job.run(entry, world_size=world, directory=_HERE) From 37232566f783832af569f2bf496e1dce473ae435 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:56:07 -0800 Subject: [PATCH 48/51] [None][fix] modeling_v2 qwen4_exp: follow main's sigmoid-mul and gated-norm launches in the family direct-launch copies The sigmoid-mul kernel now takes a scale pointer before the strides, and the token-major gated norm launch tiles large row counts in groups with contiguity and evenness hints; the qwen4_exp copies launch the same way, which their bitwise pins against the registered ops check. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../activation/fused_sigmoid_mul_inplace.py | 1 + .../ops/norm/rms_norm_gated_token_major.py | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py index 63fd5a69d27d..5a733488bf40 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py @@ -47,6 +47,7 @@ def _launch(attention_output: torch.Tensor, gate: torch.Tensor) -> bool: attention_output, attention_output, gate, + attention_output, # scale_ptr, read only for an FP8 output attention_output.stride(0), attention_output.stride(0), gate.stride(0), diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py index 2ae4a233728b..853fa2409eb5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py @@ -50,20 +50,29 @@ def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> def _direct_launch(x, z, weight, eps, gate_activation): - """The registered op's own multi-row launch (same kernel, 4 rows a program, PDL by the - op's policy), skipping only the custom-op dispatch.""" + """The registered op's own multi-row launch (same kernel, 4 rows a tile, its tiles per + program and fast-path hints, PDL by the op's policy), skipping only the custom-op dispatch.""" rows, n = x.shape out = torch.empty_like(x) - grid = triton.cdiv(rows, _layernorm_gated._MULTIROW_ROWS) + rows_per_tile = _layernorm_gated._MULTIROW_ROWS + groups = ( + _layernorm_gated._MULTIROW_GROUPS + if rows >= _layernorm_gated._MULTIROW_GROUPS_MIN_ROWS + else 1 + ) + grid = triton.cdiv(rows, rows_per_tile * groups) device = x.device.index pdl = _layernorm_gated._multirow_pdl(grid, device) with torch.cuda.device(device): _layernorm_gated._rms_norm_gated_fwd_multirow_kernel[(grid,)]( x, out, weight, z, None, None, x.stride(0), out.stride(0), z.stride(0), rows, eps, - N=n, ROWS=_layernorm_gated._MULTIROW_ROWS, HEADS_PER_TOK=z.shape[1], + N=n, ROWS=rows_per_tile, HEADS_PER_TOK=z.shape[1], GATE_SIGMOID=gate_activation == "sigmoid", WEIGHT_IS_DELTA=False, - LAUNCH_WITH_PDL=pdl, num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, - EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], launch_pdl=pdl, + LAUNCH_WITH_PDL=pdl, CONTIG=(x.stride(0) == n and out.stride(0) == n), + EVEN=(rows % (rows_per_tile * groups) == 0), + num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, + EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], GROUPS=groups, + launch_pdl=pdl, ) # fmt: skip return out From 30c927a99ad124bc5d224ab1c4f4c5626931f6b4 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:04:52 -0800 Subject: [PATCH 49/51] [None][test] modeling_v2 qwen4_exp: restore the FP8 target's MNNVL tp_sum and MoE all-to-all collective matrices Both op entries declare certified cells, which the modeling_v2 contract requires a test to drive; the matrices are those tests, and MNNVL tp_sum is on the default decode path at tp2 and tp4. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../_mnnvl_allreduce__tp_sum_op_matrix.py | 178 +++++++++++++++ ..._moe_a2a__dp_dispatch_combine_op_matrix.py | 208 ++++++++++++++++++ .../comm/test_fp8_unified_op_matrices.py | 7 + 3 files changed, 393 insertions(+) create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py new file mode 100644 index 000000000000..fef3add8edde --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's `mnnvl_allreduce__tp_sum` under mpirun: each sum is +bitwise the IPC op's and within `_compare`'s band of the fp32 sum, eagerly and graph-replayed. +The MPI-disabled launch (`TLLM_DISABLE_MPI=1`) is not certified here.""" + +import argparse +import functools +import os +import sys +from pathlib import Path +from typing import Any + +import torch + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: tests/unittest/_torch/modeling_v2: `_validating` and the shared `comm/` helpers. +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +# Bound inside the rank body: module scope imports only torch and the standard library, since +# `_rank_job` executes it to read `DEADLINE_S`. +entry: Any = None +ipc: Any = None +mnnvl: Any = None +modeling: Any = None +validating: Any = None +WORKSPACE: dict = {} +MPI_RANK: Any = None +RANK = 0 +WORLD = 1 + +#: Row counts a captured `tp_sum` meets inside the budget: decode batches 1..8 and the MTP-3 +#: verify steps of 1..8 requests (4..32 rows). +GRAPH_ROWS = (1, 2, 3, 4, 8, 12, 16, 24, 32) + + +def _compare(out, ref) -> None: + """`catalog/comm/allreduce`'s band: 5 ulp per element against the row scale, 1.25 ulp RMS.""" + from _validating import assert_within_ulp + + assert_within_ulp(out, ref, element_ulp=5.0, rms_ulp=1.25) + + +def _payload(rank: int, rows: int, seed: int) -> torch.Tensor: + """One rank's partial: every rank can regenerate every other rank's from the seed.""" + gen = torch.Generator(device="cuda").manual_seed(seed * 977 + rank + 1) + return torch.randn(rows, entry.HIDDEN, generator=gen, device="cuda").to(torch.bfloat16) + + +def _reference(rows: int, seed: int) -> torch.Tensor: + return entry.mnnvl_tp_sum.reference([_payload(r, rows, seed) for r in range(WORLD)]) + + +def _exact(out, ref) -> float: + return (out == ref).float().mean().item() + + +def check_certified_cells() -> None: + cells = [c for c in entry.mnnvl_tp_sum.CELLS if c.spec["world"] == WORLD] + assert cells, f"no cell is certified at world size {WORLD}" + for i, cell in enumerate(cells): + rows = cell.spec["rows"] + for s in range(3): + seed = 4000 + 31 * i + 1000 * s + x = _payload(RANK, rows, seed) + with validating(entry.mnnvl_tp_sum): + out = mnnvl(x) + twin = ipc(x) + ref = _reference(rows, seed) + torch.cuda.synchronize() + assert out.shape == x.shape and out.dtype == x.dtype, (out.shape, out.dtype) + assert torch.equal(out, twin), f"{cell.why}: not the IPC op's bits (seed {seed})" + _compare(out, ref) + if RANK == 0 and s == 0: + print( + f"[world {WORLD}] {cell.why}: bitwise the IPC op; " + f"{100 * _exact(out, ref):.4f}% bit-exact to the fp32 sum rounded once", + flush=True, + ) + MPI_RANK.barrier() + + +def check_cuda_graph_replay_matches_eager() -> None: + """Captured once per row count, refilled with fresh partials and replayed: bitwise the + eager call and the IPC op on those partials (the Lamport buffers rotate per call).""" + for rows in GRAPH_ROWS: + static = _payload(RANK, rows, 900 + rows) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + for _ in range(3): + mnnvl(static) + torch.cuda.current_stream().wait_stream(side) + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = mnnvl(static) + MPI_RANK.barrier() + for step in range(3): + fresh = _payload(RANK, rows, 1900 + 10 * rows + step) + static.copy_(fresh) + graph.replay() + eager = mnnvl(fresh) + twin = ipc(fresh) + torch.cuda.synchronize() + assert torch.equal(captured, eager), f"{rows} rows step {step}: replay != eager" + assert torch.equal(captured, twin), f"{rows} rows step {step}: replay != IPC op" + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] captured {rows} rows: replay == eager == IPC op", flush=True) + del graph + + +CHECKS = (check_certified_cells, check_cuda_graph_replay_matches_eager) + + +def _run_one_mpi_rank() -> int: + global entry, ipc, mnnvl, modeling, validating, WORKSPACE, MPI_RANK, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm._torch.modeling_v2.catalog.comm import ( + allreduce as allreduce_mod, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _modeling, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + mnnvl_allreduce__tp_sum as _entry, + ) + from tensorrt_llm.functional import AllReduceFusionOp + + entry, modeling, validating = _entry, _modeling, _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + group = allreduce_mod.allreduce_group_args(MPI_RANK.mapping) + ipc = functools.partial( + allreduce_mod.allreduce, + residual=None, + norm_weight=None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=1e-6, + trigger_completion_at_end=False, + **group, + ) + WORKSPACE = entry.mnnvl_workspace_args( + MPI_RANK.mapping, modeling._TP_SUM_MNNVL_MAX_ROWS, entry.HIDDEN, torch.bfloat16 + ) + mnnvl = functools.partial(entry.mnnvl_tp_sum, **WORKSPACE) + + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + parser = argparse.ArgumentParser() + parser.add_argument("--world-size", type=int, default=None) + world_size = parser.parse_args().world_size or len(_launch.visible_devices()) + _launch.mpirun_ranks( + __file__, world_size, [_MPI_WORKER_FLAG, "--world-size", str(world_size)], DEADLINE_S + ) + print("OK: MPI session (mnnvl_fusion_allreduce, fusion NONE)", flush=True) + print("OK") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py new file mode 100644 index 000000000000..819bf0d36b7c --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's MoE all-to-all round trip (`moe_a2a_dispatch` and +`moe_a2a_combine` on the stock NVLink one-sided all-to-all) at dep2 and dep4 under mpirun, eagerly +and graph-replayed. Fails where the platform has no MNNVL (the collected test skips there first).""" + +import os +import sys +from pathlib import Path +from typing import Any, List, Tuple + +import torch + +assert torch.cuda.is_available(), "the MoE all-to-all requires CUDA devices" + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: The attention-DP group sizes this target serves (dep2, dep4). +CERTIFIED_WORLDS: Tuple[int, ...] = (2, 4) + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] +_TARGET = ( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." + "qwen3_8_flash_next_fp8__sm_103__unified" +) + +# Bound inside the rank body: importing the entry pulls in tensorrt_llm, which calls MPI_Init at +# import, and the launcher must stay free of MPI and CUDA state. +entry: Any = None +validating: Any = None +MPI_RANK: Any = None +STRATEGY: Any = None +RANK = 0 +WORLD = 1 + + +def _gen(*key: int) -> torch.Generator: + """Seeded alike on every rank for a key, so each rank can rebuild what any other rank holds.""" + seed = 0 + for k in key: + seed = seed * 1000003 + int(k) + return torch.Generator(device="cuda").manual_seed(seed % (2**62)) + + +def _inputs(rows: int, src: int, salt: int) -> tuple: + """Rank `src`'s rows and routing: bf16 rows, ten distinct experts a row, softmax weights.""" + g = _gen(salt, src) + x = torch.randn(rows, HIDDEN, generator=g, device="cuda").to(torch.bfloat16) + ids = torch.rand(rows, NUM_EXPERTS, generator=g, device="cuda").topk(TOPK, dim=1).indices + w = torch.softmax(torch.randn(rows, TOPK, generator=g, device="cuda"), dim=1) + return x, ids.to(torch.int32), w.to(torch.bfloat16) + + +def _window(rows, ids, w, rank: int) -> torch.Tensor: + """Rank `rank`'s stand-in expert window: each row scaled by its in-window experts' weights times + their ids, in fp32, rounded once; a row with no expert there (an empty slot too) is zero.""" + epr = NUM_EXPERTS // WORLD + keep = (ids >= rank * epr) & (ids < (rank + 1) * epr) + scale = (w.float() * keep * (ids.float() + 1) / NUM_EXPERTS).sum(-1, keepdim=True) + out = scale * rows.float() + return torch.where(keep.any(-1, keepdim=True), out, torch.zeros_like(out)).to(torch.bfloat16) + + +def _cells() -> list: + return [c for c in entry.moe_a2a_dispatch.CELLS if c.spec["world"] == WORLD] + + +def _check_received(got, everyone, counts, what: str) -> None: + """Received token slots equal the reference rows as a set: the kernel orders a block itself.""" + want = entry.moe_a2a_dispatch.reference(everyone, counts, NUM_EXPERTS // WORLD, rank=RANK) + by_key = {} + for x, ids, w in want: + for i in range(ids.shape[0]): + by_key[tuple(ids[i].tolist())] = (x[i], w[i]) + valid = (got.topk_ids != entry.NO_EXPERT).any(1) + assert got.rows.shape[0] == WORLD * max(counts), (what, got.rows.shape, counts) + assert int(valid.sum()) == len(by_key), (what, int(valid.sum()), len(by_key)) + for i in torch.nonzero(valid).flatten().tolist(): + key = tuple(got.topk_ids[i].tolist()) + assert key in by_key, f"{what}: slot {i} holds a row no source routed here" + x, w = by_key[key] + assert torch.equal(got.rows[i], x) and torch.equal(got.topk_weights[i], w), (what, i) + + +def _check_combined(out, mine, what: str) -> None: + x, ids, w = mine + want = entry.moe_a2a_combine.reference([_window(x, ids, w, r) for r in range(WORLD)]) + assert out.shape == want.shape, (what, out.shape, want.shape) + torch.testing.assert_close(out.float(), want.float(), rtol=2 * 2**-8, atol=1e-3, msg=what) + + +def _round_trip(x, ids, w, counts): + got = entry.moe_a2a_dispatch(STRATEGY, x, ids, w, counts) + partials = _window(got.rows, got.topk_ids, got.topk_weights, RANK) + return got, entry.moe_a2a_combine(STRATEGY, partials) + + +def check_cells() -> None: + """Every cell of this world size: dispatch as a set against its reference, then the round + trip through the stand-in window against the combine reference.""" + cells = _cells() + assert cells, f"no cell at world {WORLD}" + for i, cell in enumerate(cells): + counts = list(cell.spec["counts"]) + everyone = [_inputs(n, r, 10 + i) for r, n in enumerate(counts)] + with validating(entry.moe_a2a_dispatch): + got, out = _round_trip(*everyone[RANK], counts) + torch.cuda.synchronize() + _check_received(got, everyone, counts, cell.why) + _check_combined(out, everyone[RANK], cell.why) + MPI_RANK.barrier() + + +def check_graph() -> None: + """The round trip captured once at a uniform cell and replayed with fresh rows and routing.""" + cell = max( + (c for c in _cells() if len(set(c.spec["counts"])) == 1), key=lambda c: c.spec["counts"][0] + ) + counts = list(cell.spec["counts"]) + static = [t.clone() for t in _inputs(counts[RANK], RANK, 50)] + _round_trip(*static, counts) # eager first: the graph replays what the warm call set up + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got, out = _round_trip(*static, counts) + for salt in (51, 52): + everyone = [_inputs(n, r, salt) for r, n in enumerate(counts)] + for dst, src in zip(static, everyone[RANK]): + dst.copy_(src) + MPI_RANK.barrier() + graph.replay() + torch.cuda.synchronize() + _check_received(got, everyone, counts, f"replay {salt}") + _check_combined(out, everyone[RANK], f"replay {salt}") + MPI_RANK.barrier() + del graph + torch.cuda.synchronize() + MPI_RANK.barrier() + + +CHECKS = (check_cells, check_graph) + + +def _run_one_mpi_rank() -> int: + """One MPI rank's checks; a failing check aborts the MPI job, which ends its siblings.""" + global entry, validating, MPI_RANK, STRATEGY, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import importlib + + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm.mapping import Mapping + + entry = importlib.import_module(f"{_TARGET}.ops.comm.moe_a2a__dp_dispatch_combine") + validating = _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + mapping = Mapping( + world_size=WORLD, + rank=RANK, + gpus_per_node=WORLD, + tp_size=WORLD, + moe_tp_size=1, + moe_ep_size=WORLD, + enable_attention_dp=True, + ) + max_rows = max(max(c.spec["counts"]) for c in entry.moe_a2a_dispatch.CELLS) + STRATEGY = entry.build_strategy(mapping, max_rows) + if STRATEGY is None: + print(f"[rank {RANK}] no MNNVL on this platform: nothing certified", flush=True) + MPI_RANK.MPI.Finalize() + os._exit(1) + _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + print(f"[rank {RANK}] {len(CHECKS)} checks passed over the MPI session", flush=True) + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + # Skip static destructors: the symmetric workspace's teardown adds nothing to the verdict. + os._exit(0) + + +def _world_size(argv: List[str]) -> int: + if "--world-size" in argv: + return int(argv[argv.index("--world-size") + 1]) + return len([d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()]) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + WORLD_SIZE = _world_size(sys.argv) + assert WORLD_SIZE in CERTIFIED_WORLDS, (WORLD_SIZE, CERTIFIED_WORLDS) + _launch.mpirun_ranks( + __file__, WORLD_SIZE, [_MPI_WORKER_FLAG, "--world-size", str(WORLD_SIZE)], DEADLINE_S + ) + print(f"OK: {WORLD_SIZE} ranks over the MPI session", flush=True) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py index b3c5539e5576..a8256b92fef8 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -23,9 +23,11 @@ #: The group sizes the target runs each entry at (each launcher's `CERTIFIED_WORLDS`, restated #: so that collecting this module on a machine without GPUs loads no launcher). _WORLDS = { + "mnnvl_allreduce__tp_sum": (2, 4), "ce_exchange__sp_pair": (2,), "lamport_allgather__index_split": (2, 4), "hc_ar_inject_norm": (2, 4), + "moe_a2a__dp_dispatch_combine": (2, 4), } _CASES = [(entry, world) for entry, worlds in _WORLDS.items() for world in worlds] @@ -44,4 +46,9 @@ def _sm103_devices(world: int) -> int: def test_op_matrix(entry: str, world: int) -> None: if _sm103_devices(world) < world: pytest.skip(f"the {entry} matrix at world {world} runs on {world} sm_103 GPUs in one node") + if entry == "moe_a2a__dp_dispatch_combine": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided + + if not NVLinkOneSided.is_platform_supported(): + pytest.skip(f"the {entry} matrix needs MNNVL, which this platform lacks") _rank_job.run(entry, world_size=world, directory=_HERE) From 5969704ca85ad864a7254e3f293e7815b56a813e Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:12:56 -0800 Subject: [PATCH 50/51] [None][chore] modeling_v2 qwen4_exp: re-sort imports and fix a package path after the modeling_v2 move Main moved modeling_v2 out of _torch/_experimental; ruff re-sorts the shortened import paths in the FP8 target's and the family's files, and the hc sync test builds its package path without the old directory. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../ops/attention/qsa_paged_index_scores.py | 7 ++--- .../attention/qsa_paged_sparse_gqa_nvfp4.py | 3 +- .../ops/hc/qwen4_exp_hc_inject_norm.py | 6 ++-- .../qwen4_exp/ops/ple/ple_ngram_embed.py | 8 ++--- .../modeling.py | 31 +++++++------------ .../qsa_index_scores_groups_cute__decode.py | 2 +- .../qsa_sparse_gqa_grouped_cute__decode.py | 2 +- .../qsa_sparse_gqa_rowtile__prefill.py | 4 +-- .../qsa_sparse_gqa_splitk_cute__decode.py | 4 +-- .../qsa_topk_expand_cluster__decode.py | 2 +- .../attention/qsa_topk_rows_cute__prefill.py | 4 +-- .../qsa_triton_kernels__stable_jit.py | 8 ++--- .../qwen4_exp_attn_decode_prologue__decode.py | 2 +- .../comm/lamport_allgather__index_split.py | 2 +- .../ops/gemm/low_m_bf16_mm__decode.py | 7 +---- .../hc/qwen4_exp_hc_decode_rows_tc__decode.py | 2 +- .../qwen4_exp_hc_decode_tc_first__decode.py | 2 +- .../ops/moe/fp8_block_scale_moe_runner.py | 7 +---- .../ops/moe/moe_front_router_quant__decode.py | 7 +---- .../ops/moe/moe_tpb_decode__decode.py | 2 +- .../ops/ssm/gdn_in_proj_conv__verify.py | 7 +---- .../vision.py | 16 +++------- ...ing_v2_qwen4_exp_hc_decode_rows_tc_sync.py | 2 +- .../_mnnvl_allreduce__tp_sum_op_matrix.py | 4 +-- ..._moe_a2a__dp_dispatch_combine_op_matrix.py | 5 +-- .../test_attention_ops.py | 6 ++-- .../test_routing_and_layout.py | 4 +-- .../test_vision.py | 2 +- .../test_weights.py | 2 +- .../test_routing.py | 2 +- 30 files changed, 55 insertions(+), 107 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py index fca40bd492b2..64c9908ab2e7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py @@ -8,11 +8,6 @@ import triton import triton.language as tl -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( - warm_variants, -) - # Approved exception: QSA's selection path is Triton called as Python, with no # torch.ops boundary. The engine's own sparse hook (`select_qsa_paged_tokens` # in `_torch/attention/backends/sparse/qsa/module.py`) calls this same callable. @@ -20,6 +15,8 @@ from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( triton_qsa_paged_index_scores as _triton_qsa_paged_index_scores, ) +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber _INTEGER_DTYPES = (torch.int32, torch.int64) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py index 5dd8b2f41c3e..f57e0659843e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py @@ -10,12 +10,11 @@ import triton import triton.language as tl -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper - # Approved exception (as for `qsa_paged_sparse_gqa`): QSA's sparse path has no torch.ops # surface; its row-count PDL policy is reached as Python. The split-K buckets are this # entry's own: an NVFP4 tile needs 16 tokens where the bf16 one takes 64. from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import _qsa_pdl_enabled +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Base-2 online softmax: scores are scaled by log2(e) once. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py index 31598e6c1199..a4b8e7aa6c07 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py @@ -31,12 +31,10 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( - warm_variants, -) from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Trigger dependents at kernel start (`griddepcontrol.launch_dependents`); a no-op diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py index ba3f8a97ec75..98e37d40a03d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py @@ -31,12 +31,8 @@ import triton.language as tl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ( - ngram_hash, -) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( - warm_variants, -) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ngram_hash +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, TEP4_SLOTS, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py index fdde49544360..6574adcadfda 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -97,29 +97,29 @@ from transformers import PretrainedConfig from tensorrt_llm._ipc_utils import can_access_peer +from tensorrt_llm._torch.attention.backends.fmha.phased import get_spec_decoding_position_offsets +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.distributed.allreduce_helper import CustomAllReduceHelper +from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modeling_v2._router_index import ( NoTarget, engine_kv_cache_quant_algo, step_contract_enabled, ) -from tensorrt_llm._torch.modeling_v2.catalog.attention.fused_qk_norm_rope import ( - fused_qk_norm_rope, -) +from tensorrt_llm._torch.modeling_v2.catalog.attention.fused_qk_norm_rope import fused_qk_norm_rope from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import ( QM_FP8_KV_CACHE, QM_NVFP4_KV_CACHE, thop_attention, ) from tensorrt_llm._torch.modeling_v2.catalog.comm.allgather import allgather -from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import ( - allreduce, - allreduce_group_args, -) +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import allreduce, allreduce_group_args from tensorrt_llm._torch.modeling_v2.catalog.comm.reducescatter import reducescatter from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.catalog.graph.eager_on_graph import ( - eager_on_graph, -) +from tensorrt_llm._torch.modeling_v2.catalog.graph.eager_on_graph import eager_on_graph from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( fused_sigmoid_mul_inplace, ) @@ -222,9 +222,7 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_gate_conv_inject_norm import ( ple_gate_conv_inject_norm, ) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed import ( - ple_ngram_embed, -) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed import ple_ngram_embed from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( bf16_to_fp32, ) @@ -244,13 +242,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( gdn_prefill_conv_post, ) -from tensorrt_llm._torch.attention.backends.fmha.phased import get_spec_decoding_position_offsets -from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.distributed.allreduce_helper import CustomAllReduceHelper -from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl -from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM from tensorrt_llm._torch.modules.fla.cached_replay import CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE from tensorrt_llm._torch.modules.linear import TensorParallelMode diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py index 125124bd1bf2..fd76c8732f6f 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py @@ -25,11 +25,11 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( qsa_paged_index_scores as _family_index_scores, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from .qsa_sparse_gqa_splitk_cute__decode import ( _asm, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py index fd2e6eb20b33..5a1385019612 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py @@ -39,8 +39,8 @@ from cutlass.cute.typing import Float32, Int32, Int64 from cutlass.cutlass_dsl import T, dsl_user_op -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from .qsa_sparse_gqa_splitk_cute__decode import ( MAX_SPLITS, diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py index fc88da9ec6a5..acb07f85d796 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py @@ -24,6 +24,8 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( qsa_paged_sparse_gqa as _per_row_entry, @@ -41,8 +43,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( rowtile_attention_cute as _rowtile_attention_cute, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl -from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr _INTEGER_DTYPES = (torch.int32, torch.int64) # The per-row kernel's scan tile: the prefill bound is rounded up to it, as that kernel does. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py index 296c9785387d..d267f0d3aaee 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py @@ -28,12 +28,12 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import T, dsl_user_op +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( qsa_paged_sparse_gqa as _composition, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl -from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr TILE = 64 WARPS = 8 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py index bc0da743883e..cb124b86c423 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py @@ -17,9 +17,9 @@ from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import dsl_user_op -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper #: Consecutive scores a thread loads at once (one 16-byte load): a group. KPT = 4 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py index 63f4819d6bcf..6f9262cf830b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py @@ -18,12 +18,12 @@ from cutlass import const_expr from cutlass.cute.runtime import from_dlpack +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( qsa_topk_expand_decode as _family_topk, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl -from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr #: Consecutive scores a thread holds per slab (one 16-byte load). KPT = 4 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py index 96d8198eb3c7..0265172be74a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py @@ -19,16 +19,16 @@ import triton import triton.language as tl +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_paged_kv_store, + triton_qsa_paged_sparse_gqa, +) from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( compile_only, first_time, mark_warmed, ) -from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( - triton_qsa_paged_kv_store, - triton_qsa_paged_sparse_gqa, -) from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber from tensorrt_llm._utils import get_sm_version diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py index bbe60a559c65..95fa342e654c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py @@ -23,8 +23,8 @@ import triton import triton.language as tl -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py index a6eeb7949056..c8246c7a77cd 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py @@ -26,8 +26,8 @@ import triton import triton.language as tl -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber SLOTS = 3 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py index c071facbdeba..0f5a1e65b502 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py @@ -19,12 +19,7 @@ from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fi_warp import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* -from tensorrt_llm._torch.modeling_v2.catalog._op import ( - Arch, - Cell, - OpWrapper, - true_fp32_matmul, -) +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul class _Plan(NamedTuple): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py index 7e8f07f0ba72..22b9e89025f7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py @@ -31,13 +31,13 @@ from cutlass.cutlass_dsl import T, dsl_user_op from cutlass.experimental import primitives as prims +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( _asm, _mapa, _sigmoid, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl #: This checkpoint's hyper-connection bottleneck width. RANK = 320 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py index 6560f8f6706b..9bf11039fcd2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py @@ -27,10 +27,10 @@ from cutlass.cutlass_dsl import T, dsl_user_op from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait -from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.custom_pipeline import PipelineTmaUmma, PipelineUmmaAsync from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper #: Largest row count this entry is certified for: five 128-row MMA tiles. MAX_ROWS = 640 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py index 2bf7d75d6e1c..e7b4b40f2bd2 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py @@ -13,14 +13,9 @@ import torch import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* -from tensorrt_llm._torch.modeling_v2.catalog._op import ( - Arch, - Cell, - OpWrapper, - true_fp32_matmul, -) from tensorrt_llm._torch.autotuner import AutoTuner from tensorrt_llm._torch.custom_ops.trtllm_gen_custom_ops import FP8BlockScaleMoERunner +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul from ... import native_ops diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py index 9057d2fac40f..a6315a2eff9e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py @@ -13,13 +13,8 @@ import triton import triton.language as tl -from tensorrt_llm._torch.modeling_v2.catalog._op import ( - Arch, - Cell, - OpWrapper, - true_fp32_matmul, -) from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Columns per activation scale. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py index acb1faafccaf..b7759c784273 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py @@ -22,11 +22,11 @@ import torch +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( renorm_moe_routing_op, ) -from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from ... import native_ops from .fp8_block_scale_moe_runner import dequantize_block_weight, requantize_1x128 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py index 8051c6608283..765a5b882b4e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py @@ -27,12 +27,7 @@ from cutlass.experimental import primitives as prims from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fw -from tensorrt_llm._torch.modeling_v2.catalog._op import ( - Arch, - Cell, - OpWrapper, - true_fp32_matmul, -) +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul from ..gemm import low_m_bf16_mm__decode as _lowm diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py index 855eb6d54fbb..9066d448bf26 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -16,19 +16,11 @@ import torch -from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import ( - thop_attention, -) +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_erf import ( - gelu_erf, -) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_tanh import ( - gelu_tanh, -) -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.layer_norm import ( - layer_norm, -) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_erf import gelu_erf +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_tanh import gelu_tanh +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.layer_norm import layer_norm from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.vision.patch_embed_conv3d import ( patch_embed_conv3d, ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py index 9136af1e7868..19235c5748e4 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py @@ -23,7 +23,7 @@ import tensorrt_llm _QWEN4_EXP = Path(tensorrt_llm.__file__).parent.joinpath( - "_torch", "_experimental", "modeling_v2", "models", "qwen4_exp" + "_torch", "modeling_v2", "models", "qwen4_exp" ) _FAMILY = _QWEN4_EXP / "ops" / "hc" / "qwen4_exp_hc_decode_rows_tc.py" _FP8_TARGET = ( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py index fef3add8edde..cc871b086ffe 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py @@ -125,9 +125,7 @@ def _run_one_mpi_rank() -> int: from _validating import validating as _validating_cm MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) - from tensorrt_llm._torch.modeling_v2.catalog.comm import ( - allreduce as allreduce_mod, - ) + from tensorrt_llm._torch.modeling_v2.catalog.comm import allreduce as allreduce_mod from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 modeling as _modeling, ) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py index 819bf0d36b7c..e84444c1cec2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py @@ -24,10 +24,7 @@ TOPK = 10 _MODELING_V2_TESTS = Path(__file__).resolve().parents[4] -_TARGET = ( - "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." - "qwen3_8_flash_next_fp8__sm_103__unified" -) +_TARGET = "tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified" # Bound inside the rank body: importing the entry pulls in tensorrt_llm, which calls MPI_Init at # import, and the launcher must stay free of MPI and CUDA state. diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py index 3dea0e43d24b..6c32927aadc2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -14,6 +14,9 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_generation_token_mapping, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( # noqa: E501 qsa_topk_expand_decode as family_op, ) @@ -87,9 +90,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qwen4_exp_attn_decode_prologue__decode import ( # noqa: E501 qwen4_exp_attn_decode_prologue as _pro_op, ) -from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( - triton_qsa_generation_token_mapping, -) __extra_import_path__ = ["../../.."] from _validating import ( # noqa: E402,F401 — needs the path declared above; fixture by name diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py index 5ca89d5b65a2..743d60c7b5c4 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -12,6 +12,8 @@ import torch from transformers import PretrainedConfig +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig +from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 @@ -23,8 +25,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 weights as W, ) -from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig -from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache from tensorrt_llm.functional import AllReduceStrategy diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py index dbaf67474efc..ee9145103fb5 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -20,6 +20,7 @@ import torch.nn.functional as F from utils.llm_data import llm_models_root +from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 modeling as M, ) @@ -37,7 +38,6 @@ from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.vision.triton_pos_embed_interpolate import ( # noqa: E501 triton_pos_embed_interpolate, ) -from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig from tensorrt_llm.inputs.multimodal import ( MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalItemMetadata, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py index 24767f759759..be23081f81a1 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -16,10 +16,10 @@ import torch from utils.llm_data import llm_models_root +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 weights as WT, ) -from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight from tensorrt_llm.mapping import Mapping diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py index eae8d4159190..d320852b53c3 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py @@ -15,12 +15,12 @@ import torch from transformers import PretrainedConfig +from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 modeling as M, ) -from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm.llmapi.llm_args import ( KvCacheConfig, MoeConfig, From 2fa681ccb02707c00ccc115a335748243f4121e4 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:51:20 -0800 Subject: [PATCH 51/51] [None][fix] modeling_v2 qwen4_exp: join the PLE broker's serve thread before its answer threads on close Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../_torch/modeling_v2/models/qwen4_exp/chunk_broker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py index 865cbe724c61..885d8033770c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py @@ -76,7 +76,9 @@ def close(self) -> None: except OSError: pass self._server.close() - for thread in [self._thread, *self._answers]: + # `_serve` alone starts answers: once it has returned, the list of answers is final. + self._thread.join(_POLL_SECONDS * 5) + for thread in self._answers: thread.join(_POLL_SECONDS * 5) def _serve(self) -> None: