diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py new file mode 100644 index 000000000000..885d8033770c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/chunk_broker.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hand one process's exportable allocation to the other processes of the host that ask for it. + +A segment is named by a key. The first process to bind the key's abstract Unix socket +(`\\0trtllm-ple--`) owns it: it creates and fills the allocation, then serves the +allocation's file descriptor to every process that connects (`SCM_RIGHTS`). Abstract names carry +no permissions, so both sides check the other's uid through `SO_PEERCRED`: the owner refuses a +peer of another uid, and a waiter whose name is held by another uid's socket gets +`PermissionError` at once. A process that cannot bind connects and waits, at most `timeout_s` +(also when the name is bound but nothing listens on it). When the owner exits or aborts before +publishing, the kernel closes its sockets and the waiter tries to own the segment itself. +Nothing is written to the filesystem, so a crash leaves nothing behind. + +Abstract socket names belong to a network namespace: processes in different namespaces do not +meet, and each keeps its own copy. +""" + +from __future__ import annotations + +import errno +import json +import os +import socket +import struct +import threading +import time +from typing import Callable, Optional + +OWNER = "owner" +ATTACHED = "attached" + +_CRED = struct.Struct("iII") # struct ucred: pid_t pid, uid_t uid, gid_t gid +_POLL_SECONDS = 1.0 + + +def socket_name(key: str) -> str: + """The abstract socket name of `key` (per uid).""" + return f"\0trtllm-ple-{os.getuid()}-{key}" + + +def _peer_uid(conn: socket.socket) -> int: + """The uid of the process at the other end of the connected Unix socket `conn`.""" + _, uid, _ = _CRED.unpack(conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, _CRED.size)) + return uid + + +class OwnedSegment: + """The owner's side: serves `publish`ed `(fd, meta)` to every same-uid peer, until `close`.""" + + def __init__(self, server: socket.socket, key: str) -> None: + self.key = key + self._server = server + self._ready = threading.Event() + self._closed = threading.Event() + self._fd = -1 + self._meta: dict = {} + self._answers: list[threading.Thread] = [] + self._thread = threading.Thread( + target=self._serve, name=f"ple-broker-{key[:12]}", daemon=True + ) + self._thread.start() + + def publish(self, fd: int, meta: dict) -> None: + """Make the allocation available. `fd` stays owned by the caller and must stay open.""" + self._fd = fd + self._meta = dict(meta) + self._ready.set() + + def close(self) -> None: + """Stop serving; peers still waiting see the connection close and retry.""" + self._closed.set() + try: + # Closing alone does not wake the thread blocked in `accept`; shutting down does. + self._server.shutdown(socket.SHUT_RDWR) + except OSError: + pass + self._server.close() + # `_serve` alone starts answers: once it has returned, the list of answers is final. + self._thread.join(_POLL_SECONDS * 5) + for thread in self._answers: + thread.join(_POLL_SECONDS * 5) + + def _serve(self) -> None: + while not self._closed.is_set(): + try: + conn, _ = self._server.accept() + except OSError: + # `close`, or `accept` failing (EMFILE, ENFILE, ENOMEM, ...). Either way the name + # is freed with the thread: connected peers already answered keep their answer, + # and every other peer sees its connection reset and owns the key itself. + self._server.close() + return + answer = threading.Thread(target=self._answer, args=(conn,), daemon=True) + self._answers = [t for t in self._answers if t.is_alive()] + [answer] + answer.start() + + def _answer(self, conn: socket.socket) -> None: + with conn: + try: + if _peer_uid(conn) != os.getuid(): + return + while not self._ready.wait(_POLL_SECONDS): + if self._closed.is_set(): + return + if self._closed.is_set(): + return + socket.send_fds(conn, [json.dumps(self._meta).encode()], [self._fd]) + except OSError: + return + + +def try_own(key: str) -> Optional[OwnedSegment]: + """Own `key` if no process does (non-blocking), else None.""" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) + try: + server.bind(socket_name(key)) + server.listen(64) + except OSError as error: + server.close() + if error.errno == errno.EADDRINUSE: + return None + raise + return OwnedSegment(server, key) + + +def acquire( + key: str, timeout_s: float, log: Callable[[str], None] = print, report_s: float = 60.0 +) -> tuple: + """`(OWNER, OwnedSegment)` when no process owns `key`, else `(ATTACHED, fd, meta)` once + the owner publishes; an owner that exits, aborts or stops accepting first hands `key` to its + waiters. Raises `TimeoutError` when no owner has published within `timeout_s`, and + `PermissionError` when a process of another uid holds the name.""" + name = socket_name(key) + deadline = time.monotonic() + timeout_s + next_report = time.monotonic() + report_s + while True: + owned = try_own(key) + if owned is not None: + return OWNER, owned + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) + try: + conn.connect(name) + except (ConnectionRefusedError, FileNotFoundError): + # The owner just exited, or is between `bind` and `listen` (or never listens). + conn.close() + if time.monotonic() >= deadline: + raise TimeoutError(f"no owner of {key!r} listens within {timeout_s:.0f} s") + time.sleep(0.05) + continue + got: Optional[tuple] = None + try: + owner_uid = _peer_uid(conn) + if owner_uid != os.getuid(): + raise PermissionError(f"the owner of {key!r} runs as uid {owner_uid}") + while got is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"no allocation for {key!r} within {timeout_s:.0f} s") + conn.settimeout(min(remaining, report_s)) + try: + got = socket.recv_fds(conn, 4096, 1) + except socket.timeout: + if time.monotonic() >= next_report: + log(f"ple host table: waiting for the owner of {key!r} to publish it") + next_report = time.monotonic() + report_s + except ConnectionResetError: + # The owner closed its listener before accepting this connection. + got = (b"", [], 0, None) + finally: + conn.close() + msg, fds, _, _ = got + if fds: + return ATTACHED, fds[0], json.loads(msg.decode()) + # The connection closed without a descriptor: the owner exited or aborted. Retry. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt new file mode 100644 index 000000000000..a9f9a0c1eb8f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/csrc/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# qwen4_exp native code. Each target keeps its own in /csrc/; see the +# modeling_v2 README. +add_modeling_v2_family(qwen4_exp) + +# The FP8 target routes on SM103 only (its kernels need only SM90+ features). +# The hyper-connection all-reduce uses the stock all-reduce's IPC workspace, +# which exists only in multi-device builds. +add_modeling_v2_target( + qwen4_exp qwen3_8_flash_next_fp8__sm_103__unified + ARCHS 103a + SOURCES *.cu *.cpp + REQUIRES MULTI_DEVICE) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py index 641965755582..5a733488bf40 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/activation/fused_sigmoid_mul_inplace.py @@ -4,21 +4,63 @@ The gated attention's output gate, between the attention core and o_proj, in one Triton kernel (`torch.ops.trtllm.fused_sigmoid_mul_inplace`). Called by the -qwen4_exp target (`qwen3_8_flash_next_nvfp4__sm_103__default`) at every layout -- its -trunk, gate off the fused input-projection row, and its MTP layer, gate as the strided -half of the q projection. +qwen4_exp targets at every layout -- the trunk, gate off the fused input-projection row, +and the MTP layer, gate as the strided half of the q projection. """ from __future__ import annotations import torch +import triton # Registered by the module that defines it: after `import # tensorrt_llm._torch.custom_ops` alone, torch.ops.trtllm has no # `fused_sigmoid_mul_inplace` (measured). -import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate # noqa: F401 +import tensorrt_llm._torch.attention.kernels.fused_qk_norm_rope_gate as _gate_mod from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +def _launch(attention_output: torch.Tensor, gate: torch.Tensor) -> bool: + """The registered op's launch, made here when its preconditions hold: the same kernel, + grid, block, warps and PDL policy, without the custom-op dispatch around it. False + (nothing launched) when they do not, for the op to refuse loudly.""" + if attention_output.dim() != 2 or attention_output.stride(-1) != 1 or gate.stride(-1) != 1: + return False + num_tokens, hidden = attention_output.shape + if gate.dim() == 3: + if gate.shape[0] != num_tokens or gate.shape[1] * gate.shape[2] != hidden: + return False + head_dim, gate_stride_head = gate.shape[2], gate.stride(1) + elif gate.dim() == 2 and gate.shape == attention_output.shape: + head_dim, gate_stride_head = hidden, hidden + else: + return False + if num_tokens == 0: + return True + block = min(triton.next_power_of_2(hidden), 1024 if num_tokens < 1024 else 2048) + pdl = ( + dict(LAUNCH_WITH_PDL=True, launch_pdl=True) + if _gate_mod._sigmoid_mul_pdl(attention_output.device.index) + else {} + ) + _gate_mod._fused_sigmoid_mul_kernel[(num_tokens, triton.cdiv(hidden, block))]( + attention_output, + attention_output, + gate, + attention_output, # scale_ptr, read only for an FP8 output + attention_output.stride(0), + attention_output.stride(0), + gate.stride(0), + gate_stride_head, + hidden_size=hidden, + head_dim=head_dim, + block_size=block, + num_warps=4, + **pdl, + ) + return True + + #: The checkpoint's gated attention: 24 q heads of 256, q projection [T, 24, 512]. _CKPT = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") @@ -135,7 +177,12 @@ class FusedSigmoidMulInplace(OpWrapper): the running device at the first call of each (hidden, head_dim, block) specialization, no autotuner: grid `(T, cdiv(hidden, block))`, `block = min(next_pow2(hidden), 1024 if T < 1024 else 2048)`, 4 warps. The - cells ran on sm_103. + cells ran on sm_103. With `direct_launch=True` (the FP8 target), `_launch` makes + that same launch itself where the preconditions below hold and skips only the + custom-op dispatch (15 instead of 28 us of host time a call on GB300); + otherwise the op is called and refuses the call. `_launch` is a copy of the + op's launch: the test holds it bitwise equal to the op at the direct-launch + cells and more shapes, so a change to the op's launch fails there. Call sites, each the attention output gate between the attention core and o_proj: @@ -192,7 +239,10 @@ class FusedSigmoidMulInplace(OpWrapper): gate) at 24 heads (`[T, 24, 512]`, hidden 6144) at 1 token, and at 6 heads (a compact `[T, 6, 512]`, hidden 1536) at 1, 128, 512, 1024 and 8192 rows; CUDA-graph capture and replay of the 1536-wide fused-row form at 8 rows and - of the 6-head 3-D gate at 1, 128 and 1024 rows. + of the 6-head 3-D gate at 1, 128 and 1024 rows. With `direct_launch` (the FP8 + target's cells in the test): the 2-D window at hidden 3072 (row stride 7424, + gate column 3584) at 1 and 8192 rows, the 24-head 3-D gate at 128 and 8192 + rows, and graph capture of the 1536-wide fused-row form. Hard limits T * row stride token offset computed in int64 boundary 2^63 not probed @@ -202,19 +252,24 @@ class FusedSigmoidMulInplace(OpWrapper): Not certified: 0 tokens; rows past 8192; the 3-D gate at a token stride other than its own q_proj's (heads x 2 x head_dim, e.g. a gate inside a - fused q|k|v row); the 24-head strided q_proj half above 1 token (a dep4 - rank's MTP layer: drafter 1..32 rows and its verify); a strided - `attention_output`; a gate dtype other than `attention_output.dtype`; - fp16/fp32 operands; hidden sizes other than 1536 and 6144; CUDA-graph - capture of the 6144-wide forms; torch.compile. + fused q|k|v row); the 24-head strided q_proj half above 1 token without + `direct_launch`; a strided `attention_output`; a gate dtype other than + `attention_output.dtype`; fp16/fp32 operands; hidden sizes other than 1536, + 3072 and 6144; CUDA-graph capture of the 6144-wide forms; torch.compile. """ - def __call__(self, attention_output: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: - torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) + def __call__( + self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False + ) -> torch.Tensor: + if not (direct_launch and _launch(attention_output, gate)): + torch.ops.trtllm.fused_sigmoid_mul_inplace(attention_output, gate) return attention_output - def reference(self, attention_output: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + def reference( + self, attention_output: torch.Tensor, gate: torch.Tensor, direct_launch: bool = False + ) -> torch.Tensor: """What `attention_output` holds after the call; neither input is mutated. + `direct_launch` does not change it. `gate.reshape(T, hidden)` is the per-head `(t, i // D, i % D)` read, for the 2-D and 3-D forms alike. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py index 4084ca92ddd9..64c9908ab2e7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_index_scores.py @@ -11,10 +11,13 @@ # Approved exception: QSA's selection path is Triton called as Python, with no # torch.ops boundary. The engine's own sparse hook (`select_qsa_paged_tokens` # in `_torch/attention/backends/sparse/qsa/module.py`) calls this same callable. +from tensorrt_llm._torch.attention.backends.sparse.qsa import kernels as _qsa_kernels from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( triton_qsa_paged_index_scores as _triton_qsa_paged_index_scores, ) from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber _INTEGER_DTYPES = (torch.int32, torch.int64) @@ -31,6 +34,10 @@ #: 1024 tiles of a 262143-position window; wider batches stream several requests at once. _SPLIT_PROGRAMS_PER_SM = 16 _SPLIT_PROGRAMS_PER_SM_ONE_TILE = 32 +#: With `wide_split`, from this many rows four times as many programs, so the longest windows' +#: chains of column tiles do not set the launch's time. The scores do not depend on the split. +_SPLIT_WIDE_MIN_ROWS = 32 +_SPLIT_PROGRAMS_PER_SM_WIDE = 64 _SPLIT_NUM_WARPS = 2 _SM_COUNT: dict = {} @@ -81,7 +88,8 @@ def _qsa_index_scores_visible_split_kernel( row of their request in the row tile, and different requests run side by side. """ if USE_PDL: - tl.extra.cuda.gdc_wait() + # The memory clobber keeps loads of the predecessor's outputs below the wait. + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() row_base = tl.program_id(0) * ROWS_PER_TILE group_slot = tl.program_id(1) @@ -210,6 +218,7 @@ def _qsa_index_scores_visible_split( programs_per_sm: int | None = None, num_warps: int = _SPLIT_NUM_WARPS, num_stages: int = 2, + wide: bool = False, ) -> torch.Tensor: """One launch of the visible-split kernel after the callable's own checks.""" if index_cache.ndim != 4 or index_cache.shape[2] != 1: @@ -242,7 +251,11 @@ def _qsa_index_scores_visible_split( row_tiles = triton.cdiv(rows, rows_per_tile) if programs_per_sm is None: programs_per_sm = ( - _SPLIT_PROGRAMS_PER_SM_ONE_TILE if row_tiles == 1 else _SPLIT_PROGRAMS_PER_SM + _SPLIT_PROGRAMS_PER_SM_ONE_TILE + if row_tiles == 1 + else _SPLIT_PROGRAMS_PER_SM_WIDE + if wide and rows >= _SPLIT_WIDE_MIN_ROWS + else _SPLIT_PROGRAMS_PER_SM ) # Every (row tile, group slot) pair gets `split` programs; slots past a tile's request # count exit at once. @@ -276,7 +289,9 @@ def _qsa_index_scores_visible_split( TOKENS_PER_BLOCK=tokens_per_block, COMPRESS_RATIO=compress_ratio, MAX_COMPRESSED_BLOCKS=max_compressed_blocks, - ROWS_PER_TILE=rows_per_tile, + # One row tile masks its rows by `num_rows` alone: compile it at the full tile so every + # row count up to `query_tile` shares one variant. + ROWS_PER_TILE=query_tile if row_tiles == 1 else rows_per_tile, QUERY_TILE=query_tile, BLOCK_H=dot_width, BLOCK_N=block_n, @@ -288,7 +303,11 @@ def _qsa_index_scores_visible_split( return output -@triton.jit +# The row count follows the batch; the metadata views begin at any row of a mixed step. +@triton.jit( + do_not_specialize=["rows"], + do_not_specialize_on_alignment=["query_positions", "request_indices"], +) def _qsa_index_scores_group_kernel( q, index_cache, @@ -327,7 +346,8 @@ def _qsa_index_scores_group_kernel( the row-wise kernel's, bit for bit.""" BLOCK_H: tl.constexpr = BLOCK_G * NUM_INDEX_HEADS if USE_PDL: - tl.extra.cuda.gdc_wait() + # The memory clobber keeps loads of the predecessor's outputs below the wait. + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() group = tl.program_id(0) slots = tl.arange(0, BLOCK_G) @@ -453,7 +473,34 @@ def _takes_request_groups( ) -def _qsa_index_scores_request_groups( +#: With `prefill_query_tile`, a prefill-sized, visible-only context-row call at 4 index heads of +#: 128 runs the engine's query-tiled kernel at a deeper launch than the engine's (32 rows a dot +#: tile, 8 column tiles a program): each gathered key tile serves twice the rows. Bitwise the +#: engine launch's scores (a row's dot and head sum do not depend on the tile shape). +_PREFILL_QUERY_TILE = 64 +_PREFILL_TILES_PER_PROGRAM = 16 +_PREFILL_BLOCK_N = 64 +_PREFILL_HEADS = 4 +_PREFILL_HEAD_DIM = 128 + + +def _takes_prefill_query_tile( + q: torch.Tensor, only_visible_blocks: bool, context_rows: bool, prefetch: bool +) -> bool: + """A context-row call the deeper query-tiled launch serves (`_PREFILL_QUERY_TILE`).""" + return ( + context_rows + and only_visible_blocks + and not prefetch + and q.ndim == 3 + and q.shape[0] > _DECODE_ROW_LIMIT + and q.shape[1] == _PREFILL_HEADS + and q.shape[2] == _PREFILL_HEAD_DIM + and q.dtype == torch.bfloat16 + ) + + +def _qsa_index_scores_prefill_query_tile( q: torch.Tensor, index_cache: torch.Tensor, block_table: torch.Tensor, @@ -461,15 +508,77 @@ def _qsa_index_scores_request_groups( request_indices: torch.Tensor, tokens_per_block: int, compress_ratio: int, - use_pdl: bool, - rows_per_request: int, ) -> torch.Tensor: - """One launch of the request-group kernel after the callable's own checks.""" + """The engine's query-tiled kernel at the deeper launch. Same contract as the engine's + launcher with `context_rows` and `only_visible_blocks`: columns at and past a row's + visible bound are unspecified.""" + rows = q.shape[0] + max_blocks = _check_index_scores_operands( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) + output = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + grid = ( + triton.cdiv(rows, _PREFILL_QUERY_TILE), + triton.cdiv(max_blocks, _PREFILL_BLOCK_N * _PREFILL_TILES_PER_PROGRAM), + ) + _qsa_kernels._qsa_paged_index_scores_qtile_kernel[grid]( + q, + index_cache, + block_table, + query_positions, + request_indices, + output, + q.shape[2] ** -0.5, + rows, + q.stride(0), + q.stride(1), + q.stride(2), + index_cache.stride(0), + index_cache.stride(1), + index_cache.stride(2), + index_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + output.stride(0), + output.stride(1), + NUM_INDEX_HEADS=q.shape[1], + INDEX_HEAD_DIM=q.shape[2], + BLOCK_D=q.shape[2], + TOKENS_PER_BLOCK=tokens_per_block, + COMPRESS_RATIO=compress_ratio, + MAX_COMPRESSED_BLOCKS=max_blocks, + QUERY_TILE=_PREFILL_QUERY_TILE, + BLOCK_H=_PREFILL_QUERY_TILE * q.shape[1], + BLOCK_N=_PREFILL_BLOCK_N, + TILES_PER_PROGRAM=_PREFILL_TILES_PER_PROGRAM, + ONLY_VISIBLE_BLOCKS=True, + num_warps=4, + ) + return output + + +def _check_index_scores_operands( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, +) -> int: + """The engine launcher's operand checks for a `[rows, heads, dim]` query; returns the + output's compressed-block count.""" if index_cache.ndim != 4 or index_cache.shape[2] != 1: raise ValueError("QSA index cache must have shape [pages, tokens, 1, dim]") if block_table.ndim != 2 or block_table.dtype != torch.int32: raise ValueError("QSA index scoring requires a two-dimensional int32 block table") - rows, num_index_heads, index_head_dim = q.shape + rows, _, index_head_dim = q.shape if index_head_dim <= 0 or index_cache.shape[3] != index_head_dim: raise ValueError("QSA index query and cache dimensions do not match") if q.dtype != index_cache.dtype: @@ -486,6 +595,31 @@ def _qsa_index_scores_request_groups( max_compressed_blocks = block_table.shape[1] * tokens_per_block // compress_ratio if max_compressed_blocks <= 0: raise ValueError("QSA index scoring requires a non-empty block table") + return max_compressed_blocks + + +def _qsa_index_scores_request_groups( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, +) -> torch.Tensor: + """One launch of the request-group kernel after the callable's own checks.""" + rows, num_index_heads, index_head_dim = q.shape + max_compressed_blocks = _check_index_scores_operands( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) output = torch.empty((rows, max_compressed_blocks), dtype=torch.float32, device=q.device) if rows == 0: return output @@ -494,7 +628,7 @@ def _qsa_index_scores_request_groups( groups = triton.cdiv(rows, rows_per_request) tiles = 1 if groups <= 1 else 2 if groups <= 4 else 4 grid = (groups, triton.cdiv(max_compressed_blocks, _GROUP_BLOCK_N * tiles)) - _qsa_index_scores_group_kernel[grid]( + args = ( q, index_cache, block_table, @@ -513,6 +647,8 @@ def _qsa_index_scores_request_groups( block_table.stride(1), output.stride(0), output.stride(1), + ) + constexprs = dict( NUM_INDEX_HEADS=num_index_heads, INDEX_HEAD_DIM=index_head_dim, BLOCK_D=max(16, triton.next_power_of_2(index_head_dim)), @@ -528,6 +664,28 @@ def _qsa_index_scores_request_groups( num_warps=4, launch_pdl=use_pdl, ) + # `TILES_PER_PROGRAM` follows the request count: compile every depth at the first call. + warm_variants( + _qsa_index_scores_group_kernel, + ( + args[8:18], # the strides + q.dtype, + q.data_ptr() % 16 == 0, + query_positions.dtype, + request_indices.dtype, + num_index_heads, + index_head_dim, + tokens_per_block, + compress_ratio, + max_compressed_blocks, + rows_per_request, + use_pdl, + ), + args, + [{"TILES_PER_PROGRAM": t} for t in (1, 2, 4)], + **constexprs, + ) + _qsa_index_scores_group_kernel[grid](*args, **constexprs) return output @@ -645,9 +803,13 @@ class QsaPagedIndexScores(OpWrapper): from row 0 (a verify step's `predicted_tokens_per_seq`): with `only_visible_blocks`, no `prefetch` and 2, 4 or 8 index heads, the entry's request-group kernel shares one key gather across each run (a run whose rows name different requests is gathered - row by row). Each cell names the kernel it was - measured on. The request-group kernel is bitwise the row-wise one; the - visible-split and query-tiled kernels are not. + row by row). With `prefill_query_tile`, a visible-only context-row call of more + than 256 rows at 4 bf16 index heads of 128 runs the query-tiled kernel at a deeper + launch, bitwise the engine launch's scores. With `wide_split`, a visible-split call + of `_SPLIT_WIDE_MIN_ROWS` rows or more past one row tile takes more programs an SM, + bitwise the narrow split. Both default off. Each cell names the kernel it was + measured on. The request-group kernel is bitwise the row-wise one; the visible-split + and query-tiled kernels are not. Fusion boundary: the table walk, the gather, the dots, relu, head sum, scale and causal bound are inside. Producing `q` (norm, RoPE), writing and @@ -1071,6 +1233,8 @@ def __call__( use_pdl: bool = False, prefetch: bool = False, rows_per_request: int = 1, + prefill_query_tile: bool = False, + wide_split: bool = False, ) -> torch.Tensor: if rows_per_request < 1: raise ValueError("QSA index scoring requires rows_per_request >= 1") @@ -1084,6 +1248,7 @@ def __call__( tokens_per_block, compress_ratio, use_pdl, + wide=wide_split, ) if _takes_request_groups(q, only_visible_blocks, context_rows, prefetch, rows_per_request): return _qsa_index_scores_request_groups( @@ -1097,6 +1262,18 @@ def __call__( use_pdl, rows_per_request, ) + if prefill_query_tile and _takes_prefill_query_tile( + q, only_visible_blocks, context_rows, prefetch + ): + return _qsa_index_scores_prefill_query_tile( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) return _triton_qsa_paged_index_scores( q=q, index_cache=index_cache, @@ -1125,10 +1302,13 @@ def reference( use_pdl: bool = False, prefetch: bool = False, rows_per_request: int = 1, + prefill_query_tile: bool = False, + wide_split: bool = False, ) -> torch.Tensor: """fp32 scores, `-inf` past each row's bound -- or NaN there, meaning - "unspecified", when `only_visible_blocks` is True. `context_rows` and - `rows_per_request` are ignored: they pick a kernel, not a result. + "unspecified", when `only_visible_blocks` is True. `context_rows`, + `rows_per_request`, `prefill_query_tile` and `wide_split` are ignored: they pick a + kernel or a launch, not a result. Walked in row chunks only to bound the gathered keys, which reach 8.7 GiB in fp32 at 8191 rows x 2080 columns. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py index 19aebdf34d6d..f57e0659843e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa_nvfp4.py @@ -15,6 +15,7 @@ # entry's own: an NVFP4 tile needs 16 tokens where the bf16 one takes 64. from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import _qsa_pdl_enabled from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Base-2 online softmax: scores are scaled by log2(e) once. _LOG2_E = tl.constexpr(1.4426950408889634) @@ -499,7 +500,7 @@ def _nvfp4_sparse_gqa_splitk_kernel( kv_head = tl.program_id(1) split = tl.program_id(2) if USE_PDL: - tl.extra.cuda.gdc_wait() + gdc_wait_with_memory_clobber() request = tl.load(request_indices + row).to(tl.int64) valid_request = (request >= 0) & (request < num_requests) safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) @@ -618,7 +619,7 @@ def _nvfp4_merge_splitk_kernel( row = tl.program_id(0).to(tl.int64) head = tl.program_id(1) if USE_PDL: - tl.extra.cuda.gdc_wait() + gdc_wait_with_memory_clobber() tl.extra.cuda.gdc_launch_dependents() split_offsets = tl.arange(0, BLOCK_SPLITS) dim_offsets = tl.arange(0, HEAD_DIM) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py index 9f3364ac5eff..38fb558a9a43 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/qsa_topk_expand_decode.py @@ -106,12 +106,13 @@ def __call__( len_ptr: cute.Pointer, topk_ptr: cute.Pointer, sel_ptr: cute.Pointer, + order_ptr: cute.Pointer, rows: cutlass.Int32, stream, ): # The keys arrive as a tensor, as the TVM-FFI entry takes its stream from a GPU # tensor; their layout is rebuilt with the static width and row stride the 16-byte - # loads need. The row count is the call's. + # loads need. The row count is the call's. `order_ptr` is the wide kernel's (unread). mKeys = cute.make_tensor( keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) ) @@ -471,6 +472,7 @@ def __init__( threads=_WIDE_THREADS, ctas=1, prefilter=True, + order=False, ): self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( width, @@ -486,6 +488,9 @@ def __init__( self.warps = threads // 32 self.slab = KPT * threads self.ctas = ctas + # `order`: CTA b takes row `row_order[b]` (longest window first), so the CTAs launched + # last get the shortest windows. Every row's selection is the same. + self.order = bool(order) self.cache_slabs = _cache_slabs(width, threads, ctas) self.per_warp = -(-block_topk // self.warps) self.prefilter = bool(prefilter) and self.per_warp <= 32 @@ -502,6 +507,7 @@ def __call__( len_ptr: cute.Pointer, topk_ptr: cute.Pointer, sel_ptr: cute.Pointer, + order_ptr: cute.Pointer, rows: cutlass.Int32, stream, ): @@ -516,11 +522,12 @@ def __call__( mSel = cute.make_tensor( sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) ) + mOrder = cute.make_tensor(order_ptr, per_row) ld_keys = cute.make_copy_atom( cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT ) g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.Int32, num_bits_per_copy=32 * KPT) - self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys, g2s).launch( + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, mOrder, ld_keys, g2s).launch( grid=[rows, 1, 1], block=[self.threads, 1, 1], smem=self.smem_bytes(), @@ -876,7 +883,16 @@ def _select_window( @cute.kernel def kernel( - self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom, g2s: cute.CopyAtom + self, + mKeys, + mEnd, + mPos, + mLen, + mTopk, + mSel, + mOrder, + ld_keys: cute.CopyAtom, + g2s: cute.CopyAtom, ): tidx, _, _ = cute.arch.thread_idx() row, _, _ = cute.arch.block_idx() @@ -928,16 +944,22 @@ def kernel( cutlass.Int32, cute.make_layout((KPT, nt), stride=(1, KPT)), byte_alignment=128 ) - # 0. Shared state and the row's host-filled metadata, before the wait. + # 0. Shared state and the row's host-filled metadata, before the wait. Under `order` + # the row is `row_order`'s, a device sort earlier in the step: read after the wait. for b in cutlass.range_constexpr(-(-_HIST // nt)): if tidx + b * nt < _HIST: hist[tidx + b * nt] = cutlass.Int32(0) if tidx == 0: state[5] = cutlass.Int32(0) # candidates compacted + if const_expr(self.order): + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + row = mOrder[row] n = mEnd[row] seqlen = mLen[row] visible = cutlass.Int32(mPos[row]) + 1 - if const_expr(self.pdl): + if const_expr(self.pdl and not self.order): cute.arch.griddepcontrol_wait() cute.arch.griddepcontrol_launch_dependents() @@ -1128,14 +1150,29 @@ def kernel( _SMS: dict = {} -def _wide_config(rows: int, device) -> tuple: - """The wide kernel's `(threads, CTAs an SM, prefilter)` for a call of `rows` rows: one - 1024-thread CTA an SM (the largest window cache) while the rows fit one wave, two once - they do not, so one CTA's walks overlap the other's loads.""" +def _sms(device) -> int: sms = _SMS.get(device.index) if sms is None: sms = _SMS[device.index] = torch.cuda.get_device_properties(device).multi_processor_count - return (_WIDE_THREADS, 1 if rows <= sms else 2, True) + return sms + + +def _wide_config(rows: int, device, ordered: bool = False) -> tuple: + """The wide kernel's `(threads, CTAs an SM, prefilter, order)` for a call of `rows` rows: + one 1024-thread CTA an SM (the largest window cache) while the rows fit one wave, two once + they do not, so one CTA's walks overlap the other's loads. With `ordered` (a `row_order` + given), past one wave the CTAs take the rows in that order, so the last wave takes the + shortest windows: one CTA an SM up to two waves, two past them. The same selection either + way.""" + sms = _sms(device) + if ordered and rows > sms: + return (_WIDE_THREADS, 1 if rows <= 2 * sms else 2, True, True) + return (_WIDE_THREADS, 1 if rows <= sms else 2, True, False) + + +def orders_rows(scores) -> bool: + """Whether a call over `scores` reads `row_order`: the wide kernel past one wave of rows.""" + return scores.shape[1] > MAX_WIDTH and scores.shape[0] > _sms(scores.device) _INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} @@ -1151,9 +1188,11 @@ def launch( token_topk, pdl=True, out=None, + row_order=None, ): """The launch behind `QsaTopkExpandDecode.__call__`: writes `topk_indices` and returns the - token selection `[rows, token_topk + compress_ratio - 1]` int32 (`out` when given).""" + token selection `[rows, token_topk + compress_ratio - 1]` int32 (`out` when given). + `row_order`: int32 `[rows]`, a permutation of the rows, longest window first.""" rows, width = scores.shape block_topk = topk_indices.shape[1] sel = out @@ -1174,7 +1213,7 @@ def launch( query_positions.dtype, sequence_lengths.dtype, pdl, - None if width <= MAX_WIDTH else _wide_config(rows, scores.device), + None if width <= MAX_WIDTH else _wide_config(rows, scores.device, row_order is not None), ) ptrs = ( row_ends.data_ptr(), @@ -1182,6 +1221,7 @@ def launch( sequence_lengths.data_ptr(), topk_indices.data_ptr(), sel.data_ptr(), + (row_ends if row_order is None else row_order).data_ptr(), ) fn = _COMPILED.get(key) if fn is None: @@ -1197,9 +1237,9 @@ def launch( if width <= MAX_WIDTH: kernel = _TopkExpandKernel(**geometry) else: - threads, ctas, prefilter = key[-1] + threads, ctas, prefilter, order = key[-1] kernel = _TopkExpandWideKernel( - **geometry, threads=threads, ctas=ctas, prefilter=prefilter + **geometry, threads=threads, ctas=ctas, prefilter=prefilter, order=order ) fn = _COMPILED[key] = cute.compile( kernel, @@ -1219,6 +1259,7 @@ def launch( ), make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[5], gm, assumed_align=4), cutlass.Int32(rows), cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi", @@ -1409,6 +1450,9 @@ class QsaTopkExpandDecode(OpWrapper): kernel, `_TopkExpandWideKernel`); `topk_indices` int32 `[rows, K]` contiguous; `row_ends` int32, the positions and lengths int32 or int64, each a contiguous `[rows]`. Ties: the higher block at the threshold, as `qsa_canonical_selection` rewrites the CUDA top-k's pick. + `row_order` (optional): int32 `[rows]`, a permutation of the rows, longest window first, + read by the wide kernel past one wave of rows (`orders_rows`), after the wait; a caller + sorts once a step for every layer. Compiled once per score-table width and row stride (the block table's, fixed for an engine), k, compress ratio, token budget, index dtypes and PDL: the row count is a @@ -1441,7 +1485,7 @@ class QsaTopkExpandDecode(OpWrapper): The widths are derived from `max_seq_len` (288, 512 and 8192 (+4) pages of 32 tokens), not read off an engine. 4096 is `MAX_WIDTH`: 1024 threads, the largest CTA; wider tables take the wide kernel, compiled per row-count class (one CTA an SM up to the - SM count, two past it). + SM count, two past it; with a `row_order`, one up to twice the SM count, two past it). `is_valid` refuses `token_topk != k * compress_ratio`. It sizes the selection's rows while `k * compress_ratio` places the tokens, so a smaller budget writes past each row. @@ -1469,6 +1513,7 @@ def __call__( compress_ratio, token_topk, out=None, + row_order=None, ): return launch( scores, @@ -1479,6 +1524,7 @@ def __call__( compress_ratio, token_topk, out=out, + row_order=row_order, ) def reference( @@ -1491,9 +1537,11 @@ def reference( compress_ratio, token_topk, out=None, + row_order=None, ): """`(topk_indices_after, selection)`: a stable descending sort per row (ties to the - higher block), the k first in block order, then the expansion entry's reference.""" + higher block), the k first in block order, then the expansion entry's reference. + `row_order` picks a launch, not a result.""" rows, _ = scores.shape k = topk_indices.shape[1] out = torch.full_like(topk_indices, -1) @@ -1525,6 +1573,7 @@ def is_valid( compress_ratio, token_topk, out=None, + row_order=None, ): assert scores.dtype == torch.float32 and scores.stride(1) == 1, ( "fp32 scores, unit column stride" @@ -1551,6 +1600,15 @@ def is_valid( and out.is_contiguous() and out.shape == (scores.shape[0], token_topk + compress_ratio - 1) ), "out: contiguous int32 [rows, token_topk + compress_ratio - 1]" + assert row_order is None or ( + row_order.dtype == torch.int32 + and row_order.is_contiguous() + and row_order.shape == (rows,) + and torch.equal( + torch.sort(row_order).values, + torch.arange(rows, dtype=torch.int32, device=row_order.device), + ) + ), "row_order: a contiguous int32 permutation of the rows" qsa_topk_expand_decode = QsaTopkExpandDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py index 5a69fd373f72..9161717308b3 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/attention/spec_rollback_commit.py @@ -63,17 +63,21 @@ def _stage_kernel( STATE: tl.constexpr, STATE_BLOCK: tl.constexpr, BLOCK_C: tl.constexpr, + STAGE_CONV: tl.constexpr = True, ): i = tl.program_id(0) cb = tl.program_id(1) slot = tl.load(slots + i).to(tl.int64) - c = cb * BLOCK_C + tl.arange(0, BLOCK_C) - s = tl.arange(0, STATE_BLOCK) - m = (c < channels)[:, None] & (s < STATE)[None, :] - vals = tl.load(conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, mask=m) - tl.store( - conv_prev + (i.to(tl.int64) * channels + c[:, None]) * STATE + s[None, :], vals, mask=m - ) + if STAGE_CONV: + c = cb * BLOCK_C + tl.arange(0, BLOCK_C) + s = tl.arange(0, STATE_BLOCK) + m = (c < channels)[:, None] & (s < STATE)[None, :] + vals = tl.load( + conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, mask=m + ) + tl.store( + conv_prev + (i.to(tl.int64) * channels + c[:, None]) * STATE + s[None, :], vals, mask=m + ) if cb == 0: w = tl.arange(0, W_BLOCK) hist = tl.load(ngram_pool + slot * ngram_stride + w, mask=w < CTX) @@ -118,6 +122,7 @@ def _commit_kernel( STATE_BLOCK: tl.constexpr, BLOCK_C: tl.constexpr, ROW: tl.constexpr, + PREV_IN_POOL: tl.constexpr = False, ): i = tl.program_id(0) cb = tl.program_id(1) @@ -131,13 +136,22 @@ def _commit_kernel( j = a + s m = (c < channels)[:, None] & (s < STATE)[None, :] from_prev = (j < STATE)[None, :] - prev = tl.load( - conv_prev + (i64 * channels + c[:, None]) * STATE + j[None, :], mask=m & from_prev - ) + if PREV_IN_POOL: + # The pool row still holds the pre-step state: every read lands before any store. + prev = tl.load( + conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + j[None, :] * conv_s2, + mask=m & from_prev, + ) + else: + prev = tl.load( + conv_prev + (i64 * channels + c[:, None]) * STATE + j[None, :], mask=m & from_prev + ) xin = tl.load( conv_x + i64 * x_s0 + (j - STATE).to(tl.int64)[None, :] * x_s1 + c[:, None] * x_s2, mask=m & ~from_prev, ) + if PREV_IN_POOL: + tl.debug_barrier() tl.store( conv_pool + slot * conv_s0 + c[:, None] * conv_s1 + s[None, :] * conv_s2, tl.where(from_prev, prev, xin), @@ -219,16 +233,21 @@ class SpecRollbackStage(OpWrapper): int32 as the call site hands them, slots 0 and 1873 among those read. """ - def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): + def __call__(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True): + """`conv=False` stages the n-gram stream only and returns `(stream, None)`, for a caller + whose PLE launch left the conv pool rows unpublished (the commit then reads them in + place).""" n = slots.shape[0] ctx = ngram_pool.shape[1] channels, state = conv_pool.shape[1], conv_pool.shape[2] stream = torch.empty([n, ctx + p], dtype=ngram_pool.dtype, device=ngram_pool.device) - conv_prev = torch.empty( - [n, channels, state], dtype=conv_pool.dtype, device=conv_pool.device + conv_prev = ( + torch.empty([n, channels, state], dtype=conv_pool.dtype, device=conv_pool.device) + if conv + else None ) if n: - _stage_kernel[(n, triton.cdiv(channels, _BLOCK_C))]( + _stage_kernel[(n, triton.cdiv(channels, _BLOCK_C) if conv else 1)]( ngram_pool, ngram_pool.stride(0), conv_pool, @@ -238,7 +257,7 @@ def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): slots, ids, stream, - conv_prev, + conv_prev if conv else stream, channels, P=p, CTX=ctx, @@ -246,17 +265,18 @@ def __call__(self, ngram_pool, conv_pool, slots, ids, p: int): STATE=state, STATE_BLOCK=_pow2(state), BLOCK_C=_BLOCK_C, + STAGE_CONV=bool(conv), num_warps=4, ) return stream, conv_prev - def reference(self, ngram_pool, conv_pool, slots, ids, p: int): + def reference(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True): n = slots.shape[0] slots = slots.to(torch.long) rows = torch.reshape(ids[: n * p], [n, p]).to(ngram_pool.dtype) - return torch.cat([ngram_pool[slots], rows], 1), conv_pool[slots] + return torch.cat([ngram_pool[slots], rows], 1), (conv_pool[slots] if conv else None) - def is_valid(self, ngram_pool, conv_pool, slots, ids, p: int) -> None: + def is_valid(self, ngram_pool, conv_pool, slots, ids, p: int, conv: bool = True) -> None: assert ngram_pool.dim() == 2 and ngram_pool.stride(1) == 1 assert ngram_pool.dtype == torch.int64 assert conv_pool.dim() == 3 @@ -335,6 +355,10 @@ def __call__( if not n: return blocks = max(triton.cdiv(channels, _BLOCK_C), index_old.shape[0]) + # `conv_prev` None: the conv pool rows still hold the pre-step state (read in place). + prev_in_pool = conv_prev is None + if prev_in_pool: + conv_prev = conv_pool _commit_kernel[(n, blocks)]( accepted, slots, @@ -371,6 +395,7 @@ def __call__( STATE_BLOCK=_pow2(state), BLOCK_C=_BLOCK_C, ROW=ROW, + PREV_IN_POOL=prev_in_pool, num_warps=4, ) @@ -404,6 +429,8 @@ def reference( """ n, p, _ = conv_x.shape context, state_len = ngram_pool.shape[1], conv_pool.shape[2] + if conv_prev is None: + conv_prev = conv_pool[slots.to(torch.long)] a = accepted.to(torch.long) req = torch.arange(n, device=a.device) # Window k of the unfold is the id history after k rows. @@ -454,11 +481,13 @@ def is_valid( f"stream must be the contiguous [n, context + p] = [{n}, {ngram_pool.shape[1] + p}] " f"spec_rollback_stage wrote; got {tuple(stream.shape)}" ) - assert conv_pool.shape[1] == channels and conv_prev.is_contiguous(), ( + assert conv_pool.shape[1] == channels and ( + conv_prev is None or conv_prev.is_contiguous() + ), ( "conv_prev must be contiguous (read as packed [n, C, state_len] rows), and conv_pool " "as wide as conv_x" ) - assert conv_prev.shape == (n, channels, conv_pool.shape[2]), ( + assert conv_prev is None or conv_prev.shape == (n, channels, conv_pool.shape[2]), ( f"conv_prev is {tuple(conv_prev.shape)}, expected {(n, channels, conv_pool.shape[2])}" ) assert position.dim() == 3 and position.shape[2] == 3, ( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py index 679479d54a4c..a4b8e7aa6c07 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_inject_norm.py @@ -34,6 +34,8 @@ from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber #: Trigger dependents at kernel start (`griddepcontrol.launch_dependents`); a no-op #: unless the next kernel is launched with programmatic dependent launch. @@ -74,12 +76,16 @@ def _hc_inject_norm_kernel( TOPK: tl.constexpr, STREAMS_PER_PROGRAM: tl.constexpr, PDL_TRIGGER: tl.constexpr, + PDL_WAIT: tl.constexpr = False, ): if PDL_TRIGGER: # Release a programmatic-dependent-launch consumer (the decode mixer's # first projection) at once: it streams its static weights while this # kernel runs and waits for the grid to finish before reading `normed`. tl.extra.cuda.gdc_launch_dependents() + if PDL_WAIT: + # Every input may be the previous kernel's output: nothing is read before its grid ends. + gdc_wait_with_memory_clobber() # One program per `STREAMS_PER_PROGRAM` streams of a row; a stream's `HIDDEN` # columns are one RMS group. A row's programs are adjacent in launch order, so # the row's block output (and the combine's operands) come from HBM once and @@ -954,7 +960,12 @@ def __call__( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: + # `num_warps` (None: 8, or 2 from `_FEW_WARPS_MIN_ROWS` rows, at hidden >= 2048; else 4) + # and `pdl_wait` (launch with PDL and wait for the predecessor before any read) set the + # Triton launch; the finalize fold's CuTe path ignores both. rows, width = hyper_input.shape hidden = width // hc_count normed = torch.empty((rows, width), dtype=hyper_input.dtype, device=hyper_input.device) @@ -1000,7 +1011,7 @@ def __call__( # The fold gathers TOPK rows to form a row's block output: two streams a # program form it half as often (when hc_count is even). spp = 2 if top_k and hc_count % 2 == 0 else 1 - _hc_inject_norm_kernel[(rows * hc_count // spp,)]( + args = ( hyper_input, block_out if has_inject else hyper_input, inject if has_inject else hyper_input, @@ -1021,6 +1032,8 @@ def __call__( hc_out.stride(0), normed.stride(0), eps, + ) + constexprs = dict( HC_COUNT=hc_count, HIDDEN=hidden, BLOCK=block, @@ -1029,8 +1042,34 @@ def __call__( TOPK=top_k, STREAMS_PER_PROGRAM=spp, PDL_TRIGGER=_PDL_TRIGGER, - num_warps=(2 if rows >= _FEW_WARPS_MIN_ROWS else 8) if block >= 2048 else 4, ) + if pdl_wait: + constexprs.update(PDL_WAIT=True, launch_pdl=True) + if num_warps is not None: + warps = num_warps + elif block >= 2048: + # Both warp counts compile at a geometry's first call. + warps = 2 if rows >= _FEW_WARPS_MIN_ROWS else 8 + warm_variants( + _hc_inject_norm_kernel, + ( + args[10:19], + hyper_input.dtype, + hc_count, + hidden, + has_inject, + has_shared, + top_k, + spp, + pdl_wait, + ), + args, + [{"num_warps": 8}, {"num_warps": 2}], + **constexprs, + ) + else: + warps = 4 + _hc_inject_norm_kernel[(rows * hc_count // spp,)](*args, **constexprs, num_warps=warps) return hc_out, normed def reference( @@ -1045,11 +1084,14 @@ def reference( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: """float64 throughout. The finalize (with `topk_rows`) and the combine (with `shared`) are each rounded once to `block_out`'s dtype; `hc_out` is the float64 accumulate rounded once; `normed` is the float64 norm of - that rounded residual, rounded once. No other intermediate rounding.""" + that rounded residual, rounded once. No other intermediate rounding. + `num_warps` and `pdl_wait` (launch options) are ignored.""" dtype = hyper_input.dtype rows, width = hyper_input.shape hidden = width // hc_count @@ -1084,6 +1126,8 @@ def is_valid( gate_logits: Optional[torch.Tensor] = None, topk_weights: Optional[torch.Tensor] = None, topk_rows: Optional[torch.Tensor] = None, + num_warps: Optional[int] = None, + pdl_wait: bool = False, ) -> None: # The kernel indexes with raw row strides and unit column stride, and # reads `inject` / `block_out` / `shared` / the gate per row: a broadcast diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py index 2e36fc757808..d8158f9212a7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_lowrank_act.py @@ -65,7 +65,8 @@ class Qwen4ExpHcLowrankAct(OpWrapper): ARCHS = frozenset({Arch.SM_103}) - # spec: rows, inject (True), hc_count (4), rank (320), logits_width (384), dtype (bf16). + # spec: rows, inject (True), hc_count (4), rank (320), logits_width (384), dtype (bf16), + # capture (False: eager; True: replayed from a captured CUDA graph over two steps' logits). # The default target's HC mixers run this form on the fresh fp32 `cublas_mm` output # over the first `hc_first_cols` = ceil((320 + 4) / 32) * 32 = 352 columns of the # first projection: hence the 352-wide cells. @@ -93,6 +94,21 @@ class Qwen4ExpHcLowrankAct(OpWrapper): why="default stack-final mixer (no inject): one decode token", spec=dict(rows=1, inject=False, logits_width=352), ), + # The FP8 unified target's HC mixers in its captured graphs at 641..3071 rows (e.g. the + # breakable prefill buckets): fewer rows run `qwen4_exp_hc_decode_tc_first`, more the + # rstd form. + Cell( + why="FP8 unified captured prefill HC mixer, 641 rows: 352-wide logits", + spec=dict(rows=641, logits_width=352, capture=True), + ), + Cell( + why="FP8 unified captured prefill HC mixer, 1024 rows: 352-wide logits", + spec=dict(rows=1024, logits_width=352, capture=True), + ), + Cell( + why="FP8 unified captured prefill HC mixer, 3071 rows (the largest): 352-wide", + spec=dict(rows=3071, logits_width=352, capture=True), + ), ) note = """ @@ -107,20 +123,25 @@ class Qwen4ExpHcLowrankAct(OpWrapper): Preconditions: `logits` fp32, unit column stride, at least `rank (+ hc_count)` columns; `hc_count` a power of two. Graph-capturable. + CUDA-graph capture (the `capture` cells): the replay is bitwise the eager + call, and a second step's logits replay bitwise their own eager call. + Source: a Triton kernel JIT-compiled at first call for the running device; the cells ran on sm_103. One program per row with an int64 row offset; 2^31 elements is 6.1e6 rows of 352, far past the envelope. Certified: bf16 outputs from fp32 logits 352 wide (the call site) at 1, 32, 1024 and 8192 rows with the injection and at 1 row without it, and 384 wide - at 128 and 8192 rows (without the injection at 8192); graph replay. + at 128 and 8192 rows (without the injection at 8192); graph replay, and + captured at 641, 1024 and 3071 rows 352 wide. Against the float64 reference: bitwise at 1, 32 and 128 rows; at 1024 and 8192 rows at most 0.99 ulp on a few `low` elements, relative L2 <= 1.6e-5. Not certified: zero rows; rows past 8192; hc_count other than 4; rank other than 320; float16 outputs. The kernel has no row-count switch, so row counts between the cells take the same path, and the logits width enters - only as the row stride (128 rows is driven at 384 wide only). + only as the row stride. Not driven under capture: the form without the + injection, and the other captured batches. """ def __call__( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py index 5cd56ddc0fe8..445a7d9faf66 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/hc/qwen4_exp_hc_up_gate_mean.py @@ -195,11 +195,22 @@ class HcUpGateMeanKernel: """ def __init__( - self, block_h: int, hc: int, hidden: int, rank: int, num_n_stage: int = 2, fp4: bool = False + self, + block_h: int, + hc: int, + hidden: int, + rank: int, + num_n_stage: int = 2, + fp4: bool = False, + wide: bool = False, ): assert block_h in (32, 64) and hidden % block_h == 0 and rank % 64 == 0 + assert not (wide and fp4), "the NVFP4 epilogue stores 32-column rows" self.bh, self.hc, self.hidden, self.rank = block_h, hc, hidden, rank self.fp4 = fp4 + # `wide`: eight epilogue warps; warps w and w + 4 read the same TMEM lane quarter and + # take the two 16-column halves of each stream's `block_h` columns. + self.wide = wide self.acc_dtype = cutlass.Float32 self.io_dtype = cutlass.BFloat16 self.mma_tiler = (128, hc * block_h, 1) @@ -211,6 +222,12 @@ def __init__( self.tma_warp_id = 5 self.nrm_warp_id = 6 self.threads_per_cta = 32 * 7 + if wide: + assert block_h == 32 + self.epi_cols = 16 + self.epilogue_warp_id = tuple(range(8)) + self.mma_warp_id, self.tma_warp_id, self.nrm_warp_id = 8, 9, 10 + self.threads_per_cta = 32 * 11 self.num_n_stage = num_n_stage self.epilog_sync_bar_id = 1 self.tmem_alloc_sync_bar_id = 2 @@ -235,13 +252,15 @@ def _setup(self, a_major, b_major): smem_capacity = utils.get_smem_capacity_in_bytes() # One normed stage is the 32-column slice of every stream for one tile half. self.n_box = (self.mma_tiler[0], self.epi_cols) - self.num_n_bytes = self.hc * self.n_box[0] * self.n_box[1] * 2 + # A normed stage holds `n_slots` boxes: every stream, with `wide` both halves of each. + self.n_slots = self.hc * (self.bh // self.epi_cols if self.wide else 1) + self.num_n_bytes = self.n_slots * self.n_box[0] * self.n_box[1] * 2 self.num_ab_stage = min( 8, (smem_capacity - 2048 - self.num_n_stage * self.num_n_bytes) // ab_bytes ) self.n_pre = min(self.num_ab_stage, self.rank // self.mma_tiler[2]) self.n_smem_layout_staged = utils.sm100.make_smem_layout_epi( - self.io_dtype, utils.LayoutEnum.ROW_MAJOR, self.n_box, self.num_n_stage * self.hc + self.io_dtype, utils.LayoutEnum.ROW_MAJOR, self.n_box, self.num_n_stage * self.n_slots ) self.a_smem_layout_staged = utils.sm100.make_smem_layout_a( tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage @@ -573,15 +592,27 @@ class SharedStorage: col_blocks = self.hidden // self.epi_cols while work_tile.is_valid_tile: crd = work_tile.tile_idx - for half in cutlass.range_constexpr(halves): + if cutlass.const_expr(self.wide): + # One stage a tile: box (half, stream) in slot half * C + stream. handle = n_producer.acquire_and_advance() - for s in cutlass.range_constexpr(self.hc): - cute.copy( - tma_atom_n, - tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], - tNsN[(None, handle.index * self.hc + s)], - tma_bar_ptr=handle.barrier, - ) + for half in cutlass.range_constexpr(halves): + for s in cutlass.range_constexpr(self.hc): + cute.copy( + tma_atom_n, + tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], + tNsN[(None, handle.index * self.n_slots + half * self.hc + s)], + tma_bar_ptr=handle.barrier, + ) + else: + for half in cutlass.range_constexpr(halves): + handle = n_producer.acquire_and_advance() + for s in cutlass.range_constexpr(self.hc): + cute.copy( + tma_atom_n, + tNgN[(None, crd[0], s * col_blocks + crd[1] * halves + half)], + tNsN[(None, handle.index * self.hc + s)], + tma_bar_ptr=handle.barrier, + ) tile_sched.advance_to_next_work() work_tile = tile_sched.get_current_work() n_producer.tail() @@ -634,7 +665,11 @@ class SharedStorage: ) tAcc_epi = cute.flat_divide(tCtAcc_base[((None, None), 0, 0, None)], epi_tile) tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) - thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + # With `wide`, warp w + 4 reads warp w's TMEM lanes (rows); it takes the other half. + etid = tidx + if cutlass.const_expr(self.wide): + etid = tidx % 128 + thr_copy_t2r = tiled_copy_t2r.get_slice(etid) tTR_tAcc_base = thr_copy_t2r.partition_S(tAcc_epi) tTR_sN = thr_copy_t2r.partition_D(sN) tTR_gX = thr_copy_t2r.partition_D( @@ -676,7 +711,35 @@ class SharedStorage: row_ok = cX_t[(0, 0, 0, 0)][0] < m acc_pipeline.consumer_wait(acc_consumer_state) - for half in cutlass.range_constexpr(halves): + if cutlass.const_expr(self.wide): + # This warp's half of every stream's columns, in the 4-warp form's arithmetic. + g = warp_idx // 4 + n_handle = n_consumer.wait_and_advance() + for s in cutlass.range_constexpr(self.hc): + cute.autovec_copy( + tTR_sN[ + (None, None, None, n_handle.index * self.n_slots + g * self.hc + s) + ], + rN, + ) + cute.copy( + tiled_copy_t2r, + tTR_tAcc[(None, None, None, s * halves + g)], + tTR_rAcc, + ) + for i in cutlass.range_constexpr(nfrag): + gate = _sigmoid(tTR_rAcc[i]) + if cutlass.const_expr(s == 0): + tTR_rSum[i] = gate * rN[i].to(self.acc_dtype) + else: + tTR_rSum[i] = tTR_rSum[i] + gate * rN[i].to(self.acc_dtype) + cute.arch.sync_warp() + n_handle.release() + for i in cutlass.range_constexpr(nfrag): + tTR_rX[i] = (tTR_rSum[i] * inv_hc).to(self.io_dtype) + if row_ok: + cute.autovec_copy(tTR_rX, gX_t[(None, None, None, g)]) + for half in cutlass.range_constexpr(0 if self.wide else halves): n_handle = n_consumer.wait_and_advance() for s in cutlass.range_constexpr(self.hc): cute.autovec_copy( @@ -736,14 +799,16 @@ class SharedStorage: _COMPILED: dict = {} -def _launcher(block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = False): +def _launcher( + block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = False, wide: bool = False +): """Compile once per static configuration; later calls reuse the TVM-FFI entry, which launches on the caller's current stream (graph-capturable). `ptrs` is `(low, up, normed, mixed tensor, xq, sf, gs)`.""" - key = (block_h, hc, hidden, rank, fp4) + key = (block_h, hc, hidden, rank, fp4, wide) fn = _COMPILED.get(key) if fn is None: - kern = HcUpGateMeanKernel(block_h, hc, hidden, rank, fp4=fp4) + kern = HcUpGateMeanKernel(block_h, hc, hidden, rank, fp4=fp4, wide=wide) cptrs = [ make_ptr(cutlass.BFloat16, p, cute.AddressSpace.gmem, assumed_align=16) for p in ptrs[:3] @@ -769,7 +834,12 @@ def _launcher(block_h: int, hc: int, hidden: int, rank: int, ptrs, fp4: bool = F def hc_up_gate_mean( - low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int, block_h: int + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + block_h: int, + wide: bool = False, ) -> torch.Tensor: """Launch the fused kernel; `up` is the `[R, C*H]` transposed view of the `[C*H, R]` up weight, as `cublas_mm(low, up)` takes it.""" @@ -780,7 +850,7 @@ def hc_up_gate_mean( ptrs = (low.data_ptr(), up.data_ptr(), normed.data_ptr()) # The NVFP4 pointers are never read without `fp4`. unused = (mixed.data_ptr(),) * 3 - _launcher(block_h, hc_count, hidden, rank, (*ptrs, mixed, *unused))( + _launcher(block_h, hc_count, hidden, rank, (*ptrs, mixed, *unused), wide=wide)( rows, *ptrs, mixed, *unused ) return mixed @@ -869,6 +939,9 @@ class Qwen4ExpHcUpGateMean(OpWrapper): turns it off): each CTA's first tile streams its up-weight stages before `griddepcontrol.wait` and reads `low` / `normed` after it. + `wide=True` runs the epilogue on eight warps instead of four, each element in the + four-warp form's arithmetic, so the bits are the same (bf16 output only). + Certified: bf16 at the checkpoint's 4 x 2560, rank 320, at 1 and 32 rows (part of one tile), 128 (one full tile), 1024, 8192, 16384 and 16390 rows (a partial last tile); graph replay. @@ -889,14 +962,24 @@ class Qwen4ExpHcUpGateMean(OpWrapper): """ def __call__( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> torch.Tensor: - return hc_up_gate_mean(low, up, normed, hc_count, _BLOCK_H) + return hc_up_gate_mean(low, up, normed, hc_count, _BLOCK_H, wide) def reference( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> torch.Tensor: - """float64, one rounding.""" + """float64, one rounding. `wide` does not change the result.""" hidden = normed.shape[1] // hc_count gate = torch.sigmoid(F.linear(low.double(), up.double().t())).unflatten( -1, (hc_count, hidden) @@ -905,8 +988,14 @@ def reference( return (gate * x).mean(dim=-2).to(normed.dtype) def is_valid( - self, low: torch.Tensor, up: torch.Tensor, normed: torch.Tensor, hc_count: int + self, + low: torch.Tensor, + up: torch.Tensor, + normed: torch.Tensor, + hc_count: int, + wide: bool = False, ) -> None: + # `wide` needs 32-column blocks and bf16: always true of this entry. rows, rank = low.shape width = normed.shape[1] assert low.dtype == up.dtype == normed.dtype == torch.bfloat16, "bf16 operands" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py new file mode 100644 index 000000000000..47fb6841536f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/jit_variants.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compile a Triton kernel's launch variants before the step that would first need one. + +A launcher that picks constexprs or launch options from the step's shape compiles one variant per +value; a served step whose value the engine's warm-up did not reach would compile inside a timed +request. `warm_variants` compiles the launcher's whole finite set at its first eager call for a +geometry, with that call's arguments and only the bucketed values changed. It only compiles +(`JITFunction.warmup`): nothing is launched, so results cannot move. Calls inside a CUDA-graph +capture defer to the next eager call. +""" + +from __future__ import annotations + +from typing import Any, Hashable, Iterable, Mapping, Sequence + +import torch + +_WARMED: set = set() + + +def _warm_key(name: str, key: Hashable) -> tuple: + """`(name, key, device)`: Triton compiles per device, so each device warms on its own.""" + device = torch.cuda.current_device() if torch.cuda.is_available() else None + return name, key, device + + +def first_time(name: str, key: Hashable) -> bool: + """True while `(name, key)` is not yet warmed on the current device; False inside a + CUDA-graph capture (the next eager call warms it). The caller marks the key with + `mark_warmed` once its compiles succeed, so a failed compile is retried.""" + if _warm_key(name, key) in _WARMED: + return False + return not torch.cuda.is_current_stream_capturing() + + +def mark_warmed(name: str, key: Hashable) -> None: + """Record `(name, key)` as compiled on the current device.""" + _WARMED.add(_warm_key(name, key)) + + +def compile_only(kernel, args: Sequence[Any], **kwargs: Any) -> None: + """Compile `kernel`'s variant for `args` and `kwargs` without launching it. A stand-in + without `warmup` (a test's launch recorder) is skipped.""" + warmup = getattr(kernel, "warmup", None) + if warmup is not None: + warmup(*args, grid=(1,), **kwargs) + + +def warm_variants( + kernel, + key: Hashable, + args: Sequence[Any], + variants: Iterable[Mapping[str, Any]], + **kwargs: Any, +) -> None: + """Compile `kernel` once per `key` and device for every mapping of `variants` laid over + `kwargs` (the call's constexprs and launch options); the key is marked only after every + variant compiled.""" + name = getattr(getattr(kernel, "fn", None), "__name__", None) or repr(kernel) + if first_time(name, key): + for variant in variants: + compile_only(kernel, args, **{**kwargs, **variant}) + mark_warmed(name, key) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py index 7a454035db8c..92e97f2c1030 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/moe/renorm_moe_routing_op.py @@ -16,8 +16,9 @@ from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper -def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16) -> dict: - """The checkpoint's router: 512 experts, top-10, bf16 logits.""" +def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16, capture: bool = False) -> dict: + """The checkpoint's router: 512 experts, top-10, bf16 logits; `capture`: replayed from a + captured CUDA graph over two steps' logits.""" return dict( tokens=tokens, num_experts=512, @@ -25,6 +26,7 @@ def _ckpt(tokens: int, output_dtype: torch.dtype = torch.bfloat16) -> dict: logits_dtype=torch.bfloat16, output_dtype=output_dtype, dist="randn", + capture=capture, ) @@ -86,6 +88,20 @@ class RenormMoeRoutingOp(OpWrapper): why="coarse logits (3 values): ties decide most of the ranking", spec=dict(_ckpt(64), output_dtype=torch.float32, dist="coarse"), ), + # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). + Cell( + why="FP8 unified captured attention-DP decode: 8 rows (dep2 2 x 4, dep4 4 x 2)", + spec=_ckpt(8, capture=True), + ), + Cell( + why="FP8 unified captured decode: 72 rows (batch 72)", + spec=_ckpt(72, capture=True), + ), + Cell( + why="FP8 unified captured decode: 128 rows, the largest (batch 128; dep2 2 x 64, " + "dep4 4 x 32 gathered)", + spec=_ckpt(128, capture=True), + ), ) note = """ @@ -111,7 +127,8 @@ class RenormMoeRoutingOp(OpWrapper): every multi-row cell. The bf16 weights of a row sum to 1 within 2.44e-3 (the rounding of ten weights). CUDA-graph replay at 128 rows is bit-exact against the reference on - refilled logits (the test holds it). + refilled logits (the test holds it), and so is a second step's replay at + the `capture` cells. Preconditions, enforced loudly by the op: `router_logits` 2-D CUDA bf16 / fp16 / fp32; `output_dtype` fp32, bf16 or None (fp16 raises even for fp16 @@ -149,8 +166,9 @@ class RenormMoeRoutingOp(OpWrapper): NaN and an all `-inf` row poison the row to NaN; a `-inf` among finite logits is a soft mask). Row counts between the cells -- dep4's smallest gathered decode (4 rows), the MTP3 verify (128 x 4) -- run the same kernel - instance on a different grid size and are not cells. Router logits from - the checkpoint's own router GEMM were not driven; the cells use randn and + instance on a different grid size and are not cells. Not driven under + capture: the other captured row counts. Router logits from the + checkpoint's own router GEMM were not driven; the cells use randn and coarse logits. """ diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py index 59bc3712f697..853fa2409eb5 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/norm/rms_norm_gated_token_major.py @@ -29,12 +29,12 @@ _PREFILL_ROWS = 8 -def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: - # A `z` that does not cover `x` goes to the registered op, which rejects it. +def _multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: + """The registered op's multi-row branch (its `eligible`, the gates it knows). A `z` + that does not cover `x` goes to the registered op, which rejects it.""" n = x.shape[-1] return ( - x.shape[0] >= _PREFILL_MIN_ROWS - and x.shape[0] == z.shape[0] * z.shape[1] + x.shape[0] == z.shape[0] * z.shape[1] and z.shape[2] == n and x.stride(-1) == 1 and z.stride(2) == 1 @@ -45,6 +45,38 @@ def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> ) +def _prefill_multirow(x: torch.Tensor, z: torch.Tensor, gate_activation: str) -> bool: + return x.shape[0] >= _PREFILL_MIN_ROWS and _multirow(x, z, gate_activation) + + +def _direct_launch(x, z, weight, eps, gate_activation): + """The registered op's own multi-row launch (same kernel, 4 rows a tile, its tiles per + program and fast-path hints, PDL by the op's policy), skipping only the custom-op dispatch.""" + rows, n = x.shape + out = torch.empty_like(x) + rows_per_tile = _layernorm_gated._MULTIROW_ROWS + groups = ( + _layernorm_gated._MULTIROW_GROUPS + if rows >= _layernorm_gated._MULTIROW_GROUPS_MIN_ROWS + else 1 + ) + grid = triton.cdiv(rows, rows_per_tile * groups) + device = x.device.index + pdl = _layernorm_gated._multirow_pdl(grid, device) + with torch.cuda.device(device): + _layernorm_gated._rms_norm_gated_fwd_multirow_kernel[(grid,)]( + x, out, weight, z, None, None, x.stride(0), out.stride(0), z.stride(0), rows, eps, + N=n, ROWS=rows_per_tile, HEADS_PER_TOK=z.shape[1], + GATE_SIGMOID=gate_activation == "sigmoid", WEIGHT_IS_DELTA=False, + LAUNCH_WITH_PDL=pdl, CONTIG=(x.stride(0) == n and out.stride(0) == n), + EVEN=(rows % (rows_per_tile * groups) == 0), + num_warps=_layernorm_gated._MULTIROW_NUM_WARPS, + EARLY_TRIGGER=_layernorm_gated._pdl_device_policy(device)[0], GROUPS=groups, + launch_pdl=pdl, + ) # fmt: skip + return out + + def _prefill_launch(x, z, weight, eps, gate_activation): """The multi-row kernel at `_PREFILL_ROWS` rows a program (no PDL: the grid covers every SM).""" rows, n = x.shape @@ -130,7 +162,13 @@ class RmsNormGatedTokenMajor(OpWrapper): copy of `z`. PDL on the 4-row path only when the grid is smaller than the SM count and `TRTLLM_ENABLE_PDL != 0`; results are bitwise identical with it off. The 8-row programs reduce a row's mean square in another order - than the 4-row ones (one-ulp differences on some outputs). + than the 4-row ones (one-ulp differences on some outputs). With + `direct_launch=True` (the FP8 target) below the 8-row threshold, the 4-row + launch is made here (`_direct_launch`, a copy of the registered op's own), + skipping only the custom-op dispatch: 24 instead of 39 us of host time a + call on GB300. The test holds it bitwise equal to the op at the + direct-launch cells and more shapes, so a change to the op's launch fails + there. Loud (measured), so not guarded: 3-D `x` (ValueError), a `z` not covering `x` (AssertionError "does not match x shape"; 2-D `z` ValueError), an unknown @@ -154,6 +192,9 @@ class RmsNormGatedTokenMajor(OpWrapper): 12 heads, token stride 4120 (a tp4/tep4 rank): 1 and 4 tokens on the 4-row kernel with PDL (grids of 3 and 12 programs); 128 and 1024 tokens on it without; 8192 tokens (98304 rows) on the 8-row kernel. + 24 heads, token stride 8240 (the FP8 target's tp2/tep2 rank, with + `direct_launch`, cells in the test): 1 and 683 tokens on the 4-row + kernel, 8192 tokens on the 8-row kernel. A one-token `z` view reports the dense token stride (1536 at 12 heads); only token 0 is read, so it computes the same. Not certified: fp8 output, the silu gate, the generic fallback kernel, @@ -186,9 +227,19 @@ def __call__( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> torch.Tensor: if fp8_scale is None and z.dim() == 3 and _prefill_multirow(x, z, gate_activation): return _prefill_launch(x, z, weight, eps, gate_activation) + if ( + direct_launch + and fp8_scale is None + and x.dim() == 2 + and z.dim() == 3 + and weight.is_contiguous() + and _multirow(x, z, gate_activation) + ): + return _direct_launch(x, z, weight, eps, gate_activation) return torch.ops.trtllm.rms_norm_gated_token_major( x, z, weight, eps, fp8_scale, gate_activation ) @@ -201,8 +252,10 @@ def reference( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> torch.Tensor: - """fp32 throughout; row `t * heads + h` pairs with `z[t, h]`. + """fp32 throughout; row `t * heads + h` pairs with `z[t, h]`. `direct_launch` does + not change the result. With `fp8_scale` the gated value is rounded to `x.dtype` first, then multiplied by the fp32 reciprocal of the scale, as the kernel does. @@ -227,6 +280,7 @@ def is_valid( eps: float, fp8_scale: Optional[torch.Tensor] = None, gate_activation: str = "silu", + direct_launch: bool = False, ) -> None: assert weight.shape == (x.shape[-1],), ( f"weight must be 1-D of length x.shape[-1]={x.shape[-1]}; got " diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py index 46548cc3f797..0a8f3d8a93e3 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_gate_conv_inject_norm.py @@ -31,6 +31,11 @@ import triton.language as tl from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + compile_only, + first_time, + mark_warmed, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, TEP4_SLOTS, @@ -39,6 +44,7 @@ PER_SEQ_MAX_ROWS, _seq_of_rows, ) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber @triton.jit @@ -47,6 +53,40 @@ def _rms(x, w, HIDDEN: tl.constexpr, eps): return x * tl.rsqrt(tl.sum(x * x, axis=0) / HIDDEN + eps) * (1.0 + w) +@triton.jit +def _injected_stream( + hc_ptr, + row, + stride_hc, + ch, + cols, + m, + stream, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, +): + """One stream of the residual, fp32. With `HAS_INJECT` the previous MoE block's injection + comes first, rounded as `qwen4_exp_hc_inject_norm` rounds it.""" + x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) + if HAS_INJECT: + b = tl.load(block_ptr + row * stride_block + cols, mask=m, other=0.0).to(tl.float32) + if HAS_SHARED: + sg = tl.sigmoid(tl.load(gate_ptr + row * stride_gate).to(tl.float32)) + sh = tl.load(sh_ptr + row * stride_sh + cols, mask=m, other=0.0) + b = (b + sg * sh.to(tl.float32)).to(block_ptr.dtype.element_ty).to(tl.float32) + g = tl.load(inj_ptr + row * stride_inj + stream).to(tl.float32) + x = (x + b * g).to(hc_ptr.dtype.element_ty).to(tl.float32) + return x + + @triton.jit def _ple_gate_kernel( hc_ptr, @@ -64,22 +104,39 @@ def _ple_gate_kernel( eps, clamp_min, sqrt_d, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, HIDDEN: tl.constexpr, BLOCK: tl.constexpr, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (row, stream): the key and query grouped norms, the gate, the gated - value and the conv-input grouped norm. Writes `gated` and `xc`.""" + value and the conv-input grouped norm. Writes `gated` and `xc`. With `PDL` the norm + weights are read before the wait, everything else after it.""" row = tl.program_id(0).to(tl.int64) stream = tl.program_id(1) cols = tl.arange(0, BLOCK) m = cols < HIDDEN ch = stream * HIDDEN + cols - x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) - kp = tl.load(key_ptr + row * stride_key + ch, mask=m, other=0.0).to(tl.float32) - v = tl.load(value_ptr + row * stride_value + cols, mask=m, other=0.0).to(tl.float32) wk = tl.load(nk_ptr + ch, mask=m, other=0.0).to(tl.float32) wq = tl.load(nq_ptr + ch, mask=m, other=0.0).to(tl.float32) wc = tl.load(nc_ptr + ch, mask=m, other=0.0).to(tl.float32) + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + x = _injected_stream(hc_ptr, row, stride_hc, ch, cols, m, stream, block_ptr, inj_ptr, sh_ptr, + gate_ptr, stride_block, stride_inj, stride_sh, stride_gate, HAS_INJECT, + HAS_SHARED) # fmt: skip + kp = tl.load(key_ptr + row * stride_key + ch, mask=m, other=0.0).to(tl.float32) + v = tl.load(value_ptr + row * stride_value + cols, mask=m, other=0.0).to(tl.float32) kn = _rms(kp, wk, HIDDEN, eps).to(tl.bfloat16).to(tl.float32) qn = _rms(x, wq, HIDDEN, eps).to(tl.bfloat16).to(tl.float32) g = tl.sum(kn * qn, axis=0) / sqrt_d @@ -91,7 +148,11 @@ def _ple_gate_kernel( tl.store(xc_ptr + row * stride_out + ch, xc, mask=m) -@triton.jit +# The per-sequence metadata may begin at any row of a mixed step's tensors. +@triton.jit( + do_not_specialize=["publish_end"], + do_not_specialize_on_alignment=["slots_ptr", "carry_ptr", "qsl_ptr"], +) def _ple_conv_kernel( xc_ptr, gated_ptr, @@ -104,6 +165,7 @@ def _ple_conv_kernel( stride_out, stride_state, pad_slot, + publish_end, KSIZE: tl.constexpr, DIL: tl.constexpr, STATE: tl.constexpr, @@ -111,16 +173,21 @@ def _ple_conv_kernel( MAX_P: tl.constexpr, ROWS: tl.constexpr, HAS_CARRY: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (sequence, block of ROWS channels): the dilated depthwise conv of each of the sequence's rows against `[state | conv inputs]`, its SiLU added - to the gated value (`ple`), and the state advanced in place. + to the gated value (`ple`), and the state advanced in place for the + sequences below `publish_end` (the others' states are left as they were). A channel's state is `STATE` contiguous columns. The publish moves it as a `[ROWS, HEAD]` tile of the first HEAD (a power of two) columns plus, when `STATE == HEAD + 1`, one `[ROWS]` vector for the last, so no lane is spent on padding and every index is a plain 2-D offset. Every load is issued before the first store.""" + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() s = tl.program_id(0) ch0 = tl.program_id(1) * ROWS begin = tl.load(qsl_ptr + s).to(tl.int64) @@ -181,12 +248,17 @@ def _ple_conv_kernel( # Every read of these state rows -- `keep` and the taps -- lands before # the first store over them. tl.debug_barrier() - tl.store(block + r[:, None] * STATE + i[None, :], head, mask=live) + publish = live & (s < publish_end) + tl.store(block + r[:, None] * STATE + i[None, :], head, mask=publish) if STATE > HEAD: - tl.store(block + r * STATE + HEAD, tail, mask=live) + tl.store(block + r * STATE + HEAD, tail, mask=publish) -@triton.jit +# The packed row and sequence counts follow the step. +@triton.jit( + do_not_specialize=["num_rows", "num_seqs"], + do_not_specialize_on_alignment=["slots_ptr", "carry_ptr", "qsl_ptr"], +) def _ple_conv_rows_kernel( xc_ptr, gated_ptr, @@ -262,25 +334,62 @@ def _ple_residual_norm_kernel( stride_hc_out, stride_normed, eps, + block_ptr, + inj_ptr, + sh_ptr, + gate_ptr, + stride_block, + stride_inj, + stride_sh, + stride_gate, HIDDEN: tl.constexpr, BLOCK: tl.constexpr, + HAS_INJECT: tl.constexpr, + HAS_SHARED: tl.constexpr, + PDL: tl.constexpr = False, ): """Per (row, stream): the PLE output added into the stream, then the next - site's grouped norm of the result.""" + site's grouped norm of the result. With `PDL` the norm weight is read before the wait.""" row = tl.program_id(0).to(tl.int64) stream = tl.program_id(1) cols = tl.arange(0, BLOCK) m = cols < HIDDEN ch = stream * HIDDEN + cols - x = tl.load(hc_ptr + row * stride_hc + ch, mask=m, other=0.0).to(tl.float32) - ple = tl.load(ple_ptr + row * stride_ple + ch, mask=m, other=0.0).to(tl.float32) wx = tl.load(nx_ptr + ch, mask=m, other=0.0).to(tl.float32) + if PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + x = _injected_stream(hc_ptr, row, stride_hc, ch, cols, m, stream, block_ptr, inj_ptr, sh_ptr, + gate_ptr, stride_block, stride_inj, stride_sh, stride_gate, HAS_INJECT, + HAS_SHARED) # fmt: skip + ple = tl.load(ple_ptr + row * stride_ple + ch, mask=m, other=0.0).to(tl.float32) hn = (x + ple).to(tl.bfloat16) tl.store(hc_out_ptr + row * stride_hc_out + ch, hn, mask=m) nrm = _rms(hn.to(tl.float32), wx, HIDDEN, eps) tl.store(normed_ptr + row * stride_normed + ch, nrm.to(tl.bfloat16), mask=m) +def _inject_args(hc: torch.Tensor, inject_block: Optional[tuple]) -> tuple: + """`(block, inject, shared, gate)`, their row strides, `HAS_INJECT`, `HAS_SHARED` for the gate + and residual kernels (unused operands point at `hc`).""" + if inject_block is None: + return (hc, hc, hc, hc, 0, 0, 0, 0, False, False) + block_out, inject, shared, gate_logits = inject_block + has_shared = shared is not None + return ( + block_out, + inject, + shared if has_shared else hc, + gate_logits if has_shared else hc, + block_out.stride(0), + inject.stride(0), + shared.stride(0) if has_shared else 0, + gate_logits.stride(0) if has_shared else 0, + True, + has_shared, + ) + + def _state_head(state_len: int) -> int: """The largest power of two not above `state_len`: the conv kernel's tile width; the one column past it, if any, moves as a vector.""" @@ -493,6 +602,9 @@ def __call__( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: rows, width = hc.shape hidden = width // hc_count @@ -514,6 +626,7 @@ def __call__( ) block = triton.next_power_of_2(hidden) warps = 8 if block >= 2048 else 4 + inj = _inject_args(hc, inject_block) _ple_gate_kernel[(rows, hc_count)]( hc, key, @@ -530,9 +643,14 @@ def __call__( eps, clamp_min, math.sqrt(hidden), + *inj[:-2], HIDDEN=hidden, BLOCK=block, + HAS_INJECT=inj[-2], + HAS_SHARED=inj[-1], + PDL=pdl, num_warps=warps, + launch_pdl=pdl, ) # `ple` reuses `gated`'s storage row for row: each element is read by # the program that then writes it. @@ -540,35 +658,9 @@ def __call__( has_carry = has_initial_states is not None carry = has_initial_states if has_carry else state_indices per_seq = max_rows <= PER_SEQ_MAX_ROWS - if not per_seq: - tile_ch = min(_ROWS_CHANNELS, width) - _ple_conv_rows_kernel[(triton.cdiv(rows, _ROWS_BLOCK_T), width // tile_ch)]( - xc, - gated, - gated, - conv_weight, - conv_states, - state_indices, - carry, - query_start_loc, - rows, - num_seqs, - gated.stride(0), - conv_states.stride(0), - pad_slot_id, - KSIZE=ksize, - DIL=dilation, - STATE=state_len, - BLOCK_T=_ROWS_BLOCK_T, - ROWS=tile_ch, - LOG_S=max(_MIN_LOG_S, (num_seqs - 1).bit_length()), - HAS_CARRY=has_carry, - num_warps=_ROWS_WARPS, - ) - # Per sequence: the rows (decode) and the state publish. On the - # context-shaped path it carries no rows and only publishes, after - # every program above has read the state. - _ple_conv_kernel[(num_seqs, width // conv_rows)]( + tile_ch = min(_ROWS_CHANNELS, width) + log_s = max(_MIN_LOG_S, (num_seqs - 1).bit_length()) + rows_args = ( xc, gated, gated, @@ -577,9 +669,37 @@ def __call__( state_indices, carry, query_start_loc, + rows, + num_seqs, gated.stride(0), conv_states.stride(0), pad_slot_id, + ) + rows_constexprs = dict( + KSIZE=ksize, + DIL=dilation, + STATE=state_len, + BLOCK_T=_ROWS_BLOCK_T, + ROWS=tile_ch, + LOG_S=log_s, + HAS_CARRY=has_carry, + num_warps=_ROWS_WARPS, + ) + conv_args = ( + xc, + gated, + gated, + conv_weight, + conv_states, + state_indices, + carry, + query_start_loc, + gated.stride(0), + conv_states.stride(0), + pad_slot_id, + num_seqs if publish_end is None else int(publish_end), + ) + conv_constexprs = dict( KSIZE=ksize, DIL=dilation, STATE=state_len, @@ -587,8 +707,42 @@ def __call__( MAX_P=max_rows if per_seq else 0, ROWS=conv_rows, HAS_CARRY=has_carry, + PDL=pdl, num_warps=_CONV_WARPS, + launch_pdl=pdl, ) + # `MAX_P` follows the step's longest sequence (0 past PER_SEQ_MAX_ROWS): every value, and + # the context-shaped rows kernel, compile at a geometry's first call. + warm_key = ( + xc.dtype, + conv_states.dtype, + carry.dtype, + state_indices.dtype, + query_start_loc.dtype, + gated.stride(0), + conv_states.stride(0), + pad_slot_id, + ksize, + dilation, + state_len, + conv_rows, + has_carry, + log_s, + pdl, + ) + if first_time("ple_gate_conv_inject_norm", warm_key): + for max_p in range(PER_SEQ_MAX_ROWS + 1): + compile_only(_ple_conv_kernel, conv_args, **{**conv_constexprs, "MAX_P": max_p}) + compile_only(_ple_conv_rows_kernel, rows_args, **rows_constexprs) + mark_warmed("ple_gate_conv_inject_norm", warm_key) + if not per_seq: + _ple_conv_rows_kernel[(triton.cdiv(rows, _ROWS_BLOCK_T), width // tile_ch)]( + *rows_args, **rows_constexprs + ) + # Per sequence: the rows (decode) and the state publish. On the + # context-shaped path it carries no rows and only publishes, after + # every program above has read the state. + _ple_conv_kernel[(num_seqs, width // conv_rows)](*conv_args, **conv_constexprs) _ple_residual_norm_kernel[(rows, hc_count)]( hc, gated, @@ -600,9 +754,14 @@ def __call__( hc_out.stride(0), normed.stride(0), eps, + *inj[:-2], HIDDEN=hidden, BLOCK=block, + HAS_INJECT=inj[-2], + HAS_SHARED=inj[-1], + PDL=pdl, num_warps=warps, + launch_pdl=pdl, ) return hc_out, normed @@ -628,6 +787,9 @@ def reference( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """`(hc_out, normed, conv_states)` as the call leaves them: float64 throughout, rounded to the activation dtype exactly where the op (and @@ -640,6 +802,17 @@ def reference( def rnd(x): return x.to(dtype).double() + if inject_block is not None: + # The previous MoE block injected first (`qwen4_exp_hc_inject_norm`'s rounding). + block_out, inject, shared, gate_logits = inject_block + b = block_out.double() + if shared is not None: + b = rnd(b + torch.sigmoid(gate_logits.reshape(rows, 1).double()) * shared.double()) + hc = hc.double().view(rows, hc_count, hidden) + b.unsqueeze( + 1 + ) * inject.double().unsqueeze(-1) + hc = hc.reshape(rows, width).to(dtype) + def norm(x, w): w = w.double().view(hc_count, hidden) return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + w) @@ -677,7 +850,7 @@ def norm(x, w): for k in range(ksize): acc += w[:, k : k + 1] * window[:, k * dilation : k * dilation + n] conv[begin:end] = rnd(acc.t()) - if live: + if live and (publish_end is None or s < publish_end): pool[slot] = window[:, end - begin : end - begin + state_len].to(dtype) ple = rnd(gated.reshape(rows, width) + rnd(torch.nn.functional.silu(conv))) hc_out = (hc.double() + ple).to(dtype) @@ -706,8 +879,25 @@ def is_valid( has_initial_states: Optional[torch.Tensor] = None, conv_input: Optional[torch.Tensor] = None, out: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + inject_block: Optional[tuple] = None, + pdl: bool = False, + publish_end: Optional[int] = None, ) -> None: rows, width = hc.shape + if inject_block is not None: + block_out, inject, shared, gate_logits = inject_block + hidden_ = width // hc_count + assert block_out.shape == (rows, hidden_) and block_out.stride(1) == 1, tuple( + block_out.shape + ) + assert inject.shape == (rows, hc_count) and inject.stride(1) == 1, tuple(inject.shape) + assert block_out.dtype == inject.dtype == hc.dtype + if shared is not None: + assert shared.shape == (rows, hidden_) and shared.stride(1) == 1, tuple( + shared.shape + ) + assert shared.dtype == hc.dtype + assert gate_logits is not None and gate_logits.numel() == rows assert width % hc_count == 0, f"width {width} not divisible by hc_count {hc_count}" hidden = width // hc_count for name, t in (("hc", hc), ("key", key), ("value", value)): diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py index d09f220581a3..98e37d40a03d 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ple/ple_ngram_embed.py @@ -32,6 +32,7 @@ from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ngram_hash +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import warm_variants from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( DEP4_SLOTS, TEP4_SLOTS, @@ -52,13 +53,18 @@ #: Above this many rows per sequence (MTP verify carries at most 4) the call #: takes the context-shaped path: packed row tiles plus a separate publish. PER_SEQ_MAX_ROWS = 4 +#: `BLOCK_P` values a per-sequence call can pick (`next_power_of_2` of 1..PER_SEQ_MAX_ROWS), +#: all compiled at a geometry's first call. +_PER_SEQ_BLOCK_PS = (1, 2, 4) _ROWS_BLOCK_T = 32 #: The row-to-sequence search covers at least 2^8 sequences, so one compiled #: variant serves every batch up to that size. _MIN_LOG_S = 8 -@triton.jit +# `row_lo`, `row_hi`, `row_base`: the gather reads table rows `[row_lo, row_hi)` from the view whose +# first row is `row_base` and writes zeros for the others (a rank's share of a row-sharded table). +@triton.jit(do_not_specialize=["row_lo", "row_hi", "row_base"]) def _ple_ngram_embed_kernel( ids_ptr, pool_ptr, @@ -75,6 +81,9 @@ def _ple_ngram_embed_kernel( stride_table, stride_out, eos, + row_lo, + row_hi, + row_base, NGRAM: tl.constexpr, HEADS_PER: tl.constexpr, HEADS: tl.constexpr, @@ -127,7 +136,12 @@ def _ple_ngram_embed_kernel( d = tl.arange(0, BLOCK_D) m3 = tmask[:, None, None] & hmask[None, :, None] & (d < HEAD_DIM)[None, None, :] - q = tl.load(table_ptr + rows[:, :, None] * stride_table + d[None, None, :], mask=m3, other=0.0) + own = ((rows >= row_lo) & (rows < row_hi))[:, :, None] + q = tl.load( + table_ptr + (rows - row_base)[:, :, None] * stride_table + d[None, None, :], + mask=m3 & own, + other=0.0, + ) val = q.to(tl.float32) * tl.load(scale_ptr).to(tl.float32) cols = heads[None, :, None] * HEAD_DIM + d[None, None, :] tl.store( @@ -165,7 +179,7 @@ def _seq_of_rows(qsl_ptr, t, num_seqs, LOG_S: tl.constexpr): return s -@triton.jit +@triton.jit(do_not_specialize=["row_lo", "row_hi", "row_base"]) def _ple_ngram_embed_rows_kernel( ids_ptr, pool_ptr, @@ -184,6 +198,9 @@ def _ple_ngram_embed_rows_kernel( stride_table, stride_out, eos, + row_lo, + row_hi, + row_base, NGRAM: tl.constexpr, HEADS_PER: tl.constexpr, HEADS: tl.constexpr, @@ -241,7 +258,12 @@ def _ple_ngram_embed_rows_kernel( scale = tl.load(scale_ptr).to(tl.float32) for h in tl.static_range(HEADS): row_h = tl.sum(tl.where((heads == h)[None, :], rows, 0), axis=1) - q = tl.load(table_ptr + row_h[:, None] * stride_table + d[None, :], mask=m2, other=0.0) + own = ((row_h >= row_lo) & (row_h < row_hi))[:, None] + q = tl.load( + table_ptr + (row_h - row_base)[:, None] * stride_table + d[None, :], + mask=m2 & own, + other=0.0, + ) val = q.to(tl.float32) * scale tl.store( out_ptr + t[:, None] * stride_out + (h * HEAD_DIM + d)[None, :], @@ -302,7 +324,10 @@ class PleNgramEmbed(OpWrapper): out[t, h * D : (h + 1) * D] = to(scale.dtype)(float(table[row[t, h]]) * float(scale)) - with `D = table.shape[1]` -- one rounding of the scaled fp8 value. + with `D = table.shape[1]` -- one rounding of the scaled fp8 value. With + `row_window = (lo, hi, base)` the view `table` holds global rows from `base` + on: rows in `[lo, hi)` are read from `table[row - base]` and the others + give zeros, so a row-sharded table's shares sum to the whole table's output. Then, **in place**, `context_pool[slot]` becomes the last `ngram_size - 1` ids of the stream (a zero-row sequence republishes its carry). Unnamed slots are untouched. @@ -453,8 +478,10 @@ def __call__( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> torch.Tensor: num_tokens = input_ids.shape[0] + row_lo, row_hi, row_base = row_window or (0, table.shape[0], 0) num_seqs = state_indices.shape[0] ngram_size = layer_multipliers.numel() heads = head_vocab_sizes.numel() @@ -483,6 +510,9 @@ def __call__( table.stride(0), out.stride(0), eos_token_id, + row_lo, + row_hi, + row_base, NGRAM=ngram_size, HEADS_PER=heads // (ngram_size - 1), HEADS=heads, @@ -508,7 +538,7 @@ def __call__( num_warps=1, ) elif num_seqs > 0: - _ple_ngram_embed_kernel[(num_seqs,)]( + embed_args = ( input_ids, context_pool, state_indices, @@ -524,6 +554,11 @@ def __call__( table.stride(0), out.stride(0), eos_token_id, + row_lo, + row_hi, + row_base, + ) + embed_constexprs = dict( NGRAM=ngram_size, HEADS_PER=heads // (ngram_size - 1), HEADS=heads, @@ -535,6 +570,20 @@ def __call__( HAS_CARRY=has_initial_states is not None, num_warps=4, ) + # `BLOCK_P` follows the step's longest sequence: every value compiles at a + # geometry's first call, so a served step never compiles. + warm_variants( + _ple_ngram_embed_kernel, + ( + tuple(t.dtype for t in embed_args[:11]), + embed_args[11:], + tuple(embed_constexprs[k] for k in ("NGRAM", "HEADS", "HEAD_DIM", "HAS_CARRY")), + ), + embed_args, + [{"BLOCK_P": p} for p in _PER_SEQ_BLOCK_PS], + **embed_constexprs, + ) + _ple_ngram_embed_kernel[(num_seqs,)](*embed_args, **embed_constexprs) return out def reference( @@ -551,6 +600,7 @@ def reference( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> tuple[torch.Tensor, torch.Tensor]: """`(out, context_pool)` as the call leaves them, from the scalar references of the two entries it fuses -- `ngram_token_history` @@ -574,7 +624,14 @@ def reference( rows = ngram_hash.reference( window, layer_multipliers, head_vocab_sizes, head_offsets, eos_token_id, 1 ).reshape(window.shape[0], -1) - gathered = table[rows.reshape(-1)].double().reshape(rows.shape[0], -1) + flat = rows.reshape(-1) + lo, hi, base = row_window or (0, table.shape[0], 0) + own = (flat >= lo) & (flat < hi) + picked = torch.zeros( + (flat.numel(), table.shape[1]), dtype=torch.float64, device=table.device + ) + picked[own] = table[flat[own] - base].double() + gathered = picked.reshape(rows.shape[0], -1) out = (gathered * scale.double().reshape(())).to(scale.dtype) return out, pool @@ -592,6 +649,7 @@ def is_valid( head_offsets: torch.Tensor, eos_token_id: int, max_rows: int, + row_window: Optional[tuple[int, int, int]] = None, ) -> None: num_seqs = state_indices.shape[0] ngram_size = layer_multipliers.numel() @@ -619,6 +677,12 @@ def is_valid( assert table.dim() == 2 and table.stride(1) == 1, "table [rows, D], unit column stride" assert scale.numel() == 1, "one per-tensor scale" assert max_rows >= 1, "max_rows >= 1" + if row_window is not None: + lo, hi, base = row_window + assert 0 <= base <= lo <= hi and hi - base <= table.shape[0], ( + f"row_window {row_window}: rows [lo, hi) must lie in the view of " + f"{table.shape[0]} rows starting at row `base`" + ) ple_ngram_embed = PleNgramEmbed() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py index 77531fa97feb..e3ac3a0103c7 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/quantization/fp8_quantize_1x128.py @@ -69,6 +69,8 @@ class Fp8Quantize1x128(OpWrapper): ARCHS = frozenset({Arch.SM_103}) + # spec: tokens, hidden, use_ue8m0, capture (False: eager; True: replayed from a captured + # CUDA graph over two steps' activations). CELLS: tuple[Cell, ...] = ( Cell(why="MTPLayer._moe decode: one token at hidden 2560", spec=dict(tokens=1, **_CKPT)), Cell( @@ -91,6 +93,20 @@ class Fp8Quantize1x128(OpWrapper): why="8192 rows: the MTP layer over one max_num_tokens context chunk", spec=dict(tokens=8192, **_CKPT), ), + # The FP8 unified target's routed experts in its captured decode graph (1..128 rows). + Cell( + why="FP8 unified captured attention-DP decode: 8 rows (dep2 2 x 4, dep4 4 x 2)", + spec=dict(tokens=8, **_CKPT, capture=True), + ), + Cell( + why="FP8 unified captured decode: 72 rows (batch 72)", + spec=dict(tokens=72, **_CKPT, capture=True), + ), + Cell( + why="FP8 unified captured decode: 128 rows, the largest (batch 128; dep2 2 x 64, " + "dep4 4 x 32 gathered)", + spec=dict(tokens=128, **_CKPT, capture=True), + ), ) note = """ @@ -110,6 +126,8 @@ class Fp8Quantize1x128(OpWrapper): stop at `T`. At a `T` that is a multiple of 4 (128, 1024, 8192) nothing is padded and `scales` is the kernel's own buffer, not a compacting copy. CUDA-graph capture and replay are certified at 64, 128 and 1024 rows. + At the `capture` cells the replay is bitwise the eager call, and a second + step's activations replay bitwise their own eager call. Source: compiled-in C++, `torch.ops.trtllm.fp8_quantize_1x128` from `thop/fp8Quantize.cpp` (`fp8_1x128_cs` -> `launch_scale_1x128`). The @@ -129,14 +147,17 @@ class Fp8Quantize1x128(OpWrapper): T certified at 1, 20, 128, 1024, 8192 and 16384 at H 2560 (128: the drafter's decode at max_batch_size 128; 1024: the first draft over mtp7's 128 x 8 verify rows; 8192: a max_num_tokens context chunk; - 16384: max_num_tokens of the tuned config); above that not probed + 16384: max_num_tokens of the tuned config), captured at 8, 72 and + 128; above that not probed H multiple of 16 (raises otherwise); certified at 2560, the draft layer's width Not certified: `T == 0` (the op accepts it and returns `[0, H]` / `[NB, 0]`); a short tail block (`H % 128 != 0`); all-zero blocks (the `AMAX_FLOOR` scale), `inf` / NaN inputs and saturation; `use_ue8m0=True`; - the element-wise fallback kernel; the registered fake. + the element-wise fallback kernel; the registered fake. Not driven under + capture: the other captured row counts, and a count that is not a multiple + of 4 (the compacting copy). """ def __call__( diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py new file mode 100644 index 000000000000..bd7b896f98eb --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ple_host_table.py @@ -0,0 +1,577 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Where the PLE n-gram table lives in host memory, for both Qwen3.8-Flash-Next targets. + +The table's rows are cut into `CHUNKS` fixed chunks (`chunk_rows`); a rank of a server of +`world` ranks is responsible for chunks `[rank * CHUNKS / world, (rank + 1) * CHUNKS / world)`. +`PLE_HOST_TABLE_ENV` picks how the rows are stored and looked up: + +* `rank` (default): every rank holds the whole table in private host memory. +* `server`: under attention TP every rank holds only the rows of its chunks; the n-gram gather + reads those rows and writes zeros for the others (`window`), and the caller sums the gathered + rows over the attention-TP group (`needs_sum`). One copy per server, no inter-process state. + Exact: every element has one nonzero term. +* `node`: every chunk is one CUDA VMM allocation on host NUMA memory, shared by every process of + the host through `chunk_broker`; every rank maps all chunks back to back and gathers locally. + A rank fills the chunks it is responsible for and attaches to the others. One copy per node. + Processes share a chunk when their tables have one `table_identity`: the bytes of a strided + sample of every chunk's rows, whatever directory the checkpoint was read from. A rank that + attaches compares the chunk with its own checkpoint at other sampled rows. + +`server` needs attention TP with a size that divides `CHUNKS`, else it runs as `rank`. When a +rank cannot take part in `node` (no VMM host NUMA allocation, no table identity, no owner +within the timeout), every rank of the server runs `server` instead, or `rank`. A requested +mode that does not run is a warning, once per process. +""" + +from __future__ import annotations + +import hashlib +import math +import os +from typing import Callable, Optional + +import torch +from cuda.bindings import driver as cuda_driver +from cuda.bindings import runtime as cudart + +from tensorrt_llm._dlpack_utils import pack_strided_memory +from tensorrt_llm._utils import CUASSERT +from tensorrt_llm.logger import logger + +from . import chunk_broker as _broker + +#: `rank`, `server` or `node` (see the module docstring); default `rank`. +PLE_HOST_TABLE_ENV = "TRTLLM_MODELING_V2_PLE_HOST_TABLE" +RANK = "rank" +SERVER = "server" +NODE = "node" +_MODES = (RANK, SERVER, NODE) + +#: Fixed chunks of the table; every server size that divides it maps whole chunks. +CHUNKS = 8 +#: Chunk byte sizes are multiples of this: the VMM host NUMA granularity, so `node` can map the +#: chunks back to back as one table. +CHUNK_ALIGN = 2 << 20 +#: How long a rank waits for another process to publish a chunk before giving up on `node`. +_ATTACH_TIMEOUT_S = 3600.0 +#: A chunk's sample (`sample_blocks`): this many blocks of `_SAMPLE_ROWS` rows at an even stride. +_SAMPLE_BLOCKS = 256 +_SAMPLE_ROWS = 8 +_CUDA_MEMORY_TYPE_HOST = 1 +#: `(requested, running)` fallbacks this process has warned about. +_WARNED: set = set() + +Copy = Callable[[torch.Tensor, int, int, int], None] +Read = Callable[[int, int], torch.Tensor] +Private = Callable[[int, int, torch.dtype, torch.device], tuple] + + +def host_table_mode() -> str: + """The requested mode from `PLE_HOST_TABLE_ENV`.""" + value = os.environ.get(PLE_HOST_TABLE_ENV, RANK).strip().lower() or RANK + if value not in _MODES: + raise ValueError(f"{PLE_HOST_TABLE_ENV}={value!r}: expected one of {', '.join(_MODES)}") + return value + + +def _warn_fallback(requested: str, running: str, why: str) -> None: + """Warn, once per process and pair, that the requested mode runs as `running`, and why: the + table then takes more host memory than the deployment asked for.""" + if (requested, running) in _WARNED: + return + _WARNED.add((requested, running)) + logger.warning( + f"ple host table: {PLE_HOST_TABLE_ENV}={requested} requested, running {running}: {why}" + ) + + +def chunk_rows(rows: int, cols: int, align: int = CHUNK_ALIGN) -> int: + """Rows of one chunk: an even split of `rows` over `CHUNKS`, rounded up so a chunk's bytes + (`cols` one-byte columns a row) are a multiple of `align`.""" + unit = align // math.gcd(align, cols) + per = -(-rows // CHUNKS) + return -(-per // unit) * unit + + +def chunk_range(k: int, rows: int, cols: int, align: int = CHUNK_ALIGN) -> tuple[int, int]: + """Global rows `[lo, hi)` of chunk `k` (empty past the table's end).""" + per = chunk_rows(rows, cols, align) + return min(k * per, rows), min((k + 1) * per, rows) + + +def owned_chunks(rank: int, world: int) -> range: + """The chunks rank `rank` of `world` is responsible for.""" + assert world > 0 and CHUNKS % world == 0, (rank, world) + per = CHUNKS // world + return range(rank * per, (rank + 1) * per) + + +def owned_rows(rank: int, world: int, rows: int, cols: int, align: int = CHUNK_ALIGN) -> tuple: + """Global rows `[lo, hi)` of rank `rank`'s chunks.""" + chunks = owned_chunks(rank, world) + return chunk_range(chunks[0], rows, cols, align)[0], chunk_range(chunks[-1], rows, cols, align)[ + 1 + ] + + +def sample_blocks(k: int, rows: int, cols: int, align: int = CHUNK_ALIGN, phase: int = 0) -> list: + """Row blocks `[lo, hi)` of chunk `k` that `node` compares: `_SAMPLE_BLOCKS` blocks of + `_SAMPLE_ROWS` rows at an even stride, `phase` 1 halfway between phase 0's; the whole chunk + when it is too short to stride.""" + lo, hi = chunk_range(k, rows, cols, align) + if hi - lo <= 2 * _SAMPLE_BLOCKS * _SAMPLE_ROWS: + return [(lo, hi)] if hi > lo else [] + stride = (hi - lo) // _SAMPLE_BLOCKS + first = lo + phase * (stride // 2) + return [ + (start, min(start + _SAMPLE_ROWS, hi)) + for start in range(first, hi, stride)[:_SAMPLE_BLOCKS] + ] + + +def _sample(read: Read, blocks) -> torch.Tensor: + """The checkpoint's rows at `blocks`, in order, as bytes on the CPU.""" + return torch.cat([read(lo, hi).contiguous().view(torch.uint8).cpu() for lo, hi in blocks]) + + +def table_identity(read: Read, rows: int, cols: int, *parts, align: int = CHUNK_ALIGN) -> str: + """The `node` key of a `rows` x `cols` table that `read` returns rows of: a digest of + `parts` and of every chunk's phase-0 sample (`sample_blocks`). Two checkpoints share it only + if their tables agree at those rows, wherever each was read from.""" + digest = hashlib.sha256(f"{rows}x{cols}".encode()) + for part in parts: + digest.update(f"\0{part}".encode()) + for k in range(CHUNKS): + blocks = sample_blocks(k, rows, cols, align) + if blocks: + digest.update(_sample(read, blocks).numpy()) + return digest.hexdigest()[:32] + + +def shard_rows(weights, keys, rows_per_shard: int, read_entry) -> tuple[Copy, Read]: + """`(copy, read)` over a table stored as equal contiguous row shards `keys`. + + `copy(dst, lo, hi, base)` writes global rows `[lo, hi)` into `dst`, whose row 0 is global row + `base`, one shard at a time; `read(lo, hi)` returns those rows. `read_entry(entry, cut)` reads + rows `cut` of a checkpoint entry.""" + + def pieces(lo: int, hi: int): + s = lo // rows_per_shard + while lo < hi: + a = lo - s * rows_per_shard + b = min(hi - s * rows_per_shard, rows_per_shard) + yield lo, read_entry(weights[keys[s]], (slice(a, b),)) + lo += b - a + s += 1 + + def copy(dst: torch.Tensor, lo: int, hi: int, base: int) -> None: + for start, src in pieces(lo, hi): + dst[start - base : start - base + src.shape[0]].copy_(src.to(dst.device)) + + def read(lo: int, hi: int) -> torch.Tensor: + return torch.cat([src for _, src in pieces(lo, hi)]) + + return copy, read + + +def _parse_cpulist(text: str) -> set: + cpus = set() + for part in filter(None, text.strip().split(",")): + lo, _, hi = part.partition("-") + cpus.update(range(int(lo), int(hi or lo) + 1)) + return cpus + + +def device_numa_node(device) -> Optional[int]: + """The NUMA node of `device`'s local CPUs, or None.""" + try: + props = torch.cuda.get_device_properties(device) + pci = f"{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:{props.pci_device_id:02x}.0" + with open(f"/sys/bus/pci/devices/{pci}/local_cpulist") as fh: + first = min(_parse_cpulist(fh.read())) + for name in sorted(os.listdir("/sys/devices/system/node")): + if name.startswith("node") and name[4:].isdigit(): + with open(f"/sys/devices/system/node/{name}/cpulist") as fh: + if first in _parse_cpulist(fh.read()): + return int(name[4:]) + except (OSError, AttributeError, RuntimeError, ValueError): + return None + return None + + +def _cu(result): + err, *values = result + if err != cuda_driver.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"CUDA driver call failed: {err}") + return values[0] if len(values) == 1 else tuple(values) + + +class _VmmChunks: + """`count` chunk allocations of `chunk_bytes` on host NUMA memory, mapped back to back for + one device, each created here or imported from a POSIX fd another process exported.""" + + def __init__(self, device_index: int, chunk_bytes: int, count: int, node: int) -> None: + self.device_index = device_index + self.chunk_bytes = chunk_bytes + self.count = count + self.node = node + granularity = int( + _cu( + cuda_driver.cuMemGetAllocationGranularity( + self._prop(node), + cuda_driver.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED, + ) + ) + ) + if chunk_bytes % granularity: + raise RuntimeError( + f"chunks of {chunk_bytes} bytes are not a multiple of the {granularity}-byte " + "VMM host NUMA granularity" + ) + self.va = int(_cu(cuda_driver.cuMemAddressReserve(count * chunk_bytes, granularity, 0, 0))) + self._handles: dict[int, object] = {} + + @staticmethod + def _prop(node: int): + D = cuda_driver + prop = D.CUmemAllocationProp() + prop.type = D.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + prop.location.id = node + prop.requestedHandleTypes = ( + D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + return prop + + def _map(self, k: int, handle, read_only: bool) -> None: + address = self.va + k * self.chunk_bytes + _cu(cuda_driver.cuMemMap(address, self.chunk_bytes, 0, handle, 0)) + try: + self._set_access(address, read_only) + except BaseException: + # The caller releases `handle`; nothing stays mapped to it. + cuda_driver.cuMemUnmap(address, self.chunk_bytes) + raise + self._handles[k] = handle + + def _set_access(self, address: int, read_only: bool) -> None: + D = cuda_driver + desc = D.CUmemAccessDesc() + desc.location.type = D.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = self.device_index + desc.flags = ( + D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READ + if read_only + else D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + ) + try: + _cu(D.cuMemSetAccess(address, self.chunk_bytes, [desc], 1)) + except RuntimeError: + if not read_only: + raise + desc.flags = D.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + _cu(D.cuMemSetAccess(address, self.chunk_bytes, [desc], 1)) + + def create(self, k: int) -> None: + handle = _cu(cuda_driver.cuMemCreate(self.chunk_bytes, self._prop(self.node), 0)) + try: + self._map(k, handle, read_only=False) + except BaseException: + cuda_driver.cuMemRelease(handle) + raise + + def export(self, k: int) -> int: + D = cuda_driver + return int( + _cu( + D.cuMemExportToShareableHandle( + self._handles[k], + D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, + 0, + ) + ) + ) + + def attach(self, k: int, fd: int) -> None: + D = cuda_driver + handle = _cu( + D.cuMemImportFromShareableHandle( + fd, D.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + ) + ) + try: + self._map(k, handle, read_only=True) + except BaseException: + D.cuMemRelease(handle) + raise + + def complete(self) -> bool: + return len(self._handles) == self.count + + def view(self, rows: int, cols: int, dtype, device) -> torch.Tensor: + index = device.index if device.index is not None else torch.cuda.current_device() + flat = pack_strided_memory(self.va, rows * cols, rows * cols, 1, torch.uint8, index) + return flat.view(rows, cols).view(dtype) + + def resident_on_host(self) -> bool: + host = cuda_driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA + return all( + _cu(cuda_driver.cuMemGetAllocationPropertiesFromHandle(h)).location.type == host + for h in self._handles.values() + ) + + def close(self) -> None: + D = cuda_driver + for k, handle in self._handles.items(): + D.cuMemUnmap(self.va + k * self.chunk_bytes, self.chunk_bytes) + D.cuMemRelease(handle) + self._handles.clear() + if self.va: + D.cuMemAddressFree(self.va, self.count * self.chunk_bytes) + self.va = 0 + + +def _all_agree(ok: bool) -> bool: + """Whether every rank of the server reports `ok` (a collective: every rank calls it).""" + from tensorrt_llm._utils import mpi_allgather, mpi_disabled + + if not mpi_disabled(): + result = mpi_allgather(bool(ok)) + return all(result) if isinstance(result, list) else bool(result) + import torch.distributed as dist + + if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1: + return ok + flag = torch.tensor([1 if ok else 0], dtype=torch.int32, device="cuda") + dist.all_reduce(flag, op=dist.ReduceOp.MIN) + return bool(flag.item()) + + +class PleHostTable: + """One rank's storage of the PLE n-gram table (`rows` x `cols` one-byte elements). + + After `fill`: `table` is the view the gather reads, `window` its `row_window` (None for the + whole table), `needs_sum` whether the caller sums the gathered rows over the attention-TP + group, and `mode` the mode that runs. `rank` / `world` are this rank's place in the TP group + and `dp` the attention-DP size; `private(rows, cols, dtype, device)` returns + `(view, owner)` of the target's private host allocation.""" + + def __init__( + self, + mode: str, + rows: int, + cols: int, + dtype: torch.dtype, + device: torch.device, + *, + rank: int, + world: int, + dp: int, + private: Private, + identity: Optional[str] = None, + align: int = CHUNK_ALIGN, + log: Callable[[str], None] = logger.info, + ) -> None: + assert mode in _MODES, mode + assert torch.empty(0, dtype=dtype).element_size() == 1, dtype + self.requested = mode + self.rows, self.cols, self.dtype, self.device = rows, cols, dtype, device + self.rank, self.world, self.dp = rank, world, dp + self.align = align + self.identity = identity + self._private = private + self._log = log + self.mode = RANK + self.window: Optional[tuple[int, int, int]] = None + self.table: Optional[torch.Tensor] = None + self._owner = None + self._vmm: Optional[_VmmChunks] = None + self._segments: dict[int, _broker.OwnedSegment] = {} + self._fds: list[int] = [] + self._node_error: Optional[str] = None + self._attached = 0 + if device.type != "cuda": + self.table = torch.empty(rows, cols, dtype=dtype, device=device) + elif mode == NODE: + self._declare_node() + else: + self.mode = SERVER if mode == SERVER and self._shards() else RANK + if mode == SERVER and self.mode == RANK and world > 1: + _warn_fallback( + mode, + self.mode, + f"{SERVER} needs attention TP over a size that divides {CHUNKS} (tp {world}, " + f"attention DP {dp})", + ) + self._declare_private() + + @property + def needs_sum(self) -> bool: + return self.mode == SERVER + + def _shards(self) -> bool: + return self.dp == 1 and self.world > 1 and CHUNKS % self.world == 0 + + def _key(self, k: int) -> str: + return f"{self.identity}-c{k}" + + # -------------------------------------------------------------------- declare -- + + def _declare_private(self) -> None: + if self.mode == SERVER: + lo, hi = owned_rows(self.rank, self.world, self.rows, self.cols, self.align) + self.window = (lo, hi, lo) + else: + lo, hi = 0, self.rows + if hi > lo: + self.table, self._owner = self._private(hi - lo, self.cols, self.dtype, self.device) + else: + self.table = torch.empty(0, self.cols, dtype=self.dtype, device=self.device) + + def _declare_node(self) -> None: + """Map nothing yet; own (create) the chunks this rank is responsible for if no process + does. A failure is kept for `fill`, where every rank agrees on the mode.""" + if self.identity is None: + self._node_error = "no table identity" + return + try: + chunk_bytes = chunk_rows(self.rows, self.cols, self.align) * self.cols + node = device_numa_node(self.device) + index = ( + self.device.index if self.device.index is not None else torch.cuda.current_device() + ) + self._vmm = _VmmChunks(index, chunk_bytes, CHUNKS, 0 if node is None else node) + if CHUNKS % self.world == 0: + for k in owned_chunks(self.rank, self.world): + segment = _broker.try_own(self._key(k)) + if segment is not None: + # Recorded first: if `create` fails, `_abort_node` closes it, and the + # processes waiting on it take the chunk over. + self._segments[k] = segment + self._vmm.create(k) + except (RuntimeError, OSError) as error: + self._node_error = str(error) + self._abort_node() + + # ----------------------------------------------------------------------- fill -- + + def fill(self, copy: Copy, read: Read) -> None: + """Fill (and in `node`, share and gather) the rows this rank stores; ends with `table`, + `window` and `mode` set. Every rank of the server calls it.""" + if self.device.type != "cuda": + copy(self.table, 0, self.rows, 0) + return + if self.requested != NODE: + lo, hi, base = self.window or (0, self.rows, 0) + if hi > lo: + copy(self.table, lo, hi, base) + return + ok = self._node_error is None and self._fill_node(copy, read) + if _all_agree(ok): + self.mode = NODE + return + reason = self._node_error or "another rank of this server could not share the table" + self.mode = SERVER if self._shards() else RANK + _warn_fallback(NODE, self.mode, reason) + if ok: + # Keep the full shared view; `server` reads only this rank's rows from it. + if self.mode == SERVER: + lo, hi = owned_rows(self.rank, self.world, self.rows, self.cols, self.align) + self.window = (lo, hi, 0) + return + self._abort_node() + self._declare_private() + lo, hi, base = self.window or (0, self.rows, 0) + if hi > lo: + copy(self.table, lo, hi, base) + + def _fill_node(self, copy: Copy, read: Read) -> bool: + vmm = self._vmm + self.table = vmm.view(self.rows, self.cols, self.dtype, self.device) + try: + for k in list(self._segments): + self._fill_and_publish(k, copy) + for k in range(CHUNKS): + if k in self._segments: + continue + got = _broker.acquire(self._key(k), _ATTACH_TIMEOUT_S, self._log) + if got[0] == _broker.OWNER: + self._segments[k] = got[1] + vmm.create(k) + self._fill_and_publish(k, copy) + continue + _, fd, meta = got + try: + if meta.get("chunk_bytes") != vmm.chunk_bytes: + raise RuntimeError(f"chunk {k} is {meta.get('chunk_bytes')} bytes") + vmm.attach(k, fd) + finally: + os.close(fd) + self._attached += 1 + self._check_chunk(k, read) + except (RuntimeError, OSError, TimeoutError) as error: + self._node_error = str(error) + self._abort_node() + return False + return vmm.complete() + + def _fill_and_publish(self, k: int, copy: Copy) -> None: + lo, hi = chunk_range(k, self.rows, self.cols, self.align) + if hi > lo: + copy(self.table, lo, hi, 0) + torch.cuda.synchronize(self.device) + fd = self._vmm.export(k) + self._fds.append(fd) + self._segments[k].publish(fd, {"chunk": k, "chunk_bytes": self._vmm.chunk_bytes}) + + def _check_chunk(self, k: int, read: Read) -> None: + """An attached chunk holds the checkpoint's bytes at its phase-1 sample, rows the key does + not hash; raises `RuntimeError` otherwise.""" + blocks = sample_blocks(k, self.rows, self.cols, self.align, phase=1) + if not blocks: + return + rows = torch.cat([torch.arange(lo, hi) for lo, hi in blocks]) + got = self.table.view(torch.uint8).index_select(0, rows.to(self.device)).cpu() + want = _sample(read, blocks) + if not torch.equal(got, want): + first = int(rows[(got != want).any(1)][0]) + raise RuntimeError(f"shared chunk {k} differs from the checkpoint at row {first}") + + def _abort_node(self) -> None: + for segment in self._segments.values(): + segment.close() + self._segments.clear() + for fd in self._fds: + os.close(fd) + self._fds.clear() + if self._vmm is not None: + self._vmm.close() + self._vmm = None + self.table = None + + # ----------------------------------------------------------------- inspection -- + + def resident_on_host(self) -> bool: + """Whether the table's pages are host memory, from the driver.""" + if self._vmm is not None: + return self._vmm.resident_on_host() + if self.table is None or self.table.numel() == 0: + return True + (attributes,) = CUASSERT(cudart.cudaPointerGetAttributes(self.table.data_ptr())) + return int(attributes.type) == _CUDA_MEMORY_TYPE_HOST + + def describe(self) -> str: + if self.mode == NODE: + return ( + f"{NODE}: {CHUNKS} shared chunks, this process created {len(self._segments)} and " + f"attached {self._attached}" + ) + if self.mode == SERVER: + lo, hi, _ = self.window + return f"{SERVER}: rows [{lo}, {hi}) of {self.rows}, summed over {self.world} ranks" + return f"{RANK}: the whole table" + + def close(self) -> None: + self._abort_node() + self._owner = None diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..366a4aa96e8b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Qwen3.8-Flash-Next-FP8 / sm_103 / unified: tp1, tp2, tep2, dep2, tp4, tep4 and dep4. + +The collective matrices (`tests/unittest/.../qwen3_8_flash_next_fp8__sm_103__unified/comm`) run in +the pre-merge GB300 multi-GPU stage (`l0_gb300.yml`), which CI schedules only when a file it lists +as multi-GPU changes (`getMultiGpuFileChanged` in `jenkins/L0_MergeRequest.groovy`); this +directory is not among them. A change to `csrc/` or `ops/comm/` is therefore run with +`/bot run --add-multi-gpu-test` (`docs/source/developer-guide/ci-overview.md`, Multi-GPU Tests). +""" + +from tensorrt_llm._torch.modeling_v2._native import load_target_ops + +# torch.ops.mv2_qwen4_exp_qwen3_8_flash_next_fp8_sm_103_unified +native_ops = load_target_ops(__name__) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu new file mode 100644 index 000000000000..83df1e66a134 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.cu @@ -0,0 +1,1257 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A block (one expert's 128 intermediate rows) closes its FC1 -> activation -> FC2 chain inside one cluster. Every CTA +// routes every row (top-10, ties to the lower id, fp32 softmax over the ten rounded to bf16). Cluster k takes blocks +// k, k + K, ...: a producer warp streams weights into a shared-memory ring, five consumer warps run the MMAs, and an +// epilogue warp sends each FC1 item to every cluster CTA by DSMEM bulk copy, completing that CTA's block barrier. +// Past one grid-wide counter every CTA sums its share of the K cluster partials in a fixed order and stores bf16. + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h" + +#include +#include + +#include +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_moe_tpb +{ +namespace +{ + +constexpr int kH = 2560; +constexpr int kE = 512; +constexpr int kK = 10; +constexpr int kRows = 8; +constexpr int kKb = kH / 128; // 20 k-blocks of FC1 +constexpr int kBlkRows = 256; // FC1 rows of a block: 128 up, 128 gate +constexpr int kItemFc2Rows = 320; // FC2 rows of one ring item +constexpr int kMt = kItemFc2Rows / 16; // 20 m-tiles of 16 FC2 rows an item +constexpr int kConsumerWarps = 5; +constexpr int kEpiWarp = 5; +constexpr int kProducerWarp = 6; +constexpr int kThreads = 7 * 32; +constexpr int kConsumerThreads = kConsumerWarps * 32; +constexpr int kMtPerWarp = kMt / kConsumerWarps; // 4 +constexpr int kItemRowBytes = 16 * kH; // 40960: 16 FC1 rows, or 320 FC2 rows x 128 +constexpr int kStageBytes = kItemRowBytes + 128; // + the item's 20 fp32 scales +constexpr int kMaxStages = 8; +constexpr int kMaxSlots = kRows * kK; // 80 +constexpr int kMaxBlk = 2 * kMaxSlots + kShItems; // up to two blocks a routed expert, plus the shared items +constexpr int kActRow = 128 + 16; +constexpr int kMaxClusters = 40; +constexpr float kE4m3Max = 448.f; +constexpr float kAmaxFloor = 1.0e-10f; + +// Workspace: the u64 grid counter at 0, which grows by kStride a call and never needs a reset, then the fp32 partials +// [kMaxClusters][8][2560] at 4096. +constexpr unsigned long long kStride = 1024; +constexpr size_t kWsPart = 4096; +constexpr size_t kWsBytes = kWsPart + static_cast(kMaxClusters) * kRows * kH * 4; +constexpr int kDefaultStages = 4; // as fast as five, and leaves ~40 KB of shared memory an SM to side-stream kernels + +struct alignas(16) Smem +{ + float4 red[2][kConsumerWarps][32]; + uint64_t fullBar[kMaxStages]; + uint64_t emptyBar[kMaxStages]; + uint64_t xbar[2]; + float topkW[kRows][kK]; + int topkId[kRows][kK]; + int expertOf[kMaxSlots]; + uint32_t bitmap[kE / 32]; + int wordBase[kE / 32]; + unsigned long long barTarget; + int numDistinct; + int numBlk; + int8_t slotOf[kMaxSlots][kRows]; + uint16_t blkJ[kMaxBlk]; + uint8_t blkD[kMaxBlk]; + alignas(16) uint8_t actC[2][kRows][kActRow]; // 16-byte rows: the FC2 B fragments are 128-bit loads + alignas(16) float stage[2][4][kRows][16]; // an FC1 item's 16 rows x T outputs, the DSMEM copy's source + float actF[2][kRows]; +}; + +struct alignas(16) SmemSh : Smem +{ + alignas(16) __nv_bfloat16 actS[2][kRows][32]; // the slice item's 32 activations of each row (FC2 B operand) + float shScale[kRows]; // sigmoid(bf16 gate logit) of each row +}; + +template +using SmemT = std::conditional_t; + +template +constexpr int kStaticBytes = (static_cast(sizeof(SmemT)) + 127) / 128 * 128; + +constexpr int kShMark = 1024; // block-list entries >= this are shared-slice items (kShMark + item) +constexpr int kShCols = 32; // intermediate columns of a shared item +constexpr int kShRowBytes = kH * 2; // one bf16 weight / activation row + +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) +__device__ __forceinline__ uint32_t smemAddr(void const* p) +{ + return static_cast(__cvta_generic_to_shared(p)); +} + +__device__ __forceinline__ void mbarInit(uint64_t* bar, uint32_t count) +{ + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(smemAddr(bar)), "r"(count) : "memory"); +} + +__device__ __forceinline__ void mbarArriveExpectTx(uint64_t* bar, uint32_t bytes) +{ + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(smemAddr(bar)), "r"(bytes) : "memory"); +} + +__device__ __forceinline__ void mbarArrive(uint64_t* bar) +{ + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" ::"r"(smemAddr(bar)) : "memory"); +} + +__device__ __forceinline__ void mbarWait(uint64_t* bar, uint32_t parity) +{ + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smemAddr(bar)), + "r"(parity) + : "memory"); +} + +// Wait with cluster-scope acquire: the peers' DSMEM stores released by their arrivals are visible after it. +__device__ __forceinline__ void mbarWaitCluster(uint64_t* bar, uint32_t parity) +{ + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smemAddr(bar)), + "r"(parity) + : "memory"); +} + +__device__ __forceinline__ uint32_t clusterCtaRank() +{ + uint32_t r; + asm volatile("mov.u32 %0, %%cluster_ctarank;" : "=r"(r)); + return r; +} + +__device__ __forceinline__ uint32_t mapaShared(uint32_t addr, uint32_t rank) +{ + uint32_t r; + asm volatile("mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(r) : "r"(addr), "r"(rank)); + return r; +} + +__device__ __forceinline__ void clusterSync() +{ + asm volatile("barrier.cluster.arrive.release.aligned;\nbarrier.cluster.wait.acquire.aligned;" ::: "memory"); +} + +__device__ __forceinline__ void fenceAcqRelGpu() +{ + asm volatile("fence.acq_rel.gpu;" ::: "memory"); +} + +__device__ __forceinline__ void fenceBarrierInit() +{ + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +} + +__device__ __forceinline__ void bulkG2S(void* dst, void const* src, uint32_t bytes, uint64_t* bar) +{ + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(smemAddr(dst)), + "l"(src), "r"(bytes), "r"(smemAddr(bar)) + : "memory"); +} + +// DSMEM bulk copy into another cluster CTA, completing `bytes` on its mbarrier (`dst`, `bar`: shared::cluster). +__device__ __forceinline__ void bulkS2Cluster(uint32_t dst, void const* src, uint32_t bytes, uint32_t bar) +{ + asm volatile( + "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(dst), + "r"(smemAddr(src)), "r"(bytes), "r"(bar) + : "memory"); +} + +__device__ __forceinline__ void bulkCommit() +{ + asm volatile("cp.async.bulk.commit_group;" ::: "memory"); +} + +template +__device__ __forceinline__ void bulkWaitRead() +{ + asm volatile("cp.async.bulk.wait_group.read %0;" ::"n"(N) : "memory"); +} + +__device__ __forceinline__ void fenceProxyAsyncShared() +{ + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +__device__ __forceinline__ void namedBarSync(int id, int count) +{ + asm volatile("bar.sync %0, %1;" ::"r"(id), "r"(count) : "memory"); +} + +__device__ __forceinline__ void namedBarArrive(int id, int count) +{ + asm volatile("bar.arrive %0, %1;" ::"r"(id), "r"(count) : "memory"); +} + +__device__ __forceinline__ void gridDepWait() +{ + asm volatile("griddepcontrol.wait;" ::: "memory"); +} + +__device__ __forceinline__ void gridDepLaunchDependents() +{ + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); +} + +__device__ __forceinline__ unsigned long long ldAcquireGpu64(unsigned long long const* p) +{ + unsigned long long v; + asm volatile("ld.acquire.gpu.global.u64 %0, [%1];" : "=l"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void redAddRelease64(unsigned long long* p, unsigned long long v) +{ + asm volatile("red.release.gpu.global.add.u64 [%0], %1;" ::"l"(p), "l"(v) : "memory"); +} + +__device__ __forceinline__ uint4 lds128(uint8_t const* p) +{ + return *reinterpret_cast(p); +} + +__device__ __forceinline__ uint4 ldcg128(void const* p) +{ + return __ldcg(reinterpret_cast(p)); +} + +__device__ __forceinline__ void mmaE4m3( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm("mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ void mmaBf16( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, " + "{%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +// A 32-wide k step from the 8 bf16 a lane holds per operand, as two k16 MMAs that permute k alike in A and B. +__device__ __forceinline__ void mma32Bf16(float (&acc)[4], uint4 const& ag, uint4 const& ag8, uint4 const& b) +{ + mmaBf16(acc, ag.x, ag8.x, ag.y, ag8.y, b.x, b.y); + mmaBf16(acc, ag.z, ag8.z, ag.w, ag8.w, b.z, b.w); +} + +__device__ __forceinline__ float bf16Round(float x) +{ + return __bfloat162float(__float2bfloat16_rn(x)); +} + +__device__ __forceinline__ float warpMaxAbs(float v) +{ + uint32_t bits = __float_as_uint(fabsf(v)); + return __uint_as_float(__reduce_max_sync(0xffffffffu, bits)); +} + +__device__ __forceinline__ float e4m3RoundTrip(float x) +{ + __nv_fp8_storage_t q = __nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3); + return __half2float(__half(__nv_cvt_fp8_to_halfraw(q, __NV_E4M3))); +} + +__device__ __forceinline__ uint32_t e4m3Code(float x) +{ + return static_cast(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3)); +} + +__device__ __forceinline__ float siluFast(float x) +{ + return __fdividef(x, 1.0f + __expf(-x)); +} + +// Row t's top-10 logits (ties to the lower id) and their softmax rounded to bf16; lane l holds experts 16l..16l+15. +__device__ __forceinline__ void routeRow(TpbParams const& p, int t, int lane, Smem& s) +{ + uint8_t const* row = static_cast(p.logits) + static_cast(t) * kE * 2 + lane * 32; + uint4 r0 = ldcg128(row); + uint4 r1 = ldcg128(row + 16); + uint32_t words[8] = {r0.x, r0.y, r0.z, r0.w, r1.x, r1.y, r1.z, r1.w}; + uint32_t key[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) + { + uint32_t b = (words[i >> 1] >> ((i & 1) * 16)) & 0xffffu; + uint32_t ord = (b & 0x8000u) ? (~b & 0xffffu) : (b | 0x8000u); + key[i] = (ord << 16) | static_cast(kE - 1 - (lane * 16 + i)); + } + uint32_t mine = 0; +#pragma unroll 1 + for (int k = 0; k < kK; ++k) + { + uint32_t m8[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + m8[i] = max(key[2 * i], key[2 * i + 1]); + } + uint32_t const m4a = max(m8[0], m8[1]), m4b = max(m8[2], m8[3]); + uint32_t const m4c = max(m8[4], m8[5]), m4d = max(m8[6], m8[7]); + uint32_t const best = max(max(m4a, m4b), max(m4c, m4d)); + uint32_t const top = __reduce_max_sync(0xffffffffu, best); +#pragma unroll + for (int i = 0; i < 16; ++i) + { + key[i] = key[i] == top ? 0u : key[i]; + } + mine = lane == k ? top : mine; + } + float v = -INFINITY; + if (lane < kK) + { + uint32_t ord = mine >> 16; + uint32_t b = (ord & 0x8000u) ? (ord & 0x7fffu) : (~ord & 0xffffu); + v = __uint_as_float(b << 16); + s.topkId[t][lane] = kE - 1 - static_cast(mine & 0xffffu); + } + float const mx = __shfl_sync(0xffffffffu, v, 0); + float e = lane < kK ? expf(v - mx) : 0.f; + float sum = e; +#pragma unroll + for (int o = 16; o > 0; o >>= 1) + { + sum += __shfl_xor_sync(0xffffffffu, sum, o); + } + if (lane < kK) + { + s.topkW[t][lane] = __bfloat162float(__float2bfloat16_rn(__fdiv_rn(e, sum))); + } +} + +struct Ring +{ + int stage; + uint32_t phase; + int numStages; + + __device__ __forceinline__ void advance() + { + if (++stage == numStages) + { + stage = 0; + phase ^= 1u; + } + } +}; + +#endif // !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) + +// CS: CTAs a cluster (4 or 8); kSh: also fold in this rank's shared-expert slice. +template +__global__ void __launch_bounds__(kThreads, 1) fp8BlockMoeTpbKernel( + TpbParams const p, int const numStages, int const xbufOffset, int const ringOffset, int const shXOffset) +{ +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) + static_assert(!kSh || CS == 4, "the shared-expert fold takes 4-CTA clusters"); + constexpr int kCs = CS; + constexpr int kUpRows = 128 / kCs; // a CTA's up rows (and gate rows) of a block + constexpr int kNh = kUpRows / 16; // items of each half, and FC2 items + constexpr int kFc2Rows = kH / kCs; // a CTA's FC2 rows of a block + extern __shared__ __align__(128) uint8_t smemRaw[]; + SmemT& s = *reinterpret_cast*>(smemRaw); + float* xbuf = reinterpret_cast(smemRaw + xbufOffset); // [2][T][256] raw FC1 outputs of a block + uint8_t* ring = smemRaw + ringOffset; + + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int const T = p.numRows; + int const G = gridDim.x; + int const cta = blockIdx.x; + int const nSlots = T * kK; + int const crank = static_cast(clusterCtaRank()); + int const cl = cta / kCs; + int const NC = G / kCs; + + unsigned long long* counter = static_cast(p.workspace); + float* part = reinterpret_cast(static_cast(p.workspace) + kWsPart); + + if (threadIdx.x == 0) + { + for (int i = 0; i < numStages; ++i) + { + mbarInit(&s.fullBar[i], 1); + mbarInit(&s.emptyBar[i], kConsumerWarps); + } + mbarInit(&s.xbar[0], 1); // the local arrive.expect_tx; the peers' copies complete the bytes + mbarInit(&s.xbar[1], 1); + fenceBarrierInit(); + } + for (int i = threadIdx.x; i < kMaxSlots * kRows; i += kThreads) + { + (&s.slotOf[0][0])[i] = -1; + } + if (threadIdx.x < kE / 32) + { + s.bitmap[threadIdx.x] = 0; + } + // The peers' barriers are initialized before any CTA of the cluster arrives on them. + clusterSync(); + + gridDepWait(); + if (threadIdx.x == 0) + { + // Read past the wait, so the previous call's arrivals have all landed, and before arriving: the counter is a + // multiple of kStride plus fewer than all of this call's arrivals. + s.barTarget = (ldAcquireGpu64(counter) / kStride + 1) * kStride; + } + + for (int t = warp; t < T; t += kThreads / 32) + { + routeRow(p, t, lane, s); + } + __syncthreads(); + int myExpert = -1; + if (threadIdx.x < nSlots) + { + int const t = threadIdx.x / kK; + int const k = threadIdx.x % kK; + myExpert = s.topkId[t][k]; + atomicOr(&s.bitmap[myExpert >> 5], 1u << (myExpert & 31)); + } + __syncthreads(); + if (warp == 0) + { + int const c = lane < kE / 32 ? __popc(s.bitmap[lane]) : 0; + int inc = c; +#pragma unroll + for (int o = 1; o < 32; o <<= 1) + { + int const v = __shfl_up_sync(0xffffffffu, inc, o); + inc += lane >= o ? v : 0; + } + if (lane < kE / 32) + { + s.wordBase[lane] = inc - c; + } + if (lane == 31) + { + s.numDistinct = inc; + } + } + __syncthreads(); + if (threadIdx.x < nSlots) + { + int const t = threadIdx.x / kK; + int const wd = myExpert >> 5; + int const bit = myExpert & 31; + int const d = s.wordBase[wd] + __popc(s.bitmap[wd] & ((1u << bit) - 1u)); + s.expertOf[d] = myExpert; + s.slotOf[d][t] = static_cast(threadIdx.x); + } + __syncthreads(); + int const D = s.numDistinct; + // This rank's blocks: entry d is block (rank - e) % 4 of expert e = expertOf[d], then block-4 entries, then shared. + if (warp == 0) + { + int base = 0; + for (int d0 = 0; d0 < D; d0 += 32) + { + int const d = d0 + lane; + int extra = 0; + if (d < D) + { + int const e = s.expertOf[d]; + int const b0 = (p.rank - e) & (kTpbRanks - 1); + s.blkJ[d] = static_cast(e); + s.blkD[d] = static_cast(d); + extra = b0 == 0; + } + uint32_t const bal = __ballot_sync(0xffffffffu, extra); + if (extra) + { + int const pos = D + base + __popc(bal & ((1u << lane) - 1u)); + s.blkJ[pos] = static_cast(kE + (s.expertOf[d] >> 2)); + s.blkD[pos] = static_cast(d); + } + base += __popc(bal); + } + if constexpr (kSh) + { + if (lane < p.shBlocks) + { + s.blkJ[D + base + lane] = static_cast(kShMark + lane); + s.blkD[D + base + lane] = 0; + } + } + if (lane == 0) + { + s.numBlk = D + base + (kSh ? p.shBlocks : 0); + } + } + __syncthreads(); + // Every CTA is resident by now: the next kernel may start its prologue. + gridDepLaunchDependents(); + int const nB = s.numBlk; + if (nB == 0) + { + __nv_bfloat162* o = static_cast<__nv_bfloat162*>(p.out); + for (int q = cta * kThreads + threadIdx.x; q < T * kH / 2; q += G * kThreads) + { + o[q] = __floats2bfloat162_rn(0.f, 0.f); + } + return; + } + int const NCa = min(NC, nB); + int const nMy = cl < NCa ? (nB - cl + NC - 1) / NC : 0; + + if (warp == kProducerWarp) + { + // ---- producer: per block the up / gate items (16 FC1 rows + 20 scales), then FC2 items (320 rows + 20 scales) + Ring rs{0, 0u, numStages}; + for (int n = 0; n < nMy; ++n) + { + int const j = s.blkJ[cl + n * NC]; + if (kSh && j >= kShMark) + { + // Shared item: 2 x 8 bf16 FC1 rows ([gate 4 ; up 4]), then this CTA's 640 down rows x 32 columns. + int const sb = j - kShMark; + for (int q = 0; q < 3; ++q) + { + mbarWait(&s.emptyBar[rs.stage], rs.phase ^ 1u); + if (lane == 0) + { + uint8_t* dst = ring + rs.stage * kStageBytes; + mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes); + uint8_t const* src = q < 2 ? static_cast(p.shGu) + + (static_cast(sb) * 64 + crank * 16 + q * 8) * kShRowBytes + : static_cast(p.shDn) + + (static_cast(sb) * kH + crank * kFc2Rows) * kShCols * 2; + bulkG2S(dst, src, kItemRowBytes, &s.fullBar[rs.stage]); + } + __syncwarp(); + rs.advance(); + } + continue; + } + for (int q = 0; q < 3 * kNh; ++q) + { + int const kind = q / kNh; // 0 up, 1 gate, 2 FC2 + int const u = q % kNh; + mbarWait(&s.emptyBar[rs.stage], rs.phase ^ 1u); + if (lane == 0) + { + uint8_t* dst = ring + rs.stage * kStageBytes; + mbarArriveExpectTx(&s.fullBar[rs.stage], kItemRowBytes + kKb * 4); + if (kind < 2) + { + size_t const row = static_cast(j) * kBlkRows + kind * 128 + crank * kUpRows + u * 16; + bulkG2S(dst, p.w1 + row * kH, kItemRowBytes, &s.fullBar[rs.stage]); + bulkG2S(dst + kItemRowBytes, p.s1 + (static_cast(j) * 2 + kind) * kKb, kKb * 4, + &s.fullBar[rs.stage]); + } + else + { + size_t const row = static_cast(j) * kH + crank * kFc2Rows + u * kItemFc2Rows; + bulkG2S(dst, p.w2 + row * 128, kItemRowBytes, &s.fullBar[rs.stage]); + bulkG2S( + dst + kItemRowBytes, p.s2 + static_cast(j) * kKb, kKb * 4, &s.fullBar[rs.stage]); + } + } + __syncwarp(); + rs.advance(); + } + } + __syncwarp(); + } + else if (warp == kEpiWarp) + { + // ---- epilogue: per FC1 item, sum the consumer warps' partials in a fixed order and send them to every CTA + int const g = lane >> 2; + int const tig = lane & 3; + int const t0 = 2 * tig, t1 = 2 * tig + 1; + auto fc1Items = [&](int n) { return kSh && s.blkJ[cl + n * NC] >= kShMark ? 2 : 2 * kNh; }; + int nFc1 = 0; + for (int n = 0; n < nMy; ++n) + { + nFc1 += fc1Items(n); + } + uint32_t const xbase = smemAddr(xbuf); + uint32_t const xbarBase = smemAddr(&s.xbar[0]); + int n = 0, q = 0, nq = nMy > 0 ? fc1Items(0) : 0, prevItems = 2 * kNh; + for (int jf = 0; jf < nFc1; ++jf) + { + if (q == nq) + { + prevItems = nq; + ++n; + q = 0; + nq = fc1Items(n); + } + int const buf = jf & 1; + namedBarSync(1 + buf, kConsumerThreads + 32); + float4 v = s.red[buf][0][lane]; +#pragma unroll + for (int w = 1; w < kConsumerWarps; ++w) + { + float4 const o = s.red[buf][w][lane]; + v.x += o.x; + v.y += o.y; + v.z += o.z; + v.w += o.w; + } + if (jf + 2 < nFc1) + { + namedBarArrive(3 + buf, kConsumerThreads + 32); + } + // Stage 16 rows x T, then copy into each CTA's exchange buffer [CS][2 kNh][T][16] at (this CTA, item). + if (q == 0 && n >= 2) + { + // Block n - 2's copies (this parity's slots) must have read their source; block n - 1's may pend. + if (prevItems == 2) + { + bulkWaitRead<2>(); + } + else + { + bulkWaitRead<2 * kNh>(); + } + // Only the issuing lanes waited; the whole warp overwrites the slots below. + __syncwarp(); + } + float* stg = &s.stage[n & 1][q][0][0]; + if (t0 < T) + { + stg[t0 * 16 + g] = v.x; + stg[t0 * 16 + g + 8] = v.z; + } + if (t1 < T) + { + stg[t1 * 16 + g] = v.y; + stg[t1 * 16 + g + 8] = v.w; + } + fenceProxyAsyncShared(); + __syncwarp(); + if (lane < kCs) + { + uint32_t const off = static_cast((n & 1) * T * kBlkRows + ((crank * nq + q) * T) * 16) * 4u; + bulkS2Cluster(mapaShared(xbase + off, static_cast(lane)), stg, + static_cast(T * 16 * 4), + mapaShared(xbarBase + static_cast(n & 1) * 8u, static_cast(lane))); + } + if (lane < kCs) + { + bulkCommit(); + } + ++q; + } + bulkWaitRead<0>(); // the DSMEM copies have read this CTA's staging slots + } + else + { + // ---- consumers (warps 0..4) + int const g = lane >> 2; + int const tig = lane & 3; + uint4 bx[4][2]; + float xs[4][2]; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + int const kb = warp * 4 + q; +#pragma unroll + for (int m = 0; m < 2; ++m) + { + int const kpos = kb * 128 + m * 64 + tig * 16; + bx[q][m] = g < T ? ldcg128(p.xq + static_cast(g) * kH + kpos) : make_uint4(0, 0, 0, 0); + } + xs[q][0] = 2 * tig < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig) : 0.f; + xs[q][1] = 2 * tig + 1 < T ? __ldcg(p.xsf + kb * p.xsfStride + 2 * tig + 1) : 0.f; + } + uint8_t* xsh = smemRaw + shXOffset; // [T][2560] bf16 MoE-site rows (shared fold only) + if constexpr (kSh) + { + bool anySh = false; + for (int n = 0; n < nMy; ++n) + { + anySh |= s.blkJ[cl + n * NC] >= kShMark; + } + if (anySh) + { + for (int q = threadIdx.x; q < T * kH / 8; q += kConsumerThreads) + { + *reinterpret_cast(xsh + q * 16) = ldcg128(static_cast(p.shX) + q * 16); + } + // Gate logit of each row: bf16(x . w) as the unfused GEMM outputs it, then its sigmoid in fp32. + for (int t = warp; t < T; t += kConsumerWarps) + { + float acc = 0.f; + for (int c = lane; c < kH / 8; c += 32) + { + uint4 const xv = ldcg128( + static_cast(p.shX) + static_cast(t) * kShRowBytes + c * 16); + uint4 const wv = ldcg128(static_cast(p.shGl) + c * 16); + __nv_bfloat162 const* xa = reinterpret_cast<__nv_bfloat162 const*>(&xv); + __nv_bfloat162 const* wa = reinterpret_cast<__nv_bfloat162 const*>(&wv); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 const xf = __bfloat1622float2(xa[i]); + float2 const wf = __bfloat1622float2(wa[i]); + acc = fmaf(xf.x, wf.x, acc); + acc = fmaf(xf.y, wf.y, acc); + } + } +#pragma unroll + for (int o = 16; o > 0; o >>= 1) + { + acc += __shfl_xor_sync(0xffffffffu, acc, o); + } + if (lane == 0) + { + s.shScale[t] = 1.0f / (1.0f + expf(-bf16Round(acc))); + } + } + namedBarSync(9, kConsumerThreads); + } + } + Ring rs{0, 0u, numStages}; + uint32_t xphase = 0; // bit p: the phase of buffer p's barrier + float out[kNh][kMtPerWarp][4]; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { +#pragma unroll + for (int r = 0; r < 4; ++r) + { + out[u][mi][r] = 0.f; + } + } + } + int jf = 0; + for (int n = 0; n < nMy; ++n) + { + int const pb = n & 1; + int const d = s.blkD[cl + n * NC]; + if constexpr (kSh) + { + if (s.blkJ[cl + n * NC] >= kShMark) + { + // ---- shared item: bf16 FC1 of 2 x 8 rows, warp w over k [512 w, 512 w + 512) + for (int q = 0; q < 2; ++q, ++jf) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + uint4 const zero = make_uint4(0, 0, 0, 0); +#pragma unroll 4 + for (int m = 0; m < 16; m += 2) + { + int const k0 = warp * 512 + m * 32 + tig * 8; + uint4 const a0 = lds128(st + g * kShRowBytes + k0 * 2); + uint4 const a1 = lds128(st + g * kShRowBytes + (k0 + 32) * 2); + uint4 const b0 = g < T ? lds128(xsh + g * kShRowBytes + k0 * 2) : zero; + uint4 const b1 = g < T ? lds128(xsh + g * kShRowBytes + (k0 + 32) * 2) : zero; + mma32Bf16(acc, a0, zero, b0); + mma32Bf16(acc1, a1, zero, b1); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + int const buf = jf & 1; + if (jf >= 2) + { + namedBarSync(3 + buf, kConsumerThreads + 32); + } + s.red[buf][warp][lane] = make_float4(acc[0] + acc1[0], acc[1] + acc1[1], 0.f, 0.f); + namedBarArrive(1 + buf, kConsumerThreads + 32); + } + if (warp == 0 && lane == 0) + { + mbarArriveExpectTx(&s.xbar[pb], static_cast(kCs * 2 * T * 16 * 4)); + } + mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); + xphase ^= 1u << pb; + // Column c comes from CTA c / 8, item (c % 8) / 4, rows c % 4 (gate) and 4 + c % 4 (up). + float const* xb = xbuf + pb * T * kBlkRows; + for (int i = threadIdx.x; i < T * kShCols; i += kConsumerThreads) + { + int const t = i / kShCols; + int const c = i % kShCols; + float const* src = xb + (((c / 8) * 2 + (c % 8) / 4) * T + t) * 16 + (c % 4); + float const ga = bf16Round(src[0]); + float const up = bf16Round(src[4]); + s.actS[pb][t][c] = __float2bfloat16_rn(ga / (1.0f + expf(-ga)) * up); + } + namedBarSync(8, kConsumerThreads); + // FC2 of this CTA's 640 down rows, scaled by sigmoid(gate logit), into the routed accumulators. + uint4 const zero = make_uint4(0, 0, 0, 0); + uint4 const bsh = g < T ? lds128(reinterpret_cast(&s.actS[pb][g][tig * 8])) : zero; + float const sc0 = 2 * tig < T ? s.shScale[2 * tig] : 0.f; + float const sc1 = 2 * tig + 1 < T ? s.shScale[2 * tig + 1] : 0.f; + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const row = u * kItemFc2Rows + (warp + kConsumerWarps * mi) * 16 + g; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + uint4 const ag = lds128(st + row * kShCols * 2 + tig * 16); + uint4 const ag8 = lds128(st + (row + 8) * kShCols * 2 + tig * 16); + mma32Bf16(acc, ag, ag8, bsh); + out[u][mi][0] = fmaf(acc[0], sc0, out[u][mi][0]); + out[u][mi][1] = fmaf(acc[1], sc1, out[u][mi][1]); + out[u][mi][2] = fmaf(acc[2], sc0, out[u][mi][2]); + out[u][mi][3] = fmaf(acc[3], sc1, out[u][mi][3]); + } + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + continue; + } + } + // FC1: the up items, then the gate items + for (int q = 0; q < 2 * kNh; ++q, ++jf) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float const* sc = reinterpret_cast(st + kItemRowBytes); + float t0 = 0.f, t1 = 0.f, t2 = 0.f, t3 = 0.f; +#pragma unroll + for (int qq = 0; qq < 4; ++qq) + { + int const kb = warp * 4 + qq; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + int const koff = kb * 128 + tig * 16; + uint4 const a0 = lds128(st + g * kH + koff); + uint4 const a08 = lds128(st + (g + 8) * kH + koff); + uint4 const a1 = lds128(st + g * kH + koff + 64); + uint4 const a18 = lds128(st + (g + 8) * kH + koff + 64); + mmaE4m3(acc, a0.x, a08.x, a0.y, a08.y, bx[qq][0].x, bx[qq][0].y); + mmaE4m3(acc1, a1.x, a18.x, a1.y, a18.y, bx[qq][1].x, bx[qq][1].y); + mmaE4m3(acc, a0.z, a08.z, a0.w, a08.w, bx[qq][0].z, bx[qq][0].w); + mmaE4m3(acc1, a1.z, a18.z, a1.w, a18.w, bx[qq][1].z, bx[qq][1].w); +#pragma unroll + for (int r = 0; r < 4; ++r) + { + acc[r] += acc1[r]; + } + float const sw = sc[kb]; + t0 = fmaf(acc[0], sw * xs[qq][0], t0); + t1 = fmaf(acc[1], sw * xs[qq][1], t1); + t2 = fmaf(acc[2], sw * xs[qq][0], t2); + t3 = fmaf(acc[3], sw * xs[qq][1], t3); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + int const buf = jf & 1; + if (jf >= 2) + { + namedBarSync(3 + buf, kConsumerThreads + 32); + } + s.red[buf][warp][lane] = make_float4(t0, t1, t2, t3); + namedBarArrive(1 + buf, kConsumerThreads + 32); + } + + if (warp == 0 && lane == 0) + { + mbarArriveExpectTx(&s.xbar[pb], static_cast(kCs * 2 * kNh * T * 16 * 4)); + } + mbarWaitCluster(&s.xbar[pb], (xphase >> pb) & 1u); + xphase ^= 1u << pb; + // Activation, one routed token a warp: requantize up and gate (1x128), siluFast(gate) * up, requantize. + float const* xb = xbuf + pb * T * kBlkRows; + for (int t = warp; t < T; t += kConsumerWarps) + { + int const sl = s.slotOf[d][t]; + if (sl < 0) + { + continue; + } + int const col = 4 * lane; + int const c = col / kUpRows; + int const u = (col % kUpRows) / 16; + float const* up = xb + ((c * 2 * kNh + u) * T + t) * 16 + (col & 15); + float const* gt = xb + ((c * 2 * kNh + kNh + u) * T + t) * 16 + (col & 15); + float4 const upv = *reinterpret_cast(up); + float4 const gtv = *reinterpret_cast(gt); + float const ux[4] = {upv.x, upv.y, upv.z, upv.w}; + float const gx[4] = {gtv.x, gtv.y, gtv.z, gtv.w}; + float amU = 0.f, amG = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + amU = fmaxf(amU, fabsf(ux[i])); + amG = fmaxf(amG, fabsf(gx[i])); + } + float const sU = __fdiv_rn(fmaxf(warpMaxAbs(amU), kAmaxFloor), kE4m3Max); + float const sG = __fdiv_rn(fmaxf(warpMaxAbs(amG), kAmaxFloor), kE4m3Max); + float const rU = __frcp_rn(sU); + float const rG = __frcp_rn(sG); + float a[4]; + float amA = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float const x1 = e4m3RoundTrip(ux[i] * rU) * sU; + float const x2 = e4m3RoundTrip(gx[i] * rG) * sG; + a[i] = siluFast(x2) * x1; + amA = fmaxf(amA, fabsf(a[i])); + } + float const sA = __fdiv_rn(fmaxf(warpMaxAbs(amA), kAmaxFloor), kE4m3Max); + float const rA = __frcp_rn(sA); + uint32_t packed = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + packed |= e4m3Code(a[i] * rA) << (8 * i); + } + *reinterpret_cast(&s.actC[pb][t][lane * 4]) = packed; + if (lane == 0) + { + s.actF[pb][t] = sA * s.topkW[t][sl % kK]; + } + } + namedBarSync(8, kConsumerThreads); + + int const sg = g < T ? s.slotOf[d][g] : -1; + int const sa = 2 * tig < T ? s.slotOf[d][2 * tig] : -1; + int const sb = 2 * tig + 1 < T ? s.slotOf[d][2 * tig + 1] : -1; + float const fa = sa >= 0 ? s.actF[pb][2 * tig] : 0.f; + float const fb = sb >= 0 ? s.actF[pb][2 * tig + 1] : 0.f; + uint4 bm0 = make_uint4(0, 0, 0, 0), bm1 = make_uint4(0, 0, 0, 0); + if (sg >= 0) + { + bm0 = lds128(&s.actC[pb][g][tig * 16]); + bm1 = lds128(&s.actC[pb][g][64 + tig * 16]); + } +#pragma unroll + for (int u = 0; u < kNh; ++u) + { + mbarWait(&s.fullBar[rs.stage], rs.phase); + uint8_t const* st = ring + rs.stage * kStageBytes; + float const* sc2 = reinterpret_cast(st + kItemRowBytes); +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const mt = warp + kConsumerWarps * mi; + float acc[4] = {0.f, 0.f, 0.f, 0.f}; + float acc1[4] = {0.f, 0.f, 0.f, 0.f}; + uint8_t const* pb2 = st + mt * 16 * 128; + int const koff = tig * 16; + uint4 const a0 = lds128(pb2 + g * 128 + koff); + uint4 const a08 = lds128(pb2 + (g + 8) * 128 + koff); + uint4 const a1 = lds128(pb2 + g * 128 + koff + 64); + uint4 const a18 = lds128(pb2 + (g + 8) * 128 + koff + 64); + mmaE4m3(acc, a0.x, a08.x, a0.y, a08.y, bm0.x, bm0.y); + mmaE4m3(acc1, a1.x, a18.x, a1.y, a18.y, bm1.x, bm1.y); + mmaE4m3(acc, a0.z, a08.z, a0.w, a08.w, bm0.z, bm0.w); + mmaE4m3(acc1, a1.z, a18.z, a1.w, a18.w, bm1.z, bm1.w); +#pragma unroll + for (int r = 0; r < 4; ++r) + { + acc[r] += acc1[r]; + } + float const s2 = sc2[(crank * kFc2Rows + u * kItemFc2Rows + mt * 16) >> 7]; + float const f0 = s2 * fa; + float const f1 = s2 * fb; + out[u][mi][0] = fmaf(acc[0], f0, out[u][mi][0]); + out[u][mi][1] = fmaf(acc[1], f1, out[u][mi][1]); + out[u][mi][2] = fmaf(acc[2], f0, out[u][mi][2]); + out[u][mi][3] = fmaf(acc[3], f1, out[u][mi][3]); + } + __syncwarp(); + if (lane == 0) + { + mbarArrive(&s.emptyBar[rs.stage]); + } + rs.advance(); + } + } + if (nMy > 0) + { + int const t0 = 2 * tig, t1 = 2 * tig + 1; +#pragma unroll + for (int u = 0; u < kNh; ++u) + { +#pragma unroll + for (int mi = 0; mi < kMtPerWarp; ++mi) + { + int const row = crank * kFc2Rows + u * kItemFc2Rows + (warp + kConsumerWarps * mi) * 16 + g; + float* pp = part + static_cast(cl) * kRows * kH + row; + if (t0 < T) + { + __stcg(pp + t0 * kH, out[u][mi][0]); + __stcg(pp + t0 * kH + 8, out[u][mi][2]); + } + if (t1 < T) + { + __stcg(pp + t1 * kH, out[u][mi][1]); + __stcg(pp + t1 * kH + 8, out[u][mi][3]); + } + } + } + } + } + + // ---- past the grid counter, each CTA sums its share of the NCa cluster partials: 16 threads a (token, 4 rows) + // unit, thread q summing partials q, q + 16, ... in order, then a fixed shuffle tree. + __syncthreads(); + if (threadIdx.x == 0) + { + // Every CTA arrives, inactive ones too, so past the wait every CTA is past its prologue. + fenceAcqRelGpu(); + redAddRelease64(counter, cta == 0 ? kStride - static_cast(G - 1) : 1ull); + unsigned long long const target = s.barTarget; + while (ldAcquireGpu64(counter) < target) + { + } + } + __syncthreads(); + { + int const nUnits = T * (kH / 4); + int const u0 = static_cast(static_cast(cta) * nUnits / G); + int const u1 = static_cast(static_cast(cta + 1) * nUnits / G); + int const grp = threadIdx.x >> 4; + int const q = threadIdx.x & 15; + constexpr int kGroups = kThreads / 16; + __nv_bfloat16* o = static_cast<__nv_bfloat16*>(p.out); + for (int ub = u0; ub < u1; ub += kGroups) + { + int const u = ub + grp; + bool const live = u < u1; + int const t = live ? u / (kH / 4) : 0; + int const row = live ? (u % (kH / 4)) * 4 : 0; + float const* src = part + static_cast(t) * kH + row; + float4 v = make_float4(0.f, 0.f, 0.f, 0.f); + if (live) + { + for (int k = q; k < NCa; k += 16) + { + float4 const x = __ldcg(reinterpret_cast(src + static_cast(k) * kRows * kH)); + v.x += x.x; + v.y += x.y; + v.z += x.z; + v.w += x.w; + } + } +#pragma unroll + for (int off = 8; off > 0; off >>= 1) + { + v.x += __shfl_xor_sync(0xffffffffu, v.x, off); + v.y += __shfl_xor_sync(0xffffffffu, v.y, off); + v.z += __shfl_xor_sync(0xffffffffu, v.z, off); + v.w += __shfl_xor_sync(0xffffffffu, v.w, off); + } + if (live && q == 0) + { + __nv_bfloat162 const lo = __floats2bfloat162_rn(v.x, v.y); + __nv_bfloat162 const hi = __floats2bfloat162_rn(v.z, v.w); + uint2 pk; + pk.x = *reinterpret_cast(&lo); + pk.y = *reinterpret_cast(&hi); + *reinterpret_cast(o + static_cast(t) * kH + row) = pk; + } + } + } +#endif // !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 900) +} + +int smemOptinTpb(int device) +{ + int v = 0; + cudaDeviceGetAttribute(&v, cudaDevAttrMaxSharedMemoryPerBlockOptin, device); + return v; +} + +template +int activeClustersT(int device) +{ + static int cache[16] = {}; + static std::mutex mu; + std::lock_guard lock(mu); + if (device >= 0 && device < 16 && cache[device] > 0) + { + return cache[device]; + } + int const maxSmemBytes = smemOptinTpb(device); + cudaError_t err = cudaFuncSetAttribute( + fp8BlockMoeTpbKernel, cudaFuncAttributeMaxDynamicSharedMemorySize, maxSmemBytes); + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: smem attribute: ") + cudaGetErrorString(err)); + } + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(CS * kMaxClusters); + cfg.blockDim = dim3(kThreads); + cfg.dynamicSmemBytes = static_cast(maxSmemBytes); + cudaLaunchAttribute attr[1]; + attr[0].id = cudaLaunchAttributeClusterDimension; + attr[0].val.clusterDim.x = CS; + attr[0].val.clusterDim.y = 1; + attr[0].val.clusterDim.z = 1; + cfg.attrs = attr; + cfg.numAttrs = 1; + int n = 0; + err = cudaOccupancyMaxActiveClusters(&n, fp8BlockMoeTpbKernel, &cfg); + if (err != cudaSuccess || n <= 0) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: occupancy: ") + cudaGetErrorString(err)); + } + n = std::min(n, kMaxClusters); + if (device >= 0 && device < 16) + { + cache[device] = n; + } + return n; +} + +// Few rows have few blocks, so give each more CTAs; from 3 rows, 4-CTA clusters keep more clusters resident. +int clusterSize(int rows) +{ + return rows <= 2 ? 8 : 4; +} + +template +void launchTpbT(TpbParams const& params, bool pdl, cudaStream_t stream) +{ + int device = 0; + cudaGetDevice(&device); + int const clusters = activeClustersT(device); + int const xbufOffset = kStaticBytes; + int const xbufBytes = (2 * params.numRows * kBlkRows * 4 + 127) / 128 * 128; + int const shXOffset = xbufOffset + xbufBytes; + int const shXBytes = kSh ? (params.numRows * kShRowBytes + 127) / 128 * 128 : 0; + int const ringOffset = shXOffset + shXBytes; + int const fit = (smemOptinTpb(device) - ringOffset) / kStageBytes; + int const stages = std::min(std::min(kDefaultStages, fit), kMaxStages); + if (stages < 3) + { + throw std::invalid_argument("fp8_block_moe_tpb: three ring stages do not fit in shared memory"); + } + size_t const smem = static_cast(ringOffset) + static_cast(stages) * kStageBytes; + cudaLaunchConfig_t cfg{}; + cfg.gridDim = dim3(clusters * CS); + cfg.blockDim = dim3(kThreads); + cfg.dynamicSmemBytes = smem; + cfg.stream = stream; + cudaLaunchAttribute attr[2]; + attr[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr[0].val.programmaticStreamSerializationAllowed = pdl ? 1 : 0; + attr[1].id = cudaLaunchAttributeClusterDimension; + attr[1].val.clusterDim.x = CS; + attr[1].val.clusterDim.y = 1; + attr[1].val.clusterDim.z = 1; + cfg.attrs = attr; + cfg.numAttrs = 2; + cudaError_t const err + = cudaLaunchKernelEx(&cfg, fp8BlockMoeTpbKernel, params, stages, xbufOffset, ringOffset, shXOffset); + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("fp8_block_moe_tpb: launch: ") + cudaGetErrorString(err)); + } +} + +} // namespace + +size_t tpbWorkspaceBytes() +{ + return kWsBytes; +} + +void launchTpb(TpbParams const& params, bool pdl, cudaStream_t stream) +{ + if (params.numRows < 1 || params.numRows > kRows) + { + throw std::invalid_argument("fp8_block_moe_tpb: 1..8 rows"); + } + if (params.rank < 0 || params.rank >= kTpbRanks) + { + throw std::invalid_argument("fp8_block_moe_tpb: rank 0..3 of 4"); + } + if (params.shBlocks > 0) + { + if (params.shBlocks != kShItems || clusterSize(params.numRows) != 4 || params.shX == nullptr + || params.shGu == nullptr || params.shGl == nullptr || params.shDn == nullptr) + { + throw std::invalid_argument( + "fp8_block_moe_tpb: the shared-expert fold takes 5 slice items and 4-CTA clusters (3..8 rows)"); + } + launchTpbT<4, true>(params, pdl, stream); + } + else if (clusterSize(params.numRows) == 8) + { + launchTpbT<8, false>(params, pdl, stream); + } + else + { + launchTpbT<4, false>(params, pdl, stream); + } +} + +} // namespace kernels::fp8_block_moe_tpb + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h new file mode 100644 index 000000000000..f4ef92ebbdf4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_moe_tpb +{ + +//! Decode routed experts over 128-row intermediate blocks (e4m3, fp32 128x128 weight and 1x128 activation scales). +//! Block j of a rank is one (expert, intermediate block) pair, block-major: +//! w1 [blocks, 256, 2560] e4m3 [128 up rows ; 128 gate rows], s1 [blocks, 2, 20] fp32 their block scales; +//! w2 [blocks, 2560, 128] e4m3 the block's down_proj columns, s2 [blocks, 20] fp32 their block scales. +//! Expert e's block b lives on attention-TP rank (e + b) % 4: rank r stores block b < 4 of e at j = e and block 4 +//! (when e % 4 == r) at j = 512 + e / 4, so each rank holds 640 blocks and at least one block of every expert. +constexpr int kHidden = 2560; +constexpr int kExperts = 512; +constexpr int kMaxRows = 8; +constexpr int kTpbRanks = 4; +constexpr int kTpbBlocks = 640; //!< blocks a rank holds (512 * 5 / 4) +constexpr int kShItems = 5; //!< work items of the folded shared-expert slice +constexpr int kShItemCols = 32; //!< intermediate columns of one shared item + +//! Arguments of one TPB launch (layouts above; T = numRows). +struct TpbParams +{ + void const* logits; //!< [T, 512] bf16 + uint8_t const* xq; //!< [T, 2560] e4m3 + float const* xsf; //!< [20, xsfStride] fp32: the 1x128 scales of xq, transposed + int xsfStride; //!< row stride of xsf, in elements + uint8_t const* w1; //!< [640, 256, 2560] e4m3 + float const* s1; //!< [640, 2, 20] fp32 + uint8_t const* w2; //!< [640, 2560, 128] e4m3 + float const* s2; //!< [640, 20] fp32 + int rank; //!< this rank of kTpbRanks + int numRows; //!< 1..8 + //! Optional fold of this rank's TP slice of the bf16 shared expert (shBlocks = 5 items of 32 columns; 3..8 rows): + //! the partial also accumulates sigmoid(gate logit) * the slice's down projection in fp32 before the bf16 rounding. + void const* shX; //!< [T, 2560] bf16: the MoE site's rows + void const* shGu; //!< [5, 64, 2560] bf16: item sb, 8 groups of 4 columns, each [gate rows (4) ; up rows (4)] + void const* shGl; //!< [2560] bf16: the gate-logit row + void const* shDn; //!< [5, 2560, 32] bf16: down_proj's columns of item sb + int shBlocks; //!< kShItems with the shared-expert fold, else 0 + void* out; //!< [T, 2560] bf16: this rank's partial (the TP sum of the ranks' partials is the layer) + void* workspace; //!< tpbWorkspaceBytes(), zeroed once; its grid counter only grows (one per device) +}; + +//! Bytes of the per-device workspace launchTpb takes. +size_t tpbWorkspaceBytes(); + +//! One launch of 8-CTA clusters (up to 2 rows) or 4-CTA clusters (3..8 rows). +void launchTpb(TpbParams const& params, bool pdl, cudaStream_t stream); + +} // namespace kernels::fp8_block_moe_tpb + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp new file mode 100644 index 000000000000..6caf72083a7a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpbOp.cpp @@ -0,0 +1,165 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockMoeTpb.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace torch_ext +{ + +namespace +{ + +namespace kd = TRTLLM_MV2_NS::kernels::fp8_block_moe_tpb; + +void checkAligned(at::Tensor const& t, char const* name) +{ + TORCH_CHECK(reinterpret_cast(t.data_ptr()) % 16 == 0, name, " must be 16-byte aligned"); +} + +// Returns this rank's [T, 2560] bf16 partial of the routed experts for T <= 8 rows. `xsf` holds the transposed 1x128 +// scales of `xq`; `w1`..`s2` are this `rank`'s 640 blocks (layouts in fp8BlockMoeTpb.h). `workspace` is a zeroed +// uint8 buffer of at least `fp8_block_moe_tpb_workspace_bytes()`, one per device. The optional `sh_*` operands +// (3..8 rows) fold in this rank's TP slice of the bf16 shared expert (see TpbParams). +at::Tensor fp8BlockMoeTpb(at::Tensor const& logits, at::Tensor const& xq, at::Tensor const& xsf, at::Tensor const& w1, + at::Tensor const& s1, at::Tensor const& w2, at::Tensor const& s2, int64_t rank, at::Tensor& workspace, + bool enablePdl, std::optional const& shX, std::optional const& shGu, + std::optional const& shGl, std::optional const& shDn) +{ + int64_t const rows = logits.size(0); + TORCH_CHECK(logits.is_cuda() && logits.scalar_type() == at::kBFloat16 && logits.dim() == 2 + && logits.size(1) == kd::kExperts && logits.is_contiguous(), + "logits must be contiguous bf16 [T, 512]"); + TORCH_CHECK(rows >= 1 && rows <= kd::kMaxRows, "1..8 rows"); + TORCH_CHECK( + xq.scalar_type() == at::kFloat8_e4m3fn && xq.is_contiguous() && xq.size(0) == rows && xq.size(1) == kd::kHidden, + "xq must be contiguous e4m3 [T, 2560]"); + TORCH_CHECK(xsf.scalar_type() == at::kFloat && xsf.dim() == 2 && xsf.size(0) == kd::kHidden / 128 + && xsf.size(1) >= rows && xsf.stride(1) == 1, + "xsf must be fp32 [20, >= T] with unit column stride"); + int64_t const blocks = w1.size(0); + TORCH_CHECK(w1.scalar_type() == at::kFloat8_e4m3fn && w1.is_contiguous() && w1.dim() == 3 && w1.size(1) == 256 + && w1.size(2) == kd::kHidden, + "w1 must be contiguous e4m3 [blocks, 256, 2560]"); + TORCH_CHECK( + s1.scalar_type() == at::kFloat && s1.is_contiguous() && s1.size(0) == blocks && s1.numel() == blocks * 40, + "s1 must be contiguous fp32 [blocks, 2, 20]"); + TORCH_CHECK(w2.scalar_type() == at::kFloat8_e4m3fn && w2.is_contiguous() && w2.size(0) == blocks + && w2.size(1) == kd::kHidden && w2.size(2) == 128, + "w2 must be contiguous e4m3 [blocks, 2560, 128]"); + TORCH_CHECK( + s2.scalar_type() == at::kFloat && s2.is_contiguous() && s2.size(0) == blocks && s2.numel() == blocks * 20, + "s2 must be contiguous fp32 [blocks, 20]"); + TORCH_CHECK(blocks == kd::kTpbBlocks, "a rank holds 640 blocks"); + TORCH_CHECK(rank >= 0 && rank < kd::kTpbRanks, "rank must be in [0, 4)"); + TORCH_CHECK(workspace.is_cuda() + && workspace.numel() * workspace.element_size() >= static_cast(kd::tpbWorkspaceBytes()), + "workspace too small"); + for (at::Tensor const* t : std::initializer_list{&logits, &xq, &w1, &s1, &w2, &s2, &workspace}) + { + checkAligned(*t, "operand"); + } + for (at::Tensor const* t : std::initializer_list{&xq, &xsf, &w1, &s1, &w2, &s2, &workspace}) + { + TORCH_CHECK(t->device() == logits.device(), "every operand must be on the logits' CUDA device"); + } + c10::cuda::CUDAGuard const guard(logits.device()); + int const device = logits.get_device(); + auto out = at::empty({rows, kd::kHidden}, logits.options()); + kd::TpbParams p{}; + p.logits = logits.data_ptr(); + p.xq = static_cast(xq.data_ptr()); + p.xsf = xsf.data_ptr(); + p.xsfStride = static_cast(xsf.stride(0)); + p.w1 = static_cast(w1.data_ptr()); + p.s1 = s1.data_ptr(); + p.w2 = static_cast(w2.data_ptr()); + p.s2 = s2.data_ptr(); + p.rank = static_cast(rank); + p.numRows = static_cast(rows); + if (shGu.has_value()) + { + TORCH_CHECK( + shX.has_value() && shGl.has_value() && shDn.has_value(), "the shared fold takes sh_x, sh_gu, sh_gl, sh_dn"); + TORCH_CHECK(shX->scalar_type() == at::kBFloat16 && shX->is_contiguous() && shX->size(0) == rows + && shX->size(1) == kd::kHidden, + "sh_x must be contiguous bf16 [T, 2560]"); + TORCH_CHECK(shGu->scalar_type() == at::kBFloat16 && shGu->is_contiguous() && shGu->dim() == 3 + && shGu->size(0) == kd::kShItems && shGu->size(1) == 64 && shGu->size(2) == kd::kHidden, + "sh_gu must be contiguous bf16 [5, 64, 2560]"); + TORCH_CHECK(shGl->scalar_type() == at::kBFloat16 && shGl->is_contiguous() && shGl->numel() == kd::kHidden, + "sh_gl must be contiguous bf16 [2560]"); + TORCH_CHECK(shDn->scalar_type() == at::kBFloat16 && shDn->is_contiguous() && shDn->dim() == 3 + && shDn->size(0) == kd::kShItems && shDn->size(1) == kd::kHidden && shDn->size(2) == kd::kShItemCols, + "sh_dn must be contiguous bf16 [5, 2560, 32]"); + for (at::Tensor const* t : std::initializer_list{&*shX, &*shGu, &*shGl, &*shDn}) + { + checkAligned(*t, "shared operand"); + TORCH_CHECK(t->device() == logits.device(), "every shared operand must be on the logits' CUDA device"); + } + p.shX = shX->data_ptr(); + p.shGu = shGu->data_ptr(); + p.shGl = shGl->data_ptr(); + p.shDn = shDn->data_ptr(); + p.shBlocks = kd::kShItems; + } + p.out = out.data_ptr(); + p.workspace = workspace.data_ptr(); + kd::launchTpb(p, enablePdl, c10::cuda::getCurrentCUDAStream(device).stream()); + return out; +} + +int64_t fp8BlockMoeTpbWorkspaceBytes() +{ + return static_cast(kd::tpbWorkspaceBytes()); +} + +} // namespace + +} // namespace torch_ext + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "fp8_block_moe_tpb(Tensor logits, Tensor xq, Tensor xsf, Tensor w1, Tensor s1, Tensor w2, Tensor s2, int rank, " + "Tensor(a!) workspace, bool enable_pdl=False, Tensor? sh_x=None, Tensor? sh_gu=None, Tensor? sh_gl=None, " + "Tensor? sh_dn=None) -> Tensor"); + m.def("fp8_block_moe_tpb_workspace_bytes() -> int"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("fp8_block_moe_tpb", &TRTLLM_MV2_NS::torch_ext::fp8BlockMoeTpb); +} + +TRTLLM_MV2_LIBRARY_IMPL(CompositeExplicitAutograd, m) +{ + m.impl("fp8_block_moe_tpb_workspace_bytes", &TRTLLM_MV2_NS::torch_ext::fp8BlockMoeTpbWorkspaceBytes); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu new file mode 100644 index 000000000000..a833a795b913 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.cu @@ -0,0 +1,378 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h" +#include "tensorrt_llm/common/cudaTypeUtils.cuh" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/kernels/moe/trtllmGen/routing/RoutingKernel.cuh" +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_scale_moe_front +{ + +namespace +{ + +namespace cg = cooperative_groups; +namespace routing = ::moe::dev::routing; +using routing::WarpSize; + +constexpr int kNumThreads = 256; +constexpr int kNumWarps = kNumThreads / WarpSize; +constexpr int kVecSize = kMaxNumExperts / WarpSize; +constexpr int kScanItemsPerThread = kMaxNumExperts / kNumThreads; +static_assert(kMaxNumExperts % kNumThreads == 0); + +// Per-device state, zeroed at allocation: two banks of per-local-expert counters (a launch counts in bank +// generation % 2 and zeroes the other), then the grid barrier's arrival count and generation flag. +constexpr int kCounterInts = 2 * kMaxNumExperts + 64; +constexpr int kBarrierCount = 2 * kMaxNumExperts; +constexpr int kBarrierFlag = 2 * kMaxNumExperts + 32; +constexpr int kQuantIters = kQuantHidden / 256; // two 1x128 groups a warp iteration + +__device__ __forceinline__ int32_t ldAcquire(int32_t const* p) +{ + int32_t v; + asm volatile("ld.acquire.gpu.global.b32 %0, [%1];\n" : "=r"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void stRelease(int32_t* p, int32_t v) +{ + asm volatile("st.release.gpu.global.b32 [%0], %1;\n" ::"l"(p), "r"(v) : "memory"); +} + +__device__ __forceinline__ int32_t atomAddAcqRel(int32_t* p, int32_t v) +{ + int32_t old; + asm volatile("atom.acq_rel.gpu.global.add.s32 %0, [%1], %2;\n" : "=r"(old) : "l"(p), "r"(v) : "memory"); + return old; +} + +// Bitwise `quantize_1x128_lane` (fp8_blockscale_gemm_kernel.cuh) for one 1x128 group over 16 lanes of 8 bf16 each. +__device__ __forceinline__ void quantizeGroup(uint4 in, __nv_fp8_e4m3* out, float* scale, int laneInGroup) +{ + union + { + uint4 pack; + __nv_bfloat16 v[8]; + } x; + + x.pack = in; + __nv_bfloat16 laneAmax = __nv_bfloat16(0); +#pragma unroll + for (int i = 0; i < 8; i++) + { + laneAmax = __hmax(laneAmax, __habs(x.v[i])); + } + float groupAmax = static_cast(laneAmax); +#pragma unroll + for (int mask = 8; mask > 0; mask /= 2) + { + groupAmax = std::max(groupAmax, __shfl_xor_sync(0xFFFFFFFF, groupAmax, mask)); + } + __nv_bfloat16 amax = ::tensorrt_llm::common::cuda_max(__nv_bfloat16(groupAmax), __nv_bfloat16(1e-10f)); + float quantScale = 448.f / static_cast(amax); + float dequantScale = 1.f / quantScale; + if (laneInGroup == 0) + { + *scale = dequantScale; + } + + union + { + uint2 pack; + __nv_fp8_e4m3 v[8]; + } y; + +#pragma unroll + for (int i = 0; i < 8; i++) + { + y.v[i] = __nv_fp8_e4m3(static_cast(x.v[i]) * quantScale); + } + *reinterpret_cast(out) = y.pack; +} + +// One warp a token selects its experts with the trtllm-gen routing device functions and takes each slot's offset in +// its local expert from a global atomic counter. After a grid barrier of the routing CTAs, every CTA scans the counts +// into expert offsets and writes its own slots' permutation; CTA 0 writes the grouped-GEMM launch tables. +__global__ void __launch_bounds__(kNumThreads) moeFrontKernel(Data params, int32_t* state) +{ +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + using InputT = __nv_bfloat16; + using BaseType = __nv_bfloat16; + using TypePacked = routing::PackedScoreIdx; + using Scan = cub::BlockScan; + + __shared__ typename Scan::TempStorage tempStorage; + __shared__ int32_t smemExpertOffset[kMaxNumExperts]; + + int32_t const warpIdx = __shfl_sync(0xffffffff, threadIdx.x / WarpSize, 0); + int32_t const laneIdx = threadIdx.x % WarpSize; + int32_t const tokenIdx = blockIdx.x * kNumWarps + warpIdx; + bool const validToken = tokenIdx < params.numTokens; + auto block = cg::this_thread_block(); + auto warp = cg::tiled_partition(block); + int32_t* barrierCount = state + kBarrierCount; + int32_t* barrierFlag = state + kBarrierFlag; + + if (params.usePdl) + { + cudaGridDependencySynchronize(); + } + + int32_t const routeCtas = (params.numTokens + kNumWarps - 1) / kNumWarps; + if (static_cast(blockIdx.x) >= routeCtas) + { + // Quantization CTAs, outside the routing CTAs' barrier: half a hidden row a warp. + int32_t const half = (blockIdx.x - routeCtas) * kNumWarps + warpIdx; + int32_t const tokenQ = half / 2; + if (tokenQ < params.numTokens) + { + int32_t const it0 = (half % 2) * (kQuantIters / 2); + auto const* in + = static_cast<__nv_bfloat16 const*>(params.ptrHiddenBf16) + static_cast(tokenQ) * kQuantHidden; + uint4 part[kQuantIters / 2]; +#pragma unroll + for (int it = 0; it < kQuantIters / 2; it++) + { + part[it] = *reinterpret_cast(in + (it0 + it) * 256 + laneIdx * 8); + } + auto* out = static_cast<__nv_fp8_e4m3*>(params.ptrHiddenFp8) + static_cast(tokenQ) * kQuantHidden; +#pragma unroll + for (int it = 0; it < kQuantIters / 2; it++) + { + int32_t const group = 2 * (it0 + it) + laneIdx / 16; + quantizeGroup(part[it], out + (it0 + it) * 256 + laneIdx * 8, + params.ptrHiddenScale + group * params.numTokens + tokenQ, laneIdx % 16); + } + } + __syncthreads(); + if (params.usePdl) + { + cudaTriggerProgrammaticLaunchCompletion(); + } + return; + } + + BaseType const minScore = BaseType{-INFINITY}; + BaseType score[kVecSize]; + int32_t idx[kVecSize]; + if (validToken) + { + InputT const* scores = static_cast(params.ptrScores) + tokenIdx * params.numExperts; +#pragma unroll + for (int i = 0; i < kVecSize; i++) + { + auto expertIdx = i * WarpSize + laneIdx; + score[i] = expertIdx < params.numExperts ? static_cast(scores[expertIdx]) : minScore; + idx[i] = expertIdx; + } + } + + // No CTA flips the generation before every CTA, this one included, has arrived; the previous launch flipped it + // before it completed. + int32_t const generation = *reinterpret_cast(barrierFlag); + int32_t* counters = state + (generation & 1) * kMaxNumExperts; + if (blockIdx.x == 0) + { + // The other bank is the previous launch's, which is complete (this grid depends on it transitively). + int32_t* other = state + ((generation + 1) & 1) * kMaxNumExperts; + for (int e = threadIdx.x; e < kMaxNumExperts; e += kNumThreads) + { + other[e] = 0; + } + } + + int32_t slotExpert = -1; // this lane's slot (lane < topK): local expert, or -1 + int32_t slotOffset = 0; + if (validToken) + { + BaseType warpTopKScore[kMaxTopK]; + int32_t warpTopKExpertIdx[kMaxTopK]; + routing::topk::reduceTopK(warp, warpTopKScore, warpTopKExpertIdx, score, idx, minScore, params.topK); + BaseType const softmaxScore = routing::calcSoftmax( + warp, laneIdx < params.topK ? warpTopKScore[laneIdx] : minScore, laneIdx, params.topK); + if (laneIdx < params.topK) + { + // Round through the packed (score, idx) as the runner's cluster kernel does. + TypePacked const packed{softmaxScore, static_cast(warpTopKExpertIdx[laneIdx])}; + int32_t const expandedIdx = tokenIdx * params.topK + laneIdx; + static_cast<__nv_bfloat16*>(params.ptrTopKWeights)[expandedIdx] = __nv_bfloat16{packed.score}; + int32_t const localExpertIdx = static_cast(packed.idx) - params.localExpertsStartIdx; + if (localExpertIdx >= 0 && localExpertIdx < params.numLocalExperts) + { + slotExpert = localExpertIdx; + slotOffset = atomicAdd(counters + localExpertIdx, 1); + } + } + } + + // Grid barrier of the routing CTAs, all resident (a few dozen CTAs, and no awaited kernel holds the SMs): one + // acq_rel arrival releases this CTA's counter atomics and acquires the others', so the counts are final after it. + __syncthreads(); + if (threadIdx.x == 0) + { + if (atomAddAcqRel(barrierCount, 1) == routeCtas - 1) + { + *barrierCount = 0; + stRelease(barrierFlag, generation + 1); + } + else + { + while (ldAcquire(barrierFlag) == generation) + { + } + } + } + __syncthreads(); + + int32_t const tileN = params.tileTokensDim; + int32_t count[kScanItemsPerThread]; + int32_t numCta[kScanItemsPerThread]; +#pragma unroll + for (int e = 0; e < kScanItemsPerThread; e++) + { + int32_t const expert = threadIdx.x * kScanItemsPerThread + e; + count[e] = expert < params.numLocalExperts ? __ldcg(counters + expert) : 0; + } +#pragma unroll + for (int e = 0; e < kScanItemsPerThread; e++) + { + numCta[e] = (count[e] + tileN - 1) / tileN; + } + int32_t ctaOffset[kScanItemsPerThread]; + int32_t numNonExitingCtas; + Scan(tempStorage).ExclusiveSum(numCta, ctaOffset, numNonExitingCtas); +#pragma unroll + for (int e = 0; e < kScanItemsPerThread; e++) + { + int32_t const expert = threadIdx.x * kScanItemsPerThread + e; + smemExpertOffset[expert] = ctaOffset[e] * tileN; + if (blockIdx.x == 0 && expert < params.numLocalExperts) + { + for (int32_t cta = 0; cta < numCta[e]; cta++) + { + params.ptrCtaIdxXyToBatchIdx[ctaOffset[e] + cta] = expert; + int32_t const mnLimit1 = (ctaOffset[e] + cta + 1) * tileN; + int32_t const mnLimit2 = ctaOffset[e] * tileN + count[e]; + params.ptrCtaIdxXyToMnLimit[ctaOffset[e] + cta] = min(mnLimit1, mnLimit2); + } + } + } + if (blockIdx.x == 0 && threadIdx.x == 0) + { + params.ptrPermutedIdxSize[0] = numNonExitingCtas * tileN; + params.ptrNumNonExitingCtas[0] = numNonExitingCtas; + } + __syncthreads(); + + if (validToken && laneIdx < params.topK) + { + int32_t const expandedIdx = tokenIdx * params.topK + laneIdx; + if (slotExpert >= 0) + { + int32_t const permutedIdx = smemExpertOffset[slotExpert] + slotOffset; + params.ptrExpandedIdxToPermutedIdx[expandedIdx] = permutedIdx; + params.ptrPermutedIdxToTokenIdx[permutedIdx] = tokenIdx; + } + else + { + params.ptrExpandedIdxToPermutedIdx[expandedIdx] = -1; + } + } + + // The consumers (activation, both grouped GEMMs, finalize) read every table written above. + __syncthreads(); + if (params.usePdl) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif +} + +int32_t* deviceCounters(int device) +{ + static std::mutex mutex; + static std::array buffers{}; + std::lock_guard lock(mutex); + TLLM_CHECK_WITH_INFO(device >= 0 && device < static_cast(buffers.size()), "device %d out of range", device); + if (buffers[device] == nullptr) + { + // May run during stream capture: allocate in relaxed mode and zero on a private stream outside the capture. + cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; + TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); + int32_t* ptr = nullptr; + TLLM_CUDA_CHECK(cudaMalloc(&ptr, kCounterInts * sizeof(int32_t))); + cudaStream_t side; + TLLM_CUDA_CHECK(cudaStreamCreateWithFlags(&side, cudaStreamNonBlocking)); + TLLM_CUDA_CHECK(cudaMemsetAsync(ptr, 0, kCounterInts * sizeof(int32_t), side)); + TLLM_CUDA_CHECK(cudaStreamSynchronize(side)); + TLLM_CUDA_CHECK(cudaStreamDestroy(side)); + TLLM_CUDA_CHECK(cudaThreadExchangeStreamCaptureMode(&mode)); + buffers[device] = ptr; + } + return buffers[device]; +} + +} // namespace + +bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim) +{ + return numTokens > 0 && numTokens <= kMaxNumTokens && numExperts > 0 && numExperts <= kMaxNumExperts && topK > 0 + && topK <= kMaxTopK && numLocalExperts > 0 && numLocalExperts <= kMaxNumExperts && hiddenDim % 128 == 0; +} + +void run(Data const& data, cudaStream_t stream) +{ + TLLM_CHECK_WITH_INFO(supports(data.numTokens, data.numExperts, data.topK, data.numLocalExperts, data.hiddenDim), + "fp8_block_scale_moe_front: unsupported problem (tokens %d, experts %d, topK %d, local %d, hidden %d)", + data.numTokens, data.numExperts, data.topK, data.numLocalExperts, data.hiddenDim); + TLLM_CHECK_WITH_INFO(data.ptrHiddenBf16 == nullptr + || (data.ptrHiddenFp8 != nullptr && data.ptrHiddenScale != nullptr && data.hiddenDim == kQuantHidden), + "fp8_block_scale_moe_front: quantization needs both outputs and hidden %d", kQuantHidden); + int device = 0; + TLLM_CUDA_CHECK(cudaGetDevice(&device)); + int32_t* counters = deviceCounters(device); + + cudaLaunchConfig_t config{}; + int32_t const routeCtas = (data.numTokens + kNumWarps - 1) / kNumWarps; + int32_t const quantCtas = data.ptrHiddenBf16 != nullptr ? (2 * data.numTokens + kNumWarps - 1) / kNumWarps : 0; + config.gridDim = routeCtas + quantCtas; + config.blockDim = kNumThreads; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attributes[1] = {}; + attributes[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attributes[0].val.programmaticStreamSerializationAllowed = static_cast(data.usePdl); + config.attrs = attributes; + config.numAttrs = 1; + TLLM_CUDA_CHECK(cudaLaunchKernelEx(&config, moeFrontKernel, data, counters)); +} + +} // namespace kernels::fp8_block_scale_moe_front + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h new file mode 100644 index 000000000000..945e57ef7fd4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::fp8_block_scale_moe_front +{ + +//! Renormalize routing (top-k of the bf16 logits, softmax over those k) in front of the trtllm-gen FP8 block-scale MoE +//! runner for a decode-sized batch: one launch writes the runner's routing tables, optionally fused with the 1x128 e4m3 +//! quantization of the bf16 hidden states. Top-k ids, order and bf16 weights match the runner bitwise; a slot's offset +//! within its expert may differ (atomic order), which no consumer depends on. Launches on one device must not overlap. +struct Data +{ + void const* ptrScores{nullptr}; //!< [numTokens, numExperts] bf16 logits + void const* ptrHiddenBf16{nullptr}; //!< optional [numTokens, hiddenDim] bf16 hidden states to quantize + int32_t numTokens{0}; + int32_t numExperts{0}; + int32_t topK{0}; + int32_t tileTokensDim{0}; //!< token tile of the grouped GEMMs + int32_t localExpertsStartIdx{0}; + int32_t numLocalExperts{0}; + int32_t hiddenDim{0}; + bool usePdl{false}; + + //! Outputs: the runner's routing tables. + void* ptrTopKWeights{nullptr}; //!< [numTokens, topK] bf16 + int32_t* ptrExpandedIdxToPermutedIdx{nullptr}; + int32_t* ptrPermutedIdxToTokenIdx{nullptr}; + int32_t* ptrPermutedIdxSize{nullptr}; + int32_t* ptrCtaIdxXyToBatchIdx{nullptr}; + int32_t* ptrCtaIdxXyToMnLimit{nullptr}; + int32_t* ptrNumNonExitingCtas{nullptr}; + void* ptrHiddenFp8{nullptr}; //!< [numTokens, hiddenDim] e4m3 (with ptrHiddenBf16) + float* ptrHiddenScale{nullptr}; //!< [hiddenDim / 128, numTokens] fp32 (with ptrHiddenBf16) +}; + +constexpr int kMaxNumExperts = 512; +constexpr int kMaxTopK = 16; +constexpr int kMaxNumTokens = 1024; + +constexpr int kQuantHidden = 2560; //!< hidden width the fused quantization is compiled for + +//! Whether `run` takes this problem (bf16 logits, no fused shared experts, stride-1 local window). +bool supports(int32_t numTokens, int32_t numExperts, int32_t topK, int32_t numLocalExperts, int32_t hiddenDim); + +//! Writes the routing tables (and, with ptrHiddenBf16, the quantized hidden states) of `data` on `stream`. +void run(Data const& data, cudaStream_t stream); + +} // namespace kernels::fp8_block_scale_moe_front + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp new file mode 100644 index 000000000000..276e181916c8 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFrontOp.cpp @@ -0,0 +1,325 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The stock `fp8_block_scale_moe_runner` logits path (thop/moe/trtllmGen/fp8BlockScaleMoe.cpp) at an explicit tactic +// (tile N, config), routed by fp8BlockScaleMoeFront.h; no routing bias, precomputed top-k, shared experts or groups. + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/fp8BlockScaleMoeFront.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/kernels/moe/trtllmGen/runner.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace torch_ext +{ + +namespace +{ + +namespace btg = ::batchedGemm::trtllm::gen; +namespace trtllmGenMoe = ::tensorrt_llm::kernels::trtllmGenFp8BlockScaleMoe; +namespace front = TRTLLM_MV2_NS::kernels::fp8_block_scale_moe_front; + +using trtllmGenMoe::Routing::RoutingMethodType; + +constexpr std::array kSupportedTileN{8, 16, 32, 64, 128}; // FP8BlockScaleMoERunner's mSupportedTileN + +// Built as FP8BlockScaleMoERunner builds its runner for the tile, so a tactic (tile N, config) tuned through that +// class names the same FC1 / FC2 kernels here. One per tile, built on first use and kept for the process. +trtllmGenMoe::MoE::Runner& moeRunner(int64_t tileN) +{ + static std::mutex mutex; + static std::array, kSupportedTileN.size()> runners; + auto const it = std::find(kSupportedTileN.begin(), kSupportedTileN.end(), tileN); + TORCH_CHECK(it != kSupportedTileN.end(), "fp8_block_scale_moe_front: unsupported tile_n ", tileN); + std::lock_guard lock(mutex); + auto& runner = runners[std::distance(kSupportedTileN.begin(), it)]; + if (runner == nullptr) + { + runner = std::make_unique( + /*dtypeElt=*/btg::Dtype::E4m3, /*useDeepSeekFp8=*/true, /*tileTokensDim=*/static_cast(tileN)); + } + return *runner; +} + +void checkOperands(at::Tensor const& routing_logits, std::optional const& hidden_states_bf16, + at::Tensor const& hidden_states, at::Tensor const& hidden_states_scale, at::Tensor const& gemm1_weights, + at::Tensor const& gemm1_weights_scale, at::Tensor const& gemm2_weights, at::Tensor const& gemm2_weights_scale, + int64_t const num_experts, int64_t const top_k, int64_t const intermediate_size, int64_t const local_num_experts, + int64_t const routing_method_type) +{ + TORCH_CHECK(routing_logits.scalar_type() == at::ScalarType::BFloat16 && routing_logits.is_contiguous(), + "fp8_block_scale_moe_front: contiguous bf16 routing_logits required"); + TORCH_CHECK(routing_logits.dim() == 2, "routing_logits must be 2D."); + TORCH_CHECK(routing_logits.sizes()[1] == num_experts, "routing_logits dim1 must match num_experts."); + TORCH_CHECK(hidden_states.dim() == 2, "hidden_states must be 2D."); + TORCH_CHECK(routing_logits.sizes()[0] == hidden_states.sizes()[0], + "routing_logits and hidden_states must have the same number of tokens."); + TORCH_CHECK(routing_logits.device() == hidden_states.device(), + "routing_logits and hidden_states must be on the same device"); + // The kernel computes Renormalize (softmax over the top-k logits), so RenormalizeNaive is not taken. + TORCH_CHECK(static_cast(routing_method_type) == RoutingMethodType::Renormalize, + "fp8_block_scale_moe_front: logits-routed Renormalize without bias or fused shared experts only"); + TORCH_CHECK(top_k <= 32 && top_k > 0, + "Current routing kernel (no groups, renormalize) only supports top_k<=32 && top_k>0."); + TORCH_CHECK(num_experts > top_k, "num_experts must be greater than top_k"); + + int64_t const num_tokens = hidden_states.sizes()[0]; + int64_t const hidden_size = hidden_states.sizes()[1]; + TORCH_CHECK(hidden_states.scalar_type() == at::ScalarType::Float8_e4m3fn, "hidden_states must be fp8."); + TORCH_CHECK(hidden_states_scale.scalar_type() == at::ScalarType::Float, "hidden_states_scale must be float."); + TORCH_CHECK(hidden_states_scale.dim() == 2, "hidden_states_scale must be 2D."); + TORCH_CHECK(hidden_states_scale.sizes()[0] == hidden_size / 128, + "hidden_states_scale dim0 must match hidden_states dim1 / 128."); + TORCH_CHECK(hidden_states_scale.sizes()[1] == num_tokens, "hidden_states_scale dim1 must match num_tokens."); + TORCH_CHECK(gemm1_weights.scalar_type() == at::ScalarType::Float8_e4m3fn, "gemm1_weights must be fp8."); + TORCH_CHECK(gemm1_weights.dim() == 3, "gemm1_weights must be 3D."); + TORCH_CHECK(gemm1_weights.sizes()[0] == local_num_experts, "gemm1_weights has incorrect shape."); + TORCH_CHECK(gemm1_weights.sizes()[1] % 2 == 0, "the second dimension of weights must be even."); + TORCH_CHECK(intermediate_size == gemm1_weights.sizes()[1] / 2, "intermediate_size has incorrect shape."); + TORCH_CHECK( + gemm1_weights.sizes()[2] == hidden_size, "the third dimension of weights must be equal to hidden_size."); + TORCH_CHECK(gemm1_weights_scale.scalar_type() == at::ScalarType::Float, "gemm1_weights_scale must be float."); + TORCH_CHECK(gemm1_weights_scale.dim() == 3, "gemm1_weights_scale must be 3D."); + TORCH_CHECK(gemm1_weights_scale.sizes()[0] == local_num_experts, "gemm1_weights_scale has incorrect dim 0."); + TORCH_CHECK(intermediate_size % 128 == 0, "the second dimension of weights must be a multiple of 128."); + TORCH_CHECK( + gemm1_weights_scale.sizes()[1] == 2 * intermediate_size / 128, "gemm1_weights_scale has incorrect shape."); + TORCH_CHECK(gemm1_weights_scale.sizes()[2] == hidden_size / 128, "gemm1_weights_scale has incorrect shape."); + TORCH_CHECK(gemm2_weights.scalar_type() == at::ScalarType::Float8_e4m3fn, "gemm2_weights must be fp8."); + TORCH_CHECK(gemm2_weights.dim() == 3, "gemm2_weights must be 3D."); + TORCH_CHECK(gemm2_weights.sizes()[0] == local_num_experts, "gemm2_weights has incorrect shape."); + TORCH_CHECK(gemm2_weights.sizes()[2] == intermediate_size, + "the third dimension of weights must be equal to intermediate_size."); + TORCH_CHECK(gemm2_weights_scale.scalar_type() == at::ScalarType::Float, "gemm2_weights_scale must be float."); + TORCH_CHECK(gemm2_weights_scale.dim() == 3, "gemm2_weights_scale must be 3D."); + TORCH_CHECK(gemm2_weights_scale.sizes()[0] == local_num_experts, "gemm2_weights_scale has incorrect dim 0."); + TORCH_CHECK(gemm2_weights_scale.sizes()[1] == hidden_size / 128, "gemm2_weights_scale has incorrect shape."); + TORCH_CHECK(gemm2_weights_scale.sizes()[2] == intermediate_size / 128, "gemm2_weights_scale has incorrect shape."); + + TORCH_CHECK( + front::supports(static_cast(num_tokens), static_cast(num_experts), + static_cast(top_k), static_cast(local_num_experts), static_cast(hidden_size)), + "fp8_block_scale_moe_front: unsupported problem size"); + if (hidden_states_bf16.has_value()) + { + TORCH_CHECK(hidden_states_bf16->scalar_type() == at::ScalarType::BFloat16 && hidden_states_bf16->is_contiguous() + && hidden_states_bf16->sizes() == hidden_states.sizes(), + "fp8_block_scale_moe_front: hidden_states_bf16 must be contiguous bf16 of hidden_states' shape"); + TORCH_CHECK(hidden_states.is_contiguous() && hidden_states_scale.is_contiguous(), + "fp8_block_scale_moe_front: the quantization outputs must be contiguous"); + TORCH_CHECK(hidden_states_bf16->device() == hidden_states.device(), + "hidden_states_bf16 and hidden_states must be on the same device"); + } +} + +// Returns [T, H] bf16. Given `hidden_states_bf16` [T, H], the routing launch first writes its 1x128 e4m3 quantization +// (bitwise `fp8_quantize_1x128`) into `hidden_states` [T, H] and `hidden_states_scale` [H / 128, T] fp32. +at::Tensor fp8BlockScaleMoeFront(at::Tensor const& routing_logits, std::optional const& hidden_states_bf16, + at::Tensor& hidden_states, at::Tensor& hidden_states_scale, at::Tensor const& gemm1_weights, + at::Tensor const& gemm1_weights_scale, at::Tensor const& gemm2_weights, at::Tensor const& gemm2_weights_scale, + int64_t const num_experts, int64_t const top_k, int64_t const intermediate_size, int64_t const local_expert_offset, + int64_t const local_num_experts, int64_t const routing_method_type, int64_t const tile_n, int64_t const config) +{ + c10::cuda::CUDAGuard const deviceGuard{hidden_states.device()}; + TORCH_CHECK(::tensorrt_llm::common::isSM100Family(), "Only SM100f is supported by FP8 block scale MOE"); + TORCH_CHECK(config >= 0, "fp8_block_scale_moe_front: needs an explicit config, got ", config); + checkOperands(routing_logits, hidden_states_bf16, hidden_states, hidden_states_scale, gemm1_weights, + gemm1_weights_scale, gemm2_weights, gemm2_weights_scale, num_experts, top_k, intermediate_size, + local_num_experts, routing_method_type); + trtllmGenMoe::MoE::Runner& runner = moeRunner(tile_n); + int64_t const tile_tokens_dim = tile_n; + int64_t const num_tokens = hidden_states.sizes()[0]; + int64_t const hidden_size = hidden_states.sizes()[1]; + auto const device = hidden_states.device(); + auto const stream = at::cuda::getCurrentCUDAStream(hidden_states.get_device()); + + trtllmGenMoe::MoE::MoERunnerArgs args; + trtllmGenMoe::MoE::MoEWorkspace workspace; + + // The output is bf16 (MoERunnerArgs' default mDtypeOut). + args.mDtypeElt = btg::Dtype::E4m3; + args.mDtypeBias = btg::Dtype::Bfloat16; + args.routing_logits = routing_logits.data_ptr(); + args.hidden_states = hidden_states.data_ptr(); + args.hidden_states_scale = hidden_states_scale.data_ptr(); + args.gemm1_weights = gemm1_weights.data_ptr(); + args.gemm1_weights_scale = gemm1_weights_scale.data_ptr(); + args.gemm2_weights = gemm2_weights.data_ptr(); + args.gemm2_weights_scale = gemm2_weights_scale.data_ptr(); + args.num_tokens = static_cast(num_tokens); + args.num_experts = static_cast(num_experts); + args.hidden_size = static_cast(hidden_size); + args.top_k = static_cast(top_k); + args.num_fused_shared_experts = 0; + args.n_group = 0; + args.topk_group = 0; + args.local_expert_offset = static_cast(local_expert_offset); + args.local_num_experts = static_cast(local_num_experts); + args.routed_scaling_factor = 1.0; + args.intermediate_size = static_cast(intermediate_size); + args.mUseDeepSeekFp8 = true; + + int32_t const max_num_padded_tokens = trtllmGenMoe::Routing::getMaxPermutedPaddedCount( + args.num_tokens, args.top_k, args.num_experts, static_cast(tile_tokens_dim)); + int32_t const max_num_padded_tokens_gemm1 = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, 2 * args.intermediate_size, btg::dtypeGetNumBits(args.mDtypeElt)); + // maybeGetMinTokenCount pads to a 128 KiB floor at the given row width, so each buffer passes its own width. + int32_t const max_num_padded_tokens_activation = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, args.intermediate_size, btg::dtypeGetNumBits(args.mDtypeElt)); + int32_t const max_num_padded_tokens_gemm2 = trtllmGenMoe::Routing::maybeGetMinTokenCount( + max_num_padded_tokens, args.hidden_size, btg::dtypeGetNumBits(args.mDtypeOut)); + at::Tensor total_num_padded_tokens = at::empty({}, at::TensorOptions().device(device).dtype(at::ScalarType::Int)); + at::Tensor expanded_idx_to_permuted_idx + = at::detail::empty_cuda({num_tokens * top_k}, at::ScalarType::Int, device, std::nullopt); + // getRouteMapAllocCount: the gemm1 cubins read one element past the end of the route map. + at::Tensor permuted_idx_to_token_idx + = at::detail::empty_cuda({trtllmGenMoe::Routing::getRouteMapAllocCount(max_num_padded_tokens)}, + at::ScalarType::Int, device, std::nullopt); + // The routing writes bf16 weights, which the finalize reads as mDtypeExpW against the bf16 gemm2_output. + at::Tensor expert_weights + = at::detail::empty_cuda({num_tokens, top_k}, at::ScalarType::BFloat16, device, std::nullopt); + + // The rgTma TMA-OOB descriptors use OOB_ADDR_GEN_MODE_BASE_128kB on Blackwell, so TMA may probe up to a 128 KiB + // boundary past the base: gemm1_output, activation_output and gemm2_output need 128 KiB alignment and >= 128 KiB + // mapped headroom (empty_cuda aligns to 512 B). The SF buffers are loaded via LDGSTS and exempt. + static constexpr int64_t kTmaOob128k = 128 * 1024; + std::vector tmaOobKeepAlive; // owns the backing storage for the aligned pointers + auto allocTmaOobWorkspace = [&](int64_t dataBytes) -> void* + { + int64_t const allocBytes = dataBytes + 2 * kTmaOob128k; // up-align slack + tail headroom + at::Tensor buf = at::detail::empty_cuda({allocBytes}, at::ScalarType::Byte, device, std::nullopt); + uint64_t const base = reinterpret_cast(buf.data_ptr()); + uint64_t const aligned = (base + kTmaOob128k - 1) & ~(static_cast(kTmaOob128k) - 1); + tmaOobKeepAlive.push_back(std::move(buf)); + return reinterpret_cast(aligned); + }; + void* gemm1_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_gemm1) * (2 * intermediate_size)); // e4m3 + at::Tensor gemm1_output_scale = at::detail::empty_cuda( + {2 * intermediate_size / 128, max_num_padded_tokens_gemm1}, at::ScalarType::Float, device, std::nullopt); + void* activation_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_activation) * intermediate_size); // e4m3 + at::Tensor activation_output_scale = at::detail::empty_cuda( + {intermediate_size / 128, max_num_padded_tokens_activation}, at::ScalarType::Float, device, std::nullopt); + void* gemm2_output_ptr + = allocTmaOobWorkspace(static_cast(max_num_padded_tokens_gemm2) * args.hidden_size * 2); // bf16 + + int32_t const max_num_ctas = trtllmGenMoe::Routing::getMaxNumCtasInBatchDim( + args.num_tokens, args.top_k, args.num_experts, static_cast(tile_tokens_dim)); + at::Tensor cta_idx_xy_to_batch_idx + = at::detail::empty_cuda({max_num_ctas}, at::ScalarType::Int, device, std::nullopt); + at::Tensor cta_idx_xy_to_mn_limit + = at::detail::empty_cuda({max_num_ctas}, at::ScalarType::Int, device, std::nullopt); + at::Tensor num_non_exiting_ctas = at::empty({}, at::TensorOptions().device(device).dtype(at::ScalarType::Int)); + + front::Data data; + data.ptrScores = routing_logits.data_ptr(); + data.numTokens = args.num_tokens; + data.numExperts = args.num_experts; + data.topK = args.top_k; + data.tileTokensDim = static_cast(tile_tokens_dim); + data.localExpertsStartIdx = args.local_expert_offset; + data.numLocalExperts = args.local_num_experts; + data.hiddenDim = args.hidden_size; + data.usePdl = ::tensorrt_llm::common::getEnvEnablePDL(); + data.ptrTopKWeights = expert_weights.data_ptr(); + data.ptrExpandedIdxToPermutedIdx = expanded_idx_to_permuted_idx.data_ptr(); + data.ptrPermutedIdxToTokenIdx = permuted_idx_to_token_idx.data_ptr(); + data.ptrPermutedIdxSize = total_num_padded_tokens.data_ptr(); + data.ptrCtaIdxXyToBatchIdx = cta_idx_xy_to_batch_idx.data_ptr(); + data.ptrCtaIdxXyToMnLimit = cta_idx_xy_to_mn_limit.data_ptr(); + data.ptrNumNonExitingCtas = num_non_exiting_ctas.data_ptr(); + if (hidden_states_bf16.has_value()) + { + data.ptrHiddenBf16 = hidden_states_bf16->data_ptr(); + data.ptrHiddenFp8 = hidden_states.data_ptr(); + data.ptrHiddenScale = hidden_states_scale.data_ptr(); + } + front::run(data, stream); + + at::Tensor output + = at::detail::empty_cuda({num_tokens, hidden_size}, at::ScalarType::BFloat16, device, std::nullopt); + + workspace.total_num_padded_tokens = total_num_padded_tokens.data_ptr(); + workspace.total_max_padded_tokens + = std::max({max_num_padded_tokens_gemm1, max_num_padded_tokens_activation, max_num_padded_tokens_gemm2}); + workspace.permuted_idx_size = total_num_padded_tokens.data_ptr(); + workspace.expanded_idx_to_permuted_idx = expanded_idx_to_permuted_idx.data_ptr(); // activation / finalize + workspace.permuted_idx_to_token_idx = permuted_idx_to_token_idx.data_ptr(); // permute + gemm1 + workspace.expert_weights = expert_weights.data_ptr(); // finalize + workspace.cta_idx_xy_to_batch_idx = cta_idx_xy_to_batch_idx.data_ptr(); + workspace.cta_idx_xy_to_mn_limit = cta_idx_xy_to_mn_limit.data_ptr(); + workspace.num_non_exiting_ctas = num_non_exiting_ctas.data_ptr(); + workspace.gemm1_output = gemm1_output_ptr; + workspace.gemm1_output_scale = gemm1_output_scale.data_ptr(); + workspace.activation_output = activation_output_ptr; + workspace.activation_output_scale = activation_output_scale.data_ptr(); + workspace.gemm2_output = gemm2_output_ptr; + workspace.gemm2_output_scale = nullptr; + args.output = output.data_ptr(); + args.output_scale = nullptr; + + auto const workspace_sizes = runner.getWorkspaceSizeInBytes(args, config); + at::Tensor workspace_fc1 + = at::detail::empty_cuda({std::get<0>(workspace_sizes)}, at::ScalarType::Char, device, std::nullopt); + at::Tensor workspace_fc2 + = at::detail::empty_cuda({std::get<1>(workspace_sizes)}, at::ScalarType::Char, device, std::nullopt); + workspace.bmm1_workspace = workspace_fc1.data_ptr(); + workspace.bmm2_workspace = workspace_fc2.data_ptr(); + + runner.run(args, workspace, hidden_states.get_device(), stream, config); + return output; +} + +} // namespace + +} // namespace torch_ext + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "fp8_block_scale_moe_front(Tensor routing_logits, Tensor? hidden_states_bf16, Tensor(a!) hidden_states, " + "Tensor(b!) hidden_states_scale, Tensor gemm1_weights, Tensor gemm1_weights_scale, Tensor gemm2_weights, " + "Tensor gemm2_weights_scale, int num_experts, int top_k, int intermediate_size, int local_expert_offset, " + "int local_num_experts, int routing_method_type, int tile_n, int config) -> Tensor"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("fp8_block_scale_moe_front", &TRTLLM_MV2_NS::torch_ext::fp8BlockScaleMoeFront); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu new file mode 100644 index 000000000000..d32cd7d252fd --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.cu @@ -0,0 +1,544 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ +namespace hc_inject +{ +// This kernel shares the stock IPC all-reduce workspace with the Lamport kernels of +// communicationKernels/allReduceFusionKernels.cu and runs between their calls, so LamportComm, the helpers below and +// the push / clear / poll / update steps must follow that file's Lamport protocol: +// - workspace[2 * NRanks + r] is rank r's buffer of three sub-buffers of workspace[3 * NRanks + 1][1] bytes; with +// flag = workspace[3 * NRanks][2], a call writes and polls sub-buffer flag % 3 and clears its own (flag + 2) % 3. +// - workspace[3 * NRanks + 1][0] holds the bytes the previous call wrote, cleared as 16-byte stores of -0.0f. +// - A 16-byte slot has arrived once none of its 32-bit words is -0.0f, so senders store -0.0f as +0.0f. +// - Every CTA increments workspace[3 * NRanks][0] after reading the flag; once all have, CTA 0 advances the flag to +// (flag + 1) % 3, stores the bytes this call wrote and resets the counter. +// The comm op matrix (tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/ +// _hc_ar_inject_norm_op_matrix.py) pins this: it interleaves this kernel with stock all-reduces on one workspace and +// checks both bitwise at 2 and 4 ranks. +template +struct LamportComm +{ + __device__ __forceinline__ LamportComm(void** workspace, int rank) + { + counter_ptr = &reinterpret_cast(workspace[NRanks * 3])[0]; + flag_ptr = &reinterpret_cast(workspace[NRanks * 3])[2]; + // Bytes the previous Lamport kernel wrote to its sub-buffer. Every Lamport kernel on this workspace reads and + // writes it, whatever its dtype, so it stays in bytes. Readers round it up to whole 16-byte stores: the scalar + // MiniMax kernel writes 4-byte words. + clear_ptr = &reinterpret_cast(workspace[NRanks * 3 + 1])[0]; + flag_value = *flag_ptr; + auto comm_size = reinterpret_cast(workspace[NRanks * 3 + 1])[1]; + clear_size = *clear_ptr; + int data_offset = flag_value % 3; + int clear_offset = (flag_value + 2) % 3; + for (int r = 0; r < NRanks; ++r) + { + data_bufs[r] = reinterpret_cast(workspace[2 * NRanks + r]) + data_offset * comm_size; + } + clear_buf = reinterpret_cast(workspace[2 * NRanks + rank]) + clear_offset * comm_size; + __syncthreads(); + if (threadIdx.x == 0) + { + atomicAdd(counter_ptr, 1); + } + } + + __device__ __forceinline__ void update(int64_t new_clear_size) + { + if (blockIdx.x == 0 && threadIdx.x == 0) + { + while (*reinterpret_cast(counter_ptr) != gridDim.x) + { + } + *flag_ptr = (flag_value + 1) % 3; + *clear_ptr = new_clear_size; + *counter_ptr = 0; + } + } + + int* counter_ptr; + int* flag_ptr; + int64_t* clear_ptr; + uint8_t* data_bufs[NRanks]; + uint8_t* clear_buf; + int64_t clear_size; + int flag_value; +}; + +//! -0.0f is the not-arrived marker: a 16-byte slot has arrived once none of its words is -0.0f. +__device__ __forceinline__ bool is_neg_zero(float v) +{ + return *reinterpret_cast(&v) == 0x80000000; +} + +__device__ __forceinline__ bool is_neg_zero(float4 v) +{ + return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || is_neg_zero(v.w); +} + +__device__ __forceinline__ float4 get_neg_zero() +{ + float4 vec; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + reinterpret_cast(&vec)[i] = 0x80000000; + } + return vec; +} + +__device__ __forceinline__ float4 ld_global_volatile(float4* addr) +{ + float4 val; + asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" + : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) + : "l"(addr)); + return val; +} + +//! The current device's SM count, read once per process. +inline int get_sm_count() +{ + static int sm_count = 0; + if (sm_count == 0) + { + int device_id; + TLLM_CUDA_CHECK(cudaGetDevice(&device_id)); + cudaDeviceProp device_prop; + TLLM_CUDA_CHECK(cudaGetDeviceProperties(&device_prop, device_id)); + sm_count = device_prop.multiProcessorCount; + } + return sm_count; +} +} // namespace hc_inject + +// Thread t holds columns [8t, 8t + 8) of a row's block output and of each of its 4 streams. The Triton program runs a +// stream on G = 32 * programWarps threads over a 4096-wide block, its thread u in chunk c holding this CTA's thread +// c * G + u. Its sum of squares, reproduced bitwise: per thread and chunk the fp32 pair (fma(x0, x0, x2 * x2) + +// fma(x4, x4, x6 * x6), fma(x1, x1, x3 * x3) + fma(x5, x5, x7 * x7)), a pairwise tree over chunks (masked ones are +// zero), low + high, the warp's xor butterfly 16..1, then ((w0 + w2) + (w1 + w3)) at 4 warps or (w0 + w1) at 2. +namespace hc_inject +{ +static constexpr int kHidden = 2560; +static constexpr int kStreams = 4; +static constexpr int kThreads = kHidden / 8; + +__device__ __forceinline__ float bf16Lo(uint32_t w) +{ + return __uint_as_float(w << 16); +} + +__device__ __forceinline__ float bf16Hi(uint32_t w) +{ + return __uint_as_float(w & 0xFFFF0000u); +} + +__device__ __forceinline__ uint32_t packBf16x2(float lo, float hi) +{ + uint32_t r; + asm("cvt.rn.bf16x2.f32 %0, %1, %2;" : "=r"(r) : "f"(hi), "f"(lo)); + return r; +} + +__device__ __forceinline__ float roundBf16(float x) +{ + return bf16Lo(packBf16x2(x, 0.f)); +} + +__device__ __forceinline__ float ldBf16(void const* base, int64_t elem) +{ + uint16_t v = reinterpret_cast(base)[elem]; + return __uint_as_float(static_cast(v) << 16); +} + +//! `tl.sigmoid` as the Triton kernel compiles it. +__device__ __forceinline__ float sigmoidTriton(float x) +{ + float y; + asm("{ .reg .f32 t; mul.f32 t, %1, 0fBFB8AA3B; ex2.approx.f32 t, t; add.f32 t, t, 0f3F800000; " + "div.full.f32 %0, 0f3F800000, t; }" + : "=f"(y) + : "f"(x)); + return y; +} + +//! `rsqrt(ss / 2560 + eps)` as the Triton kernel compiles it (`div.full`, `add`, `rsqrt.approx.ftz`). +__device__ __forceinline__ float rstdTriton(float ss, float eps) +{ + float y; + asm("{ .reg .f32 t; div.full.f32 t, %1, 0f45200000; add.f32 t, t, %2; rsqrt.approx.ftz.f32 %0, t; }" + : "=f"(y) + : "f"(ss), "f"(eps)); + return y; +} + +__device__ __forceinline__ float2 add2(float2 a, float2 b) +{ + return make_float2(__fadd_rn(a.x, b.x), __fadd_rn(a.y, b.y)); +} + +//! A program thread's sum of squares: the pairwise tree over its unmasked chunk pairs, then low + high. +template +__device__ __forceinline__ float threadSsq(float2 const* leaves, int u) +{ + if constexpr (G == 128) + { + // 4 chunks of 1024 columns: chunk 2 real for u < 64, chunk 3 masked. + float2 l = add2(leaves[u], leaves[G + u]); + if (u < 64) + { + l = add2(l, leaves[2 * G + u]); + } + return __fadd_rn(l.x, l.y); + } + else + { + static_assert(G == 64, "4- or 2-warp programs"); + // 8 chunks of 512 columns: chunks 0..4 real, 5..7 masked. + float2 a = add2(leaves[u], leaves[G + u]); + float2 b = add2(leaves[2 * G + u], leaves[3 * G + u]); + float2 l = add2(add2(a, b), leaves[4 * G + u]); + return __fadd_rn(l.x, l.y); + } +} + +//! Launch forms with the same arithmetic: Var 2 asks for three CTAs an SM (one wave of one-row CTAs up to 3 x SMs), +//! Var 3 has no occupancy bound. launch() takes 3 while the rows fit its resident grid, else 2 if they fit 2's, else 3. +template +struct HcVariant +{ + static constexpr int kMinBlocks = Var == 2 ? 3 : 1; +}; + +template +__global__ void __launch_bounds__(kThreads, HcVariant::kMinBlocks) + hcAllreduceInjectNormKernel(HcInjectNormParams params) +{ + static_assert(kHidden % (8 * G) == 0 || G == 128, "the chunk layout above"); + __shared__ float2 sLeaf[kStreams][kThreads]; + __shared__ float sRed[kStreams][G / 32]; + __shared__ uint4 sW[kStreams][kThreads]; + int const tid = threadIdx.x; + int const lane = tid & 31; + + // No kernel of the step writes the norm weights: stage them before the PDL wait (each thread reads back its own). +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + sW[s][tid] = reinterpret_cast(params.normWeight)[s * kThreads + tid]; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + if constexpr (!TriggerCompletionAtEnd) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif + LamportComm comm(params.workspace, params.rank); + int const rows = params.rows; + int const totAccess = rows * kThreads; + int const clearAccess = static_cast((comm.clear_size + sizeof(float4) - 1) / sizeof(float4)); + + // Push this rank's partial rows, -0.0f stored as +0.0f. + for (int row = blockIdx.x; row < rows; row += gridDim.x) + { + int const idx = row * kThreads + tid; + alignas(16) float val[4]; + *reinterpret_cast(val) = reinterpret_cast(params.partial)[idx]; + +#pragma unroll + for (int i = 0; i < 4; ++i) + { + if (is_neg_zero(val[i])) + { + val[i] = 0.f; + } + } +#pragma unroll + for (int r = 0; r < NRanks; ++r) + { + reinterpret_cast(comm.data_bufs[r])[params.rank * totAccess + idx] + = *reinterpret_cast(val); + } + } + float4 const clearVec = get_neg_zero(); + for (int idx = blockIdx.x * kThreads + tid; idx < clearAccess; idx += gridDim.x * kThreads) + { + reinterpret_cast(comm.clear_buf)[idx] = clearVec; + } + + for (int row = blockIdx.x; row < rows; row += gridDim.x) + { + int const idx = row * kThreads + tid; + // The site's operands (issued before the poll, consumed after it). + uint4 hcw[kStreams]; + float inj[kStreams]; + auto const* hcRow = reinterpret_cast(params.hc) + row * params.hcStride * 2; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + hcw[s] = reinterpret_cast(hcRow + s * kHidden * 2)[tid]; + inj[s] = ldBf16(params.inject, row * params.injectStride + s); + } + [[maybe_unused]] uint4 shw = make_uint4(0, 0, 0, 0); + [[maybe_unused]] float gate = 0.f; + if constexpr (HasShared) + { + shw = reinterpret_cast( + reinterpret_cast(params.shared) + row * params.sharedStride * 2)[tid]; + gate = ldBf16(params.gate, row * params.gateStride); + } + + float4 vals[NRanks]; + bool done = false; + while (!done) + { + done = true; +#pragma unroll + for (int r = 0; r < NRanks; ++r) + { + vals[r] + = ld_global_volatile(&reinterpret_cast(comm.data_bufs[params.rank])[r * totAccess + idx]); + done &= !is_neg_zero(vals[r]); + } + } + // The all-reduce's bits: fp32 sum in rank order, rounded once. + float b[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + float acc = static_cast(reinterpret_cast<__nv_bfloat16 const*>(&vals[0])[i]); +#pragma unroll + for (int r = 1; r < NRanks; ++r) + { + acc += static_cast(reinterpret_cast<__nv_bfloat16 const*>(&vals[r])[i]); + } + b[i] = roundBf16(acc); + } + if constexpr (HasShared) + { + float const sg = sigmoidTriton(gate); + uint32_t const* sw = reinterpret_cast(&shw); +#pragma unroll + for (int p = 0; p < 4; ++p) + { + b[2 * p] = roundBf16(__fmaf_rn(sg, bf16Lo(sw[p]), b[2 * p])); + b[2 * p + 1] = roundBf16(__fmaf_rn(sg, bf16Hi(sw[p]), b[2 * p + 1])); + } + } + // Injection (rounded once, stored) and the per-thread sum-of-squares pairs. + uint32_t ow[kStreams][4]; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + uint32_t const* xw = reinterpret_cast(&hcw[s]); + float e[8]; +#pragma unroll + for (int p = 0; p < 4; ++p) + { + ow[s][p] = packBf16x2( + __fmaf_rn(inj[s], b[2 * p], bf16Lo(xw[p])), __fmaf_rn(inj[s], b[2 * p + 1], bf16Hi(xw[p]))); + e[2 * p] = bf16Lo(ow[s][p]); + e[2 * p + 1] = bf16Hi(ow[s][p]); + } + reinterpret_cast( + reinterpret_cast(params.hcOut) + (row * params.hcOutStride + s * kHidden) * 2)[tid] + = make_uint4(ow[s][0], ow[s][1], ow[s][2], ow[s][3]); + float const lo + = __fadd_rn(__fmaf_rn(e[0], e[0], __fmul_rn(e[2], e[2])), __fmaf_rn(e[4], e[4], __fmul_rn(e[6], e[6]))); + float const hi + = __fadd_rn(__fmaf_rn(e[1], e[1], __fmul_rn(e[3], e[3])), __fmaf_rn(e[5], e[5], __fmul_rn(e[7], e[7]))); + sLeaf[s][tid] = make_float2(lo, hi); + } + __syncthreads(); + if (tid < G) + { + float v[kStreams]; +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + v[s] = threadSsq(sLeaf[s], tid); + } +#pragma unroll + for (int off = 16; off >= 1; off >>= 1) + { +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + v[s] = __fadd_rn(v[s], __shfl_xor_sync(0xffffffffu, v[s], off)); + } + } + if (lane == 0) + { +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + sRed[s][tid / 32] = v[s]; + } + } + } + __syncthreads(); +#pragma unroll + for (int s = 0; s < kStreams; ++s) + { + float tot; + if constexpr (G == 128) + { + tot = __fadd_rn(__fadd_rn(sRed[s][0], sRed[s][2]), __fadd_rn(sRed[s][1], sRed[s][3])); + } + else + { + tot = __fadd_rn(sRed[s][0], sRed[s][1]); + } + float const rstd = rstdTriton(tot, params.eps); + uint4 const wq = sW[s][tid]; + uint32_t const* ww = reinterpret_cast(&wq); + uint32_t nw[4]; +#pragma unroll + for (int p = 0; p < 4; ++p) + { + float const ylo = __fmul_rn(__fmul_rn(bf16Lo(ow[s][p]), rstd), __fadd_rn(1.f, bf16Lo(ww[p]))); + float const yhi = __fmul_rn(__fmul_rn(bf16Hi(ow[s][p]), rstd), __fadd_rn(1.f, bf16Hi(ww[p]))); + nw[p] = packBf16x2(ylo, yhi); + } + reinterpret_cast( + reinterpret_cast(params.normed) + (row * params.normedStride + s * kHidden) * 2)[tid] + = make_uint4(nw[0], nw[1], nw[2], nw[3]); + } + } + + comm.update(static_cast(rows) * kHidden * NRanks * sizeof(__nv_bfloat16)); + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + if constexpr (TriggerCompletionAtEnd) + { + cudaTriggerProgrammaticLaunchCompletion(); + } +#endif +} + +template +int maxGrid() +{ + static int const grid = []() + { + int perSm = 0; + TLLM_CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &perSm, hcAllreduceInjectNormKernel, kThreads, 0)); + return std::max(1, perSm) * get_sm_count(); + }(); + return grid; +} + +template +void launchVar(HcInjectNormParams const& params) +{ + auto kernel = hcAllreduceInjectNormKernel; + int const maxGrid = hc_inject::maxGrid(); + cudaLaunchConfig_t cfg; + cudaLaunchAttribute attribute[1]; + cfg.gridDim = std::min(params.rows, maxGrid); + cfg.blockDim = kThreads; + cfg.dynamicSmemBytes = 0; + cfg.stream = params.stream; + attribute[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attribute[0].val.programmaticStreamSerializationAllowed = tensorrt_llm::common::getEnvEnablePDL() ? 1 : 0; + cfg.attrs = attribute; + cfg.numAttrs = 1; + TLLM_CUDA_CHECK(cudaLaunchKernelEx(&cfg, kernel, params)); +} + +template +void launch(HcInjectNormParams const& params) +{ + int const g3 = hc_inject::maxGrid(); + int const g2 = hc_inject::maxGrid(); + if (params.rows > g3 && params.rows <= g2) + { + launchVar(params); + } + else + { + launchVar(params); + } +} + +template +void dispatchFlags(HcInjectNormParams const& params) +{ + bool const shared = params.shared != nullptr; + if (shared && params.triggerCompletionAtEnd) + { + launch(params); + } + else if (shared) + { + launch(params); + } + else if (params.triggerCompletionAtEnd) + { + launch(params); + } + else + { + launch(params); + } +} +} // namespace hc_inject + +void launchHcAllreduceInjectNorm(HcInjectNormParams const& params) +{ + TLLM_CHECK_WITH_INFO(params.hiddenDim == hc_inject::kHidden && params.hcCount == hc_inject::kStreams, + "hc_allreduce_inject_norm: hidden 2560 and 4 streams only"); + TLLM_CHECK_WITH_INFO(params.rows > 0, "hc_allreduce_inject_norm: no rows"); + TLLM_CHECK_WITH_INFO(params.programWarps == 4 || params.programWarps == 2, + "hc_allreduce_inject_norm: the Triton program's warps must be 4 or 2"); + TLLM_CHECK_WITH_INFO((params.shared == nullptr) == (params.gate == nullptr), + "hc_allreduce_inject_norm: shared and gate come together"); + TLLM_CHECK_WITH_INFO(params.partial != nullptr, "hc_allreduce_inject_norm: no partial"); + if (params.nranks == 2) + { + params.programWarps == 4 ? hc_inject::dispatchFlags<2, 128>(params) : hc_inject::dispatchFlags<2, 64>(params); + } + else if (params.nranks == 4) + { + params.programWarps == 4 ? hc_inject::dispatchFlags<4, 128>(params) : hc_inject::dispatchFlags<4, 64>(params); + } + else + { + TLLM_CHECK_WITH_INFO(false, "hc_allreduce_inject_norm: 2 or 4 ranks only"); + } +} +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h new file mode 100644 index 000000000000..93014f52cfc7 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ + +//! IPC one-shot Lamport all-reduce of a block output (stock `allreduce_fusion_kernel_oneshot_lamport` workspace and +//! protocol) fused with a hyper-connection site's `hcOut = bf16(fma(inject, block, hc))` and grouped RMS norm +//! `normed = bf16((hcOut * rstd) * (1 + w))`; bitwise that all-reduce followed by the Triton `_hc_inject_norm_kernel`. +//! With `shared` and `gate`, the block is first `bf16(fma(sigmoid(gate), shared, routed))`. +struct HcInjectNormParams +{ + int nranks; + int rank; + int rows; + int hiddenDim; //!< 2560 + int hcCount; //!< 4 + int programWarps; //!< 4 or 2: warps of the Triton program whose reduction order is reproduced + void** workspace; //!< the stock all-reduce fusion IPC workspace + void const* partial; //!< [rows, hidden] bf16, contiguous + void const* hc; //!< [rows, hcCount * hidden] bf16 + int64_t hcStride; //!< row stride of hc, in elements + void const* inject; //!< [rows, >= hcCount] bf16 + int64_t injectStride; //!< row stride of inject, in elements + void const* shared; //!< optional [rows, hidden] bf16 + int64_t sharedStride; //!< row stride of shared, in elements + void const* gate; //!< optional [rows, 1] bf16 + int64_t gateStride; //!< row stride of gate, in elements + void const* normWeight; //!< [hcCount * hidden] bf16 + void* hcOut; //!< [rows, hcCount * hidden] bf16 + int64_t hcOutStride; //!< row stride of hcOut, in elements + void* normed; //!< [rows, hcCount * hidden] bf16 + int64_t normedStride; //!< row stride of normed, in elements + float eps; + bool triggerCompletionAtEnd; //!< trigger the dependent launch at the end instead of after the grid dependency wait + cudaStream_t stream; +}; + +//! Launches the fused all-reduce + inject + norm of `params` on `params.stream`. +void launchHcAllreduceInjectNorm(HcInjectNormParams const& params); + +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp new file mode 100644 index 000000000000..c05c080ba7c5 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNormOp.cpp @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/csrc/hcAllreduceInjectNorm.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include + +#include +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace +{ + +//! Returns `(hcOut, normed)` for the block output summed over ranks; see kernels::HcInjectNormParams. +std::vector hcAllreduceInjectNorm(at::Tensor const& partial, at::Tensor workspace, int64_t const rank, + int64_t const nranks, at::Tensor const& hc, at::Tensor const& inject, std::optional const& shared, + std::optional const& gate, at::Tensor const& normWeight, double const eps, int64_t const programWarps, + bool const triggerCompletionAtEnd) +{ + auto const bf16 = at::ScalarType::BFloat16; + TORCH_CHECK(hc.is_cuda(), "hc_allreduce_inject_norm: hc must be a CUDA tensor"); + c10::cuda::CUDAGuard const guard(hc.device()); + for (at::Tensor const* t : std::initializer_list{&partial, &workspace, &inject, &normWeight}) + { + TORCH_CHECK(t->device() == hc.device(), "hc_allreduce_inject_norm: every tensor must be on hc's device"); + } + for (std::optional const* t : std::initializer_list const*>{&shared, &gate}) + { + TORCH_CHECK(!t->has_value() || (*t)->device() == hc.device(), + "hc_allreduce_inject_norm: every tensor must be on hc's device"); + } + TORCH_CHECK(workspace.scalar_type() == at::ScalarType::Long && workspace.is_contiguous(), + "hc_allreduce_inject_norm: workspace must be the stock all-reduce fusion workspace (contiguous int64)"); + TORCH_CHECK(0 <= rank && rank < nranks, "hc_allreduce_inject_norm: rank must be in [0, nranks)"); + int64_t const rows = hc.size(0); + TORCH_CHECK( + partial.dim() == 2 && partial.is_contiguous() && partial.scalar_type() == bf16 && partial.size(0) == rows, + "hc_allreduce_inject_norm: partial must be a contiguous [rows, hidden] bf16 tensor"); + int64_t const hidden = partial.size(1); + TORCH_CHECK(hc.dim() == 2 && hc.size(1) % hidden == 0 && hc.stride(1) == 1 && hc.scalar_type() == bf16, + "hc_allreduce_inject_norm: hc must be [rows, hc_count * hidden] bf16 with unit column stride"); + int64_t const hcCount = hc.size(1) / hidden; + TORCH_CHECK(hc.stride(0) % 8 == 0 && reinterpret_cast(hc.data_ptr()) % 16 == 0, + "hc_allreduce_inject_norm: hc rows must be 16-byte aligned"); + TORCH_CHECK(inject.dim() == 2 && inject.size(0) == rows && inject.size(1) >= hcCount && inject.stride(1) == 1 + && inject.scalar_type() == bf16, + "hc_allreduce_inject_norm: inject must be [rows, >= hc_count] bf16 with unit column stride"); + TORCH_CHECK( + normWeight.numel() == hcCount * hidden && normWeight.is_contiguous() && normWeight.scalar_type() == bf16, + "hc_allreduce_inject_norm: norm_weight must be [hc_count * hidden] bf16"); + TORCH_CHECK(shared.has_value() == gate.has_value(), "hc_allreduce_inject_norm: shared and gate come together"); + if (shared.has_value()) + { + TORCH_CHECK(shared->dim() == 2 && shared->size(0) == rows && shared->size(1) == hidden && shared->stride(1) == 1 + && shared->stride(0) % 8 == 0 && reinterpret_cast(shared->data_ptr()) % 16 == 0 + && shared->scalar_type() == bf16, + "hc_allreduce_inject_norm: shared must be [rows, hidden] bf16 with 16-byte aligned rows"); + TORCH_CHECK(gate->dim() == 2 && gate->size(0) == rows && gate->scalar_type() == bf16, + "hc_allreduce_inject_norm: gate must be [rows, 1] bf16"); + } + auto hcOut = at::empty({rows, hcCount * hidden}, hc.options()); + auto normed = at::empty({rows, hcCount * hidden}, hc.options()); + + kernels::HcInjectNormParams params; + params.nranks = static_cast(nranks); + params.rank = static_cast(rank); + params.rows = static_cast(rows); + params.hiddenDim = static_cast(hidden); + params.hcCount = static_cast(hcCount); + params.programWarps = static_cast(programWarps); + params.workspace = reinterpret_cast(workspace.mutable_data_ptr()); + params.partial = partial.data_ptr(); + params.hc = hc.data_ptr(); + params.hcStride = hc.stride(0); + params.inject = inject.data_ptr(); + params.injectStride = inject.stride(0); + params.shared = shared.has_value() ? shared->data_ptr() : nullptr; + params.sharedStride = shared.has_value() ? shared->stride(0) : 0; + params.gate = gate.has_value() ? gate->data_ptr() : nullptr; + params.gateStride = gate.has_value() ? gate->stride(0) : 0; + params.normWeight = normWeight.data_ptr(); + params.hcOut = hcOut.mutable_data_ptr(); + params.hcOutStride = hcOut.stride(0); + params.normed = normed.mutable_data_ptr(); + params.normedStride = normed.stride(0); + params.eps = static_cast(eps); + params.triggerCompletionAtEnd = triggerCompletionAtEnd; + params.stream = at::cuda::getCurrentCUDAStream(hc.get_device()); + kernels::launchHcAllreduceInjectNorm(params); + return {hcOut, normed}; +} + +} // namespace + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def( + "hc_allreduce_inject_norm(" + "Tensor partial," + "Tensor workspace," + "int rank," + "int nranks," + "Tensor hc," + "Tensor inject," + "Tensor? shared," + "Tensor? gate," + "Tensor norm_weight," + "float eps," + "int program_warps," + "bool trigger_completion_at_end) -> Tensor[]"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("hc_allreduce_inject_norm", &TRTLLM_MV2_NS::hcAllreduceInjectNorm); +} diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py new file mode 100644 index 000000000000..b41efcb4168b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/fake_ops.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fake (meta) implementations of this target's native ops, built from its csrc/. + +The loader imports this module right after the target is found in the family +library, so a fake for an op the library does not define fails the load. +""" + +import torch + +from tensorrt_llm._torch.modeling_v2._native import namespace_of + +_NS = namespace_of(__name__) + + +def _host_query(qualname: str): + """Ops without tensor arguments answer a host-side question at setup; tracing never runs them.""" + + def fake(*args, **kwargs): + raise NotImplementedError(f"{qualname} is a setup-time query; call it outside tracing") + + return torch.library.register_fake(qualname)(fake) + + +_host_query(f"{_NS}::fp8_block_moe_tpb_workspace_bytes") + + +@torch.library.register_fake(f"{_NS}::fp8_block_moe_tpb") +def _( + logits, + xq, + xsf, + w1, + s1, + w2, + s2, + rank, + workspace, + enable_pdl=False, + sh_x=None, + sh_gu=None, + sh_gl=None, + sh_dn=None, +): + return logits.new_empty((logits.shape[0], xq.shape[1])) + + +@torch.library.register_fake(f"{_NS}::hc_allreduce_inject_norm") +def _( + partial, + workspace, + rank, + nranks, + hc, + inject, + shared, + gate, + norm_weight, + eps, + program_warps, + trigger_completion_at_end, +): + # Two new contiguous [rows, hc_count * hidden] tensors (`hc` may carry a padded row stride). + shape = (hc.shape[0], hc.shape[1]) + return [hc.new_empty(shape), hc.new_empty(shape)] + + +@torch.library.register_fake(f"{_NS}::fp8_block_scale_moe_front") +def _( + routing_logits, + hidden_states_bf16, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + tile_n, + config, +): + # A new [rows, hidden] bf16 tensor; hidden_states / hidden_states_scale are written in place. + return hidden_states.new_empty(hidden_states.shape, dtype=torch.bfloat16) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py new file mode 100644 index 000000000000..6574adcadfda --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/modeling.py @@ -0,0 +1,7267 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""ModelingV2 target: Qwen3.8-Flash-Next-FP8 (hf-236dfdf_orig) / sm_103 / unified. + +One target for every layout `routing.py` admits for this checkpoint in one node (`_FP8_LAYOUTS`): +tp1; tp2, tep2 (attention TP2) and dep2 (attention DP) at two ranks; tp4, tep4 (attention TP4) +and dep4 (attention DP) at four. The core holds this rank's share of the weights, the derived +operands and the per-manager pool views; `PrefillTarget` (any batch with context rows) and +`DecodeTarget` (generation only) run a step over it. + +The checkpoint is block FP8 (`FP8_BLOCK_SCALES`): only the routed experts are quantized (e4m3, +one scale per 128x128 weight block); everything else is bf16 (`weights.py`). Per rank: + +* **Attention and Gated-DeltaNet heads** are split over attention TP `t` (1 at tp1 and under + attention DP; `weights.rank_dims`, `pool.pool_geometry`): 24 / t q heads, 16 / t GDN key and + 48 / t value heads, and 2 / t KV heads (one, repeated on two ranks, at t = 4). The QSA indexer + is replicated. +* **The mixer output is a partial sum** under attention TP (o_proj and out_proj read this rank's + heads); one plain sum (`catalog/comm/allreduce`) completes it before the hyper-connection + inject, which every rank runs on the same rows. Under attention DP and at tp1 nothing is summed. +* **Routed experts** are whole-expert windows: rank `r` of `t` holds experts + `[512r / t, 512(r + 1) / t)` at the whole 640-row intermediate (`weights.ExpertWindow`), + whatever the engine's `Mapping` says of MoE TP and EP, because 640 / t rows do not take the + 128-row FP8 weight block. Under attention TP the plain sum completes the window partials. + Under attention DP each rank's rows are gathered (`catalog/comm/allgather`, ragged when the + counts differ), run through the window and reduce-scattered back + (`catalog/comm/reducescatter`; `_expert_sum_op`, or an all-to-all with + `MOE_COMM_METHOD_ENV=NVLINK_ONE_SIDED`). +* **Replicated:** the router, the bf16 shared expert, the hyper-connections, PLE and the + embedding run on this rank's rows. `lm_head` is the shell's: vocab-parallel under attention + TP, whole under attention DP and at tp1. + +There is **no** `model.language_model.norm.weight`: the stack-final mixer is the final norm. + +**QSA sparse attention.** The index plane is written on every step so a request can cross +`dense_seq_len_threshold` later without rebuilding its prefix; above it the sparse core (index +scores, block top-k and expansion, sparse GQA) replaces the dense op. The raw index key is +stored before anything normalizes it: the compression applies the norm and the rotation itself. + +**KV storage.** The paged K/V pool holds bf16, fp8-e4m3 under `kv_cache_config.dtype: fp8` +(`QM_FP8_KV_CACHE`) or NVFP4 under `nvfp4` (`QM_NVFP4_KV_CACHE`: e2m1 with e4m3 block scales in +a second pool, second-level scales 1.0; paged-context attention only). The QSA index and +position planes keep their own dtypes under all three. + +**Breakable prefill CUDA graph.** With `prefill_cuda_graph_backend: breakable` the engine +captures the core's `forward` (the body, not the shell), pads a step up to its bucket and +replays it without calling the body. The shell therefore calls `begin_step` before the body on +every step: the asserts, the cache-manager bind, the target choice and the prefill target's host +bookkeeping live there. The graph breaks (the vision splice, the PLE block, each layer's +attention core and GDN recurrence) each write rows `[:n]` of a fixed output and zero rows `[n:]`. + +**Host offload.** The PLE n-gram table lives in device-addressable host memory: one copy per +rank, per server or per node (`qwen4_exp/ple_host_table.py`). + +**One-model MTP-Eagle.** With a `speculative_config` the core also holds the MTP module +(`MTPLayer`), replayed `max_draft_len` times by the spec worker; the trunk runs every generation +request's `max_draft_len + 1` verify rows. + +* The draft layer is one more full-attention QSA layer, cut like a trunk one. Under attention TP + it has a separate draft KV cache; under attention DP the trunk's manager maps it too + (`_bind_draft_cache_manager`). Its experts are the trunk's window, completed the trunk's way. +* Under attention DP `_dp_pad_rows` pads each draft step's rows by the group's row counts, so + every rank issues the same collectives at every draft step. +* Verify GDN rows run the CuTe cached replay (`ops/ssm/gdn_cached_replay_cute__verify`, the + stock Triton replay's contract): it commits the history in the kernel below + `CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE` requests and leaves it to the manager from there on. +* What a verify row writes that the runtime does not roll back (the PLE n-gram history and conv + state, every attention layer's QSA index-K plane and the shared position plane) is staged + during the forward and rewritten to the accepted prefix by `SpeculativeRollback`, which the + spec worker calls after its draft loop. +* IndexShare (`index_share_for_mtp_iteration`): the draft-extend step captures each request's + selection and the later draft steps rebuild theirs from it (`qsa_mtp_reuse_topk`). + +**Images and video.** The shell runs the replicated bf16 vision tower (`vision.VisionTower`) +over the step's media and `begin_step`'s splice break writes the rows over the token embedding; +`validate_request` refuses media the tower would assert on. + +Integration facts this file depends on: + +1. The engine rebuilds the composite config and normalizes its text tower + (`config_utils.load_pretrained_config`). The shell adopts that text config as + `model_config.pretrained_config` before building the core, because the recurrent-state and + PLE pool geometry are read from it. +2. `position_ids` arrives in the mRoPE layout (`[3, 1, T]`) even for text-only batches. The + main rope reads all three rows; the QSA index plane takes coordinate 0 only. +""" + +from __future__ import annotations + +import contextlib +import math +import os +from typing import TYPE_CHECKING, NamedTuple + +import torch +from torch import nn +from transformers import PretrainedConfig + +from tensorrt_llm._ipc_utils import can_access_peer +from tensorrt_llm._torch.attention.backends.fmha.phased import get_spec_decoding_position_offsets +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.distributed.allreduce_helper import CustomAllReduceHelper +from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2._router_index import ( + NoTarget, + engine_kv_cache_quant_algo, + step_contract_enabled, +) +from tensorrt_llm._torch.modeling_v2.catalog.attention.fused_qk_norm_rope import fused_qk_norm_rope +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import ( + QM_FP8_KV_CACHE, + QM_NVFP4_KV_CACHE, + thop_attention, +) +from tensorrt_llm._torch.modeling_v2.catalog.comm.allgather import allgather +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import allreduce, allreduce_group_args +from tensorrt_llm._torch.modeling_v2.catalog.comm.reducescatter import reducescatter +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.catalog.graph.eager_on_graph import eager_on_graph +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( + fused_sigmoid_mul_inplace, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.silu_and_mul_rows import ( + silu_and_mul_rows, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_canonical_selection import ( + qsa_canonical_selection, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_expand_block_indices import ( + qsa_expand_block_indices, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_mtp_reuse_topk import ( + qsa_mtp_reuse_topk, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + _takes_prefill_query_tile, + qsa_paged_index_scores, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + _takes_visible_split as _scores_take_visible_split, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_kv_store_nvfp4 import ( + qsa_paged_kv_store_nvfp4, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa_nvfp4 import ( + qsa_paged_sparse_gqa_nvfp4, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa_nvfp4_decode import ( # noqa: E501 + qsa_paged_sparse_gqa_nvfp4_decode, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_prefill_compress import ( + qsa_prefill_compress, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( + plane_table as _plane_table, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( + qsa_rollback_gather, + qsa_rollback_restore, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + orders_rows as _topk_expand_orders_rows, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + qsa_topk_expand_decode, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + supports as _topk_expand_supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_select import ( + qsa_topk_select, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.spec_rollback_commit import ( + spec_rollback_commit, + spec_rollback_stage, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + BK as _HC_RESCALE_BK, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + _pick_split as _hc_rescale_split, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_first_rescale_tc import ( + qwen4_exp_hc_first_rescale_tc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + _PDL_TRIGGER as _HC_INJECT_PDL_TRIGGER, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + qwen4_exp_hc_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_rstd import ( + qwen4_exp_hc_inject_rstd, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_lowrank_act import ( + qwen4_exp_hc_lowrank_act, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_up_gate_mean import ( + qwen4_exp_hc_up_gate_mean, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_up_gate_mean_rstd import ( + qwen4_exp_hc_up_gate_mean_rstd, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_post_inject import ( + qwen4_exp_post_inject, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.fused_sigmoid_gate_mul_add import ( + fused_sigmoid_gate_mul_add, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( + renorm_moe_routing_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.flashinfer_gemma_rmsnorm import ( + flashinfer_gemma_rmsnorm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( + rms_norm_gated_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_gate_conv_inject_norm import ( + ple_gate_conv_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed import ple_ngram_embed +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( + bf16_to_fp32, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.fp8_quantize_1x128 import ( + fp8_quantize_1x128, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( + causal_conv1d_update, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.cu_seqlens_from_seq_lens import ( + cu_seqlens_from_seq_lens, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_decode_rows_gated_norm import ( + gdn_decode_rows, + gdn_decode_rows_gated_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( + gdn_prefill_conv_post, +) +from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM +from tensorrt_llm._torch.modules.fla.cached_replay import CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE +from tensorrt_llm._torch.modules.linear import TensorParallelMode +from tensorrt_llm._torch.modules.logits_processor import LogitsProcessor +from tensorrt_llm._torch.pyexecutor.breakable_cuda_graph import ( + eager_on_graph as _keyed_eager_on_graph, +) +from tensorrt_llm._torch.pyexecutor.model_loader import effective_mamba_ssm_cache_dtype +from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks +from tensorrt_llm._torch.speculative import get_spec_worker, should_use_separate_draft_kv_cache +from tensorrt_llm._torch.utils import is_gdn_flashinfer_replay_enabled, is_gdn_replay_enabled +from tensorrt_llm._utils import mpi_disabled +from tensorrt_llm.functional import AllReduceFusionOp, AllReduceStrategy +from tensorrt_llm.logger import logger +from tensorrt_llm.quantization.mode import QuantAlgo + +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + +from .. import ple_host_table as _ple_host +from . import pool as _pool +from . import vision as _vision +from . import weights as _weights +from .ops.attention.draft_continue_rows__decode import draft_continue_select, draft_rows_gather +from .ops.attention.qsa_fp8_pages_to_bf16__prefill import qsa_fp8_pages_to_bf16 +from .ops.attention.qsa_index_scores_cute__prefill import qsa_index_scores_prefill +from .ops.attention.qsa_index_scores_cute__prefill import supports as _index_scores_cute_supports +from .ops.attention.qsa_index_scores_groups_cute__decode import ( + index_scores_groups as index_scores_groups_cute, +) +from .ops.attention.qsa_index_scores_groups_cute__decode import serves as _index_groups_cute_serves +from .ops.attention.qsa_index_split_plan__decode import index_split_plan +from .ops.attention.qsa_paged_index_store import qsa_paged_index_store +from .ops.attention.qsa_paged_kv_store import qsa_paged_kv_store +from .ops.attention.qsa_paged_sparse_gqa import qsa_paged_sparse_gqa +from .ops.attention.qsa_sparse_gqa_grouped_cute__decode import qsa_sparse_gqa_grouped_cute +from .ops.attention.qsa_sparse_gqa_rowtile__prefill import qsa_sparse_gqa_rowtile +from .ops.attention.qsa_sparse_gqa_splitk_cute__decode import qsa_sparse_gqa_splitk_cute +from .ops.attention.qsa_spec_step_metadata__decode import ( + SpecStepMetadataPrologue, + take_step_positions, +) +from .ops.attention.qsa_topk_expand_cluster__decode import qsa_topk_expand_cluster +from .ops.attention.qsa_topk_expand_cluster__decode import supports as _topk_expand_cluster_supports +from .ops.attention.qsa_topk_rows_cute__prefill import qsa_topk_rows +from .ops.attention.qsa_topk_rows_cute__prefill import supports as _topk_rows_supports +from .ops.attention.qsa_triton_kernels__stable_jit import splitk_launch_config as _gqa_splitk_config +from .ops.attention.qsa_triton_kernels__stable_jit import warm_decode_buckets +from .ops.attention.qwen4_exp_attn_decode_prologue__decode import qwen4_exp_attn_decode_prologue +from .ops.attention.spec_rollback_begin__verify import spec_rollback_begin +from .ops.comm.hc_allreduce_inject_norm__tp_sum import fits as _hc_ar_inject_operands_fit +from .ops.comm.hc_allreduce_inject_norm__tp_sum import hc_allreduce_inject_norm +from .ops.comm.lamport_allgather__index_split import build as _selection_lamport_build +from .ops.comm.lamport_allgather__index_split import lamport_allgather +from .ops.comm.mnnvl_allreduce__tp_sum import mnnvl_tp_sum, mnnvl_workspace_args +from .ops.comm.moe_a2a__dp_dispatch_combine import build_strategy as _build_moe_a2a +from .ops.comm.moe_a2a__dp_dispatch_combine import moe_a2a_combine, moe_a2a_dispatch +from .ops.embedding.embed_rows__decode import embed_rows +from .ops.gemm.low_m_bf16_mm__decode import low_m_bf16_mm +from .ops.hc.qwen4_exp_hc_decode_rows_tc__decode import MAX_ROWS as _HC_ROWS_TC_ROWS +from .ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( + qwen4_exp_hc_decode_rows_first, + qwen4_exp_hc_decode_rows_mix, +) +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import MAX_ROWS as _HC_TC_FIRST_ROWS +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import precompile as hc_tc_first_precompile +from .ops.hc.qwen4_exp_hc_decode_tc_first__decode import qwen4_exp_hc_decode_tc_first +from .ops.moe.fp8_block_scale_moe_runner import ( + fp8_block_scale_moe_front, + fp8_block_scale_moe_runner, +) +from .ops.moe.moe_front_router_quant__decode import MAX_ROWS as _MOE_FRONT_ROWS +from .ops.moe.moe_front_router_quant__decode import moe_front_router_quant +from .ops.moe.moe_prefill_grouped__prefill import ( + moe_prefill_grouped, + prefill_grouped_ready, + warm_prefill_grouped, +) +from .ops.moe.moe_tpb_decode__decode import MAX_ROWS as _MOE_TPB_ROWS +from .ops.moe.moe_tpb_decode__decode import BlockExperts as _BlockExperts +from .ops.moe.moe_tpb_decode__decode import SharedBlocks as _TpbShared +from .ops.moe.moe_tpb_decode__decode import moe_tpb_decode +from .ops.moe.mtp_moe_combine_inject__decode import mtp_moe_combine_inject +from .ops.ssm.causal_conv1d_token_major__verify import causal_conv1d_token_major +from .ops.ssm.flashinfer_chunk_gated_delta_rule import INPLACE_MAX_TOKENS as _GDN_INPLACE_MAX_TOKENS +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _chunk_carried as _gdn_chunk_carried +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _chunk_cp_inplace as _gdn_chunk_cp_inplace +from .ops.ssm.flashinfer_chunk_gated_delta_rule import _reaches_cp as _gdn_reaches_cp +from .ops.ssm.flashinfer_chunk_gated_delta_rule import ( + chunk_split_first_pass, + cp_chunk_len_for, + flashinfer_chunk_gated_delta_rule, + split_takes_cp, +) +from .ops.ssm.gdn_cached_replay_cute__verify import GEOMETRIES as _GDN_REPLAY_CUTE_GEOMETRIES +from .ops.ssm.gdn_cached_replay_cute__verify import conv_fold_supported as gdn_conv_fold_supported +from .ops.ssm.gdn_cached_replay_cute__verify import gdn_cached_replay_cute +from .ops.ssm.gdn_cached_replay_cute__verify import precompile as gdn_replay_precompile +from .ops.ssm.gdn_in_proj_conv__verify import CONV_DIM as _GDN_IN_PROJ_CONV_DIM +from .ops.ssm.gdn_in_proj_conv__verify import N_OUT as _GDN_IN_PROJ_N_OUT +from .ops.ssm.gdn_in_proj_conv__verify import fused_tactic as _gdn_in_proj_conv_fused_tactic +from .ops.ssm.gdn_in_proj_conv__verify import gdn_in_proj_conv +from .ops.ssm.gdn_inforward_tail import gdn_inforward_conv_swap, gdn_inforward_tail + +# The certified GPU architecture, checked again because a direct instantiation bypasses routing. +_SM = (10, 3) + +# `moe_config.backend` requests served by the trunk's routed experts (`fp8_block_scale_moe_runner`). +_TRUNK_MOE_BACKENDS = ("AUTO", "TRTLLM") + +# The engine's resolved quantization of the one checkpoint this target loads. +_QUANT_ALGO = QuantAlgo.FP8_BLOCK_SCALES + +# Qwen3.8-Flash-Next's (layers, hidden, experts, attention heads, linear value heads, vocab), and +# its vision tower's (depth, hidden, heads, merge, out_hidden, num_pos, deepstack_indexes, act). +_TEXT_SHAPE = (48, 2560, 512, 24, 48, 248320) +_VISION_SHAPE = (27, 1152, 16, 2, 2560, 2304, (), "gelu_pytorch_tanh") + +# The checkpoint's settings that change what it computes without changing a weight's shape, +# at the values the target implements: a fine-tune of the same shape that changed one would +# load and run with wrong logits and no error. Read off the text config as the target reads +# them (`_text_settings`). +_TEXT_SETTINGS = { + # The routed experts: Renormalize routing over the top 10 and a SiLU activation. + "norm_topk_prob": True, + "num_experts_per_tok": 10, + "hidden_act": "silu", + # Attention: two KV heads of 256, no projection bias, a sigmoid output gate on every head + # (absent `attn_output_gate` reads as gated), and an output head of its own. + "num_key_value_heads": 2, + "head_dim": 256, + "attention_bias": False, + "output_gate_type": "sigmoid", + "attn_output_gate": True, + "tie_word_embeddings": False, + # Rope: a quarter of the head (64 dims), theta 1e7, interleaved mRoPE over sections + # 11 / 11 / 10, unscaled. + "rope_theta": 1e7, + "partial_rotary_factor": 0.25, + "mrope_section": (11, 11, 10), + "mrope_interleaved": True, + "rope_factor": None, + # The MTP module: one full-attention layer on the trunk's embedding that reads the end of + # the stack (`mtp_use_hidden_state_from_layer: null`). The `mtp.` fields are the `mtp` + # block's, a mapping or a config object; a field it omits reads as the value here. (Its + # own rope theta the target reads.) + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": False, + "mtp.num_hidden_layers": 1, + "mtp.layer_types": ("full_attention",), + "mtp.mtp_use_hidden_state_from_layer": None, +} + +# Rows per request of the widest verify step (`max_draft_len + 1`). +_MAX_VERIFY_WIDTH = _pool.MAX_VERIFY_WIDTH + +_SSM_STATE_DTYPES = _pool.SSM_STATE_DTYPES + +# QSA backend metadata buffers rebuilt every step. Their absence means the engine did not +# select the QSA metadata and the index plane would never be written. +_QSA_FIELDS = ( + "qsa_block_table", + "qsa_req_idx_per_token", + "qsa_logical_positions", + "qsa_sequence_lengths", + "qsa_visible_blocks", + "qsa_topk_indices", + "qsa_topk_row_starts", +) + +# Fields read off `attn_metadata.mamba_metadata`. Absence is silent: a smoke run passes +# with no recurrent state at all. +_MAMBA_FIELDS = ( + "state_indices", + "has_initial_states", + "query_start_loc", + "query_start_loc_long", +) + +# thop_attention's constant arguments: in-kernel RoPE off (the rope runs before the op), +# causal mask, no MLA, sparse or cross surface. +_CALL_CONSTANTS = dict( + output_sf=None, + k=None, + v=None, + is_fused_qkv=True, + update_kv_cache=True, + attention_input_type=0, + is_mla_enable=False, + mask_type=1, + q_scaling=1.0, + attention_sinks=None, + position_embedding_type=0, + rotary_inv_freq=None, + rotary_cos_sin=None, + rope_dim=0, + rope_base=10000.0, + rope_scale_type=0, + rope_scale=1.0, + rope_short_m_scale=1.0, + rope_long_m_scale=1.0, + rope_max_positions=1024, + rope_original_max_positions=1024, + out_scale=None, + latent_cache=None, + q_pe=None, + q_lora_rank=None, + kv_lora_rank=None, + qk_nope_head_dim=None, + qk_rope_head_dim=None, + v_head_dim=None, + rope_append=None, + chunked_prefill_buffer_batch_size=1, + attention_chunk_size=None, + softmax_stats_tensor=None, + sparse_kv_indices=None, + sparse_kv_offsets=None, + sparse_attn_indices=None, + sparse_attn_offsets=None, + sparse_attn_indices_block_size=0, + mrope_rotary_cos_sin=None, + mrope_position_deltas=None, + helix_position_offsets=None, + helix_is_inactive_rank=None, + cross_kv=None, + relative_attention_bias=None, + relative_attention_max_distance=0, + kv_norm_weight=None, + kv_norm_eps=1e-6, + skip_correction_threshold=0.0, +) + +_ROUTING_METHOD_RENORM = 1 +_ACT_TYPE_SWIGLU = 0 +_PAD_SLOT_ID = -1 +#: Whether a decode step's `low_m_bf16_mm` projections launch with PDL (the FlashInfer kernels +#: stream the weight before waiting on the producer; the cuBLAS branch ignores it). Off with +#: TRTLLM_ENABLE_PDL=0. +_DECODE_PROJ_PDL = get_env_enable_pdl() +#: (request x value head) pairs a verify step up to which the CuTe cached replay launches with PDL +#: and the conv before it triggers at its start, so the replay streams the checkpoint and history +#: (nothing earlier in the step writes them) while the conv runs. +_GDN_REPLAY_PDL_MAX_HEAD_ROWS = 24 +#: A tp2/tep2 rank's GDN heads (8 key / 24 value), and the HT generation batches (from this many +#: requests: the partitioned replay) whose replay launch comes from `_GDN_REPLAY_HT_LAUNCH`. +_GDN_REPLAY_HT_GEOMETRY = (8, 24) +_GDN_REPLAY_HT_MIN_REQUESTS = 16 +#: `(max requests, pdl, CuTe replay mapping or None for its default)` for a tp2/tep2 HT verify +#: batch (PDL off throughout with TRTLLM_ENABLE_PDL=0). The mappings +#: (`gdn_cached_replay_cute.default_config`'s knobs) only move where a tile is loaded and which +#: CTA runs it, so every choice is bitwise the default launch. +_GDN_REPLAY_HT_LAUNCH = ( + (24, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (40, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=1, minb=6)), + (56, get_env_enable_pdl(), dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), + (80, get_env_enable_pdl(), None), + (1 << 30, False, dict(tpc=8, warps=4, pf=0, minb=5, early_acc=True)), +) +#: Requests up to which a tp2/tep2 HT verify batch folds the conv into the CuTe replay +#: (`conv_fold=`, bitwise `causal_conv1d_token_major`); the replay writes the per-row windows and +#: the acceptance promote advances the pool window from them. +_GDN_VERIFY_CONV_FOLD_MAX_REQUESTS = 24 +#: Whether a verify step's conv (`causal_conv1d_token_major`) is a PDL dependent of the input +#: projection: it reads the slots and the conv pool before `griddepcontrol.wait`, the q|k|v after. +#: Off with TRTLLM_ENABLE_PDL=0. +_GDN_CONV_PDL_WAIT = get_env_enable_pdl() +#: Whether a decode step's QSA index scoring (`_decode_index_selection`) launches with PDL: its +#: kernels wait before their first read and the decode prologue triggers at its start. +_INDEX_SCORES_PDL = get_env_enable_pdl() +#: Request groups (this rank's share) from which verify index scoring takes this target's CuTe +#: kernel (`qsa_index_scores_groups_cute__decode`) over the family's Triton one. It sums in another +#: order (~7e-8 relative), so a row's top-k can differ only at a near-tie. +_INDEX_SCORES_CUTE_MIN_GROUPS = 8 +_SOFTPLUS_THRESHOLD = 20.0 +_L2_NORM_EPS = 1e-6 +_PLE_CLAMP_MIN = 1e-6 +#: Rows up to which a hyper-connection mixer's first half is one tcgen05 launch +#: (`qwen4_exp_hc_decode_tc_first`, two 128-row MMA tiles) instead of cuBLAS + lowrank_act. +_HC_TC_FIRST_MAX_ROWS = _HC_TC_FIRST_ROWS +#: Row counts over which a decode or draft hyper-connection site (`_hc_site`) runs whole as the +#: two `qwen4_exp_hc_decode_rows_*` launches. A dispatch threshold on a per-capture row count, not +#: a buffer bound; zero rows keep the other path. +_HC_ROWS_TC_MIN_ROWS = 1 +_HC_ROWS_TC_MAX_ROWS = _HC_ROWS_TC_ROWS +#: Under attention TP, a decode block's partial output of `_HC_AR_INJECT_MIN_ROWS` .. +#: `_HC_RSTD_MIN_ROWS - 1` rows skips the TP sum and the next hyper-connection site sums it in its +#: injection (`ops/comm/hc_allreduce_inject_norm__tp_sum`, bitwise the all-reduce followed by +#: `_hc_inject_norm_launch`). Smaller steps use the MNNVL sum or the decode-rows site. +_HC_AR_INJECT_MIN_ROWS = 33 +#: Prefill steps of at least this many rows store the residual and a `[T, C]` fp32 rstd +#: (`qwen4_exp_hc_inject_rstd`) instead of the `[T, C*H]` normed stream, which `_hc_mixer_rstd` +#: forms on chip. A dispatch threshold on the step's rows, not a buffer bound. +_HC_RSTD_MIN_ROWS = 3072 +#: Warps a program of `_hc_inject_norm_launch` takes below `_HC_INJECT_FEW_WARPS_MIN_ROWS` rows, +#: and two from there (as `qwen4_exp_hc_inject_norm` does). Dispatch thresholds, not buffer bounds. +_HC_INJECT_WARPS = 4 +_HC_INJECT_FEW_WARPS_MIN_ROWS = 512 +#: Rows up to which `qwen4_exp_hc_up_gate_mean` runs its epilogue on eight warps instead of four +#: (at one tile a CTA the gate's MUFU work is the critical path). Same bits. +_HC_UGM_WIDE_MAX_ROWS = 384 +#: Whether `_hc_inject_norm_launch` launches with PDL and waits for its predecessor in the kernel +#: (`griddepcontrol.wait`); off with TRTLLM_ENABLE_PDL=0. +_HC_INJECT_PDL_WAIT = _HC_INJECT_PDL_TRIGGER +_QSA_POSITION_AXES = _pool.QSA_POSITION_AXES +# The compressed index keys rotate at coordinate 0 alone, as the index query does. +_QSA_MROPE_SECTION = None +# The routed experts' activation scales are the plain fp32 `amax / 448`, not the UE8M0 +# power-of-two form: the runner reads them so (`fp8_block_scale_moe_runner`). +_MOE_USE_UE8M0 = False +# Rows per FP8 runner call under attention DP: the rows an attention-DP group gathers (up to +# its size times a rank's) run in chunks of it. +_MOE_MAX_ROWS = 8192 +#: A decode step's routed experts up to this many rows enter the runner from one front-end launch +#: (`moe_front_router_quant`: router logits + 1x128 quantization). Past 8 rows the front end +#: contends with the concurrent shared expert. +_MOE_FRONT_MAX_ROWS = min(8, _MOE_FRONT_ROWS) +#: Past `_MOE_FRONT_MAX_ROWS` an attention-TP decode step hands the runner the router's bf16 +#: logits to route itself (bitwise `renorm_moe_routing_op`), forking the shared expert only once +#: the router and quantization are queued (`before_runner`) so it overlaps the runner's GEMMs. +#: Up to this many rows (the routing kernel's bound) the runner routes and quantizes in one launch +#: (`fp8_block_scale_moe_front`) instead of the 8-CTA cluster kernel; the layer's output is bitwise +#: the cluster path's (only a slot's offset within its expert may differ; nothing reads it). +_MOE_FAST_FRONT_MAX_ROWS = 1024 +#: With the block-major expert copy (`weights.MOE_TPB_ENV`, attention TP4), decode steps up to this +#: many rows take `moe_tpb_decode` over this rank's 128-row intermediate blocks. From +#: `_MOE_TPB_SHARED_MIN_ROWS` rows it also folds in this rank's shared-expert slice (`shared=`): +#: the launch holds nearly every SM, so a side-stream shared expert would wait behind it. +_MOE_TPB_MAX_ROWS = min(8, _MOE_TPB_ROWS) +#: The fold's smallest step: at a 3-row verify the side stream's shared expert mostly hides behind +#: the routed launch and folding it in costs more. +_MOE_TPB_SHARED_MIN_ROWS = 5 +#: Rows per FP8 runner call under attention TP (or at one rank): the runner is certified to 16384 +#: rows (past its last tuning bucket), and one call reads every expert's weights once. +_MOE_MAX_ROWS_TP = 16384 +#: Smallest rows of a runner call under attention TP that run the window as two grouped FP8 +#: block-scale GEMMs (`moe_prefill_grouped`: no host synchronization, CUDA graph capturable) +#: instead of the trtllm-gen runner. 0 keeps the runner at every row count (tests set it). +_MOE_PREFILL_GROUPED_MIN_ROWS = 2048 +#: The expert id a padding row's slots are routed to: outside every rank's window, so the +#: runner skips the slot (`_routed_experts`). +_NO_EXPERT = -1 +#: Rows up to which `_mtp_front` runs its embedding branch on the side stream; from 16 rows the +#: hidden branch's GEMM (4 streams a row) fills the GPU and the concurrent branch only slows it. +_MTP_FRONT_FORK_MAX_ROWS = 8 +#: Context rows from which an attention-TP rank scores and selects only its share of a prefill +#: step's rows and gathers the others' (`_split_context_selection`), instead of every rank +#: repeating the whole unsplit index selection. Every rank holds the same bits. +_SELECTION_SPLIT_MIN_ROWS = 1024 +#: A split selection cuts rows where the ranks' summed costs are equal (`_selection_cuts`); a row +#: costs its window (`position + 1` tokens) plus this many tokens of fixed work (a measured fit). +_SELECTION_BALANCE_ROW_TOKENS = 12500 +#: Rows a balanced cut is rounded to (the index scores' query tile). +_SELECTION_CUT_ALIGN = 64 +#: Pieces a rank's share of a split selection is processed in: each piece's gather runs on the +#: comm stream while the next is scored, so only the last gather is exposed. Same bits. +_SELECTION_PIECES = 4 +#: Rows from which a context chunk's selection over a wide score table takes the row top-k kernel +#: (`qsa_topk_rows`) instead of the family's wide kernel; same bits. +_TOPK_ROWS_MIN_ROWS = 256 +#: Elements of one score table: the index scores write their output rows and the selection +#: kernels address theirs with int32 arithmetic (row times the table's width), so a call scores +#: fewer rows than this over the table's columns: at `max_seq_len` 262144 (65568 columns) a +#: context part past 32752 rows is scored in chunks. +_SCORE_TABLE_ELEMENTS = 2**31 + + +def _context_index_scores(q, index_cache, block_table, positions, requests, tpb, cr, context_rows): + """`qsa_paged_index_scores` (visible blocks only) of a context part of a step's rows. + + Takes the tcgen05 kernel `qsa_index_scores_prefill` where it serves the call (same bits). + """ + if _takes_prefill_query_tile(q, True, context_rows, False) and _index_scores_cute_supports( + q, index_cache, tpb, cr + ): + return qsa_index_scores_prefill(q, index_cache, block_table, positions, requests, tpb, cr) + return qsa_paged_index_scores( + q=q, + index_cache=index_cache, + block_table=block_table, + query_positions=positions, + request_indices=requests, + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=context_rows, + prefill_query_tile=True, + wide_split=True, + ) + + +#: Rows a piece of a pipelined selection keeps at least (whole query tiles of the index scores). +_SELECTION_PIECE_MIN_ROWS = 1024 +#: Breakable-graph buckets of at most this many rows replay a context-only step's GDN and sparse +#: attention breaks as CUDA graphs keyed by what fixes their launches (`_gdn_graph_key`, +#: `_attn_graph_key`); a key the warmup did not capture runs eagerly. Same bits. +_GRAPH_BREAKS_MAX_TOKENS = 4096 +#: Attention-TP group sizes whose prefill steps split the context selection and run +#: sequence-parallel (`PrefillTarget._sp_split`). Two ranks only: at four the sums would run in +#: another order than the all-reduce's. +_PREFILL_SPLIT_DEFAULT_TP = frozenset({2}) +#: Rows from which a prefill step runs sequence-parallel: a rank's share stays past the HC +#: kernels' small-row paths, so each runs the configuration the whole step would. +_SP_MIN_ROWS = 4096 +# cuBLASLt picks a bf16 projection's kernel by its exact row count up to this many rows, and +# past them by the 8-row group the row count falls in (`_projection_warm_rows`). +_KERNEL_CHOICE_EACH_ROW = 16 +_KERNEL_CHOICE_ROW_GROUP = 8 +# The logits rows `lm_head`'s kernels are loaded for: a step's requests' last tokens and its +# verify rows, up to 256 requests at MTP-3 or 128 at MTP-7. +_LM_HEAD_WARM_ROWS = 1024 +# A context launch of at least this many rows over a bf16 or fp8 pool takes +# `qsa_sparse_gqa_rowtile`; below it the per-row kernel is faster. +_ROWTILE_MIN_ROWS = 2048 # capacity: fixed: a dispatch threshold, not a buffer size +#: Requests from which MTP verify rows over an e4m3 pool take `qsa_sparse_gqa_grouped_cute` where +#: it serves them (one load and widening of each selected K/V token for a request's rows, over the +#: union of their selections) instead of the per-row `qsa_sparse_gqa_splitk_cute`. +_GROUPED_DECODE_MIN_REQUESTS = 32 # capacity: fixed: a dispatch threshold, not a buffer size +#: Rows from which a generation step with several rows a request (MTP verify, draft-extend) scores +#: its QSA index keys over request groups (one key gather serves a request's rows); low-latency +#: steps of up to 16 rows keep the entry's own choice. +_INDEX_GROUP_MIN_ROWS = 17 # capacity: fixed: a dispatch threshold, not a buffer size +#: Requests from which an attention-TP verify step splits QSA index scoring and selection by +#: request over the TP group (`_index_split_rows`) and gathers the int32 selections in rank order. +#: Bitwise the unsplit selection; needs `n % t == 0`. +_INDEX_SPLIT_MIN_REQUESTS = 16 +#: Rows up to which the split's gather is one Lamport exchange over symmetric memory +#: (`lamport_allgather__index_split`) instead of NCCL, bitwise the same; it sizes the buffers. +_INDEX_SPLIT_LAMPORT_ROWS = 768 +# Over an fp8 pool the launch reads its requests' pages widened into a bf16 scratch; a launch +# whose scratch would pass this many bytes stays on the per-row kernel. +_FP8_ROWTILE_MAX_SCRATCH = 1 << 30 # capacity: fixed: bounds a per-call scratch allocation +#: Opt-in switch (`weights.switch`): a context step over an fp8 pool whose pages would widen +#: past `_FP8_ROWTILE_MAX_SCRATCH` runs one row-tile launch per group of requests that fits +#: (`_rowtile_scratch_groups`) instead of the per-row kernel. Off by default because such steps' +#: outputs then move by rounding. Read once, when the target is built; `check_deployment` +#: declines it without an FP8 KV cache, where it does nothing. +ROWTILE_SCRATCH_GROUPS_ENV = "TRTLLM_MODELING_V2_ROWTILE_SCRATCH_GROUPS" +#: The stock MoE communication override (`communication_factory`), compared upper-cased like the +#: stock factory and, like it, read only under attention DP with `dp_size > 1`: other layouts +#: have no MoE round trip and ignore it. There `NVLINK_ONE_SIDED` runs the round trip as an +#: all-to-all dispatch / combine (`moe_a2a__dp_dispatch_combine`): a reference path, not a faster +#: one; it needs MPI and MNNVL. Unset or `ALLGATHER` keeps the all-gather / reduce-scatter round +#: trip. `check_deployment` declines any other value, and the all-to-all where it cannot run. +#: Read once, when the target is built. +MOE_COMM_METHOD_ENV = "TRTLLM_FORCE_COMM_METHOD" + + +def _moe_comm_method() -> str: + """`MOE_COMM_METHOD_ENV` upper-cased, `ALLGATHER` when unset.""" + return os.environ.get(MOE_COMM_METHOD_ENV, "ALLGATHER").upper() + + +# Rows per FP8 runner call over an attention-DP group's gathered rows (`_routed_experts`): one +# call reads the window's weights once, bitwise the `_MOE_MAX_ROWS`-row calls; the router, its +# routing and the activation quantization still run per `_MOE_MAX_ROWS` rows. +_ADP_WINDOW_MAX_ROWS = 4 * _MOE_MAX_ROWS +# What every bind of this target prints, on every rank: the disaggregated-serving test looks for +# it in each worker's log. +_LOG_PREFIX = "modeling_v2 qwen4_exp fp8 unified target" + +# In-forward Mamba snapshots: a context chunk ending at a snapshot point P just before the prompt +# end L runs through L, leaving the GDN and PLE state at P in the live slot and staging the state +# at L (`PrefillTarget._gdn_split`). The fewest chunk tokens before P a split request takes: past +# `_GDN_INPLACE_MAX_TOKENS` (the main chunk's FlashInfer path) and covering the conv (3) and PLE +# (9 columns, 2 ids) windows that end at P. The manager plans the splits, so the bind declines one +# whose `inforward_snapshot_min_main` is lower (`_inforward_stage`). +_INFORWARD_MIN_MAIN = _GDN_INPLACE_MAX_TOKENS + 1 + +#: Rows up to which `tp_sum` takes the multicast NVLink one-shot kernel +#: (`ops/comm/mnnvl_allreduce__tp_sum`) instead of the IPC one, where the platform supports MNNVL +#: (`MNNVLAllReduce.is_mnnvl`); the two return the same bits. +_TP_SUM_MNNVL_MAX_ROWS = 32 +#: The engine `allreduce_strategy` values under which the target runs its own communication: the +#: MNNVL `tp_sum` and the IPC buffers `_own_ipc` admits. Under another strategy every TP sum is +#: the raw all-reduce at that strategy (AUTO for SYMM_MEM) and the gathers run NCCL; +#: `check_deployment` declines UB and LOWPRECISION, whose setup only the stock `AllReduce` +#: module does. +_OWN_COMM_STRATEGIES = frozenset({int(AllReduceStrategy.AUTO), int(AllReduceStrategy.MNNVL)}) +#: Strategies of the stock `AllReduce` module's own kernels, which the raw all-reduce op has no +#: branch for: the module runs the MNNVL or symmetric-memory kernel where it can and calls the op +#: at AUTO otherwise (`_torch/distributed/ops.py`), and so does `tp_sum`. +_MODULE_ONLY_STRATEGIES = frozenset({int(AllReduceStrategy.MNNVL), int(AllReduceStrategy.SYMM_MEM)}) +#: The stock IPC fusion all-reduce's accumulation switch (`use_fp32_acc` in +#: `allReduceFusionKernels.cu`): fp32 unless the value starts with `0`. `hc_allreduce_inject_norm` +#: always sums in fp32, so with the opt-out the decode sites sum apart (`_hc_ar_inject`). +AR_FP32_ACC_ENV = "ALL_REDUCE_FUSION_KERNEL_ACC_FP32" + + +def _own_ipc(core) -> bool: + """Whether the target opens its own IPC buffers over the attention-TP group: the split index + gather's Lamport buffers, the two-rank copy-engine exchange and (at AUTO only) the fused TP sum + + inject. Needs an `_OWN_COMM_STRATEGIES` strategy and peer access between the group's ranks.""" + return core.allreduce_strategy in _OWN_COMM_STRATEGIES and can_access_peer(core.mapping) + + +def _runtime_hooks() -> RuntimeHooks: + """Step-loop hooks of the fast paths: one execution stream, one-launch verify prologue.""" + return RuntimeHooks(loop_on_execution_stream=True, decode_prologue=SpecStepMetadataPrologue()) + + +class _Replay(NamedTuple): + """The GDN cached replay's history buffers and bookkeeping of one manager.""" + + old: dict # GDN layer -> (old_x, old_B, old_dt, old_dA_cumsum) + cache_buf_idx: torch.Tensor # [slots] int32: the history half each slot writes next + prev_num_accepted: torch.Tensor # [slots] int32: tokens pending since the checkpoint + history: int # rows a history half holds + width: int # the one verify width a step may carry per generation request + + +class _Bound(NamedTuple): + """What the core reads off one KV cache manager; valid only for that manager. + + The manager, this rank's pool views and, under MTP, the verify step's runtime-owned buffers: + the per-step conv windows (`int_conv`, by GDN layer), the rows `update_mamba_states` promotes + them from (`int_idx`), the cached replay and the in-forward snapshot staging rows. + """ + + cm: object + pools: _pool.RankPools + int_idx: torch.Tensor | None = None + int_conv: dict | None = None + replay: _Replay | None = None + stage: "_InforwardStage | None" = None + + +class _InforwardStage(NamedTuple): + """The manager's in-forward snapshot staging rows (`Qwen4ExpHybridCacheManager`). + + Each GDN layer's SSM `[rows, HV, V, K]` and conv `[rows, conv_dim, 3]` rows, and the PLE + layer's conv state `[rows, stream, 9]` and n-gram history `[rows, ngram_size - 1]` rows. The + forward stages each split request's prompt-end state in its row; the manager copies it back + after committing the snapshot at P. + """ + + ssm: dict + conv: dict + ple_conv: torch.Tensor + ple_ngram: torch.Tensor + + +class _Split(NamedTuple): + """One step's in-forward snapshot split, from `Mamba2Metadata.inforward_split`. + + `k` of the step's context requests end their chunk at the prompt end L with a snapshot point + P inside it; in the GDN recurrence each is two sequences, [.., P) and the tail [P, L). + + `cu` (int64) the split sequences' boundaries; `gather` (int32) each split sequence's live + slot and `carry` (bool) whether it starts from it (a tail starts from zeros, its final state + dropped); `main_rows` (int64) each request's [.., P) or whole sequence and `live` (int32) its + slot; `tail_tok` (int64) the tail rows, `cu_tail` (int64) their boundaries, `stage_rows` + (int64) the split requests' staging rows, `tail_slots` (int64) their live slots; `conv_win`, + `ple_win` and `ngram_win` (int64) the rows of the GDN conv (3), PLE conv (9) and n-gram id (2) + windows that end at P, oldest first, request by request; `whole` (`chunk_split_first_pass`) + the rows before every P (int64, or a host count when one request is split once), their + per-request boundaries and carry flags, or None where the first pass keeps the split + boundaries; `raw` the engine's `InforwardSnapshotSplit`, views of one persistent buffer at + fixed offsets that a break's CUDA graph reads. + """ + + k: int + cu: torch.Tensor + gather: torch.Tensor + carry: torch.Tensor + main_rows: torch.Tensor + live: torch.Tensor + tail_tok: torch.Tensor + cu_tail: torch.Tensor + stage_rows: torch.Tensor + tail_slots: torch.Tensor + conv_win: torch.Tensor + ple_win: torch.Tensor + ngram_win: torch.Tensor + whole: tuple | None = None + raw: object = None + + +def _inforward_stage(cm, gdn_layers, ple_layer: int) -> _InforwardStage | None: + """The in-forward snapshot staging rows of `cm`, or None when it plans no snapshots.""" + if getattr(cm, "inforward_snapshot_rows", 0) <= 0: + return None + missing = [ + name + for name in ("inforward_snapshot_stage_views", "inforward_ple_stage_views") + if not hasattr(cm, name) + ] + min_main = getattr(cm, "inforward_snapshot_min_main", 0) + if missing or min_main < _INFORWARD_MIN_MAIN: + raise NotImplementedError( + f"the KV cache manager {type(cm).__name__} plans in-forward Mamba snapshots; this " + "target serves them only from a manager that stages the GDN and PLE prompt-end " + f"state and splits at least {_INFORWARD_MIN_MAIN} tokens before the snapshot point, " + f"and this one lacks {missing} and splits from {min_main} tokens. " + "kv_cache_config.enable_block_reuse=False turns the snapshots off" + ) + ple = cm.inforward_ple_stage_views(ple_layer) + assert ple is not None, ( + f"the KV cache manager plans in-forward snapshots but stages no PLE state for decoder " + f"layer {ple_layer}" + ) + ssm, conv = {}, {} + for i in gdn_layers: + ssm[i], conv[i] = cm.inforward_snapshot_stage_views(i) + return _InforwardStage(ssm, conv, ple[0], ple[1]) + + +class _TpPartial(NamedTuple): + """A decode block's partial output whose TP sum is left to the next hyper-connection site.""" + + partial: torch.Tensor + + +class _HcArInject(NamedTuple): + """IPC all-reduce workspace and TP rank / size for the decode sites' fused TP sum + inject.""" + + workspace: torch.Tensor + rank: int + nranks: int + #: rows whose `nranks * rows * hidden * 2` bytes fit one of the workspace's sub-buffers + max_rows: int + + +def _hc_ar_inject_defers(core, rows: int) -> bool: + """Whether `_tp_partial` leaves a decode block's partial of `rows` rows to the next site.""" + fused = getattr(core, "hc_ar_inject", None) + return fused is not None and max( + _HC_AR_INJECT_MIN_ROWS, _HC_ROWS_TC_MAX_ROWS + 1 + ) <= rows < min(_HC_RSTD_MIN_ROWS, fused.max_rows + 1) + + +def _tp_partial(core, x: torch.Tensor): + """A decode block's partial output for the next hyper-connection site. + + Left unsummed (`_TpPartial`) where that site sums it itself, else `core.tp_sum(x)`. A core + without `hc_ar_inject` (another layout, or a test's stand-in core) always sums. + """ + if _hc_ar_inject_defers(core, x.shape[0]): + return _TpPartial(x) + return core.tp_sum(x) + + +class _DpPadding(NamedTuple): + """The group's per-rank row counts of one step's attention-DP MoE round trip (ragged split).""" + + counts: tuple[int, ...] + + +class _DraftBound(NamedTuple): + """The draft layer's planes in the manager its draft step is handed. + + Its QSA index plane, that manager's position plane, its paged K/V and, under IndexShare, its + row of the metadata's shared selection buffers. + """ + + cm: object + index_cache: torch.Tensor + position_cache: torch.Tensor + kv: _pool.KvPlanes + share_slot: int | None + + +class Qwen4ExpUnifiedModelingV2(DecoderModel): + def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig): + """`composite_config` carries the vision config and media ids the text config lacks.""" + super().__init__(model_config) + cfg = model_config.pretrained_config + mapping = model_config.mapping + assert torch.cuda.get_device_capability() == _SM, ( + f"target certified on sm_{_SM[0]}{_SM[1]}, running on " + f"sm_{''.join(map(str, torch.cuda.get_device_capability()))}" + ) + assert getattr(cfg, "model_type", None) == "qwen4_exp_text", ( + f"the core must be built on the normalized text config; got model_type " + f"{getattr(cfg, 'model_type', None)!r}" + ) + self.mapping = mapping + # Ranks the attention and recurrent heads are split over (1 under attention DP). + self.attn_tp_size = _pool.attention_tp_size(mapping) + # The engine's all-reduce strategy for the TP sums; `_OWN_COMM_STRATEGIES` gates the + # target's own MNNVL and IPC paths. + self.allreduce_strategy = int(model_config.allreduce_strategy) + # Under attention DP each rank serves its own requests; MoE runs over the TP group's rows. + self.dp_size = mapping.tp_size if mapping.enable_attention_dp else 1 + self.dp_rank = mapping.tp_rank + self.dp_group = list(mapping.tp_group) + # One-model MTP-Eagle is the only speculation routing admits; None without speculation. + spec_config = getattr(model_config, "spec_config", None) + self.spec_config = spec_config + self.mtp_enabled = spec_config is not None + self.max_num_tokens = model_config.max_num_tokens + self.moe_max_rows = _moe_max_rows( + self.max_num_tokens, mapping.tp_size if mapping.enable_attention_dp else 1 + ) + + self.num_layers = cfg.num_hidden_layers + self.hidden = cfg.hidden_size + self.vocab = cfg.vocab_size + self.eps = cfg.rms_norm_eps + layer_types = list(cfg.layer_types) + self.is_attn = [t == "full_attention" for t in layer_types] + self.attn_layers = [i for i, a in enumerate(self.is_attn) if a] + self.gdn_layers = [i for i, a in enumerate(self.is_attn) if not a] + + self.hc_count = cfg.hc_count + self.hc_lowrank = cfg.hc_lowrank + self.stream = self.hc_count * self.hidden + # Columns the first projection reads: the `[down; block_inject]` rows rounded up to 32. + self.hc_first_cols = -(-(self.hc_lowrank + self.hc_count) // 32) * 32 + + # RoPE: partial, neox pairs, interleaved mRoPE. + self.head_dim = cfg.head_dim + rope = getattr(cfg, "rope_parameters", None) or getattr(cfg, "rope_scaling", None) + self.theta = float(rope["rope_theta"]) + partial = float(rope.get("partial_rotary_factor", cfg.partial_rotary_factor)) + self.rotary_dim = int(self.head_dim * partial) + self.mrope_declared = str(rope.get("type", rope.get("rope_type"))) == "mrope" + section = list(rope["mrope_section"]) + self.mrope_section1 = int(section[1]) + self.mrope_section2 = int(section[2]) + self.max_positions = cfg.max_position_embeddings + + # QSA sparse index geometry, replicated on every rank. + self.index_heads = cfg.indexer_n_heads + self.index_kv_heads = cfg.indexer_kv_heads + self.index_head_dim = cfg.indexer_head_dim + self.token_topk = cfg.indexer_budget + self.compress_ratio = cfg.indexer_compress_ratio + self.block_topk = self.token_topk // self.compress_ratio + # Dense-attention cutoff; must match the engine's (graph families split on it). + self.dense_threshold = self.token_topk + sparse_cfg = getattr(model_config, "sparse_attention_config", None) + assert getattr(sparse_cfg, "algorithm", None) == "qsa", ( + f"this target is a QSA sparse-attention assembly and the engine resolved " + f"sparse_attention_config={sparse_cfg!r}; QSA is this architecture's own default" + ) + qsa_params = sparse_cfg.to_sparse_params(pretrained_config=cfg) + assert ( + qsa_params.index_n_heads, + qsa_params.index_kv_heads, + qsa_params.index_head_dim, + qsa_params.token_topk, + qsa_params.compress_ratio, + ) == ( + self.index_heads, + self.index_kv_heads, + self.index_head_dim, + self.token_topk, + self.compress_ratio, + ), ( + f"the engine lowered the QSA index geometry to {qsa_params!r}; the side caches " + "are sized from the engine's copy and the kernels are called with this one" + ) + assert qsa_params.dense_seq_len_threshold == self.dense_threshold, ( + f"the engine's resolved QSA cutoff is {qsa_params.dense_seq_len_threshold} and " + f"this forward branches at {self.dense_threshold}: the runner captures its short " + "decode graph family at the engine's number" + ) + # IndexShare for the MTP draft loop, as resolved by the engine (it sizes the buffers). + self.mtp_index_share = bool(qsa_params.mtp_index_share) + self.index_width = (self.index_heads + self.index_kv_heads) * self.index_head_dim + # `[q | k | v]` scratch for the index norm + rope; the v head is required, never read. + self.index_scratch_width = self.index_width + self.index_head_dim + self.softmax_scale = 1.0 / math.sqrt(self.head_dim) + + # The draft layer's global index (KV managers and the weight table key it there). + self.mtp_layer_idx = self.num_layers + mtp_block = getattr(cfg, "mtp", None) + self.mtp_theta = ( + float(mtp_block.get("rope_theta", self.theta)) + if isinstance(mtp_block, dict) + else self.theta + ) + + self.head_k_dim = cfg.linear_key_head_dim + self.head_v_dim = cfg.linear_value_head_dim + self.conv_kernel = cfg.linear_conv_kernel_dim + + self.num_experts = cfg.num_experts + self.topk = cfg.num_experts_per_tok + self.moe_inter = cfg.moe_intermediate_size + self.shared_inter = cfg.shared_expert_intermediate_size + requested = model_config.moe_backend_requested.upper() + assert requested in _TRUNK_MOE_BACKENDS, ( + f"moe_config.backend={model_config.moe_backend_requested!r} is not implemented by " + f"this target; the supported requests are {sorted(_TRUNK_MOE_BACKENDS)}" + ) + quant_algo = getattr(model_config.quant_config, "quant_algo", None) + assert quant_algo == _QUANT_ALGO, ( + f"this target loads the block-FP8 checkpoint ({_QUANT_ALGO}); the engine resolved " + f"quant_algo {quant_algo!r}" + ) + + # One-based in the config. + self.ple_layer = list(cfg.ple_layer_ids)[0] - 1 + self.ngram_size = cfg.ngram_size + self.ple_dilation = self.ngram_size + eos = cfg.eos_token_id + self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) + self.ple_table_dtype = _weights.PLE_TABLE_DTYPE + # How the table is stored (`ple_host_table`); the storage is `_ple_table_host` after load. + self.ple_host_mode = _ple_host.host_table_mode() + self._ple_table_host = None + + vcfg = getattr(composite_config, "vision_config", None) + assert vcfg is not None, "the composite config carries no vision_config" + self.vision_config = vcfg + self.image_token_id = int(composite_config.image_token_id) + self.video_token_id = int(composite_config.video_token_id) + + self.kv_cache_quant_algo = model_config.quant_config.kv_cache_quant_algo + assert self.kv_cache_quant_algo in (None, QuantAlgo.FP8, QuantAlgo.NVFP4), ( + f"this target stores K/V as the checkpoint's bf16, as fp8-e4m3 or as NVFP4; the " + f"engine resolved kv_cache_quant_algo {self.kv_cache_quant_algo!r}" + ) + self.quant_mode = { + None: 0, + QuantAlgo.FP8: QM_FP8_KV_CACHE, + QuantAlgo.NVFP4: QM_NVFP4_KV_CACHE, + }[self.kv_cache_quant_algo] + + # This rank's share of the heads and experts. + d = _weights.MODEL_WEIGHTS.dims(self) + self.dims = d + self.heads_q = d.heads_q + self.heads_kv = d.heads_kv + self.kv_head_start = d.kv_head_start + self.q_width = d.q_width + self.kv_width = d.kv_width + self.qkv_width = self.q_width + 2 * self.kv_width + # Fused input projection row: `[q | k | v | gate | index q, k | v slot]`. The in-place + # norm + rope only touches q and k, so every column after k counts as v to it. + self.attn_in_width = d.attn_in_rows + assert self.attn_in_width == self.qkv_width + self.q_width + self.index_scratch_width + assert self.attn_in_width % self.head_dim == 0, self.attn_in_width + self.attn_in_tail_heads = self.attn_in_width // self.head_dim - self.heads_q - self.heads_kv + self.index_col = self.qkv_width + self.q_width + self.num_k_heads = d.num_k_heads + self.num_v_heads = d.num_v_heads + self.key_dim = d.key_dim + self.value_dim = d.value_dim + self.conv_dim = d.conv_dim + # Column widths of the one GDN input projection's output: [q|k|v | z | a | b]. + self.in_proj_split = d.in_proj_split + # Its weight's rows: their sum zero-padded to `weights.IN_PROJ_ROW_ALIGN`. + self.in_proj_rows = d.in_proj_rows + # This rank's routed experts (trunk and draft): `[expert_offset, expert_offset + + # local_experts)`, each at the full `moe_inter`. + self.local_experts = d.local_experts + self.expert_offset = d.expert_offset + assert d.moe_inter == self.moe_inter, (d.moe_inter, self.moe_inter) + + # Every weight but the n-gram table; meta-init intercepts torch.empty. + self.w = _weights.MODEL_WEIGHTS.declare(d) + + # Derived once after load (`post_load_weights`). + self._gdn_gating: dict | None = None + self._cos_sin: torch.Tensor | None = None + self._hc_first: dict | None = None + self._hc_norm_w1: dict | None = None + # KV scales: fp32 [1] ones (fp8 pool), fp32 [3] ones for q, K, V (NVFP4), None (bf16). + self.kv_scale_orig_quant: torch.Tensor | None = None + self.kv_scale_quant_orig: torch.Tensor | None = None + # Streams are made after load: one created inside a graph capture is not capturable. + self._side_stream: torch.cuda.Stream | None = None + self._decode_stream: torch.cuda.Stream | None = None + self._capture_stream: torch.cuda.Stream | None = None + self._ple_stream: torch.cuda.Stream | None = None + self._side_fork: torch.cuda.Event | None = None + self._side_join: torch.cuda.Event | None = None + self.tp_sum = None + self.expert_sum = None + self._prefill: "PrefillTarget | None" = None + self._decode: "DecodeTarget | None" = None + # Everything derived from the KV cache manager (`_bind_cache_manager`). + self._bound: _Bound | None = None + self._binds = 0 + # Set by `begin_step`, cleared by `forward`. + self._step_begun = False + # Whether the step `begin_step` saw is a generation-only step inside a decode CUDA graph, + # which attention-DP ranks agree on (the engine captures only when every rank can). Only + # such a step runs `_decode_projection` (`_step_projection`) and, under attention DP, the + # decode router (`_routed_experts`); an eager step can run at any batch size and meet + # another rank's prefill step. + self._decode_graph_step = False + self._mtp: "_MtpWeights | None" = None + self._mtp_cos_sin: torch.Tensor | None = None + # The verify step's side-cache rollback, registered with the spec worker. + self.spec_rollback: SpeculativeRollback | None = ( + SpeculativeRollback() if self.mtp_enabled else None + ) + self._draft_bound: _DraftBound | None = None + + # ---------------------------------------------------------------- load --- + + def _fuse_hc_first_projections(self) -> None: + """Fuse each hyper-connection's `down` and `block_inject` into one GEMM operand. + + Each is `[rows, stream]`, zero-padded to a multiple of 128 rows; the checkpoint + parameters become row views of it.""" + w = self.w + + def fuse(*params): + rows = sum(t.shape[0] for t in params) + padded = -(-rows // 128) * 128 + pad = torch.zeros( + [padded - rows, self.stream], dtype=params[0].dtype, device=params[0].device + ) + both = torch.cat([t.data for t in params] + [pad], 0) + start = 0 + for t in params: + n = t.shape[0] + t.data = both[start : start + n] + start += n + return both + + self._hc_first = { + (i, tag): fuse(w[f"l{i}_{tag}_down"], w[f"l{i}_{tag}_inject"]) + for i in range(self.num_layers) + for tag in ("ahc", "mhc") + } + self._hc_first["fhc"] = fuse(w["fhc_down"]) + if self.mtp_enabled: + p = f"l{self.mtp_layer_idx}_mtp_" + self._hc_first["mtp_ahc"] = fuse(w[f"{p}ahc_down"], w[f"{p}ahc_inject"]) + self._hc_first["mtp_mhc"] = fuse(w[f"{p}mhc_down"], w[f"{p}mhc_inject"]) + self._hc_first["mtp_fhc"] = fuse(w[f"{p}fhc_down"]) + # Each site's fp32 `(1 + norm)`, keyed by the norm's data_ptr (`_hc_mixer_rstd`). + norms = [w[f"l{i}_{tag}_norm"] for i in range(self.num_layers) for tag in ("ahc", "mhc")] + norms.append(w["fhc_norm"]) + if self.mtp_enabled: + p = f"l{self.mtp_layer_idx}_mtp_" + norms += [w[f"{p}{tag}_norm"] for tag in ("ahc", "mhc", "fhc")] + self._hc_norm_w1 = {t.data_ptr(): (1.0 + t.float()).contiguous() for t in norms} + + def _precompile_hc_rstd(self, device) -> None: + """Pre-compile every prefill rstd kernel variant (each K split and form) over zeros.""" + hi = self.max_num_tokens + if hi < _HC_RSTD_MIN_ROWS: + return + width, hidden, rank, c = self.stream, self.hidden, self.hc_lowrank, self.hc_count + bf = torch.bfloat16 + splits = {} + for rows in range(_HC_RSTD_MIN_ROWS, hi + 1, 256): + splits.setdefault(_hc_rescale_split(rows, width // _HC_RESCALE_BK, device), rows) + splits.setdefault(_hc_rescale_split(hi, width // _HC_RESCALE_BK, device), hi) + rows = max(splits.values()) + hc = torch.zeros([rows, width], dtype=bf, device=device) + blk = torch.zeros([rows, hidden], dtype=bf, device=device) + inj = torch.zeros([rows, c], dtype=bf, device=device) + norm = torch.zeros([width], dtype=bf, device=device) + first = torch.zeros([-(-(rank + c) // 128) * 128, width], dtype=bf, device=device).t() + up = torch.zeros([width, rank], dtype=bf, device=device).t() + _, rstd = qwen4_exp_hc_inject_rstd(hc, None, None, self.eps, c) + qwen4_exp_hc_inject_rstd(hc, blk, inj, self.eps, c) + qwen4_exp_hc_inject_rstd(hc, blk, inj, self.eps, c, blk, blk[:, :1]) + for m in sorted(splits.values()): + low, _ = qwen4_exp_hc_first_rescale_tc(hc[:m], rstd[:m], norm, first, c, rank, True) + w1 = (1.0 + norm.float()).contiguous() + qwen4_exp_hc_up_gate_mean_rstd(low, up, hc[: low.shape[0]], rstd[: low.shape[0]], w1, c) + torch.cuda.synchronize(device) + + def _precompile_draft_hc_rows(self, device) -> None: + """Pre-compile the hyper-connection sites of eager draft passes at every row count. + + Decode graphs compile at capture, but an eager draft pass would compile while serving.""" + M = getattr(self, "_mtp", None) + if M is None or _HC_ROWS_TC_MIN_ROWS > _HC_ROWS_TC_MAX_ROWS: + return + width, hidden, c = self.stream, self.hidden, self.hc_count + bf = torch.bfloat16 + hi = min(_HC_ROWS_TC_MAX_ROWS, self.max_num_tokens) + hc = torch.zeros([hi, width], dtype=bf, device=device) + blk = torch.zeros([hi, hidden], dtype=bf, device=device) + inj = torch.zeros([hi, c], dtype=bf, device=device) + n_inject = self.hc_lowrank + c + for rows in range(_HC_ROWS_TC_MIN_ROWS, hi + 1): + sites = ( + (M.ahc_norm, M.ahc_first, M.ahc_up, blk[:rows], inj[:rows], n_inject), + (M.fhc_norm, M.fhc_first, M.fhc_up, None, None, self.hc_lowrank), + ) + for norm, first, up, block_out, inject, n_cols in sites: + out, low, _, rstd = qwen4_exp_hc_decode_rows_first( + hc[:rows], block_out, inject, norm, first, c, n_cols, self.eps + ) + qwen4_exp_hc_decode_rows_mix(low, rstd, up, out, norm, c) + torch.cuda.synchronize(device) + + def post_load_weights(self) -> None: + """Derive load-time tensors and build the targets; a shell hook, not an override. + + Reached twice per load (the shell's hook and the loader's module walk): the targets are + built once, until the shell's `load_weights` clears them for a reload.""" + if self._prefill is not None: + return + w = self.w + device = w["embed"].device + # The delta-rule kernels read A_log and dt_bias as contiguous fp32; the checkpoint is bf16. + self._gdn_gating = { + i: ( + w[f"l{i}_A_log"].float().contiguous(), + w[f"l{i}_dt_bias"].float().contiguous(), + ) + for i in self.gdn_layers + } + # Zero the GDN in-proj alignment rows so the decode GEMM's unread pad columns are finite. + for i in self.gdn_layers: + w[f"l{i}_in_proj_w"][sum(self.in_proj_split) :].zero_() + self._cos_sin = self._index_rope_table(self.theta, device) + # Before the draft layer's views, which read its fused first projections. + self._fuse_hc_first_projections() + if self.mtp_enabled: + # The draft layer's index rope uses the MTP module's own rope base. + self._mtp_cos_sin = ( + self._cos_sin + if self.mtp_theta == self.theta + else self._index_rope_table(self.mtp_theta, device) + ) + self._mtp = _mtp_views(self) + if self.quant_mode in (QM_FP8_KV_CACHE, QM_NVFP4_KV_CACHE): + # Built here, not at construction: that runs under meta init. + scales = 3 if self.quant_mode == QM_NVFP4_KV_CACHE else 1 + self.kv_scale_quant_orig = torch.ones(scales, dtype=torch.float32, device=device) + self.kv_scale_orig_quant = 1.0 / self.kv_scale_quant_orig + # The CPU-only load checks build the targets over host weights; they launch nothing. + if device.type == "cuda": + hc_tc_first_precompile(self.stream, self.hc_lowrank, self.hc_count, device) + self._precompile_hc_rstd(device) + if self.mtp_enabled: + self._precompile_draft_hc_rows(device) + self._check_ple_residency() + self._side_stream = torch.cuda.Stream(device=device) + # Generation-only steps run at the highest stream priority (`_on_decode_stream`). + self._decode_stream = torch.cuda.Stream(device=device, priority=-100) + self._capture_stream = torch.cuda.Stream(device=device) + self._ple_stream = torch.cuda.Stream(device=device) + # `_moe_block`'s side-stream fork and join, re-recorded at every layer. + self._side_fork = torch.cuda.Event() + self._side_join = torch.cuda.Event() + self.tp_sum = self._tp_sum_op() + self.expert_sum = self._expert_sum_op() + self.hc_ar_inject = self._hc_ar_inject() + self._prefill = PrefillTarget(self) + self._decode = DecodeTarget(self) + + def _tp_sum_op(self): + """Return the attention-TP sum of block partials, bit-identical on every rank. + + At AUTO or MNNVL, where the platform supports MNNVL, rows up to `_TP_SUM_MNNVL_MAX_ROWS` + take the MNNVL kernel; every other sum is the raw all-reduce, called at AUTO for MNNVL + and SYMM_MEM (`_MODULE_ONLY_STRATEGIES`).""" + if self.attn_tp_size == 1: + return lambda x: x + strategy = self.allreduce_strategy + if strategy in _MODULE_ONLY_STRATEGIES: + strategy = int(AllReduceStrategy.AUTO) + group = allreduce_group_args(self.mapping, strategy) + + def tp_sum(x: torch.Tensor) -> torch.Tensor: + return allreduce( + x, + None, + None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=self.eps, + trigger_completion_at_end=False, + **group, + ) + + if self.allreduce_strategy not in _OWN_COMM_STRATEGIES or not MNNVLAllReduce.is_mnnvl( + self.mapping, torch.bfloat16, explicitly_requested=True + ): + return tp_sum + # A collective over the TP group: every rank builds the workspace here, in order. + mnnvl = mnnvl_workspace_args( + self.mapping, _TP_SUM_MNNVL_MAX_ROWS, self.hidden, torch.bfloat16 + ) + ipc_sum = tp_sum + + def tp_sum(x: torch.Tensor) -> torch.Tensor: # noqa: F811 + if x.shape[0] <= _TP_SUM_MNNVL_MAX_ROWS: + return mnnvl_tp_sum(x, **mnnvl) + return ipc_sum(x) + + return tp_sum + + def _hc_ar_inject(self): + """Return the decode sites' fused TP-sum + injection state, or None where unsupported. + + The kernel sums in fp32, the stock IPC all-reduce's default; under that all-reduce's + opt-out (`AR_FP32_ACC_ENV`) the sites sum through `tp_sum`, which follows it.""" + if not ( + self.attn_tp_size in (2, 4) + and self.dp_size == 1 + and self.mapping.world_size == self.attn_tp_size + and self.hidden == 2560 + and self.hc_count == 4 + and self.allreduce_strategy == int(AllReduceStrategy.AUTO) + and not os.environ.get(AR_FP32_ACC_ENV, "").startswith("0") + and _own_ipc(self) + ): + return None + group = allreduce_group_args(self.mapping, self.allreduce_strategy) + if group["workspace"] is None: + return None + members = list(self.mapping.tp_group) + rank = members.index(self.mapping.rank) + # `get_allreduce_workspace` sized each IPC sub-buffer `size * tp_size` with this call; + # the fused kernel's span in one is `nranks * rows * hidden * 2` bytes. + size = CustomAllReduceHelper.max_workspace_size_auto( + self.mapping.tp_size, support_deterministic=False + ) + max_rows = size * self.mapping.tp_size // (self.hidden * 2 * len(members)) + return _HcArInject(group["workspace"], rank, len(members), max_rows) + + def _expert_sum_op(self): + """Return `expert_sum(run, x, padding)`: this rank's rows of a MoE block's output. + + `run(rows)` applies this rank's expert window; `padding` comes from `_dp_pad_rows`. + Attention TP: every rank holds every row and `tp_sum` completes the partial. Attention + DP: all-gather the group's rows in rank order, run the window, reduce-scatter; routing + gathered rows keeps every rank's routing of a row identical. + `MOE_COMM_METHOD_ENV=NVLINK_ONE_SIDED` (MPI and MNNVL) swaps in the NVLink all-to-all, + with `run` a `_WindowRun`. Unequal per-rank + counts (`padding` set) use the ragged collectives so the window never sees padding + rows, which would all route to the same experts and stall the group on one window.""" + if self.dp_size == 1: + tp_sum = self.tp_sum + + def window_sum(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + return tp_sum(run(x)) + + return window_sum + group = self.dp_group + # With MPI disabled the collectives run over the TP process group. + process_group = self.mapping.tp_group_pg.boxed() if mpi_disabled() else None + + def round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + sizes = None if padding is None else list(padding.counts) + gathered = allgather(x, sizes, group, process_group) + return reducescatter(run(gathered), sizes, group, process_group) + + if _moe_comm_method() != "NVLINK_ONE_SIDED": + return round_trip + strategy = None if mpi_disabled() else _build_moe_a2a(self.mapping, self.max_num_tokens) + if strategy is None: + # `check_deployment` declines this deployment; reached only when routing is bypassed. + raise RuntimeError( + f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED needs MPI and MNNVL, which this " + "deployment lacks; unset it to run the attention-DP MoE as the all-gather / " + "reduce-scatter round trip" + ) + self.moe_a2a = strategy + + def alltoall_round_trip(run, x: torch.Tensor, padding: _DpPadding | None) -> torch.Tensor: + counts = [x.shape[0]] * len(group) if padding is None else list(padding.counts) + topk_ids, topk_w = _route_rows(run.core, x, run.router, run.mm, run.pad_mask) + got = moe_a2a_dispatch(strategy, x, topk_ids, topk_w, counts) + partials = _window_prerouted( + run.core, got.rows, got.topk_ids, got.topk_weights, run.experts + ) + return moe_a2a_combine(strategy, partials) + + return alltoall_round_trip + + def _dp_pad_rows(self, md, num_tokens: int) -> _DpPadding | None: + """Return the group's per-rank token counts for the ragged MoE round trip, or None. + + None without attention DP or with equal counts, as in every captured step.""" + if self.dp_size == 1: + return None + counts = tuple(int(n) for n in md.all_rank_num_tokens) + assert len(counts) == self.dp_size, (counts, self.dp_size) + assert counts[self.dp_rank] == num_tokens, ( + f"rank {self.dp_rank} holds {num_tokens} rows but all_rank_num_tokens says " + f"{counts[self.dp_rank]} ({list(counts)}); the collectives would gather the wrong " + "split" + ) + if min(counts) == max(counts): + return None + return _DpPadding(counts) + + def _index_rope_table(self, theta: float, device) -> torch.Tensor: + """Index rope table at `theta`: `[max_positions, 2 * rotary_pairs]` fp32, cos then sin.""" + exponent = torch.arange(0, self.rotary_dim, 2, dtype=torch.float64, device=device) + inv_freq = (1.0 / (theta ** (exponent / self.rotary_dim))).to(torch.float32) + positions = torch.arange(self.max_positions, dtype=torch.float32, device=device) + angles = positions.unsqueeze(1) * inv_freq.unsqueeze(0) + return torch.cat([torch.cos(angles), torch.sin(angles)], dim=1).contiguous() + + def _check_ple_residency(self) -> None: + """Assert via the driver that the n-gram table is host-resident; outputs cannot show it.""" + host = self._ple_table_host + assert host.resident_on_host(), ( + f"the n-gram table ({host.describe()}) was declared in host memory, but the driver " + "reports device memory for the pointer the forward gathers from" + ) + logger.info(f"ple n-gram table: {host.describe()}") + + # ---------------------------------------------------------------- bind --- + + def _bind_cache_manager(self, md, position_ids) -> None: + """Bind this rank's pool views of the step's KV cache manager and check its geometry. + + Keyed on the manager, not a first-forward flag: capacity estimation builds a throwaway + manager before the real one, and a view held across the swap would dangle.""" + cm = md.kv_cache_manager + assert cm is not None, "a paged KV cache is required" + assert not (self._binds and torch.cuda.is_current_stream_capturing()), ( + f"the KV cache manager changed to {type(cm).__name__} during CUDA graph capture" + ) + self._binds += 1 + missing = [name for name in _QSA_FIELDS if not hasattr(md, name)] + assert not missing, ( + f"metadata fields missing: {missing}; the engine did not select QSA metadata" + ) + assert position_ids.dtype == torch.int32, position_ids.dtype + assert self.mrope_declared, "this target takes the mRoPE position layout" + assert position_ids.dim() == 3 and position_ids.shape[0] == _QSA_POSITION_AXES, ( + f"expected the [{_QSA_POSITION_AXES}, 1, T] mRoPE position buffer, got " + f"{tuple(position_ids.shape)}" + ) + pools = {row[0] for row in md.host_kv_cache_pool_mapping.tolist()} + assert pools <= {0, 1}, ( + f"KV addressing over {len(pools)} pools is not certified; layer->pool ids " + f"{sorted(pools)}" + ) + g = _pool.pool_geometry( + self.model_config.pretrained_config, + self.mapping, + tokens_per_block=md.tokens_per_block, + kv_cache_quant_algo=self.kv_cache_quant_algo, + dtype=self.w["embed"].dtype, + ) + assert (g.heads_kv, g.kv_head_start, g.num_k_heads, g.num_v_heads, g.conv_dim) == ( + self.heads_kv, + self.kv_head_start, + self.num_k_heads, + self.num_v_heads, + self.conv_dim, + ), ( + f"the KV cache manager's share {g} disagrees with the weights' heads " + f"(kv {self.heads_kv} from {self.kv_head_start}, GDN {self.num_k_heads} / " + f"{self.num_v_heads}, conv {self.conv_dim})" + ) + assert md.qsa_topk_indices.shape[1] == self.block_topk, ( + f"the engine sized the QSA selection buffer {md.qsa_topk_indices.shape[1]} wide " + f"and this target selects {self.block_topk} blocks per row" + ) + mm = md.mamba_metadata + assert mm is not None and mm is not False, ( + "attn_metadata.mamba_metadata is absent: the runtime allocated no recurrent-state pool" + ) + mm_missing = [name for name in _MAMBA_FIELDS if not hasattr(mm, name)] + assert not mm_missing, f"mamba metadata fields missing: {mm_missing}" + rank_pools = _pool.take_pools(cm, g, self.attn_layers, self.gdn_layers, self.ple_layer) + # No sliding window: the targets pass `max_seq_len` as the window. + windows = cm.max_attention_window_vec + attn_windows = {windows[off] for off in rank_pools.pool_idx.values()} + assert attn_windows <= {None, md.max_seq_len}, ( + f"the attention layers' pool windows are {sorted(map(repr, attn_windows))}; a " + "sliding window is not implemented here" + ) + assert md.effective_beam_width == 1, "beam search is not implemented" + assert md.cache_indirection is None, "beam search is not implemented" + assert not md.is_cross, "cross attention is not implemented" + # Printed, not logged: the executor's workers keep the info log level off. + first_attn, first_gdn = self.attn_layers[0], self.gdn_layers[0] + print( + f"{_LOG_PREFIX}: rank {self.mapping.rank} binds " + f"{rank_pools.kv[first_attn].k.shape[0]} K/V pages, " + f"{rank_pools.index_cache[first_attn].shape[0]} QSA index pages, " + f"{rank_pools.conv[first_gdn].shape[0]} recurrent-state slots and " + f"{rank_pools.ple_state[0].shape[0]} PLE slots (bind {self._binds})", + flush=True, + ) + bound = _Bound(cm, rank_pools, stage=_inforward_stage(cm, self.gdn_layers, self.ple_layer)) + if self.spec_rollback is not None: + self.spec_rollback.set_index_planes( + [rank_pools.index_cache[i] for i in self.attn_layers] + ) + if self.mtp_enabled: + bound = bound._replace(**self._bind_step_buffers(cm, g, rank_pools, md)) + self._bound = bound + + def _bind_step_buffers(self, cm, g: _pool.PoolGeometry, pools: _pool.RankPools, md) -> dict: + """Bind the verify step's per-step buffers and return the `_Bound` fields they add. + + Per-row conv windows feed `update_mamba_states`; the recurrent state uses cached replay.""" + _pool.require( + cm, + ("is_speculative", "mamba_layer_cache", "get_replay_state_update_metadata"), + "binds the verify step's per-step state buffers and replay metadata", + ) + assert cm.is_speculative(), ( + f"the KV cache manager {type(cm).__name__} reports is_speculative() False, so it " + "allocated none of the per-step state buffers the verify path writes" + ) + layer_caches = {i: cm.mamba_layer_cache(i) for i in self.gdn_layers} + int_conv = {i: cache.intermediate_conv_window for i, cache in layer_caches.items()} + int_idx = getattr(cm, "intermediate_state_indices", None) + g0 = self.gdn_layers[0] + _pool.check_step_buffers(g, int_conv[g0], pools.conv[g0], int_idx) + meta = cm.get_replay_state_update_metadata() + if meta is None: + raise NotImplementedError( + "the KV cache manager runs the legacy per-step recurrent verify path " + "(TRTLLM_USE_GDN_REPLAY=0), which this target does not assemble; leave " + "TRTLLM_USE_GDN_REPLAY unset for the cached replay" + ) + assert not getattr(cm, "use_gdn_flashinfer_ring_replay", False), ( + "the manager laid its replay history out for the FlashInfer ring kernel; this " + "target runs the double-buffered Triton cached replay" + ) + assert layer_caches[g0].intermediate_ssm is None, ( + "the cache manager runs the cached-replay verify path and also allocated a " + "per-step recurrent buffer; the runtime allocates one or the other" + ) + ssm0 = pools.ssm[g0] + history, width = meta.replay_history_size, meta.replay_step_width + # The bookkeeping has one row per replay slot; a step indexes it, and the history, + # with the metadata's `replay_state_indices`. + replay_slots = meta.prev_num_accepted_tokens.shape[0] + _pool.check_replay_state( + history, width, meta.prev_num_accepted_tokens, meta.cache_buf_idx, ssm0, replay_slots + ) + old = {} + for i, cache in layer_caches.items(): + old[i] = (cache.old_x, cache.old_B, cache.old_dt, cache.old_dA_cumsum) + _pool.check_replay_history(g, i, history, replay_slots, *old[i][:3]) + replay = _Replay(old, meta.cache_buf_idx, meta.prev_num_accepted_tokens, history, width) + if ssm0.dtype == torch.bfloat16 and ssm0.is_cuda: + # Every CuTe replay kernel a verify step launches, compiled here: a compile costs + # about 0.5 s, the padded decode graphs skip the request counts an eager step + # carries, and the generic warmup skips this cache manager. + gdn_replay_precompile( + width, + self.num_k_heads, + self.num_v_heads, + md.max_num_requests, + **_gdn_replay_launches(self, width), + work_item_width=md.mamba_metadata.replay_work_items.shape[1], + separate_replay_slots=_has_replay_slots(cm), + ) + return dict(int_idx=int_idx, int_conv=int_conv, replay=replay) + + def _bind_draft_cache_manager(self, md) -> None: + """Bind the draft layer's index plane, position plane and K/V in its step's manager. + + That is the separate draft KV cache under attention TP, else the trunk's manager (also + under disaggregated serving). The runner runs every captured shape eagerly first, so + the first bind never happens under capture.""" + cm = md.kv_cache_manager + assert not torch.cuda.is_current_stream_capturing(), ( + f"the draft layer is not bound to KV cache manager {type(cm).__name__} (first use, " + "or the manager changed since its last bind) and CUDA graph capture is active" + ) + g = _pool.pool_geometry( + self.model_config.pretrained_config, + self.mapping, + tokens_per_block=md.tokens_per_block, + kv_cache_quant_algo=self.kv_cache_quant_algo, + dtype=self.w["embed"].dtype, + ) + layer = self.mtp_layer_idx + draft = _pool.take_draft_pools( + cm, g, layer, separate=_pool.draft_kv_cache_is_separate(self.mapping, self.spec_config) + ) + index_cache, position_cache, kv = draft.index_cache, draft.position_cache, draft.kv + share_slot = None + if self.mtp_index_share: + assert md.sparse_metadata_params.mtp_index_share, ( + "this target resolved IndexShare on but the engine's QSA metadata did not" + ) + slots = cm.qsa_shared_topk_slots + _pool.check_shared_selection(md.qsa_shared_topk_indices, slots, layer, self.block_topk) + share_slot = slots[layer] + self._draft_bound = _DraftBound(cm, index_cache, position_cache, kv, share_slot) + + def _draft_step_bind(self, md) -> tuple[_DraftBound, int]: + """Return the draft layer's bind and pool offset for this draft step's manager. + + Checked every call: the worker swaps managers and page tables in for the draft loop.""" + cm = md.kv_cache_manager + assert cm is not None, "a paged KV cache is required" + offsets = cm.layer_offsets + assert self.mtp_layer_idx in offsets, ( + f"the MTP module's attention is global layer index {self.mtp_layer_idx}, which the " + f"cache manager {type(cm).__name__} does not map (it maps {len(offsets)} layers); " + "the engine allocated no KV layer for the draft module" + ) + layer_idx = offsets[self.mtp_layer_idx] + window = cm.max_attention_window_vec[layer_idx] + assert window is None or window == md.max_seq_len, ( + f"the MTP layer's pool window is {window!r} but this call passes max_seq_len " + f"{md.max_seq_len} as the attention window; a sliding window is not implemented" + ) + if self._draft_bound is None or cm is not self._draft_bound.cm: + self._bind_draft_cache_manager(md) + assert cm is self._bound.cm or md.qsa_block_table is getattr( + md, "qsa_draft_block_table", None + ), ( + "the draft step is bound to a separate draft KV cache manager but qsa_block_table " + "is not that manager's table: the engine's QSA metadata did not swap its page " + "table for the draft forward" + ) + return self._draft_bound, layer_idx + + # -------------------------------------------------------------- forward --- + + def _select_target(self, attn_metadata) -> "PrefillTarget | DecodeTarget": + return self._decode if attn_metadata.num_contexts == 0 else self._prefill + + def begin_step( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None, + position_ids: torch.IntTensor | None, + inputs_embeds: torch.FloatTensor | None = None, + lora_params: dict | None = None, + spec_metadata=None, + vision: tuple | None = None, + ) -> None: + """Validate the step, bind the cache manager and let the chosen target prepare it. + + All host reads happen here: a breakable prefill CUDA graph replays `forward` without + calling it. `vision` is `(rows, mm_token_indices)` when the step carries media.""" + assert self._prefill is not None and self._decode is not None, ( + "load_weights must run before forward" + ) + assert position_ids is not None + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert lora_params is None, "LoRA is not implemented by this target" + assert input_ids is not None, "this target's PLE layer hashes token ids" + assert inputs_embeds is None, ( + "this target computes its token embedding itself; an engine-supplied " + "inputs_embeds is not implemented" + ) + # One-model MTP-Eagle captures no trunk layer, so the trunk owes no capture hook. + assert (spec_metadata is not None) == self.mtp_enabled, ( + f"spec_metadata {'arrived' if spec_metadata is not None else 'is absent'} but the " + f"model was built with mtp_enabled={self.mtp_enabled}; the draft path and the " + "weights it loads are decided at construction" + ) + if self._bound is None or attn_metadata.kv_cache_manager is not self._bound.cm: + self._bind_cache_manager(attn_metadata, position_ids) + target = self._select_target(attn_metadata) + if target is self._prefill: + target.prepare(attn_metadata, input_ids, position_ids, vision, spec_metadata) + else: + assert vision is None, "a generation-only step carries vision rows" + self._decode_graph_step = attn_metadata.is_cuda_graph and attn_metadata.num_contexts == 0 + self._step_begun = True + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + lora_params: dict | None = None, + **kwargs, + ): + """Run the step `begin_step` prepared; nothing here may read the step from the host. + + Returns the logits input, or under speculation what `split_spec_output` takes.""" + assert self._step_begun, ( + "the shell calls begin_step before every forward; this forward has no prepared step" + ) + self._step_begun = False + target = self._select_target(attn_metadata) + step = target.forward + if isinstance(target, DecodeTarget): + return _on_decode_stream( + self, + input_ids.shape[0], + step, + attn_metadata, + input_ids=input_ids, + position_ids=position_ids, + **kwargs, + ) + return step(attn_metadata, input_ids=input_ids, position_ids=position_ids, **kwargs) + + def split_spec_output(self, output) -> tuple[torch.Tensor, torch.Tensor]: + """Split speculative `forward` output into `final` `[T, hidden]` and `hc` `[T, stream]`. + + `final` feeds `lm_head`, `hc` the MTP module; the prefill target packs both in one.""" + if isinstance(output, tuple): + return output + final, hc = torch.split(output, [self.hidden, self.stream], -1) + return final, hc + + +# --------------------------------------------------------------- weight views --- + + +class _LayerWeights(NamedTuple): + """One trunk layer's op views (GEMM operands `.t()`); unused attention/GDN fields are None.""" + + ahc_norm: torch.Tensor + ahc_first: torch.Tensor + ahc_up: torch.Tensor + mhc_norm: torch.Tensor + mhc_first: torch.Tensor + mhc_up: torch.Tensor + attn_in: torch.Tensor | None + o: torch.Tensor | None + q_norm: torch.Tensor | None + k_norm: torch.Tensor | None + idx_qn: torch.Tensor | None + idx_kn: torch.Tensor | None + in_proj: torch.Tensor | None + # With the zero alignment rows (`core.in_proj_rows` columns) the decode GEMM tiles whole; + # `in_proj` is its leading `sum(in_proj_split)` columns. + in_proj_dec: torch.Tensor | None + out: torch.Tensor | None + conv_w: torch.Tensor | None + a_log: torch.Tensor | None + dt_bias: torch.Tensor | None + gnorm: torch.Tensor | None + router: torch.Tensor + experts: _weights.ExpertWindow + # `[gate | up | gate logit | pad]` (`weights._cat_rows`), and the down projection. + sh_gug: torch.Tensor + sh_dn: torch.Tensor + + +def _layer_views(core: Qwen4ExpUnifiedModelingV2) -> list[_LayerWeights]: + w = core.w + layers = [] + for i in range(core.num_layers): + attn: dict = dict(attn_in=None, o=None, q_norm=None, k_norm=None, idx_qn=None, idx_kn=None) + gdn: dict = dict( + in_proj=None, + in_proj_dec=None, + out=None, + conv_w=None, + a_log=None, + dt_bias=None, + gnorm=None, + ) + if core.is_attn[i]: + attn = dict( + attn_in=w[f"l{i}_attn_in_w"].t(), + o=w[f"l{i}_o_w"].t(), + q_norm=w[f"l{i}_q_norm"], + k_norm=w[f"l{i}_k_norm"], + idx_qn=w[f"l{i}_idx_qn"], + idx_kn=w[f"l{i}_idx_kn"], + ) + else: + a_log, dt_bias = core._gdn_gating[i] + gdn = dict( + in_proj=w[f"l{i}_in_proj_w"][: sum(core.in_proj_split)].t(), + in_proj_dec=w[f"l{i}_in_proj_w"].t(), + out=w[f"l{i}_out_w"].t(), + # [conv_dim, 1, width] -> [conv_dim, width]: a view (unit width stride). + conv_w=w[f"l{i}_conv_w"].reshape(core.conv_dim, core.conv_kernel), + a_log=a_log, + dt_bias=dt_bias, + gnorm=w[f"l{i}_gnorm"], + ) + layers.append( + _LayerWeights( + ahc_norm=w[f"l{i}_ahc_norm"], + ahc_first=core._hc_first[(i, "ahc")].t(), + ahc_up=w[f"l{i}_ahc_up"].t(), + mhc_norm=w[f"l{i}_mhc_norm"], + mhc_first=core._hc_first[(i, "mhc")].t(), + mhc_up=w[f"l{i}_mhc_up"].t(), + **attn, + **gdn, + router=w[f"l{i}_router"].t(), + experts=_weights.expert_window(w, core.dims, i), + sh_gug=w[f"l{i}_sh_gu"].t(), + sh_dn=w[f"l{i}_sh_dn"].t(), + ) + ) + return layers + + +class _StackWeights(NamedTuple): + """Weights outside the layer loop: the stack-final mixer (final norm) and the PLE path.""" + + fhc_norm: torch.Tensor + fhc_first: torch.Tensor + fhc_up: torch.Tensor + ple_key: torch.Tensor + ple_val: torch.Tensor + ple_nk_w: torch.Tensor + ple_nq_w: torch.Tensor + ple_nc_w: torch.Tensor + ple_conv_w: torch.Tensor + ple_mult: torch.Tensor + ple_vocab: torch.Tensor + ple_offs: torch.Tensor + ple_tscale: torch.Tensor + + +def _stack_views(core: Qwen4ExpUnifiedModelingV2) -> _StackWeights: + w = core.w + return _StackWeights( + fhc_norm=w["fhc_norm"], + fhc_first=core._hc_first["fhc"].t(), + fhc_up=w["fhc_up"].t(), + ple_key=w["ple_key_w"].t(), + ple_val=w["ple_val_w"].t(), + # Whole norm weights: the fused tail normalizes all three streams in one program. + ple_nk_w=w["ple_nk"], + ple_nq_w=w["ple_nq"], + ple_nc_w=w["ple_nc"], + ple_conv_w=w["ple_conv_w"], + ple_mult=w["ple_mult"], + ple_vocab=w["ple_vocab"], + ple_offs=w["ple_offs"], + # The table's dequantize scale at the gathered rows' rank (3), all 1s. + ple_tscale=w["ple_tscale"].reshape(1, 1, 1), + ) + + +class _MtpWeights(NamedTuple): + """The MTP module's views at this rank's share, keyed at layer `num_hidden_layers`. + + Laid out as the trunk's (GEMM operands `.t()`); `unit_inject` adds `x_e` as a site injection.""" + + pre_e_norm: torch.Tensor + pre_h_norm: torch.Tensor + fc_e: torch.Tensor + fc_h: torch.Tensor + ahc_norm: torch.Tensor + ahc_first: torch.Tensor + ahc_up: torch.Tensor + mhc_norm: torch.Tensor + mhc_first: torch.Tensor + mhc_up: torch.Tensor + unit_inject: torch.Tensor + # [hidden, attn_in_width]: the trunk's fused attention input layout, at this rank's heads. + attn_in: torch.Tensor + o: torch.Tensor + q_norm: torch.Tensor + k_norm: torch.Tensor + idx_qn: torch.Tensor + idx_kn: torch.Tensor + router: torch.Tensor + experts: _weights.ExpertWindow + sh_gug: torch.Tensor + sh_dn: torch.Tensor + fhc_norm: torch.Tensor + fhc_first: torch.Tensor + fhc_up: torch.Tensor + + +def _mtp_views(core: Qwen4ExpUnifiedModelingV2) -> _MtpWeights: + """Built after `_fuse_hc_first_projections`, whose `"mtp_*"` tensors it reads.""" + w = core.w + p = f"l{core.mtp_layer_idx}_mtp_" + dt = w[f"{p}fc_h"].dtype + return _MtpWeights( + pre_e_norm=w[f"{p}pre_e_norm"], + pre_h_norm=w[f"{p}pre_h_norm"], + fc_e=w[f"{p}fc_e"].t(), + fc_h=w[f"{p}fc_h"].t(), + ahc_norm=w[f"{p}ahc_norm"], + ahc_first=core._hc_first["mtp_ahc"].t(), + ahc_up=w[f"{p}ahc_up"].t(), + mhc_norm=w[f"{p}mhc_norm"], + mhc_first=core._hc_first["mtp_mhc"].t(), + mhc_up=w[f"{p}mhc_up"].t(), + unit_inject=torch.ones( + [core.max_num_tokens, core.hc_count], dtype=dt, device=w[f"{p}fc_h"].device + ), + attn_in=w[f"{p}attn_in_w"].t(), + o=w[f"{p}o_w"].t(), + q_norm=w[f"{p}q_norm"], + k_norm=w[f"{p}k_norm"], + idx_qn=w[f"{p}idx_qn"], + idx_kn=w[f"{p}idx_kn"], + router=w[f"{p}router"].t(), + experts=_weights.expert_window(w, core.dims, core.mtp_layer_idx), + sh_gug=w[f"{p}sh_gu"].t(), + sh_dn=w[f"{p}sh_dn"].t(), + fhc_norm=w[f"{p}fhc_norm"], + fhc_first=core._hc_first["mtp_fhc"].t(), + fhc_up=w[f"{p}fhc_up"].t(), + ) + + +# ------------------------------------------------------------ shared blocks --- + + +def _verify_rows(core, spec_metadata, num_seqs: int, num_rows: int) -> tuple[int, bool]: + """Return `(p, is_verify)`: rows per generation sequence and whether the step verifies. + + `p` is 1 without speculation, else `runtime_draft_len + 1`; host ints, constant per graph.""" + p = num_rows // num_seqs if num_seqs else 1 + assert p * num_seqs == num_rows, ( + f"{num_rows} generation rows do not divide over {num_seqs} generation sequences; a " + "ragged per-sequence draft length can be expressed neither to the attention call nor " + "to the multi-token recurrence" + ) + assert core.mtp_enabled or p == 1, ( + f"{num_rows} generation rows over {num_seqs} generation sequences with speculation " + "off; the base path implements one query token per generation sequence" + ) + if spec_metadata is not None and num_seqs > 0: + declared = spec_metadata.runtime_draft_len + 1 + assert p == declared, ( + f"the batch carries {p} query rows per generation sequence but the worker's " + f"runtime_draft_len says {declared}; the per-step conv snapshots and the replay " + "history are sized from the second and written from the first" + ) + _pool.check_verify_width(p) + _pool.check_replay_width(core._bound.replay, p) + return p, spec_metadata is not None and num_seqs > 0 and p > 1 + + +def _has_replay_slots(cm) -> bool: + """Whether the manager keeps the replay history in slots of its own, which the metadata's + `replay_state_indices` then address (otherwise it aliases the recurrent-state slots).""" + return getattr(cm, "replay_layer_group_id", None) is not None + + +def _verify_state(core, mm, num_decodes: int, num_prefills: int = 0) -> tuple: + """Return what `_gdn_verify` reads besides the layer: promotion rows, replay work items, + and the generation rows' replay slots (behind `num_prefills` context rows). + + Work items and the all-layer commit follow the stock Gated-DeltaNet layer's conditions.""" + assert core._bound.int_idx is not None, "the cache-manager bind has not run" + int_idx = core._bound.int_idx[:num_decodes] + partitioned = num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + all_layer = partitioned and bool( + getattr(core._bound.cm, "use_gdn_cached_replay_all_layer_commit", False) + ) + work_items = mm.replay_work_items[:num_decodes] if partitioned else None + n_writes = mm.replay_n_writes if partitioned else None + replay_indices = ( + mm.replay_state_indices[num_prefills : num_prefills + num_decodes] + if _has_replay_slots(core._bound.cm) + else None + ) + return int_idx, work_items, n_writes, all_layer, replay_indices + + +def _gdn_in_proj_conv_tactic(core, x, L, si_d, verify): + """Return the fused GDN in-proj + conv tactic for a verify step, or None if unsupported. + + The fused kernel is bitwise `low_m_bf16_mm` + `causal_conv1d_token_major`, one launch fewer.""" + rows = x.shape[0] + w = L.in_proj_dec + if ( + core.conv_dim != _GDN_IN_PROJ_CONV_DIM + or tuple(w.shape) != (x.shape[1], _GDN_IN_PROJ_N_OUT) + or w.stride(0) != 1 + or not x.is_contiguous() + or si_d.dtype != torch.int32 + or verify[0].dtype != torch.int32 + or not L.conv_w.is_contiguous() + or tuple(L.conv_w.shape) != (_GDN_IN_PROJ_CONV_DIM, 4) + ): + return None + return _gdn_in_proj_conv_fused_tactic(rows) + + +def _gdn_verify(core, mq, a, b, i, L, si_d, n, p, verify, mq_out=None, z=None) -> torch.Tensor: + """Run the recurrent half of a verify step: `n` generation sequences of `p` rows. + + The conv also writes each row's window to the per-step buffer `update_mamba_states` + promotes from; the cached replay rebuilds each slot's state from its checkpoint and + history, runs the `p` rows and appends them, so rejected rows cost nothing to undo. `mq` is + the `[n * p, conv_dim]` q|k|v view, `a`/`b` the raw gate columns (the kernel applies + `A_log`, `dt_bias`, q/k L2 norm). Returns `[1, n * p, num_v_heads, head_v_dim]`.""" + int_idx, work_items, n_writes, all_layer, replay_indices = verify + pools = core._bound.pools + replay = core._bound.replay + ssm = pools.ssm[i] + assert (core.num_k_heads, core.num_v_heads) in _GDN_REPLAY_CUTE_GEOMETRIES, ( + f"GDN heads {core.num_k_heads} / {core.num_v_heads} are not a CuTe replay geometry" + ) + pdl, mapping = _gdn_replay_launch(core, n) + fold = None + if mq_out is not None: + # `gdn_in_proj_conv` already ran the conv and advanced the pool and windows. + pass + elif _gdn_verify_folds_conv(core, n) and gdn_conv_fold_supported( + mq, L.conv_w, pools.conv[i], core._bound.int_conv[i] + ): + # The replay computes the conv rows itself from the q|k|v columns (`mq`). + fold = dict( + x=mq, + weight=L.conv_w, + pool=pools.conv[i], + window=core._bound.int_conv[i], + window_rows=int_idx, + ) + mq_out = mq + else: + # Token-major `[n*p, C]` conv output, which the replay reads q/k/v from directly. + mq_out = causal_conv1d_token_major( + mq, + n, + pools.conv[i], + L.conv_w, + si_d, + core._bound.int_conv[i], + int_idx, + _PAD_SLOT_ID, + launch_dependent_kernels=pdl, + wait_producer=_GDN_CONV_PDL_WAIT, + ) + norm = None + if z is not None: + # The gated RMS norm folded in (`_gdn_verify_folds_norm`, decided by the caller). + norm = dict( + z=torch.reshape(z, [n * p, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + ) + old_x, old_B, old_dt, old_dA = replay.old[i] + qv, kv, vv = torch.split(mq_out, [core.key_dim, core.key_dim, core.value_dim], 1) + out = gdn_cached_replay_cute( + q=torch.reshape(qv, [n, p, core.num_k_heads, core.head_k_dim]), + k=torch.reshape(kv, [n, p, core.num_k_heads, core.head_k_dim]), + v=torch.reshape(vv, [n, p, core.num_v_heads, core.head_v_dim]), + g=torch.reshape(a, [n, p, core.num_v_heads]), + beta=torch.reshape(b, [n, p, core.num_v_heads]), + ssm_states=ssm, + state_indices=si_d, + old_u=old_x, + old_k=old_B, + old_G=old_dt, + old_beta=old_dA, + cache_buf_idx=replay.cache_buf_idx, + prev_num_accepted_tokens=replay.prev_num_accepted, + history_size=replay.history, + scale=None, + use_qk_l2norm_in_kernel=True, + A_log=L.a_log, + dt_bias=L.dt_bias, + launch_with_pdl=pdl, + replay_work_items=work_items, + n_writes=n_writes, + block_v=None, + num_warps=None, + use_l2_state_cache=None, + use_l2_streaming_inputs=None, + use_l2_shared_inputs=None, + main_num_stages=None, + packed_qkv=None if fold is not None else mq_out, + use_all_layer_commit=all_layer, + output=None, + replay_indices=replay_indices, + **({"config": mapping} if mapping is not None else {}), + **({"conv_fold": fold} if fold is not None else {}), + **({"gated_norm": norm} if norm is not None else {}), + ) + return torch.reshape(out, [1, n * p, core.num_v_heads, core.head_v_dim]) + + +def _gdn_verify_folds_norm(core, n: int, p: int) -> bool: + """Whether a verify step folds its gated RMS norm into the CuTe replay (bitwise equal). + + Only at tp2/tep2 HT batches (up to 4 rows a request); saves a launch and an HBM round trip.""" + return ( + (core.num_k_heads, core.num_v_heads) == _GDN_REPLAY_HT_GEOMETRY + and n >= _GDN_REPLAY_HT_MIN_REQUESTS + and p <= 4 + ) + + +def _gdn_verify_folds_conv(core, n: int) -> bool: + """Whether a verify step folds its conv into the CuTe replay (tp2/tep2 HT batches).""" + return ( + (core.num_k_heads, core.num_v_heads) == _GDN_REPLAY_HT_GEOMETRY + and _GDN_REPLAY_HT_MIN_REQUESTS <= n <= _GDN_VERIFY_CONV_FOLD_MAX_REQUESTS + ) + + +def _gdn_replay_launch(core, n: int) -> tuple: + """Return `(pdl, mapping)` for the CuTe cached replay over `n` requests.""" + heads = (core.num_k_heads, core.num_v_heads) + if heads == _GDN_REPLAY_HT_GEOMETRY and n >= _GDN_REPLAY_HT_MIN_REQUESTS: + for bound, pdl, mapping in _GDN_REPLAY_HT_LAUNCH: + if n <= bound: + return pdl, mapping + return get_env_enable_pdl() and n * core.num_v_heads <= _GDN_REPLAY_PDL_MAX_HEAD_ROWS, None + + +def _gdn_replay_launches(core, width: int) -> dict: + """The launch rules `gdn_replay_precompile` takes: those `_gdn_verify` and its callers apply + to a verify step of `width` rows a request (`_gdn_replay_launch`, `_gdn_verify_folds_conv`, + `_gdn_verify_folds_norm`).""" + return dict( + partition_from=CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + launch=lambda n: _gdn_replay_launch(core, n), + conv_fold=lambda n: _gdn_verify_folds_conv(core, n), + fold_norm=lambda n: _gdn_verify_folds_norm(core, n, width), + norm_eps=float(core.eps), + ) + + +def _draft_positions(position_ids: torch.Tensor) -> tuple: + """Return `(pos3, pos, coords)` from `[3, 1, N]` or `[3, N]` draft-step mRoPE positions. + + `pos3` is contiguous int32 `[3, N]` (the input itself if already so; nothing writes it + during the step), `pos` its row 0 and `coords` the `[N, 3]` transpose the plane stores.""" + assert position_ids.shape[0] == _QSA_POSITION_AXES, ( + f"expected the mRoPE position layout with {_QSA_POSITION_AXES} coordinate rows, got " + f"{tuple(position_ids.shape)}" + ) + if position_ids.dtype == torch.int32 and position_ids.is_contiguous(): + pos3 = torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]) + else: + pos3 = _copy_positions(position_ids) + return pos3, pos3[0], torch.transpose(pos3, 0, 1) + + +def _draft_embedding(input_ids: torch.Tensor, embed_tokens: torch.Tensor) -> torch.Tensor: + """Embed the MTP front end's tokens; negative ids read row 0. + + Rejection sampling fills slots past a request's accepted count with -1; those rows feed + nothing the step keeps, so clamping only keeps the gather inside the table.""" + if input_ids.dim() == 1 and input_ids.is_contiguous(): + # The id clamp and gather in one launch (bitwise the composed path). + return embed_rows(input_ids, embed_tokens, 1, True) + return nn.functional.embedding(input_ids.clamp_min(0), embed_tokens) + + +def _mtp_front(core, M: _MtpWeights, embed_tokens, input_ids, hidden_states, mm=None) -> tuple: + """Run the draft layer's front end and attention-side hyper-connection. + + `fc_embedding` of the normalized next-token embedding is added to every stream of + `fc_hidden` as the trunk site's unit-weight injection; returns `(x, mixed, inject)`. `mm` + runs dense projections here and in every draft-layer and MoE helper: `cublas_mm` when + None, `_step_projection` from `DecodeTarget`.""" + mm = mm or cublas_mm + rows = hidden_states.shape[0] + assert rows <= M.unit_inject.shape[0], ( + f"a draft step of {rows} rows exceeds max_num_tokens {M.unit_inject.shape[0]}" + ) + e = _draft_embedding(input_ids, embed_tokens) + main = torch.cuda.current_stream() + side = core._side_stream if rows <= _MTP_FRONT_FORK_MAX_ROWS else main + # Fork after the embedding is queued; `x_e` is consumed on `main` after the join. + side.wait_stream(main) + with torch.cuda.stream(side): + x_e = mm(flashinfer_gemma_rmsnorm(e, M.pre_e_norm, core.eps), M.fc_e) + _, h_n = qwen4_exp_hc_inject_norm( + hidden_states, None, None, M.pre_h_norm, core.eps, core.hc_count + ) + x_h = torch.reshape( + mm(torch.reshape(h_n, [rows * core.hc_count, core.hidden]), M.fc_h), + [rows, core.stream], + ) + main.wait_stream(side) + return _hc_site(core, x_h, x_e, M.unit_inject[:rows], M.ahc_norm, M.ahc_first, M.ahc_up, True) + + +def _mtp_attention_in(core, M: _MtpWeights, mixed, pos3, pos, with_index: bool, mm=None) -> tuple: + """Compute the draft layer's attention input as a trunk attention layer does. + + Returns `(proj, qkv, gate, k, v, index_scratch)`: `gate` is `proj`'s `[T, q_width]` gate + block, `k`/`v` the rotated heads, `index_scratch` None without `with_index`.""" + mm = mm or cublas_mm + rows = mixed.shape[0] + proj = mm(mixed, M.attn_in) + index_scratch = ( + torch.empty([rows, core.index_scratch_width], dtype=mixed.dtype, device=mixed.device) + if with_index + else None + ) + _attention_rope(core, proj, index_scratch, M, pos3, pos, theta=core.mtp_theta) + if rows == 1: + # One row: the `[q | k | v]` prefix is contiguous and not written after the rope. + qkv = proj[:, : core.qkv_width] + else: + qkv = torch.empty([rows, core.qkv_width], dtype=mixed.dtype, device=mixed.device) + qkv.copy_(proj[:, : core.qkv_width]) + _, k, v = _attn_in_heads(core, proj) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + return proj, qkv, gate, k, v, index_scratch + + +def _mtp_attention_planes( + core, M: _MtpWeights, mixed, draft: _DraftBound, qsa, rows_per_request, with_index, sparse, mm +) -> tuple: + """Compute a generation-only draft step's attention input and write its planes. + + Returns `(qkv, gate, q_index)`; `mm` runs the input projection. The K/V append happens here + only on the sparse path (the dense op appends itself). The draft-extend step runs everything + after the projection as the trunk's one-launch `qwen4_exp_attn_decode_prologue`.""" + req_idx, logical, block_table, tokens_per_block, pos3, pos, coords = qsa + if with_index and _decode_prologue(draft.kv, sparse): + rows = mixed.shape[0] + proj = mm(mixed, M.attn_in) + qkv = torch.empty([rows, core.qkv_width], dtype=mixed.dtype, device=mixed.device) + index_scratch = torch.empty( + [rows, core.index_scratch_width], dtype=mixed.dtype, device=mixed.device + ) + q_index = _index_query(core, index_scratch) + qwen4_exp_attn_decode_prologue( + proj, + qkv, + q_index, + pos3, + coords, + core._mtp_cos_sin, + M.q_norm, + M.k_norm, + M.idx_qn, + M.idx_kn, + draft.kv.k, + draft.kv.v, + draft.index_cache, + draft.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + core.compress_ratio, + core.heads_q, + core.heads_kv, + core.head_dim, + core.index_heads, + core.index_head_dim, + core.index_col, + core.rotary_dim, + core.eps, + core.mrope_section1, + core.mrope_section2, + rows_per_request=rows_per_request, + ) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + return qkv, gate, q_index + proj, qkv, gate, k_dst, v_dst, index_scratch = _mtp_attention_in( + core, M, mixed, pos3, pos, with_index, mm=mm + ) + q_index = None + if with_index: + q_index = _mtp_index_branch( + core, + M, + proj, + index_scratch, + draft, + (req_idx, logical, block_table, tokens_per_block, coords), + ) + if sparse: + _store_kv(core, draft.kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + return qkv, gate, q_index + + +def _mtp_index_branch(core, M: _MtpWeights, proj, index_scratch, draft: _DraftBound, qsa): + """Run the draft layer's QSA index branch on its own planes, as `_index_plane` does. + + Stores the raw index key, compresses completed groups, returns the rotated index query.""" + req_idx, logical, block_table, tokens_per_block, coords = qsa + rows = proj.shape[0] + token_k = torch.split( + torch.reshape( + proj[:, core.index_col : core.index_col + core.index_scratch_width], + [rows, core.index_scratch_width // core.index_head_dim, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, 1], + 1, + )[1] + qsa_paged_index_store( + token_k, + coords, + draft.index_cache, + draft.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + ) + qsa_prefill_compress( + logical_positions=logical, + request_indices=req_idx, + block_table=block_table, + index_cache=draft.index_cache, + position_cache=draft.position_cache, + k_norm_weight=M.idx_kn, + cos_sin=core._mtp_cos_sin, + eps=core.eps, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + mrope_section=_QSA_MROPE_SECTION, + ) + return _index_query(core, index_scratch) + + +def _moe_max_rows(max_num_tokens: int, dp_size: int) -> int: + """Return the rows per routed-experts call, never fewer than `_MOE_MAX_ROWS`. + + Attention DP: `_MOE_MAX_ROWS`. Attention TP: the step's rows, capped at `_MOE_MAX_ROWS_TP` + (the runner's certified bound) or, with the grouped prefill experts, `_ADP_WINDOW_MAX_ROWS` + (the most rows the runner takes elsewhere, for calls the grouped experts decline).""" + if dp_size > 1: + return _MOE_MAX_ROWS + if _MOE_PREFILL_GROUPED_MIN_ROWS and max_num_tokens >= _MOE_PREFILL_GROUPED_MIN_ROWS: + return min(max(max_num_tokens, _MOE_MAX_ROWS), _ADP_WINDOW_MAX_ROWS) + return min(max(max_num_tokens, _MOE_MAX_ROWS), _MOE_MAX_ROWS_TP) + + +def _runner_tune_rows(core) -> int: + """Return the FP8 runner's `tune_max_num_tokens`: `core.moe_max_rows`, clamped. + + Tuning at the rows large context chunks run beats reusing the 8192-row tactic; past + `_MOE_MAX_ROWS_TP` calls run the grouped experts. Every tactic writes the same bits.""" + return min(max(getattr(core, "moe_max_rows", _MOE_MAX_ROWS), _MOE_MAX_ROWS), _MOE_MAX_ROWS_TP) + + +def _window_runner( + core, + experts: _weights.ExpertWindow, + xq, + xsf, + topk_ids, + topk_w, + output=None, + fin_first=None, + after_first=None, +): + """Run this rank's whole-expert window over pre-routed rows. + + `xq` is 1x128 e4m3 rows with `[hidden / 128, rows]` scales `xsf`; top-k ids span the + whole routing space and an id outside the window (or -1) adds nothing. Context chunks run + `moe_prefill_grouped`, which finalizes `fin_first` rows first and hands them to + `after_first`; on the runner `after_first` sees the whole output.""" + if ( + output is None + and _MOE_PREFILL_GROUPED_MIN_ROWS + and _use_prefill_grouped(core, xq.shape[0], experts.count) + ): + split = {} if after_first is None else {"fin_first": fin_first, "after_first": after_first} + return moe_prefill_grouped( + xq, + xsf, + topk_ids, + topk_w, + experts.fc1_w, + experts.fc1_s, + experts.fc2_w, + experts.fc2_s, + experts.offset, + **split, + ) + if after_first is not None: + out = _window_runner(core, experts, xq, xsf, topk_ids, topk_w, output) + after_first(out) + return out + return _window_runner_call( + core, + experts, + xq, + xsf, + routing_logits=None, + topk_weights=topk_w, + topk_ids=topk_ids, + output=output, + tune_max_num_tokens=_runner_tune_rows(core), + ) + + +def _window_runner_call(core, experts: _weights.ExpertWindow, xq, xsf, **routing) -> torch.Tensor: + """`fp8_block_scale_moe_runner` over this rank's whole-expert window, renormalized SwiGLU; + `routing` is the call's routing (`routing_logits`, or `topk_ids` / `topk_weights`) and + any further runner keywords.""" + return fp8_block_scale_moe_runner( + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=experts.fc1_w, + gemm1_weights_scale=experts.fc1_s, + gemm2_weights=experts.fc2_w, + gemm2_weights_scale=experts.fc2_s, + num_experts=core.num_experts, + top_k=core.topk, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=core.moe_inter, + local_expert_offset=experts.offset, + local_num_experts=experts.count, + routed_scaling_factor=None, + routing_method_type=_ROUTING_METHOD_RENORM, + act_type=_ACT_TYPE_SWIGLU, + gemm1_clamp_limit=None, + **routing, + ) + + +def _use_prefill_grouped(core, rows: int, local: int) -> bool: + """Whether a call over `rows` rows of a `local`-expert window runs `moe_prefill_grouped`. + + A capture before the first eager call keeps the runner: capture cannot load kernels.""" + if not _MOE_PREFILL_GROUPED_MIN_ROWS or rows < _MOE_PREFILL_GROUPED_MIN_ROWS: + return False + if getattr(core, "dp_size", 1) != 1: + return False + if prefill_grouped_ready(local): + return True + if torch.cuda.is_current_stream_capturing(): + return False + warm_prefill_grouped(torch.cuda.current_device(), local) + return True + + +def _routed_experts( + core, x, router, experts: _weights.ExpertWindow, mm=None, pad_mask=None, before_runner=None +) -> torch.Tensor: + """Return this rank's partial of a MoE layer's routed experts over every row of `x`. + + Router logits, renormalized top-k (`renorm_moe_routing_op`) and 1x128 e4m3 quantization + feed this rank's window (`_window_runner`); slots outside it add nothing, so the windows + sum to the layer. Routing runs in chunks of `core.moe_max_rows` rows; an attention-DP + rank batches up to `_ADP_WINDOW_MAX_ROWS` rows of chunks per runner call (bitwise equal). + `pad_mask` (`[T, 1]` bool) routes a prefill graph bucket's padding rows to expert -1, so + they come back zero instead of all landing on the first window's ten lowest experts.""" + mm = mm or cublas_mm + if mm is _decode_projection and getattr(core, "dp_size", 1) > 1 and not core._decode_graph_step: + # Attention DP routes the same gathered rows on every rank, so every rank must compute + # their logits with one kernel: `_decode_projection`'s low-M kernels and cuBLAS differ in + # the last bits, and a reordered top-k is a different MoE sum. A graphed step is a decode + # step on every rank and keeps the decode router; an eager one can meet another rank's + # prefill step and takes the prefill router, `cublas_mm`. + mm = cublas_mm + rows = x.shape[0] + if pad_mask is None and mm is _decode_projection and 0 < rows <= _MOE_FRONT_MAX_ROWS: + if before_runner is not None: + before_runner() + return _routed_experts_front(core, x, router, experts) + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + if ( + pad_mask is None + and mm is _decode_projection + and getattr(core, "dp_size", 1) == 1 + and 0 < rows <= step + ): + logits = mm(x, router) + if rows <= _MOE_FAST_FRONT_MAX_ROWS: + out = _window_runner_front(core, experts, x, logits, before_runner) + if out is not None: + return out + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=_MOE_USE_UE8M0) + if before_runner is not None: + before_runner() + return _window_runner_logits(core, experts, xq, xsf, logits) + if before_runner is not None: + before_runner() + group = 1 + if getattr(core, "dp_size", 1) > 1 and rows > step: + group = max(1, _ADP_WINDOW_MAX_ROWS // step) + parts, pending = [], [] + for lo in range(0, rows, step): + chunk = x[lo : lo + step] + logits = mm(chunk, router) + topk_ids, topk_w = renorm_moe_routing_op(logits, topk=core.topk, output_dtype=chunk.dtype) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask[lo : lo + step], _NO_EXPERT) + xq, xsf = fp8_quantize_1x128(chunk, use_ue8m0=_MOE_USE_UE8M0) + pending.append((xq, xsf, topk_ids, topk_w)) + if len(pending) == group or lo + step >= rows: + if len(pending) == 1: + args = pending[0] + else: + q, sf, ids, w = zip(*pending) + codes = torch.cat([t.view(torch.uint8) for t in q]).view(q[0].dtype) + args = (codes, torch.cat(sf, 1), torch.cat(ids), torch.cat(w)) + parts.append(_window_runner(core, experts, *args)) + pending = [] + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _route_rows(core, x, router, mm=None, pad_mask=None) -> tuple: + """Route every row of `x` as `_routed_experts` does; return `(topk_ids, topk_weights)`. + + Used by the all-to-all round trip, which routes rows on their own rank.""" + mm = mm or cublas_mm + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + ids, weights = [], [] + for lo in range(0, x.shape[0], step): + chunk = x[lo : lo + step] + topk_ids, topk_w = renorm_moe_routing_op( + mm(chunk, router), topk=core.topk, output_dtype=chunk.dtype + ) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask[lo : lo + step], _NO_EXPERT) + ids.append(topk_ids) + weights.append(topk_w) + if not ids: + # A rank without rows this step: the routing op's empties. + logits = x.new_zeros(0, core.num_experts) + return renorm_moe_routing_op(logits, topk=core.topk, output_dtype=x.dtype) + return torch.cat(ids), torch.cat(weights) + + +def _window_prerouted(core, x, topk_ids, topk_w, experts: _weights.ExpertWindow) -> torch.Tensor: + """Run this rank's window over rows routed by `_route_rows`; rows with no local id are 0.""" + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + rows = x.shape[0] + group = max(1, _ADP_WINDOW_MAX_ROWS // step) if rows > step else 1 + parts, pending = [], [] + for lo in range(0, rows, step): + xq, xsf = fp8_quantize_1x128(x[lo : lo + step], use_ue8m0=_MOE_USE_UE8M0) + pending.append((xq, xsf, topk_ids[lo : lo + step], topk_w[lo : lo + step])) + if len(pending) == group or lo + step >= rows: + if len(pending) == 1: + args = pending[0] + else: + q, sf, ids, w = zip(*pending) + codes = torch.cat([t.view(torch.uint8) for t in q]).view(q[0].dtype) + args = (codes, torch.cat(sf, 1), torch.cat(ids), torch.cat(w)) + parts.append(_window_runner(core, experts, *args)) + pending = [] + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _window_runner_front(core, experts: _weights.ExpertWindow, x, logits, before_runner): + """Run `_window_runner_logits` through the one-launch decode routing with fused quantization. + + Calls `before_runner` once the router projection (and quantization) are queued. Returns + None, queuing nothing past the router, while the AutoTuner holds no explicit tactic.""" + rows, hidden = x.shape + fold = x.is_contiguous() + if fold: + xq = torch.empty((rows, hidden), dtype=torch.float8_e4m3fn, device=x.device) + xsf = torch.empty((hidden // 128, rows), dtype=torch.float32, device=x.device) + else: + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=_MOE_USE_UE8M0) + call = fp8_block_scale_moe_front( + logits, + xq, + xsf, + experts.fc1_w, + experts.fc1_s, + experts.fc2_w, + experts.fc2_s, + core.num_experts, + core.topk, + core.moe_inter, + experts.offset, + experts.count, + _ROUTING_METHOD_RENORM, + ) + if call is None: + if not fold: + if before_runner is not None: + before_runner() + return _window_runner_logits(core, experts, xq, xsf, logits) + return None + if before_runner is not None: + before_runner() + return call(x if fold else None) + + +def _window_runner_logits(core, experts: _weights.ExpertWindow, xq, xsf, logits) -> torch.Tensor: + """Run the window with the runner routing `logits` itself (bitwise `renorm_moe_routing_op`).""" + return _window_runner_call( + core, experts, xq, xsf, routing_logits=logits, topk_weights=None, topk_ids=None + ) + + +def _routed_experts_front( + core, x, router, experts: _weights.ExpertWindow, shared: _TpbShared | None = None +) -> torch.Tensor: + """Run `_routed_experts` for a decode step's rows (up to `_MOE_FRONT_MAX_ROWS`). + + One launch forms the bf16 router logits and the 1x128 e4m3 activation; then + `moe_tpb_decode` over block-major experts (optionally folding in `shared`) or the runner. + `router` is a `.t()` view of the checkpoint's `[512, hidden]` rows.""" + logits, xq, xsf = moe_front_router_quant(x.contiguous(), router.t()) + blocks = experts.blocks + if blocks is not None and x.shape[0] <= _MOE_TPB_MAX_ROWS: + return moe_tpb_decode( + logits, + xq, + xsf, + _BlockExperts(blocks.w1, blocks.s1, blocks.w2, blocks.s2, blocks.rank), + shared, + ) + assert shared is None, "the shared-expert fold rides the block launch" + return _window_runner_logits(core, experts, xq, xsf, logits) + + +class _WindowRun(NamedTuple): + """A MoE layer's `run` for `core.expert_sum`: `_routed_experts` over the given rows.""" + + core: object + router: torch.Tensor + experts: _weights.ExpertWindow + mm: object = None + pad_mask: torch.Tensor | None = None + before_runner: object = None + + def __call__(self, rows: torch.Tensor) -> torch.Tensor: + return _routed_experts( + self.core, + rows, + self.router, + self.experts, + self.mm, + self.pad_mask, + before_runner=self.before_runner, + ) + + +def _mtp_moe_parts(core, M: _MtpWeights, x, pad_rows: _DpPadding | None = None, mm=None) -> tuple: + """Return `_mtp_moe_block`'s uncombined `(routed, gate_logits, shared)`, side stream joined.""" + mm = mm or cublas_mm + main = torch.cuda.current_stream() + side = core._side_stream + side.wait_stream(main) + with torch.cuda.stream(side): + gug = mm(x, M.sh_gug) + shared = mm(silu_and_mul_rows(gug, core.shared_inter), M.sh_dn) + gate_logits = gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1] + routed = core.expert_sum(_WindowRun(core, M.router, M.experts, mm), x, pad_rows) + main.wait_stream(side) + return routed, gate_logits, shared + + +def _mtp_moe_block( + core, M: _MtpWeights, x, pad_rows: _DpPadding | None = None, mm=None +) -> torch.Tensor: + """Return the draft layer's MoE, `routed + sigmoid(gate logit) * shared`, as the trunk's. + + `core.expert_sum` completes the routed partial; the shared expert runs on the side stream.""" + return fused_sigmoid_gate_mul_add(*_mtp_moe_parts(core, M, x, pad_rows, mm)) + + +def _mtp_tail( + core, M: _MtpWeights, x, attn, inject, pad_rows: _DpPadding | None, mm=None +) -> torch.Tensor: + """Run the draft layer after its attention core: output projection, MoE-side site, MoE. + + `attn` is the `[T, q_width]` gated attention output; `pad_rows` is from `_draft_pad_rows`.""" + mm = mm or cublas_mm + x, mixed, inject = _hc_site( + core, x, _tp_partial(core, mm(attn, M.o)), inject, M.mhc_norm, M.mhc_first, M.mhc_up, True + ) + if _combine_inject_fits(x, inject): + # Combine and inject in one launch, bitwise equal to the composed ops below. + routed, gate_logits, shared = _mtp_moe_parts(core, M, mixed, pad_rows, mm) + if ( + routed.dtype == shared.dtype == torch.bfloat16 + and routed.stride(1) == shared.stride(1) == 1 + ): + return mtp_moe_combine_inject(x, routed, gate_logits, shared, inject) + return qwen4_exp_post_inject( + x, fused_sigmoid_gate_mul_add(routed, gate_logits, shared), inject + ) + return qwen4_exp_post_inject(x, _mtp_moe_block(core, M, mixed, pad_rows, mm=mm), inject) + + +def _ple_fold_fits(hc, routed, inject, shared) -> bool: + """Whether the PLE launches can take these operands: 2-D, unit column stride, `hc`'s dtype.""" + rows = hc.shape[0] + ok = ( + inject is not None + and routed.dim() == 2 + and inject.dim() == 2 + and routed.shape[0] == inject.shape[0] == rows + and routed.stride(1) == 1 + and inject.stride(1) == 1 + and routed.dtype == inject.dtype == hc.dtype + ) + if ok and shared is not None: + ok = shared.dim() == 2 and shared.stride(1) == 1 and shared.dtype == hc.dtype + return ok + + +def _combine_inject_fits(x, inject) -> bool: + """Whether `mtp_moe_combine_inject` takes `x` and `inject`: 2-D bf16, unit column stride.""" + return ( + x.dim() == 2 + and inject.dim() == 2 + and x.dtype == inject.dtype == torch.bfloat16 + and x.stride(1) == 1 + and inject.stride(1) == 1 + and x.shape[0] == inject.shape[0] + ) + + +class _DraftRowCounts(NamedTuple): + """The group's per-rank row counts of one draft step, in the field `_dp_pad_rows` reads.""" + + all_rank_num_tokens: list + + +def _draft_pad_rows(core, all_rank_num_tokens, rows: int) -> _DpPadding | None: + """Return `_dp_pad_rows` over the row counts the spec worker passes for this draft step. + + The counts are each rank's trunk tokens on the loop's first step, then its request count.""" + if core.dp_size == 1: + return None + assert all_rank_num_tokens is not None, ( + "a draft step under attention DP came without the group's row counts; the MoE round " + "trip cannot pad this rank's rows to the group's largest count" + ) + return core._dp_pad_rows(_DraftRowCounts(list(all_rank_num_tokens)), rows) + + +def _decode_projection(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor: + """Dense bf16 projection for graphed generation-only steps and their draft layer. + + `low_m_bf16_mm` picks FlashInfer's low-M PDL kernels (`_DECODE_PROJ_PDL`) where its table + names the shape, else cuBLAS; they compile on the engine's warmup forward before capture.""" + return low_m_bf16_mm(mat_a, mat_b, _DECODE_PROJ_PDL) + + +def _step_projection(core): + """The dense projection of a `DecodeTarget` step: `_decode_projection` in a graphed step + (`core._decode_graph_step`), else `cublas_mm`. An eager generation-only step runs at any + batch size (under attention DP whenever another rank has context rows), and the FlashInfer + kernels compile on the first call at each shape: only captured batch sizes compile before + serving.""" + return _decode_projection if core._decode_graph_step else cublas_mm + + +def _hc_first(core, normed, first, with_inject): + """Run a hyper-connection mixer's first half: first projection, then SiLU and injection. + + Returns `(low, inject)`; `inject` is None without `with_inject`.""" + if normed.shape[0] <= _HC_TC_FIRST_MAX_ROWS: + return qwen4_exp_hc_decode_tc_first( + normed, first, core.hc_lowrank, core.hc_count, with_inject + ) + return qwen4_exp_hc_lowrank_act( + cublas_mm(normed, first[:, : core.hc_first_cols], None, torch.float32), + core.hc_lowrank, + core.hc_count, + with_inject, + normed.dtype, + ) + + +def _hc_mixer(core, normed, first, up, with_inject): + """Run one hyper-connection mixer on a normed stream; return `(mixed, inject)`.""" + low, inject = _hc_first(core, normed, first, with_inject) + wide = normed.shape[0] <= _HC_UGM_WIDE_MAX_ROWS + return qwen4_exp_hc_up_gate_mean(low, up, normed, core.hc_count, wide), inject + + +def _hc_inject_norm_launch( + hc, block_out, inject, norm, eps, hc_count, shared, gate_logits, num_warps +): + """Run `qwen4_exp_hc_inject_norm` (no finalize fold) at `num_warps` warps with PDL. + + The residual is bitwise at any warp count; the normed stream varies only with the fp32 + sum-of-squares order. Programs read nothing before `griddepcontrol.wait`. Returns + `(hc_out, normed)`; `hc_out` is `hc` itself without `block_out`.""" + return qwen4_exp_hc_inject_norm( + hc, + block_out, + inject, + norm, + eps, + hc_count, + shared, + gate_logits, + num_warps=num_warps, + pdl_wait=_HC_INJECT_PDL_WAIT, + ) + + +def _hc_mixer_rstd(core, hc, rstd, norm, first, up, with_inject): + """Run `_hc_mixer` from the residual `hc` and its `[T, C]` rstd instead of a normed stream. + + The first projection forms `bf16(hc * rstd * (1 + norm))` on its A tiles; the up + projection's epilogue uses the fp32 value (`core._hc_norm_w1`).""" + low, inject = qwen4_exp_hc_first_rescale_tc( + hc, rstd, norm, first, core.hc_count, core.hc_lowrank, with_inject + ) + w1 = core._hc_norm_w1[norm.data_ptr()] + return qwen4_exp_hc_up_gate_mean_rstd(low, up, hc, rstd, w1, core.hc_count), inject + + +def _hc_site( + core, + hc, + block_out, + inject, + norm, + first, + up, + with_inject, + shared=None, + gate_logits=None, +): + """Run one hyper-connection site on the residual `hc`; return `(hc, mixed, inject)`. + + Injects the pending block (`block_out` at weights `inject`, plus + `sigmoid(gate_logits) * shared` after a MoE block), then the grouped norm and the mixer. + A `_TpPartial` block output is summed over the TP group inside the injection launch.""" + partial = None + if isinstance(block_out, _TpPartial): + block_out = partial = block_out.partial + if _HC_ROWS_TC_MIN_ROWS <= hc.shape[0] <= _HC_ROWS_TC_MAX_ROWS: + assert partial is None, "a deferred TP partial reached a site that does not sum it" + n_cols = core.hc_lowrank + (core.hc_count if with_inject else 0) + hc, low, inject, rstd = qwen4_exp_hc_decode_rows_first( + hc, + block_out, + inject, + norm, + first, + core.hc_count, + n_cols, + core.eps, + shared, + gate_logits, + ) + mixed = qwen4_exp_hc_decode_rows_mix(low, rstd, up, hc, norm, core.hc_count) + return hc, mixed, inject + if partial is not None: + fused = getattr(core, "hc_ar_inject", None) + if ( + fused is not None + and hc.shape[0] < min(_HC_RSTD_MIN_ROWS, fused.max_rows + 1) + and _hc_ar_inject_operands_fit(hc, partial, inject, shared, gate_logits) + ): + warps = 2 if hc.shape[0] >= _HC_INJECT_FEW_WARPS_MIN_ROWS else _HC_INJECT_WARPS + hc, normed = hc_allreduce_inject_norm( + partial, + fused.workspace, + fused.rank, + fused.nranks, + hc, + inject, + norm, + core.eps, + shared, + gate_logits, + warps, + ) + mixed, inject = _hc_mixer(core, normed, first, up, with_inject) + return hc, mixed, inject + block_out = core.tp_sum(block_out) + if hc.shape[0] >= _HC_RSTD_MIN_ROWS: + hc, rstd = qwen4_exp_hc_inject_rstd( + hc, block_out, inject, core.eps, core.hc_count, shared, gate_logits + ) + mixed, inject = _hc_mixer_rstd(core, hc, rstd, norm, first, up, with_inject) + return hc, mixed, inject + warps = 2 if hc.shape[0] >= _HC_INJECT_FEW_WARPS_MIN_ROWS else _HC_INJECT_WARPS + hc, normed = _hc_inject_norm_launch( + hc, block_out, inject, norm, core.eps, core.hc_count, shared, gate_logits, warps + ) + mixed, inject = _hc_mixer(core, normed, first, up, with_inject) + return hc, mixed, inject + + +def _moe_tpb_shared_folded(core, x, experts, mm, pad_rows, pad_mask, defer) -> bool: + """Whether `_moe_block` folds this rank's shared-expert slice into the block launch.""" + rows = x.shape[0] + return ( + defer + and experts.blocks is not None + and pad_rows is None + and pad_mask is None + and mm is _decode_projection + and getattr(core, "dp_size", 1) == 1 + and _MOE_TPB_SHARED_MIN_ROWS <= rows <= min(_MOE_FRONT_MAX_ROWS, _MOE_TPB_MAX_ROWS) + ) + + +def _moe_block( + core, + x, + L: _LayerWeights, + pad_rows: _DpPadding | None, + mm=None, + pad_mask=None, + defer=False, +) -> tuple: + """Run one MoE layer; return `(routed, gate_logits, shared)` uncombined. + + The next `qwen4_exp_hc_inject_norm` forms `routed + sigmoid(gate_logits) * shared`. + `routed` comes from `core.expert_sum` (`pad_rows` from `_dp_pad_rows`); the shared expert + runs on the side stream and joins before this returns. `pad_mask` keeps prefill graph + padding rows out of the window. With `defer` (attention-TP decode) `routed` may be a + `_TpPartial` the next site sums. When `_moe_tpb_shared_folded` holds, `routed` is the + whole combine and the others are None.""" + mm = mm or cublas_mm + if _moe_tpb_shared_folded(core, x, L.experts, mm, pad_rows, pad_mask, defer): + b = L.experts.blocks + shared = _TpbShared(x.contiguous(), b.sh_gu, b.sh_gl, b.sh_dn) + partial = _routed_experts_front(core, x, L.router, L.experts, shared=shared) + return _tp_partial(core, partial), None, None + main = torch.cuda.current_stream() + side = core._side_stream + forked = [] + + def fork_shared(): + # The side stream waits for `main`; its outputs are used on `main` after the join. + core._side_fork.record(main) + side.wait_event(core._side_fork) + with torch.cuda.stream(side): + gug = mm(x, L.sh_gug) + shared = mm(silu_and_mul_rows(gug, core.shared_inter), L.sh_dn) + forked.append((gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1], shared)) + + # Under attention TP a decode step forks once the router and the quantization are queued + # (`_routed_experts`' `before_runner`); every other step forks first. + late = pad_rows is None and getattr(core, "dp_size", 1) == 1 + if not late: + fork_shared() + run = _WindowRun(core, L.router, L.experts, mm, pad_mask, fork_shared if late else None) + if defer and getattr(core, "dp_size", 1) == 1: + routed = _tp_partial(core, run(x)) + else: + routed = core.expert_sum(run, x, pad_rows) + if not forked: + fork_shared() + core._side_join.record(side) + main.wait_event(core._side_join) + gate_logits, shared = forked[0] + return routed, gate_logits, shared + + +def _routed_experts_sp( + core, + x_rows, + router, + experts: _weights.ExpertWindow, + gather, + pad_mask=None, + fin_first=None, + after_first=None, +): + """This rank's window partial `[T, hidden]` over every row of a sequence-parallel step. + + Each rank routes and e4m3-quantizes only its own rows (`pad_mask` sends padding rows to + `_NO_EXPERT`); two uniform gathers carry the codes and a `[scales | ids | weights]` row.""" + topk = core.topk + logits = cublas_mm(x_rows, router) + topk_ids, topk_w = renorm_moe_routing_op(logits, topk=topk, output_dtype=x_rows.dtype) + if pad_mask is not None: + topk_ids.masked_fill_(pad_mask, _NO_EXPERT) + xq, xsf = fp8_quantize_1x128(x_rows, use_ue8m0=_MOE_USE_UE8M0) + nb = xsf.shape[0] + rows = x_rows.shape[0] + # bf16 columns: nb fp32 scales, topk int32 ids, topk bf16 weights (the width stays even). + c0, c1, c2 = 2 * nb, 2 * nb + 2 * topk, 2 * nb + 3 * topk + meta = torch.empty([rows, c2 + (c2 & 1)], dtype=torch.bfloat16, device=x_rows.device) + meta[:, :c0].view(torch.float32).copy_(xsf.t()) + meta[:, c0:c1].view(torch.int32).copy_(topk_ids) + meta[:, c1:c2].copy_(topk_w) + codes = gather(xq.view(torch.bfloat16)).view(xq.dtype) + every = gather(meta) + scales = every[:, :c0].view(torch.float32).t() + ids = every[:, c0:c1].contiguous().view(torch.int32) + weights = every[:, c1:c2].contiguous() + total = codes.shape[0] + step = getattr(core, "moe_max_rows", _MOE_MAX_ROWS) + if after_first is not None and total <= step: + return _window_runner( + core, + experts, + codes, + scales.contiguous(), + ids, + weights, + fin_first=fin_first, + after_first=after_first, + ) + parts = [ + _window_runner( + core, + experts, + codes[lo : lo + step], + scales[:, lo : lo + step].contiguous(), + ids[lo : lo + step], + weights[lo : lo + step], + ) + for lo in range(0, total, step) + ] + out = parts[0] if len(parts) == 1 else torch.cat(parts, 0) + if after_first is not None: + after_first(out) + return out + + +def _moe_block_sp(core, x_rows, L: _LayerWeights, gather, scatter, pad_mask=None) -> tuple: + """`_moe_block` under SP, returning `(routed, gate_logits, shared)` over this rank's rows. + + The shared expert runs on the side stream, overlapping the routed path's gathers.""" + main = torch.cuda.current_stream() + side = core._side_stream + core._side_fork.record(main) + side.wait_event(core._side_fork) + with torch.cuda.stream(side): + gug = cublas_mm(x_rows, L.sh_gug) + shared = cublas_mm(silu_and_mul_rows(gug, core.shared_inter), L.sh_dn) + gate_logits = gug[:, 2 * core.shared_inter : 2 * core.shared_inter + 1] + split = getattr(scatter, "split_finalize", None) + if split is not None: + # Two ranks: the peer's rows are finalized and sent first, overlapping this rank's. + start, finish, peer_rows = split + routed = finish( + _routed_experts_sp( + core, x_rows, L.router, L.experts, gather, pad_mask, peer_rows, start + ) + ) + else: + routed = scatter(_routed_experts_sp(core, x_rows, L.router, L.experts, gather, pad_mask)) + core._side_join.record(side) + main.wait_event(core._side_join) + return routed, gate_logits, shared + + +def _projection_warm_rows(lo: int, hi: int) -> list[int]: + """One row count of `[lo, hi]` for each kernel cuBLASLt can pick for a bf16 projection. + + Every row up to `_KERNEL_CHOICE_EACH_ROW`, then one per 8-row group, which shares a kernel.""" + each = min(hi, _KERNEL_CHOICE_EACH_ROW) + rows = list(range(lo, each + 1)) + first = max(lo, _KERNEL_CHOICE_EACH_ROW + 1) + if first <= hi: + group = _KERNEL_CHOICE_ROW_GROUP + ends = range(-(-first // group) * group, hi + group, group) + rows += [min(end, hi) for end in ends] + return rows + + +def _eager_projections(core, layers, stack: _StackWeights, mtp) -> list[tuple]: + """Every bf16 projection an eager step can hand `cublas_mm`, as `(mat_b, out_dtype, lo, hi)`. + + One entry per cuBLASLt kernel-choice key (K, N, strides, output dtype) with its row range.""" + rows = core.max_num_tokens + router_rows = min(getattr(core, "moe_max_rows", _MOE_MAX_ROWS), core.dp_size * rows) + hc_cols = core.hc_first_cols + found = {} + + def add(mat_b, hi, lo=1, out_dtype=None): + key = (tuple(mat_b.shape), mat_b.stride(), mat_b.dtype, out_dtype) + if key in found: + _, _, was_lo, was_hi = found[key] + lo, hi = min(lo, was_lo), max(hi, was_hi) + if lo <= hi: + found[key] = (mat_b, out_dtype, lo, hi) + + def mixer(first): + add(first[:, :hc_cols], rows, _HC_TC_FIRST_MAX_ROWS + 1, torch.float32) + + for L in layers: + mixer(L.ahc_first) + mixer(L.mhc_first) + for mat_b in (L.attn_in, L.o, L.in_proj, L.out, L.sh_gug, L.sh_dn): + if mat_b is not None: + add(mat_b, rows) + add(L.router, router_rows) + mixer(stack.fhc_first) + add(stack.ple_key, rows) + add(stack.ple_val, rows) + if mtp is not None: + for first in (mtp.ahc_first, mtp.mhc_first, mtp.fhc_first): + mixer(first) + for mat_b in (mtp.fc_e, mtp.attn_in, mtp.o, mtp.sh_gug, mtp.sh_dn): + add(mat_b, rows) + add(mtp.fc_h, core.hc_count * rows) + add(mtp.router, router_rows) + return list(found.values()) + + +def _load_projection_kernels(projections, device) -> int: + """Run each projection at every `_projection_warm_rows` count; return the calls made. + + A kernel's first lazy-loaded launch waits for queued work and stalls the next collective.""" + k_max = max(mat_b.shape[0] * hi for mat_b, _, _, hi in projections) + flat = torch.zeros(k_max, dtype=torch.bfloat16, device=device) + calls = 0 + for mat_b, out_dtype, lo, hi in projections: + k = mat_b.shape[0] + for m in _projection_warm_rows(lo, hi): + cublas_mm(flat[: m * k].view(m, k), mat_b, None, out_dtype) + calls += 1 + return calls + + +def _attn_in_heads(core, proj): + """The main q, k, v heads of `proj` `[T, attn_in_width]` as `[T, heads, head_dim]` views.""" + q, k, v, _ = torch.split( + torch.reshape(proj, [proj.shape[0], core.attn_in_width // core.head_dim, core.head_dim]), + [core.heads_q, core.heads_kv, core.heads_kv, core.attn_in_tail_heads - core.heads_kv], + 1, + ) + return q, k, v + + +def _packed_q(core, qkv): + """The query heads `[T, heads_q, head_dim]` of the compact `[T, qkv_width]` packed QKV.""" + heads = core.heads_q + 2 * core.heads_kv + return torch.reshape(qkv, [qkv.shape[0], heads, core.head_dim])[:, : core.heads_q] + + +def _attention_rope(core, proj, index_scratch, L, pos3, pos, theta=None) -> None: + """Both planes' Gemma norm + rope: main q/k in place in `proj`, index q/k in `index_scratch`. + + The index q/k are rotated in a copy so the raw index key stays in `proj` for the index store; + the heads after k pass as v only to set the row stride. `theta` defaults to the trunk's.""" + base = core.theta if theta is None else theta + fused_qk_norm_rope( + proj, + num_heads_q=core.heads_q, + num_heads_k=core.heads_kv, + num_heads_v=core.attn_in_tail_heads, + head_dim=core.head_dim, + rotary_dim=core.rotary_dim, + eps=core.eps, + q_weight=L.q_norm, + k_weight=L.k_norm, + base=base, + is_neox=True, + position_ids=pos3, + is_qk_norm=True, + use_gemma=True, + use_mrope=True, + mrope_section1=core.mrope_section1, + mrope_section2=core.mrope_section2, + ) + if index_scratch is None: + return + index_scratch[:, : core.index_width].copy_( + proj[:, core.index_col : core.index_col + core.index_width] + ) + fused_qk_norm_rope( + index_scratch, + num_heads_q=core.index_heads, + num_heads_k=core.index_kv_heads, + num_heads_v=core.index_kv_heads, + head_dim=core.index_head_dim, + rotary_dim=core.rotary_dim, + eps=core.eps, + q_weight=L.idx_qn, + k_weight=L.idx_kn, + base=base, + is_neox=True, + position_ids=pos, + is_qk_norm=True, + use_gemma=True, + use_mrope=False, + mrope_section1=0, + mrope_section2=0, + ) + + +def _index_query(core, index_scratch): + """The rotated index query `[T, index_heads, index_head_dim]`, a view of `index_scratch`.""" + rows = index_scratch.shape[0] + return torch.split( + torch.reshape( + index_scratch, + [rows, core.index_heads + 2 * core.index_kv_heads, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, core.index_kv_heads], + 1, + )[0] + + +def _index_plane(core, proj, i, L: _LayerWeights, qsa) -> None: + """Store layer `i`'s raw index keys and positions; compress the groups this step completes. + + The key is stored raw because the compression applies the norm and the rotation itself.""" + req_idx, logical, block_table, tokens_per_block, coords = qsa + rows = proj.shape[0] + token_k = torch.split( + torch.reshape( + proj[:, core.index_col : core.index_col + core.index_scratch_width], + [rows, core.index_scratch_width // core.index_head_dim, core.index_head_dim], + ), + [core.index_heads, core.index_kv_heads, 1], + 1, + )[1] + pools = core._bound.pools + index_cache = pools.index_cache[i] + qsa_paged_index_store( + token_k, + coords, + index_cache, + pools.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + ) + qsa_prefill_compress( + logical_positions=logical, + request_indices=req_idx, + block_table=block_table, + index_cache=index_cache, + position_cache=pools.position_cache, + k_norm_weight=L.idx_kn, + cos_sin=core._cos_sin, + eps=core.eps, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + mrope_section=_QSA_MROPE_SECTION, + ) + + +def _select(core, scores, row_starts, row_ends, topk_indices) -> None: + """Each row's canonical top-`block_topk` groups of its window into `topk_indices`. + + Canonical: groups above the budget's last score, then the highest-index ties, ascending, `-1` + last (row starts are zero, so window and table indices agree).""" + qsa_topk_select(scores, row_starts, row_ends, topk_indices, index_topk=core.block_topk) + qsa_canonical_selection(scores, row_starts, row_ends, topk_indices) + + +def _index_split_group(core) -> tuple | None: + """`(group, process_group, rank, lamport)` a decode step's index scoring splits over, or None. + + One small in-order gather builds the NCCL communicator now; a first call cannot be captured.""" + if core.attn_tp_size < 2 or core.dp_size != 1 or core.tp_sum is None: + return None + group = list(core.mapping.tp_group) + process_group = core.mapping.tp_group_pg.boxed() if mpi_disabled() else None + probe = torch.zeros([1, 1], dtype=torch.int32, device=core._cos_sin.device) + allgather(probe, None, group, process_group) + return group, process_group, core.mapping.tp_rank, _index_split_lamport(core) + + +def _index_split_lamport(core): + """The split gather's Lamport IPC buffers, or None where the Lamport exchange does not apply. + + Needs 2-4 attention-TP ranks, `_own_ipc`, PDL and captured batches that reach the split; + without it the split gathers over NCCL. + A collective: every rank builds the buffers here, before any capture.""" + graph_batches = tuple( + getattr(getattr(core, "model_config", None), "cuda_graph_batch_sizes", ()) or () + ) + if not ( + 2 <= core.attn_tp_size <= 4 + and get_env_enable_pdl() + and _INDEX_SPLIT_LAMPORT_ROWS > 0 + and (not graph_batches or max(graph_batches) >= _INDEX_SPLIT_MIN_REQUESTS) + and _own_ipc(core) + ): + return None + width = core.token_topk + core.compress_ratio - 1 + rows = min(_INDEX_SPLIT_LAMPORT_ROWS, core.max_num_tokens) + return _selection_lamport_build(core.mapping, rows * width) + + +def _index_split_rows(split, rows: int, rows_per_seq: int) -> tuple | None: + """This rank's `[lo, hi)` rows of a split index scoring, or None to score every row.""" + if split is None: + return None + group, rank = split[0], split[2] + t = len(group) + requests = rows // rows_per_seq + if requests < _INDEX_SPLIT_MIN_REQUESTS or requests % t or rows % rows_per_seq: + return None + share = (requests // t) * rows_per_seq + return rank * share, (rank + 1) * share + + +def _index_split_plan(split, qsa, num_tokens: int, rows_per_seq: int): + """A work-balanced request plan for the split index selection, or None to split by position. + + Made once a step for every layer, over the Lamport gather only; splitting by position puts the + CUDA graph padding on one rank. Bitwise equal to the position split.""" + if _index_split_rows(split, num_tokens, rows_per_seq) is None: + return None + lam = split[3] if len(split) > 3 else None + req_idx, logical, seq_lens_q, visible_blocks, _, row_starts = qsa[:6] + vectors = (visible_blocks, req_idx, logical, seq_lens_q, row_starts) + if lam is None or not all(v.is_contiguous() for v in vectors): + return None + return index_split_plan( + visible_blocks[:num_tokens], + req_idx[:num_tokens], + logical[:num_tokens], + seq_lens_q[:num_tokens], + row_starts[:num_tokens], + num_tokens // rows_per_seq, + rows_per_seq, + len(split[0]), + split[2], + ) + + +def _decode_index_selection( + core, split, q, qsa, num_tokens, p, device, index_cache, plan=None, q_permuted=False, order=None +): + """A sparse decode layer's int32 `[num_tokens, token_topk + compress_ratio - 1]` selection. + + Index scores of `q` (`p` rows a request), then `_selection`, over every row or, under `split`, + over this rank's rows gathered in rank order. `qsa` is `(req_idx, logical, seq_lens_q, + visible_blocks, topk_indices, topk_row_starts, block_table, tokens_per_block)`. With a `plan`, + this rank takes its slice of the permuted order (`q_permuted`: `q` is in it already). `order`: + the step's `_RowOrder`, shared by its layers.""" + req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts = qsa[:6] + block_table, tokens_per_block = qsa[6], qsa[7] + part = _index_split_rows(split, num_tokens, p) + if part is None: + plan = None + lo, hi = (0, num_tokens) if part is None else part + rows = hi - lo + if plan is not None: + q_rows = q[lo:hi] if q_permuted else q.index_select(0, plan.perm_rows[lo:hi]) + rows_of = (plan.req_idx, plan.logical, plan.seq_lens, plan.visible, plan.row_starts) + else: + q_rows = q[lo:hi] + rows_of = ( + req_idx[lo:hi], + logical[lo:hi], + seq_lens_q[lo:hi], + visible_blocks[lo:hi], + topk_row_starts[lo:hi], + ) + r_req, r_logical, r_seq, r_visible, r_starts = rows_of + rpr = _index_rows_per_request(rows, p) + if ( + rpr > 1 + and rows >= _INDEX_SCORES_CUTE_MIN_GROUPS * rpr + and logical.dtype == torch.int64 + and req_idx.dtype == torch.int32 + and _index_groups_cute_serves(q_rows, index_cache, rpr) + ): + scores = index_scores_groups_cute( + q_rows, + index_cache, + block_table, + r_logical, + r_req, + tokens_per_block, + core.compress_ratio, + _INDEX_SCORES_PDL, + rpr, + ) + else: + scores = qsa_paged_index_scores( + q=q_rows, + index_cache=index_cache, + block_table=block_table, + query_positions=r_logical, + request_indices=r_req, + tokens_per_block=tokens_per_block, + compress_ratio=core.compress_ratio, + only_visible_blocks=True, + context_rows=False, + use_pdl=_INDEX_SCORES_PDL, + rows_per_request=rpr, + wide_split=True, + ) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=device + ) + _selection( + core, + scores, + r_visible, + r_logical, + r_seq, + topk_indices[lo:hi], + r_starts, + selected, + order=order, + ) + if part is None: + return selected + group, process_group = split[0], split[1] + lam = split[3] if len(split) > 3 else None + if lam is not None and lam.fits(num_tokens * selected.shape[1]): + if plan is None: + return lamport_allgather(selected, lam) + return lamport_allgather(selected, lam, plan.perm_rows) + gathered = allgather(selected, None, group, process_group) + if plan is None: + return gathered + out = torch.empty_like(gathered) + out.index_copy_(0, plan.perm_rows.long(), gathered) + return out + + +def _index_rows_per_request(rows: int, rows_per_seq: int) -> int: + """`qsa_paged_index_scores`' `rows_per_request`: grouped from `_INDEX_GROUP_MIN_ROWS` rows.""" + return rows_per_seq if rows_per_seq > 1 and rows >= _INDEX_GROUP_MIN_ROWS else 1 + + +def _draft_continue_ok(core, md, p: int, share: bool) -> bool: + """Whether `_draft_continue_rows` returns rows rather than None.""" + if p <= 1 or core.dp_size != 1 or share: + return False + return getattr(md, "qsa_mtp_num_accepted", None) is not None + + +def _draft_continue_rows(core, md, gen_seqs: int, p: int, share: bool): + """The `[gen_seqs]` int64 rows a draft-extend step's outputs are read from, or None for all. + + Each request's last accepted row; the others only leave K/V and index keys behind. Not under + attention DP (the MoE pads by the worker's rows) or IndexShare (it keeps every selection).""" + if p <= 1 or core.dp_size != 1 or share: + return None + accepted = getattr(md, "qsa_mtp_num_accepted", None) + if accepted is None: + return None + last = torch.clamp(accepted[:gen_seqs].to(torch.int64) - 1, min=0, max=p - 1) + return torch.arange(gen_seqs, dtype=torch.int64, device=last.device) * p + last + + +class _RowOrder: + """A step's sparse decode rows, longest visible window first, for the ordered launch of + `qsa_topk_expand_decode`: sorted at the first layer that launches it, then read by the step's + other layers, which select over the same rows.""" + + def __init__(self): + self._order = None + + def __call__(self, visible_blocks): + if self._order is None: + self._order = torch.argsort(visible_blocks, descending=True, stable=True).to( + torch.int32 + ) + assert self._order.shape == visible_blocks.shape, "one row order serves one row set" + return self._order + + +def _selection( + core, + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + row_starts, + out, + context=False, + order=None, +): + """The canonical block top-k and expansion of `scores`' rows into `out`, fused if possible. + `order`: a `_RowOrder` over these rows, for the ordered wide launch.""" + if ( + context + and scores.shape[0] >= _TOPK_ROWS_MIN_ROWS + and _topk_rows_supports(scores, topk_indices) + ): + qsa_topk_rows( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + ) + return + if _topk_expand_cluster_supports(scores): + qsa_topk_expand_cluster( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + ) + return + if _topk_expand_supports(scores): + row_order = None + if order is not None and _topk_expand_orders_rows(scores): + row_order = order(visible_blocks) + qsa_topk_expand_decode( + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + core.compress_ratio, + core.token_topk, + out=out, + row_order=row_order, + ) + return + _select(core, scores, row_starts, visible_blocks, topk_indices) + out.copy_( + qsa_expand_block_indices( + topk_indices, + logical, + seq_lens_q, + compress_ratio=core.compress_ratio, + token_topk=core.token_topk, + ) + ) + + +def _score_chunk_rows(core, block_table, tokens_per_block: int) -> int: + """Rows one `_score_and_select` call scores: fewer than `_SCORE_TABLE_ELEMENTS / columns`, + in whole query tiles.""" + columns = block_table.shape[1] * tokens_per_block // core.compress_ratio + chunk = max(1, (_SCORE_TABLE_ELEMENTS - 1) // columns) + if chunk >= _SELECTION_CUT_ALIGN: + chunk -= chunk % _SELECTION_CUT_ALIGN + return chunk + + +def _score_and_select( + core, + index_cache, + q_index, + block_table, + logical, + req_idx, + tokens_per_block: int, + visible_blocks, + seq_lens_q, + topk_indices, + row_starts, + lo: int, + hi: int, + context_rows: bool, + out, + order=None, +) -> None: + """Score rows `[lo, hi)` against `index_cache` and select them into `out` (`_selection`), + in equal chunks of at most `_score_chunk_rows` (whole query tiles): a row's scores and + selection read only its query, position and request, so the chunks select what one call + would; below the bound the rows are one call. Equal chunks keep the last one past the + index scores' decode-row limit, whose launch rounds the scores differently. `order`: the + step's `_RowOrder` over rows `[lo, hi)`, read when they are one call.""" + bound = _score_chunk_rows(core, block_table, tokens_per_block) + pieces = max(1, -(-(hi - lo) // bound)) + chunk = -(-(hi - lo) // pieces) + if bound >= _SELECTION_CUT_ALIGN: + chunk = -(-chunk // _SELECTION_CUT_ALIGN) * _SELECTION_CUT_ALIGN + for a in range(lo, hi, max(1, chunk)): + b = min(a + chunk, hi) + scores = _context_index_scores( + q_index[a:b], + index_cache, + block_table, + logical[a:b], + req_idx[a:b], + tokens_per_block, + core.compress_ratio, + context_rows, + ) + _selection( + core, + scores, + visible_blocks[a:b], + logical[a:b], + seq_lens_q[a:b], + topk_indices[a:b], + row_starts[a:b], + out[a - lo : b - lo], + context=context_rows, + order=order if pieces == 1 else None, + ) + + +def _prefill_split_enabled(attn_tp_size: int, dp_size: int) -> bool: + """Whether a prefill step splits its selection and row-wise work over the attention-TP group.""" + return dp_size == 1 and attn_tp_size in _PREFILL_SPLIT_DEFAULT_TP + + +def _selection_cuts(chunks, starts, world: int, row_tokens: int = 0) -> tuple: + """Row boundaries `(0, c_1, ..., c_{world-1}, rows)` of a context step's split selection. + + A row at position `x` costs `x + 1 + row_tokens`; cut `r` takes `r / world` of the total, + rounded to `_SELECTION_CUT_ALIGN`. Host arithmetic, identical on every rank.""" + rows = sum(chunks) + base = [s + 1 + row_tokens for s in starts] + # Summed cost of a request's first `m` rows: m * base + m (m - 1) / 2. + totals = [c * b + c * (c - 1) // 2 for c, b in zip(chunks, base)] + total = sum(totals) + cuts = [0] + req, row0, done = 0, 0, 0 + for r in range(1, world): + want = total * r // world + while req < len(chunks) - 1 and done + totals[req] < want: + done += totals[req] + row0 += chunks[req] + req += 1 + # The fewest rows `m` of request `req` whose cost reaches `want - done`: the root of + # m^2 / 2 + (b - 1 / 2) m = rest, refined on the integers. + b, c, rest = base[req], chunks[req], want - done + m = int(-(b - 0.5) + ((b - 0.5) ** 2 + 2 * rest) ** 0.5) + m = max(0, min(c, m)) + while m < c and m * b + m * (m - 1) // 2 < rest: + m += 1 + while m > 0 and (m - 1) * b + (m - 1) * (m - 2) // 2 >= rest: + m -= 1 + cut = int(round((row0 + m) / _SELECTION_CUT_ALIGN)) * _SELECTION_CUT_ALIGN + cuts.append(min(rows, max(cuts[-1], cut))) + cuts.append(rows) + return tuple(cuts) + + +def _split_context_selection( + split, score_and_select, selected, lo: int, hi: int, pipeline=None, cuts=None +) -> None: + """Rows `[lo, hi)` of `selected` scored and selected over the attention-TP group. + + Each rank runs `score_and_select` on its share (equal, or by `cuts`) into its own + `[per, width]` buffer; one uniform gather lays the shares out in rank order (int32 as bf16 + pairs), bitwise the unsplit result. A collective: every rank calls it with the same `hi - lo`. + `pipeline` `(pieces, comm, fork, join[, exchange])` gathers each piece on `comm` while the + next is scored.""" + rank, world, group, process_group = split + rows = hi - lo + if cuts is None: + per = -(-rows // world) + starts = [min(lo + r * per, hi) for r in range(world)] + ends = [min(s + per, hi) for s in starts] + else: + assert len(cuts) == world + 1 and cuts[0] == 0 and cuts[-1] == rows, (cuts, rows) + starts = [lo + c for c in cuts[:-1]] + ends = [lo + c for c in cuts[1:]] + per = max(e - s for s, e in zip(starts, ends)) + a, b = starts[rank], ends[rank] + mine = torch.empty([per, selected.shape[1]], dtype=selected.dtype, device=selected.device) + pieces = 1 + if pipeline is not None: + pieces = max(1, min(pipeline[0], per // _SELECTION_PIECE_MIN_ROWS)) + if pieces == 1: + if b > a: + score_and_select(a, b, True, mine[: b - a]) + gathered = allgather(mine.view(torch.bfloat16), None, group, process_group) + gathered = gathered.view(selected.dtype) + if cuts is None: + selected[lo:hi].copy_(gathered[:rows]) + return + for r in range(world): + n = ends[r] - starts[r] + if n > 0: + selected[starts[r] : ends[r]].copy_(gathered[r * per : r * per + n]) + return + _, comm, fork, join = pipeline[:4] + exchange = pipeline[4] if len(pipeline) > 4 else None + size = -(-per // pieces) + width = selected.shape[1] * selected.element_size() + if exchange is not None and (world != 2 or size * width > exchange.capacity): + exchange = None + main = torch.cuda.current_stream() + for p0 in range(0, per, size): + p1 = min(p0 + size, per) + if a + p0 < b: + score_and_select(a + p0, min(a + p1, b), True, mine[p0 : min(p1, b - a)]) + fork.record(main) + comm.wait_event(fork) + with torch.cuda.stream(comm): + if exchange is not None: + # The peer's piece is a view of the receive buffer, copied out below. + pieces_of = {rank: mine[p0:p1], 1 - rank: exchange.exchange(mine[p0:p1])} + else: + gathered = allgather(mine[p0:p1].view(torch.bfloat16), None, group, process_group) + gathered = gathered.view(selected.dtype) + pieces_of = {r: gathered[r * (p1 - p0) : (r + 1) * (p1 - p0)] for r in range(world)} + for r in range(world): + r0 = starts[r] + p0 + r1 = min(starts[r] + p1, ends[r]) + if r1 > r0: + selected[r0:r1].copy_(pieces_of[r][: r1 - r0]) + join.record(comm) + main.wait_event(join) + + +def _decode_prologue(kv, sparse: bool) -> bool: + """Whether a decode attention layer runs the fused prologue: sparse, over a bf16 or e4m3 pool. + + The dense op appends K/V itself and an NVFP4 pool needs the quantizing store.""" + return sparse and kv.k_scale is None + + +def _decode_prologue_call( + core, proj, qkv, index_scratch, i, L, qsa, rows_per_request, index_q_rows=None +) -> None: + """`qwen4_exp_attn_decode_prologue` over layer `i`'s pools. + + `index_q_rows[t]`, if given, is row `t`'s `index_scratch` row (the split's permuted order).""" + req_idx, logical, block_table, tokens_per_block, pos3, coords = qsa + pools = core._bound.pools + kv = pools.kv[i] + qwen4_exp_attn_decode_prologue( + proj, + qkv, + _index_query(core, index_scratch), + pos3, + coords, + core._cos_sin, + L.q_norm, + L.k_norm, + L.idx_qn, + L.idx_kn, + kv.k, + kv.v, + pools.index_cache[i], + pools.position_cache, + req_idx, + logical, + block_table, + tokens_per_block, + core.compress_ratio, + core.heads_q, + core.heads_kv, + core.head_dim, + core.index_heads, + core.index_head_dim, + core.index_col, + core.rotary_dim, + core.eps, + core.mrope_section1, + core.mrope_section2, + rows_per_request=rows_per_request, + index_q_rows=index_q_rows, + ) + + +def _store_kv(core, kv, k, v, request_indices, logical_positions, block_table, tokens_per_block): + """Append this step's K/V rows to the layer's paged pool. + + An fp8 pool takes e4m3 unscaled, matching the dense op's unit KV scales.""" + if kv.k_scale is None: + qsa_paged_kv_store( + k, v, kv.k, kv.v, request_indices, logical_positions, block_table, tokens_per_block + ) + return + qsa_paged_kv_store_nvfp4( + k, + v, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + request_indices, + logical_positions, + block_table, + tokens_per_block, + core.kv_scale_orig_quant, + ) + + +def _fp8_context_pages(md, tpb: int) -> int: + """Pool pages the step's context requests hold, `ceil(kv_len / tpb)` each, from host lengths.""" + lens = md.kv_lens_runtime[: md.num_contexts] + return int(((lens + (tpb - 1)) // tpb).sum()) + + +def _rowtile_takes(kv, rows: int, fp8_pages) -> bool: + """Whether `_sparse_gqa` sends a context launch of `rows` rows to the row-tile pass. + + Mirrors `_sparse_gqa_rowtile`'s bound on the host; the pass may take q as a strided view.""" + if kv.k_scale is not None or rows < _ROWTILE_MIN_ROWS: + return False + if kv.k.dtype == torch.bfloat16: + return True + return ( + kv.k.dtype == torch.float8_e4m3fn + and fp8_pages is not None + and fp8_pages * kv.k[0].numel() * 4 <= _FP8_ROWTILE_MAX_SCRATCH + ) + + +def _rowtile_scratch_groups(md, page_bytes: int) -> tuple | None: + """Row-tile launches `(first row, end row, pages)` of an fp8 context step, or None for one. + + The fewest groups of consecutive requests whose widened pages fit `_FP8_ROWTILE_MAX_SCRATCH`, + pages spread evenly; a request never splits (its pages always fit).""" + num_contexts = md.num_contexts + tpb = md.tokens_per_block + rows = [int(r) for r in md.seq_lens[:num_contexts].tolist()] + pages = [-(-int(k) // tpb) for k in md.kv_lens_runtime[:num_contexts].tolist()] + cap = _FP8_ROWTILE_MAX_SCRATCH // page_bytes + if sum(pages) <= cap: + return None + + def fill(limit: int) -> list: + # Contiguous groups, each closed before a request that would take it past `limit`. + out, first, held = [], 0, 0 + for j, p in enumerate(pages): + if held and held + p > limit: + out.append((first, j, held)) + first, held = j, 0 + held += p + out.append((first, len(pages), held)) + return out + + fewest = len(fill(cap)) + lo, hi = max(pages), cap + while lo < hi: + mid = (lo + hi) // 2 + if len(fill(mid)) <= fewest: + hi = mid + else: + lo = mid + 1 + starts = [0] + for r in rows: + starts.append(starts[-1] + r) + return tuple((starts[a], starts[b], n) for a, b, n in fill(lo)) + + +def _sparse_gqa_rowtile( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages +): + """The row-tile pass over a context launch, or None where it does not serve it. + + An fp8 pool's reached pages are widened exactly into a bf16 scratch pool; `fp8_pages` is + their host count, without which the pass does not serve an fp8 pool (`_rowtile_takes`).""" + k_cache, v_cache, table = kv.k, kv.v, block_table + if kv.k.dtype == torch.float8_e4m3fn: + if fp8_pages is None or fp8_pages * kv.k[0].numel() * 4 > _FP8_ROWTILE_MAX_SCRATCH: + return None + # Each slot's highest position, -1 if absent: a compare-and-max over `[slots, rows]` + # (a scatter max would serialize a request's rows on one address). + slots = torch.arange(block_table.shape[0], dtype=request_indices.dtype, device=q.device) + held = request_indices[None, :] == slots[:, None] + last = torch.where(held, query_positions[None, :].long(), -1).amax(1) + counts = torch.div(last + tpb, tpb, rounding_mode="floor") + k_cache = torch.empty([fp8_pages, *kv.k.shape[1:]], dtype=q.dtype, device=q.device) + v_cache = torch.empty_like(k_cache) + table = torch.full_like(block_table, -1) + qsa_fp8_pages_to_bf16( + kv.k, + kv.v, + block_table, + slots.int(), + counts, + torch.cumsum(counts, 0) - counts, + table, + k_cache, + v_cache, + ) + elif kv.k.dtype != q.dtype: + return None + return qsa_sparse_gqa_rowtile( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + query_positions=query_positions, + compress_ratio=core.compress_ratio, + ) + + +def _sparse_gqa( + core, + kv, + q, + block_table, + selected, + request_indices, + tpb, + query_positions, + fp8_pages=None, + output_gate=None, + rows_per_request=1, +): + """Sparse GQA over the layer's paged pool. + + `query_positions` bounds each context row's scan (None for decode rows). Decode rows only: + `output_gate` `[rows, Hq * D]` or `[rows, Hq, D]` makes the result `bf16(attn) * + sigmoid(gate)`, and `rows_per_request` is the consecutive rows of each request.""" + if kv.k_scale is None and query_positions is None: + if ( + rows_per_request > 1 + and q.shape[0] >= _GROUPED_DECODE_MIN_REQUESTS * rows_per_request + and qsa_sparse_gqa_grouped_cute.serves(q, kv.k, selected, rows_per_request) + ): + return qsa_sparse_gqa_grouped_cute( + q, + kv.k, + kv.v, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + rows_per_request, + output_gate, + ) + if qsa_sparse_gqa_splitk_cute.serves(q, kv.k, selected): + # Compile the split-K fallback's row buckets at warm-up, not on a serving step. + warm_decode_buckets( + q=q, + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + compress_ratio=core.compress_ratio, + ) + return qsa_sparse_gqa_splitk_cute( + q, + kv.k, + kv.v, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + output_gate, + ) + out = _sparse_gqa_ungated( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages + ) + if output_gate is not None: + fused_sigmoid_mul_inplace( + torch.reshape(out, [out.shape[0], -1]), output_gate, direct_launch=True + ) + return out + + +def _sparse_gqa_ungated( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages=None +): + """`_sparse_gqa` without the CuTe decode kernels and without the output gate.""" + if kv.k_scale is None: + if query_positions is not None and q.shape[0] >= _ROWTILE_MIN_ROWS: + out = _sparse_gqa_rowtile( + core, kv, q, block_table, selected, request_indices, tpb, query_positions, fp8_pages + ) + if out is not None: + return out + return qsa_paged_sparse_gqa( + q=q, + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=request_indices, + tokens_per_block=tpb, + softmax_scale=core.softmax_scale, + query_positions=query_positions, + compress_ratio=None if query_positions is None else core.compress_ratio, + output_gate=None, + ) + if query_positions is None: + return qsa_paged_sparse_gqa_nvfp4_decode( + q, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + core.kv_scale_quant_orig, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + None, + ) + return qsa_paged_sparse_gqa_nvfp4( + q, + kv.k, + kv.v, + kv.k_scale, + kv.v_scale, + core.kv_scale_quant_orig, + block_table, + selected, + request_indices, + tpb, + core.softmax_scale, + query_positions=query_positions, + compress_ratio=core.compress_ratio, + ) + + +def _copy_positions(position_ids: torch.Tensor) -> torch.Tensor: + """A contiguous int32 `[3, N]` copy of the mRoPE positions; the rope rejects the engine's.""" + pos3 = torch.empty( + [_QSA_POSITION_AXES, position_ids.numel() // _QSA_POSITION_AXES], + dtype=torch.int32, + device=position_ids.device, + ) + pos3.copy_(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1])) + return pos3 + + +def _step_args(md: TrtllmAttentionMetadata, num_contexts, num_ctx_tokens, paged_context) -> dict: + """thop_attention's batch state off the prepared metadata. + + Tensors are engine-owned buffers refreshed in place; ints are per-capture constants. + `paged_context` forces the paged-context path the engine enables only for chunked prefill + or KV cache block reuse: an NVFP4 pool's context kernels exist on it alone.""" + return dict( + sequence_length=md.kv_lens_cuda_runtime, + host_past_key_value_lengths=md.kv_lens_runtime, + host_total_kv_lens=md.host_total_kv_lens, + context_lengths=md.prompt_lens_cuda_runtime, + host_context_lengths=md.prompt_lens_cpu_runtime, + host_request_types=md.host_request_types_runtime, + kv_cache_block_offsets=md.kv_cache_block_offsets, + host_kv_cache_pool_pointers=md.host_kv_cache_pool_pointers, + host_kv_cache_pool_mapping=md.host_kv_cache_pool_mapping, + workspace=md.effective_workspace, + tokens_per_block=md.tokens_per_block, + max_num_requests=md.max_num_requests, + max_context_length=md.max_context_length, + max_seq_len=md.max_seq_len, + attention_window_size=md.max_seq_len, + num_contexts=num_contexts, + num_ctx_tokens=num_ctx_tokens, + trtllm_gen_jit_warmup=md.trtllm_gen_jit_warmup, + use_paged_context_fmha=md.use_paged_context_fmha or paged_context, + beam_width=md.effective_beam_width, + cache_indirection=md.cache_indirection, + block_ids_per_seq=md.block_ids_per_seq, + max_context_q_len_override=md.max_context_q_len_override, + is_cross=md.is_cross, + is_spec_decoding_enabled=md.is_spec_decoding_enabled, + use_spec_decoding=md.use_spec_decoding, + is_spec_dec_tree=md.is_spec_dec_tree, + spec_decoding_generation_lengths=md.spec_decoding_generation_lengths, + spec_decoding_position_offsets=get_spec_decoding_position_offsets(md), + spec_decoding_packed_mask=md.spec_decoding_packed_mask, + spec_decoding_bl_tree_mask_offset=md.spec_decoding_bl_tree_mask_offset, + spec_decoding_bl_tree_mask=md.spec_decoding_bl_tree_mask, + spec_decoding_target_max_draft_tokens=md.max_total_draft_tokens, + spec_bl_tree_first_sparse_mask_offset_kv=md.spec_bl_tree_first_sparse_mask_offset_kv, + num_sparse_topk=md.num_sparse_topk, + flash_mla_tile_scheduler_metadata=md.flash_mla_tile_scheduler_metadata, + flash_mla_num_splits=md.flash_mla_num_splits, + max_num_sequences=md.max_num_sequences, + force_prepare_spec_dec_tree_mask=md.force_prepare_spec_dec_tree_mask, + ) + + +class _KvRangeProbe: + """`TRTLLM_MODELING_V2_VALIDATE` over an fp8 KV pool: reports, once, the largest K/V + magnitude the first eager forward's attention layers store. + + The pool takes K/V at the unit scale, so anything past the e4m3 maximum saturates silently + (a bf16 inf included). One host sync per attention layer of that one forward; a forward + under CUDA graph capture is passed over without a read.""" + + def __init__(self, last_layer: int) -> None: + self._last = last_layer + self._worst = 0.0 + + @classmethod + def for_target(cls, core, enabled: bool) -> "_KvRangeProbe | None": + """A probe over `core`'s attention layers when `enabled` and the pool is fp8.""" + if not enabled or core.quant_mode != QM_FP8_KV_CACHE: + return None + return cls(core.attn_layers[-1]) + + def observe(self, layer: int, k: torch.Tensor, v: torch.Tensor) -> bool: + """Fold one attention layer's K/V in; False once the forward's last layer has reported.""" + if torch.cuda.is_current_stream_capturing(): + return True + peak = float(torch.maximum(k.abs().amax(), v.abs().amax())) + # A NaN stays: `max` would drop it. + self._worst = math.nan if math.isnan(peak + self._worst) else max(self._worst, peak) + if layer != self._last: + return True + limit = torch.finfo(torch.float8_e4m3fn).max + # Written so that NaN fails too. + if not self._worst <= limit: + logger.warning( + f"FP8 KV cache: the first forward stores a K/V magnitude of {self._worst:g}, " + f"past the e4m3 maximum {limit:g}; the pool is written at the unit scale, " + "so those values saturate" + ) + return False + + +# ------------------------------------------------------------------ prefill --- + + +def _pad_window(num_tokens: int, rows: int) -> int: + """First row a break graph over a `rows` bucket zeroes from; part of the graph key. + + Half the bucket if the batch fills more than half (any power-of-two bucket ladder), else 0.""" + return rows // 2 if num_tokens > rows // 2 else 0 + + +class _PrefillStep(NamedTuple): + """One step's host-side state, built fresh by `PrefillTarget.prepare` and read by the breaks. + + `num_tokens` is the batch's rows `n`, `physical_tokens` the `T` the forward runs over (`n` or + its CUDA graph bucket), `dp_pad_rows` the MoE padding (attention DP without a CUDA graph).""" + + num_tokens: int + physical_tokens: int + dp_pad_rows: _DpPadding | None + input_ids: torch.Tensor + attn: dict + gdn: tuple + ple: tuple + qsa: tuple + vision: tuple | None + # A verify step: the generation rows' side-cache writes are staged for the rollback. + stage: bool = False + # fp8 pool only: `_fp8_context_pages`. + fp8_pages: int | None = None + # fp8 pool only: `_rowtile_scratch_groups`, None for one launch. + rowtile_groups: tuple | None = None + split: _Split | None = None + # Graph-replayable steps only: QSA per-row metadata over all `T` rows (padding = last row). + qsa_rows: tuple | None = None + # `_selection_cuts`, or None for equal row counts. + sel_cuts: tuple | None = None + # The generation rows' `_RowOrder`, shared by the step's sparse layers. + gen_order: _RowOrder | None = None + + +class PrefillTarget: + """Any batch with context rows (`num_contexts > 0`), possibly with generation rows.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: + self.core = core + self._contract_pending = step_contract_enabled() + # Dropped once it has reported. + self._kv_probe = _KvRangeProbe.for_target(core, self._contract_pending) + self._layers = _layer_views(core) + self._stack = _stack_views(core) + self._step: _PrefillStep | None = None + # `_attn_dispatch` by (rows, score-table width). + self._attn_dispatch_memo: dict = {} + self._coords_buf: torch.Tensor | None = None + # Keyed CUDA graphs on short buckets; a key without a graph runs the eager core. + self._attention_break = _keyed_eager_on_graph( + self._attention_core, + graph_key=self._attn_graph_key, + graph_body=self._attention_core_graph, + ) + self._gdn_break = _keyed_eager_on_graph( + self._gdn_core, graph_key=self._gdn_graph_key, graph_body=self._gdn_core_graph + ) + # Batch row count `n` (`[1]` int32), refreshed by `prepare` before every forward or + # replay so a CUDA graph can tell its padding rows; allocated once, at a fixed address. + self._live_rows: torch.Tensor | None = None + # `(rank, world, group, process_group)` of the attention-TP group, or None. + split = None + if _prefill_split_enabled(core.attn_tp_size, core.dp_size): + split = ( + core.mapping.tp_rank, + core.attn_tp_size, + list(core.mapping.tp_group), + core.mapping.tp_group_pg.boxed() if mpi_disabled() else None, + ) + self._selection_split = split + self._rowtile_groups = _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) + self._sequence_parallel = split + # TP sum, or reduce-scatter under SP; set by `forward`, a capture-time constant per bucket. + self._block_sum = core.tp_sum + # `(lo, hi, gather)` of a sequence-parallel forward, or None. + self._sp_rows = None + # Highest priority: in a CUDA graph a default-priority stream's gather waits for the + # main stream's persistent GEMM, which holds every SM. + self._comm_stream = None + self._comm_fork = None + self._comm_join = None + side = getattr(core, "_side_stream", None) + pipelined = self._selection_split is not None + if (self._sequence_parallel is not None or pipelined) and side is not None: + self._comm_stream = torch.cuda.Stream(device=side.device, priority=-1) + self._comm_fork = torch.cuda.Event() + self._comm_join = torch.cuda.Event() + # Two-rank sequence parallelism only: sized for half a max_num_tokens step's bf16 rows. + # Without `_own_ipc` the pair exchanges over NCCL. + self._ce_exchange = None + if ( + self._comm_stream is not None + and self._sequence_parallel is not None + and self._sequence_parallel[1] == 2 + and _own_ipc(core) + ): + from .ops.comm.ce_exchange__sp_pair import CopyEngineExchange + + rows = -(-core.max_num_tokens // 2) + self._ce_exchange = CopyEngineExchange.create(core.mapping, rows * core.hidden * 2) + + def _sp_split(self, num_tokens: int): + """`(lo, hi, gather, scatter)` of a sequence-parallel forward over `num_tokens`, or None. + + `[lo, hi)` is this rank's share; depends on `num_tokens` alone, the same on every rank.""" + sp = self._sequence_parallel + if sp is None or num_tokens < _SP_MIN_ROWS: + return None + rank, world, group, process_group = sp + if num_tokens % world: + return None + rows = num_tokens // world + + def gather(t: torch.Tensor) -> torch.Tensor: + return allgather(t, None, group, process_group) + + ce = getattr(self, "_ce_exchange", None) if world == 2 else None + + def scatter(t: torch.Tensor) -> torch.Tensor: + if ce is not None: + # Send the peer its rows, add the peer's partial of this rank's rows. + lo, hi = rank * rows, (rank + 1) * rows + peer = t[hi:] if rank == 0 else t[:lo] + return t[lo:hi] + ce.exchange(peer.contiguous()) + return reducescatter(t, None, group, process_group) + + comm = getattr(self, "_comm_stream", None) + if ce is not None and comm is not None: + # Split around the producer: `start` sends early on the comm stream, `finish` adds. + lo, hi = rank * rows, (rank + 1) * rows + peer_rows = (hi, num_tokens) if rank == 0 else (0, lo) + box = [] + + def start(t: torch.Tensor) -> None: + main = torch.cuda.current_stream() + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + with torch.cuda.stream(comm): + box.append(ce.exchange(t[peer_rows[0] : peer_rows[1]])) + + def finish(t: torch.Tensor): + self._comm_join.record(comm) + torch.cuda.current_stream().wait_event(self._comm_join) + return t[lo:hi] + box.pop() + + scatter.split_finalize = (start, finish, peer_rows) + + return rank * rows, (rank + 1) * rows, gather, scatter + + def _block_rows(self, x: torch.Tensor) -> torch.Tensor: + """A block's input over every row: `x`, or the gather of every rank's rows under SP.""" + return x if self._sp_rows is None else self._sp_rows[2](x) + + def _overlap_projection(self) -> bool: + """Whether a block's input projection runs as `_project_overlapped`.""" + return self._sp_rows is not None and self._comm_stream is not None + + def _project_overlapped(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """A sequence-parallel block's input projection over every row; `x` is this rank's rows. + + The gather on the comm stream overlaps this rank's rows; bitwise one `cublas_mm`.""" + lo, hi, gather = self._sp_rows + total = (hi - lo) * self._sequence_parallel[1] + out = torch.empty([total, w.shape[1]], dtype=x.dtype, device=x.device) + main = torch.cuda.current_stream() + comm = self._comm_stream + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + ce = getattr(self, "_ce_exchange", None) + with torch.cuda.stream(comm): + # Only the peer's rows: a view of the receive buffer, read before the next exchange. + whole = ce.exchange(x.contiguous()) if ce is not None else gather(x) + torch.mm(x, w, out=out[lo:hi]) + self._comm_join.record(comm) + main.wait_event(self._comm_join) + if ce is not None: + peer = slice(hi, total) if lo == 0 else slice(0, lo) + torch.mm(whole, w, out=out[peer]) + return out + if lo: + torch.mm(whole[:lo], w, out=out[:lo]) + if hi < total: + torch.mm(whole[hi:], w, out=out[hi:]) + return out + + def _overlap_reduce(self) -> bool: + """Whether a block's output projection reduces as `_reduce_overlapped`.""" + return ( + self._sp_rows is not None + and self._comm_stream is not None + and self._sequence_parallel[1] == 2 + ) + + def _reduce_overlapped(self, a: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """A two-rank SP block's output `sum over ranks of a @ w` on this rank's rows `[lo, hi)`. + + The peer's rows are projected first and exchanged while this rank's rows are projected.""" + lo, hi, gather = self._sp_rows + rows = hi - lo + rank = lo // rows + peer = a[hi:] if rank == 0 else a[:lo] + sent = torch.mm(peer, w) + main = torch.cuda.current_stream() + comm = self._comm_stream + self._comm_fork.record(main) + comm.wait_event(self._comm_fork) + ce = getattr(self, "_ce_exchange", None) + with torch.cuda.stream(comm): + if ce is not None: + got = ce.exchange(sent) # the peer's product of this rank's rows + else: + got = gather(sent)[(1 - rank) * rows : (2 - rank) * rows] + own = torch.mm(a[lo:hi], w) + self._comm_join.record(comm) + main.wait_event(self._comm_join) + return own + got + + def _split_context_rows(self, rows: int) -> bool: + """Whether `rows` context rows take `_split_context_selection`. + + A host decision on the batch, identical on every rank, so all take the gather or none.""" + return self._selection_split is not None and rows >= _SELECTION_SPLIT_MIN_ROWS + + def _selection_pipeline(self): + """`_split_context_selection`'s `pipeline`, or None to gather the share once.""" + if self._comm_stream is None: + return None + pipeline = (_SELECTION_PIECES, self._comm_stream, self._comm_fork, self._comm_join) + ce = getattr(self, "_ce_exchange", None) + return pipeline if ce is None else pipeline + (ce,) + + def _check_step_contract(self, md) -> None: + """Opt-in first-forward check: surfaces an upstream metadata rename before the forward.""" + if not self._contract_pending: + return + try: + self.step_args(md) + except AttributeError as exc: + raise AssertionError(f"attention metadata surface drifted: {exc}") from exc + self._contract_pending = False + + def step_args(self, md: TrtllmAttentionMetadata) -> dict: + """Project the prepared metadata onto thop_attention's batch state.""" + return _step_args( + md, md.num_contexts, md.num_ctx_tokens, self.core.quant_mode == QM_NVFP4_KV_CACHE + ) + + # --------------------------------------------------------------- blocks --- + + def _attention_block(self, x, i, L, proj_qkv, index_scratch, attn_out, pos3, pos): + """One gated GQA layer plus its QSA index branch, at this rank's heads. + + Norm + rope run over all `T` rows; caches, selection and attention run in the break.""" + core = self.core + if self._overlap_projection(): + proj = self._project_overlapped(x, L.attn_in) + else: + proj = cublas_mm(self._block_rows(x), L.attn_in) + gate = torch.split(proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1)[1] + _attention_rope(core, proj, index_scratch, L, pos3, pos) + self._attention_break(proj, proj_qkv, index_scratch, attn_out, i) + fused_sigmoid_mul_inplace(attn_out, gate, direct_launch=True) + if self._overlap_reduce(): + return self._reduce_overlapped(attn_out, L.o) + return self._block_sum(cublas_mm(attn_out, L.o)) + + def _attn_dispatch(self, rows: int, width: int) -> tuple: + """The sparse context path's launch choices that depend on its row count. + + A `T`-row graph is bitwise the eager break only where these match the `n` rows'.""" + memo = self._attn_dispatch_memo + hit = memo.get((rows, width)) + if hit is not None: + return hit + core = self.core + sel = core.token_topk + core.compress_ratio - 1 + q = torch.empty((rows, core.index_heads, core.index_head_dim), device="meta") + scores = torch.empty((rows, width), device="meta") + heads_kv = core.heads_kv + gqa = "pair" if rows * heads_kv > 512 else _gqa_splitk_config(rows, heads_kv, sel) + topk = torch.empty((rows, core.block_topk), dtype=torch.int32, device="meta") + hit = ( + _scores_take_visible_split(q, True, False), + rows >= _TOPK_ROWS_MIN_ROWS and _topk_rows_supports(scores, topk), + _topk_expand_cluster_supports(scores), + _topk_expand_supports(scores), + gqa, + ) + memo[(rows, width)] = hit + return hit + + def _attn_graph_key(self, proj, proj_qkv, index_scratch, attn_out, i): + """Graph key for `_attention_core_graph`, or None to run `_attention_core` eagerly. + + Context-only sparse steps over bf16 / fp8 pools without the row-tile pass, whose launch + choices match at `n` and `T` rows. The key includes every buffer address the graph reads.""" + step = self._step + if step is None or step.qsa_rows is None: + return None + rows = proj.shape[0] + if rows != step.physical_tokens or step.attn["num_ctx_tokens"] != step.num_tokens: + return None + core = self.core + kv = core._bound.pools.kv[i] + n = step.num_tokens + if ( + kv.k_scale is not None + or _rowtile_takes(kv, n, step.fp8_pages) + or step.rowtile_groups is not None + ): + return None + block_table = step.qsa[6] + width = block_table.shape[1] * step.qsa[7] // core.compress_ratio + dispatch = self._attn_dispatch(n, width) + if dispatch != self._attn_dispatch(rows, width): + return None + pools = core._bound.pools + return ( + step.attn["num_contexts"], + _pad_window(n, rows), + dispatch, + kv.k.data_ptr(), + kv.v.data_ptr(), + pools.index_cache[i].data_ptr(), + pools.position_cache.data_ptr(), + block_table.data_ptr(), + tuple(block_table.shape), + tuple(t.data_ptr() for t in step.qsa_rows), + self._live_rows.data_ptr(), + ) + + def _attention_core_graph(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: + """`_attention_core` of a context-only sparse step as a CUDA graph over all `T` rows. + + Padding rows reuse the last row's metadata and store nothing (position -2 is skipped by the + stores and never anchors a compression); their outputs are zeroed. Rows `[:n]` are bitwise + the eager break's.""" + core = self.core + L = self._layers[i] + step = self._step + req_idx, logical, seq_lens_q, visible_blocks, topk_indices, topk_row_starts, coords = ( + step.qsa_rows + ) + block_table, tokens_per_block = step.qsa[6], step.qsa[7] + rows = proj.shape[0] + pad = torch.arange(rows, dtype=torch.int32, device=proj.device) >= self._live_rows + store_positions = torch.where(pad, -2, logical) + q_rows, k_dst, v_dst = _attn_in_heads(core, proj) + _index_plane( + core, proj, i, L, (req_idx, store_positions, block_table, tokens_per_block, coords) + ) + kv = core._bound.pools.kv[i] + _store_kv(core, kv, k_dst, v_dst, req_idx, store_positions, block_table, tokens_per_block) + q_index = _index_query(core, index_scratch) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=proj.device + ) + scores = _context_index_scores( + q_index, + core._bound.pools.index_cache[i], + block_table, + logical, + req_idx, + tokens_per_block, + core.compress_ratio, + True, + ) + _selection( + core, + scores, + visible_blocks, + logical, + seq_lens_q, + topk_indices, + topk_row_starts, + selected, + context=True, + ) + proj_qkv.copy_(proj[:, : core.qkv_width]) + out = qsa_paged_sparse_gqa( + q=_packed_q(core, proj_qkv), + k_cache=kv.k, + v_cache=kv.v, + block_table=block_table, + selected_tokens=selected, + request_indices=req_idx, + tokens_per_block=tokens_per_block, + softmax_scale=core.softmax_scale, + query_positions=logical, + compress_ratio=core.compress_ratio, + output_gate=None, + ) + attn_out.copy_(torch.reshape(out, [rows, core.q_width])) + lo = _pad_window(step.num_tokens, rows) + attn_out[lo:].masked_fill_(torch.unsqueeze(pad[lo:], 1), 0) + + def _attention_core(self, proj, proj_qkv, index_scratch, attn_out, i) -> None: + """Break: index store and compression, then dense or sparse attention over `n` rows. + + Writes rows `[:n]` of `attn_out` and zeroes `[n:]`. The index plane advances on every + step so a request crossing the threshold finds its prefix compressed; a verify step stages + it for `SpeculativeRollback` first.""" + core = self.core + L = self._layers[i] + step = self._step + ( + req_idx, + logical, + seq_lens_q, + visible_blocks, + topk_indices, + topk_row_starts, + block_table, + tokens_per_block, + coords, + sparse, + ) = step.qsa + num_tokens = step.num_tokens + proj = proj[:num_tokens] + q_rows, k_dst, v_dst = _attn_in_heads(core, proj) + if self._kv_probe is not None and not self._kv_probe.observe(i, k_dst, v_dst): + self._kv_probe = None + qkv = proj_qkv[:num_tokens] + if not sparse: + # The dense op reads a compact `[q | k | v]`. + qkv.copy_(proj[:, : core.qkv_width]) + if step.stage: + core.spec_rollback.stage_index(core._bound.pools.position_cache) + _index_plane(core, proj, i, L, (req_idx, logical, block_table, tokens_per_block, coords)) + + if sparse: + kv = core._bound.pools.kv[i] + # Nothing else writes the pool on this path; the selection reads the current token. + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + q_index = _index_query(core, index_scratch[:num_tokens]) + # Context rows come first and score as query tiles; generation rows score row by row. + num_ctx = step.attn["num_ctx_tokens"] + parts = [(0, num_ctx, True)] + if num_ctx < num_tokens: + parts.append((num_ctx, num_tokens, False)) + selected = torch.empty( + [num_tokens, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=proj.device, + ) + + def score_and_select(lo, hi, context_rows, out, order=None): + _score_and_select( + core, + core._bound.pools.index_cache[i], + q_index, + block_table, + logical, + req_idx, + tokens_per_block, + visible_blocks, + seq_lens_q, + topk_indices, + topk_row_starts, + lo, + hi, + context_rows, + out, + order, + ) + + for lo, hi, context_rows in parts: + if context_rows and self._split_context_rows(hi - lo): + _split_context_selection( + self._selection_split, + score_and_select, + selected, + lo, + hi, + self._selection_pipeline(), + step.sel_cuts if lo == 0 and hi == num_ctx else None, + ) + else: + order = None if context_rows else step.gen_order + score_and_select(lo, hi, context_rows, selected[lo:hi], order) + # The row-tile pass reads q in place; the other kernels need a compact copy. + launches = [] + for lo, hi, context_rows in parts: + if context_rows and step.rowtile_groups is not None: + launches += [(a, b, True, pages) for a, b, pages in step.rowtile_groups] + else: + launches.append((lo, hi, context_rows, step.fp8_pages)) + for lo, hi, context_rows, pages in launches: + if context_rows and _rowtile_takes(kv, hi - lo, pages): + q = q_rows[lo:hi] + else: + qkv[lo:hi].copy_(proj[lo:hi, : core.qkv_width]) + q = _packed_q(core, qkv)[lo:hi] + out = _sparse_gqa( + core, + kv, + q, + block_table, + selected[lo:hi], + req_idx[lo:hi], + tokens_per_block, + logical[lo:hi] if context_rows else None, + pages, + ) + attn_out[lo:hi].copy_(torch.reshape(out, [hi - lo, core.q_width])) + else: + thop_attention( + q=qkv, + output=attn_out[:num_tokens], + local_layer_idx=core._bound.pools.pool_idx[i], + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step.attn, + **_CALL_CONSTANTS, + ) + attn_out[num_tokens:].zero_() + + def _gdn_block(self, x, i, L, core_out) -> torch.Tensor: + """One Gated-DeltaNet layer over a batch with a context prefix, at this rank's heads. + + The projection and epilogue run over all `T` rows; the recurrence runs in `_gdn_core`.""" + core = self.core + if self._overlap_projection(): + proj = self._project_overlapped(x, L.in_proj) + else: + proj = cublas_mm(self._block_rows(x), L.in_proj) + z = torch.split(proj, core.in_proj_split, 1)[1] + self._gdn_break(proj, core_out, i) + return self._gdn_epilogue(core_out, z, L, proj.shape[0]) + + def _gdn_graph_key(self, proj, out, i): + """Graph key for `_gdn_core_graph`, or None to run `_gdn_core` eagerly. + + Context-only steps past the in-place threshold in short buckets. The key holds the request + count, split shape, FlashInfer's CP chunk length and every buffer address read.""" + step = self._step + if step is None or step.stage: + return None + rows = proj.shape[0] + if rows != step.physical_tokens or rows > _GRAPH_BREAKS_MAX_TOKENS: + return None + n = step.num_tokens + gdn = step.gdn + num_decodes, verify = gdn[11], gdn[13] + if num_decodes or verify is not None or n <= _GDN_INPLACE_MAX_TOKENS: + return None + si_all, hi_all, qsl_all, qsl_long = gdn[3], gdn[6], gdn[8], gdn[9] + num_seqs = si_all.shape[0] + core = self.core + pools = core._bound.pools + dev = proj.device + heads = max(core.num_k_heads, core.num_v_heads) + key = ( + num_seqs, + _pad_window(n, rows), + pools.ssm[i].data_ptr(), + pools.conv[i].data_ptr(), + si_all.data_ptr(), + hi_all.data_ptr(), + qsl_all.data_ptr(), + qsl_long.data_ptr(), + self._live_rows.data_ptr(), + ) + sp = step.split + if sp is None: + cp = _gdn_reaches_cp(num_seqs, core.num_v_heads, dev) + return ("whole", cp_chunk_len_for(n, heads, dev) if cp else 0) + key + raw = sp.raw + if raw is None or sp.whole is None and split_takes_cp(num_seqs, core.num_v_heads, dev): + return None + stage = core._bound.stage + key += ( + sp.k, + raw.tail_tok.shape[0], + raw.cu.data_ptr(), + stage.ssm[i].data_ptr(), + stage.conv[i].data_ptr(), + ) + if sp.whole is None: + return ("split",) + key + lead = sp.whole[0] + if not isinstance(lead, int) or num_seqs != 1 or sp.k != 1: + return None + return ("lead", cp_chunk_len_for(lead, heads, dev)) + key + + def _gdn_core_graph(self, proj, out, i) -> None: + """`_gdn_core` of a context-only step as a CUDA graph over all `T` rows. + + Padding rows never reach the batch's rows or states and are zeroed after, so rows `[:n]` of + `out` and the pools are bitwise the eager break's.""" + core = self.core + L = self._layers[i] + step = self._step + gdn = step.gdn + si_all, hi_all, qsl_all, qsl_long = gdn[3], gdn[6], gdn[8], gdn[9] + rows = proj.shape[0] + dev = proj.device + pools = core._bound.pools + ssm_states = pools.ssm[i] + mixed_qkv, _, a, b = torch.split(proj, core.in_proj_split, 1) + # Padding rows are finite (the splice zeroes their embedding). + q, k, v, g, beta = self._gdn_prefill_prep(mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all) + heads = max(core.num_k_heads, core.num_v_heads) + sp = step.split + if sp is None: + num_seqs = si_all.shape[0] + if _gdn_reaches_cp(num_seqs, core.num_v_heads, dev): + fi = { + "_cp_chunk_len": cp_chunk_len_for(step.num_tokens, heads, dev), + "max_seqlen": rows, + } + _gdn_chunk_cp_inplace( + q, k, v, g, beta, None, ssm_states, si_all, hi_all, qsl_long, out, True, + fi_kwargs=fi, + ) # fmt: skip + else: + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, si_all, hi_all, qsl_long, out, True, + ) # fmt: skip + else: + raw = sp.raw + stage = core._bound.stage + stage_ssm, stage_conv = stage.ssm[i], stage.conv[i] + gather = raw.gather.to(torch.int32) + live = raw.live.to(torch.int32) + if sp.whole is None: + carry = torch.zeros([gather.shape[0]], dtype=torch.bool, device=dev) + carry.index_copy_(0, raw.main_rows, hi_all.to(torch.bool)) + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, gather, carry, raw.cu, out, True, + keep_rows=raw.main_rows, keep_slots=live, use_cp=False, + ) # fmt: skip + else: + # One request split once: the first pass covers its rows before P (`raw.cu[:2]`). + fi = { + "_cp_chunk_len": cp_chunk_len_for(sp.whole[0], heads, dev), + "max_seqlen": rows, + } + _gdn_chunk_carried( + q, k, v, g, beta, None, ssm_states, live, hi_all.to(torch.bool), raw.cu[:2], + out, True, fi_kwargs=fi, + ) # fmt: skip + gdn_inforward_tail( + q, + k, + v, + g, + beta, + raw.tail_tok, + raw.cu_tail, + ssm_states, + raw.tail_slots, + stage_ssm, + raw.stage_rows, + out, + ) + gdn_inforward_conv_swap( + pools.conv[i], stage_conv, mixed_qkv, raw.tail_slots, raw.stage_rows, raw.conv_win + ) + lo = _pad_window(step.num_tokens, rows) + pad = torch.arange(lo, rows, dtype=torch.int32, device=dev) >= self._live_rows + out[:, lo:].masked_fill_(pad.view(1, rows - lo, 1, 1), 0) + + def _gdn_core(self, proj, out, i) -> None: + """Break: the recurrence over `n` rows; writes rows `[:n]` of `out`, zeroes rows `[n:]`. + + A context sequence with no carried state starts from zeros without clearing its slots + (`has_initial_state`); a verify step's draft chains take `_gdn_verify`.""" + core = self.core + L = self._layers[i] + step = self._step + ( + num_prefill_tokens, + num_decode_tokens, + si_p, + si_all, + si_d, + hi_p, + hi_all, + qsl_p, + qsl_all, + qsl_long, + qsl_long_p, + num_decodes, + gen_p, + verify, + ) = step.gdn + num_tokens = step.num_tokens + pools = core._bound.pools + ssm_states = pools.ssm[i] + mixed_qkv, _, a, b = torch.split(proj[:num_tokens], core.in_proj_split, 1) + + if verify is not None: + mq_p, mq_d = torch.split(mixed_qkv, [num_prefill_tokens, num_decode_tokens], 0) + a_p, a_d = torch.split(a, [num_prefill_tokens, num_decode_tokens], 0) + b_p, b_d = torch.split(b, [num_prefill_tokens, num_decode_tokens], 0) + q, k, v, g, beta = self._gdn_prefill_prep(mq_p, a_p, b_p, L, i, qsl_p, si_p, hi_p) + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_p, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long_p, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_prefill_tokens], + state_workspace=None, + has_initial_state=hi_p, + linear_gate=True, + ) + out[:, num_prefill_tokens:num_tokens].copy_( + _gdn_verify( + core, + mq_d, + a_d, + b_d, + i, + L, + si_d, + num_decodes, + gen_p, + verify, + ) + ) + elif num_decode_tokens > 0: + mq_p, mq_d = torch.split(mixed_qkv, [num_prefill_tokens, num_decode_tokens], 0) + a_p, b_p = a[:num_prefill_tokens], b[:num_prefill_tokens] + q, k, v, g, beta = self._gdn_prefill_prep(mq_p, a_p, b_p, L, i, qsl_p, si_p, hi_p) + causal_conv1d_update( + x=mq_d, + conv_state=pools.conv[i], + weight=L.conv_w, + bias=None, + silu_activation=True, + cache_seqlens=None, + conv_state_indices=si_d, + pad_slot_id=_PAD_SLOT_ID, + ) + # The two halves use disjoint slots and write disjoint row ranges of the output. + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_p, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long_p, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_prefill_tokens], + state_workspace=None, + has_initial_state=hi_p, + linear_gate=True, + ) + # Reads the conv-updated q/k/v columns; the gated norm runs later, in the epilogue. + gdn_decode_rows( + proj[num_prefill_tokens:num_tokens], + ssm_states, + si_d, + L.a_log, + L.dt_bias, + core.num_k_heads, + core.head_k_dim, + out[0, num_prefill_tokens:num_tokens].view(num_decode_tokens, core.value_dim), + ) + elif step.split is not None: + q, k, v, g, beta = self._gdn_prefill_prep( + mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all + ) + self._gdn_split(q, k, v, g, beta, mixed_qkv, i, out[:, :num_tokens]) + else: + q, k, v, g, beta = self._gdn_prefill_prep( + mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all + ) + # q and k arrive L2-normalized; the recurrence must not repeat it. + flashinfer_chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + scale=None, + initial_state=ssm_states, + initial_state_indices=si_all, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=qsl_long, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=out[:, :num_tokens], + state_workspace=None, + has_initial_state=hi_all, + linear_gate=True, + ) + out[:, num_tokens:].zero_() + + def _gdn_split(self, q, k, v, g, beta, mixed_qkv, i, out) -> None: + """The recurrence of a context-only step with an in-forward snapshot split (`_Split`). + + Leaves each split request's state at P in its live slot, its prompt-end state in its staging + row, and every row's output in `out` `[1, n, HV, V]`. `chunk_split_first_pass` writes the + state at P; `gdn_inforward_tail` runs the tails `[P, L)` from it into the staging rows + (fp32, FlashInfer's to rounding); `gdn_inforward_conv_swap` swaps the conv windows.""" + core = self.core + sp = self._step.split + pools = core._bound.pools + stage = core._bound.stage + ssm_states, conv_states = pools.ssm[i], pools.conv[i] + stage_ssm, stage_conv = stage.ssm[i], stage.conv[i] + chunk_split_first_pass( + q, + k, + v, + g, + beta, + ssm_states, + sp.gather, + sp.carry, + sp.cu, + sp.main_rows, + sp.live, + out, + num_whole_seqs=sp.live.shape[0], + whole=sp.whole, + ) + gdn_inforward_tail( + q, + k, + v, + g, + beta, + sp.tail_tok, + sp.cu_tail, + ssm_states, + sp.tail_slots, + stage_ssm, + sp.stage_rows, + out, + ) + gdn_inforward_conv_swap( + conv_states, stage_conv, mixed_qkv, sp.tail_slots, sp.stage_rows, sp.conv_win + ) + + def _gdn_prefill_prep(self, mq, a, b, L, i, qsl, si, hi): + """The context rows' recurrence inputs `(q, k, v, g, beta)` in one pass over `mq`. + + Publishes the conv window back to the pool; q/k come out L2-normalized and the decay as the + factor `exp(g)` (`linear_gate=True`).""" + core = self.core + return gdn_prefill_conv_post( + x=mq, + weight=L.conv_w, + conv_states=core._bound.pools.conv[i], + query_start_loc=qsl, + cache_indices=si, + has_initial_state=hi, + a=a, + b=b, + A_log=L.a_log, + dt_bias=L.dt_bias, + num_k_heads=core.num_k_heads, + head_k_dim=core.head_k_dim, + num_v_heads=core.num_v_heads, + head_v_dim=core.head_v_dim, + pad_slot_id=_PAD_SLOT_ID, + l2_norm_eps=_L2_NORM_EPS, + softplus_threshold=_SOFTPLUS_THRESHOLD, + linear_gate=True, + ) + + def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: + """The gated RMS norm (sigmoid gate, non-Gemma weight) and the reduced output projection.""" + core = self.core + o = rms_norm_gated_token_major( + torch.reshape(out, [-1, core.head_v_dim]), + torch.reshape(z, [num_tokens, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + fp8_scale=None, + gate_activation="sigmoid", + direct_launch=True, + ) + o = torch.reshape(o, [num_tokens, core.value_dim]) + if self._overlap_reduce(): + return self._reduce_overlapped(o, L.out) + return self._block_sum(cublas_mm(o, L.out)) + + def _ple_block(self, hc, hc_out, normed_out) -> None: + """Break: the n-gram side path over `n` rows, added into the stream and normalized. + + Writes rows `[:n]` of `hc_out` and `normed_out`, zeroes `[n:]`. A sequence with no history + is seeded with `eos_token_id`; a verify step stages its state for `SpeculativeRollback`.""" + core = self.core + S = self._stack + step = self._step + conv_states, ngram_pool, si_all, carry_all, qsl = step.ple + num_tokens = step.num_tokens + next_norm = self._layers[core.ple_layer].ahc_norm + conv_input = None + if step.stage: + core.spec_rollback.stage_ple_verify(ngram_pool, conv_states, step.input_ids) + conv_input = torch.empty_like(hc[:num_tokens]) + elif step.split is not None: + # The conv input rows rebuild the conv state at P (the op allocates them anyway). + conv_input = torch.empty_like(hc[:num_tokens]) + ngram = ple_ngram_embed( + step.input_ids, + ngram_pool, + si_all, + carry_all, + qsl, + core.w["ple_table"], + S.ple_tscale, + S.ple_mult, + S.ple_vocab, + S.ple_offs, + core.eos_token_id, + num_tokens, + row_window=core._ple_table_host.window, + ) + if core._ple_table_host.needs_sum: + ngram = core.tp_sum(ngram) + ple_gate_conv_inject_norm( + hc[:num_tokens], + cublas_mm(ngram, S.ple_key), + cublas_mm(ngram, S.ple_val), + S.ple_nk_w, + S.ple_nq_w, + S.ple_nc_w, + S.ple_conv_w, + conv_states, + si_all, + qsl, + next_norm, + core.eps, + core.hc_count, + _PLE_CLAMP_MIN, + core.ple_dilation, + _PAD_SLOT_ID, + num_tokens, + carry_all, + conv_input=conv_input, + out=(hc_out[:num_tokens], normed_out[:num_tokens]), + ) + if step.stage: + core.spec_rollback.stage_ple_conv_input(conv_input) + elif step.split is not None: + self._split_ple_states(conv_states, ngram_pool, conv_input) + hc_out[num_tokens:].zero_() + normed_out[num_tokens:].zero_() + + def _split_ple_states(self, conv_states, ngram_pool, conv_input) -> None: + """The PLE side of an in-forward snapshot split. + + Moves each split request's prompt-end state to its staging row and writes the state at P + (the last conv-input rows and ids before P, oldest first) to its slot.""" + sp = self._step.split + stage = self.core._bound.stage + stage.ple_conv.index_copy_(0, sp.stage_rows, conv_states.index_select(0, sp.tail_slots)) + window = conv_input.index_select(0, sp.ple_win) + conv_states.index_copy_( + 0, sp.tail_slots, window.view(sp.k, -1, window.shape[-1]).transpose(1, 2) + ) + stage.ple_ngram.index_copy_(0, sp.stage_rows, ngram_pool.index_select(0, sp.tail_slots)) + ids = self._step.input_ids.index_select(0, sp.ngram_win) + ngram_pool.index_copy_(0, sp.tail_slots, ids.view(sp.k, -1).to(ngram_pool.dtype)) + + def _splice(self, h) -> None: + """Break: write vision rows over their placeholders in `h`; zero rows `[n:]`.""" + step = self._step + if step.vision is not None: + rows, mm_token_indices = step.vision + h.index_copy_(0, mm_token_indices, rows) + h[step.num_tokens :].zero_() + + # -------------------------------------------------------------- forward --- + + def prepare( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor, + position_ids: torch.IntTensor, + vision: tuple | None, + spec_metadata=None, + ) -> None: + """Build the step's host-side state (`_PrefillStep`) once, before the forward; on a + breakable prefill graph replay this and the breaks are all of this target that runs.""" + core = self.core + md = attn_metadata + self._check_step_contract(md) + attn_args = self.step_args(md) + mm = md.mamba_metadata + + physical_tokens = input_ids.shape[0] + assert physical_tokens in (md.num_tokens, md.padded_num_tokens), ( + f"the engine handed {physical_tokens} activation rows for a batch its metadata " + f"describes as {md.num_tokens} tokens (padded_num_tokens={md.padded_num_tokens})" + ) + num_tokens = md.num_tokens + if self._live_rows is None: + self._live_rows = torch.zeros([1], dtype=torch.int32, device=input_ids.device) + self._live_rows.fill_(num_tokens) + # Attention DP pads the MoE rows to the group's largest count; under a prefill CUDA + # graph every rank runs one bucket, so this is None (a per-capture constant). + dp_pad_rows = core._dp_pad_rows(md, physical_tokens) + assert physical_tokens == num_tokens or dp_pad_rows is None, ( + f"a prefill CUDA graph bucket of {physical_tokens} rows with attention-DP padding " + f"{dp_pad_rows}: the graph would freeze the group's row counts" + ) + batch = md.num_seqs + num_prefills = md.num_contexts + num_decodes = batch - num_prefills + num_prefill_tokens = md.num_ctx_tokens + num_decode_tokens = num_tokens - num_prefill_tokens + gen_p, is_verify = _verify_rows(core, spec_metadata, num_decodes, num_decode_tokens) + attn_args["predicted_tokens_per_seq"] = gen_p + + si_all = mm.state_indices[:batch] + hi_all = mm.has_initial_states[:batch] + si_p, si_d = torch.split(si_all, [num_prefills, num_decodes], 0) + # The decode kernel needs a 16-byte-aligned slot vector; the generation half starts + # `num_prefills` rows in, so copy it once per step. + si_d = torch.clone(si_d) + hi_p = hi_all[:num_prefills] + # `query_start_loc_long` spans the whole batch; its prefix segments the context rows. + qsl_all = mm.query_start_loc + qsl_p = None if qsl_all is None else qsl_all[: num_prefills + 1] + qsl_long = mm.query_start_loc_long + qsl_long_p = qsl_long[: num_prefills + 1] + gdn_state = ( + num_prefill_tokens, + num_decode_tokens, + si_p, + si_all, + si_d, + hi_p, + hi_all, + qsl_p, + qsl_all, + qsl_long, + qsl_long_p, + num_decodes, + gen_p, + _verify_state(core, mm, num_decodes, num_prefills) if is_verify else None, + ) + + # `has_initial_states` covers the context prefix only; every generation row carries. + ple_carry = nn.functional.pad(hi_p, [0, num_decodes], mode="constant", value=1.0) + # `query_start_loc_long` mis-segments PLE when `gen_p > 1`; rebuild from `seq_lens_cuda`. + ple_qsl = ( + qsl_long + if gen_p == 1 + else cu_seqlens_from_seq_lens(md.seq_lens_cuda[:batch], torch.int64) + ) + pools = core._bound.pools + ple_state = (pools.ple_state[0], pools.ple_state[1], si_all, ple_carry, ple_qsl) + + # Dense/sparse decision from a host read: a per-capture constant. + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + # `[T, 3]` view of the `[3, 1, T]` positions, as the QSA position plane stores them. + coords = torch.transpose(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]), 0, 1) + qsa = ( + md.qsa_req_idx_per_token[:num_tokens], + md.qsa_logical_positions[:num_tokens], + md.qsa_sequence_lengths[:num_tokens], + md.qsa_visible_blocks[:num_tokens], + md.qsa_topk_indices[:num_tokens], + md.qsa_topk_row_starts[:num_tokens], + md.qsa_block_table, + md.tokens_per_block, + coords[:num_tokens], + sparse, + ) + + # Verify rows write the PLE pools and QSA side caches for every draft row; reopened every + # forward, `SpeculativeRollback` stages what its commit needs to undo rejected ones. + rollback = core.spec_rollback + if rollback is not None: + rollback.discard() + if is_verify: + rollback.begin( + num_contexts=num_prefills, + prefix_tokens=num_prefill_tokens, + num_gens=num_decodes, + rows_per_gen=gen_p, + slots=si_d, + request_indices=md.qsa_req_idx_per_token[num_prefill_tokens:num_tokens], + logical_positions=md.qsa_logical_positions[num_prefill_tokens:num_tokens], + block_table=md.qsa_block_table, + tokens_per_block=md.tokens_per_block, + ) + fp8_pages = None + if ( + sparse + and core.quant_mode == QM_FP8_KV_CACHE + and num_prefill_tokens >= _ROWTILE_MIN_ROWS + ): + fp8_pages = _fp8_context_pages(md, md.tokens_per_block) + rowtile_groups = None + if fp8_pages is not None and self._rowtile_groups and num_decodes == 0: + kv0 = core._bound.pools.kv[core.attn_layers[0]] + rowtile_groups = _rowtile_scratch_groups(md, kv0.k[0].numel() * 4) + qsa_rows = None + if ( + sparse + and num_decodes == 0 + and not is_verify + and 0 < num_tokens <= physical_tokens <= _GRAPH_BREAKS_MAX_TOKENS + ): + qsa_rows = self._qsa_padded_rows(md, coords, num_tokens, physical_tokens) + sel_cuts = None + if sparse and qsa_rows is None and self._split_context_rows(num_prefill_tokens): + chunks = [int(c) for c in md.seq_lens[:num_prefills].tolist()] + kv_lens = [int(k) for k in md.kv_lens_runtime[:num_prefills].tolist()] + if sum(chunks) == num_prefill_tokens: + sel_cuts = _selection_cuts( + chunks, + [k - c for k, c in zip(kv_lens, chunks)], + self._selection_split[1], + _SELECTION_BALANCE_ROW_TOKENS, + ) + self._step = _PrefillStep( + num_tokens=num_tokens, + physical_tokens=physical_tokens, + dp_pad_rows=dp_pad_rows, + input_ids=input_ids[:num_tokens], + attn=attn_args, + gdn=gdn_state, + ple=ple_state, + qsa=qsa, + vision=vision, + stage=is_verify, + fp8_pages=fp8_pages, + rowtile_groups=rowtile_groups, + split=self._split_of(mm, num_prefills, num_decodes, hi_p, num_tokens), + qsa_rows=qsa_rows, + sel_cuts=sel_cuts, + gen_order=_RowOrder(), + ) + + def _qsa_padded_rows(self, md, coords, num_tokens: int, physical_tokens: int) -> tuple: + """QSA per-row metadata over the bucket's `T` rows: rows past `n` copy the last real row, + so padded rows read only what real rows read (their stores use position -2, which every + store kernel skips). Persistent buffers: the views keep their addresses across steps.""" + n, rows = num_tokens, physical_tokens + planes = ( + md.qsa_req_idx_per_token, + md.qsa_logical_positions, + md.qsa_sequence_lengths, + md.qsa_visible_blocks, + ) + if rows > n: + for plane in planes: + plane[n:rows].copy_(plane[n - 1 : n].expand(rows - n)) + # The coordinates are a fresh reshape each step: the graph reads them from its own buffer. + buf = self._coords_buf + if buf is None or buf.shape[0] < rows or buf.dtype != coords.dtype: + buf = torch.empty( + [max(rows, self.core.max_num_tokens), coords.shape[1]], + dtype=coords.dtype, + device=coords.device, + ) + self._coords_buf = buf + buf[:rows].copy_(coords[:rows]) + return tuple(plane[:rows] for plane in planes) + ( + md.qsa_topk_indices[:rows], + md.qsa_topk_row_starts[:rows], + buf[:rows], + ) + + def _split_of( + self, mm, num_prefills: int, num_decodes: int, hi_p, num_tokens: int + ) -> _Split | None: + """The step's `_Split` from `inforward_split` (planned for context-only batches, each split + chunk at least `_INFORWARD_MIN_MAIN` tokens before P), or None. No host read.""" + split = getattr(mm, "inforward_split", None) + if split is None: + return None + stage = self.core._bound.stage + assert stage is not None, ( + "the engine split a step for in-forward snapshots but the KV cache manager stages " + "no state for them" + ) + assert num_decodes == 0, "an in-forward snapshot split in a step with generation rows" + k = split.k + dev = split.cu.device + # Main parts carry as their request does; tails start from zeros and drop their state. + carry = torch.zeros([num_prefills + k], dtype=torch.bool, device=dev) + carry.index_copy_(0, split.main_rows, hi_p.to(torch.bool)) + # The row of P in the packed batch: one past the last row of the conv window ending at P. + p_rows = split.conv_win.view(k, -1)[:, -1:] + 1 + ple_len = stage.ple_conv.shape[-1] + ngram_len = stage.ple_ngram.shape[-1] + ple_win = p_rows + torch.arange(-ple_len, 0, dtype=p_rows.dtype, device=dev) + ngram_win = p_rows + torch.arange(-ngram_len, 0, dtype=p_rows.dtype, device=dev) + whole = None + v_heads = next(iter(stage.ssm.values())).shape[1] if stage.ssm else 0 + if v_heads and split_takes_cp(num_prefills, v_heads, dev): + if num_prefills == 1 and k == 1: + # One request split once: the rows before P are a prefix (no gather). + rows = num_tokens - split.num_tail_tokens + else: + # Every non-tail row, in order (a stable sort on the tail mask; no host read). + tail_mask = torch.zeros([num_tokens], dtype=torch.int8, device=dev) + tail_mask.index_fill_(0, split.tail_tok, 1) + rows = torch.argsort(tail_mask, stable=True)[: num_tokens - split.num_tail_tokens] + before = split.cu.index_select(0, split.main_rows + 1) - split.cu.index_select( + 0, split.main_rows + ) + cu = torch.zeros([num_prefills + 1], dtype=torch.long, device=dev) + torch.cumsum(before, 0, out=cu[1:]) + whole = (rows, cu, hi_p.to(torch.bool)) + return _Split( + k=k, + cu=split.cu, + gather=split.gather.to(torch.int32), + carry=carry, + main_rows=split.main_rows, + live=split.live.to(torch.int32), + tail_tok=split.tail_tok, + cu_tail=split.cu_tail, + stage_rows=split.stage_rows, + tail_slots=split.tail_slots, + conv_win=split.conv_win, + ple_win=ple_win.reshape(-1), + ngram_win=ngram_win.reshape(-1), + whole=whole, + raw=split, + ) + + def _pad_mask(self, physical_tokens: int, device) -> torch.Tensor | None: + """`[T, 1]` bool, True at a graph bucket's padding rows (past `_live_rows`), which the MoE + blocks keep out of the expert window. Computed on device, so a replay recomputes it; None + for an eager step and under attention DP (whose round trip drops padding itself).""" + if self.core.dp_size != 1: + return None + if ( + physical_tokens == self._step.num_tokens + and not torch.cuda.is_current_stream_capturing() + ): + return None + rows = torch.arange(physical_tokens, dtype=torch.int32, device=device) + return torch.unsqueeze(rows >= self._live_rows, 1) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor, + position_ids: torch.IntTensor, + **kwargs, + ): + """Run the trunk over `T` rows: the batch's `n`, or the prefill CUDA graph bucket's. + + A breakable graph captures this body per bucket and replays it without this Python, so + nothing here reads the step: ops are row-wise over `T`, and step-dependent work runs in + the breaks (`_splice`, `_ple_block`, `_attention_core`, `_gdn_core`), each writing rows + `[:n]` of a fixed output allocated here and zeroing rows `[n:]`.""" + core = self.core + num_tokens = input_ids.shape[0] + assert num_tokens == self._step.physical_tokens, ( + f"the forward got {num_tokens} activation rows and prepare saw " + f"{self._step.physical_tokens}" + ) + pos3 = _copy_positions(position_ids) + pos = pos3[0] + + # The token embedding (replicated) with this step's vision rows spliced in. + h = nn.functional.embedding(input_ids, core.w["embed"]) + eager_on_graph(h, inner=self._splice) + dt = h.dtype + dev = h.device + # Sequence parallelism (`_sp_split`): the stream runs on this rank's rows `[lo, hi)`; + # each block gathers all rows and leaves its sum on this rank's rows. + sp = self._sp_split(num_tokens) + if sp is None: + lo, hi, gather = 0, num_tokens, None + self._block_sum = core.tp_sum + self._sp_rows = None + else: + lo, hi, gather, self._block_sum = sp + self._sp_rows = (lo, hi, gather) + + # Scratch shared by every layer that uses it; each break rewrites its whole output. + proj_qkv = torch.empty([num_tokens, core.qkv_width], dtype=dt, device=dev) + index_scratch = torch.empty([num_tokens, core.index_scratch_width], dtype=dt, device=dev) + attn_out = torch.empty([num_tokens, core.q_width], dtype=dt, device=dev) + gdn_out = torch.empty( + [1, num_tokens, core.num_v_heads, core.head_v_dim], dtype=dt, device=dev + ) + ple_hc = torch.empty([num_tokens, core.stream], dtype=dt, device=dev) + ple_normed = torch.empty([num_tokens, core.stream], dtype=dt, device=dev) + + hc_count = core.hc_count + hc = torch.cat([h[lo:hi]] * hc_count, -1) + layers, S = self._layers, self._stack + pad_rows = self._step.dp_pad_rows + pad_mask = self._pad_mask(num_tokens, dev) + # From `_HC_RSTD_MIN_ROWS` rows (fixed per bucket) sites keep rstd and the mixer norms on + # chip; the PLE site always uses the stored-normed mixer on the PLE kernel's output. + by_rstd = hc.shape[0] >= _HC_RSTD_MIN_ROWS + normed = rstd = None + if by_rstd: + _, rstd = qwen4_exp_hc_inject_rstd(hc, None, None, core.eps, hc_count) + else: + _, normed = qwen4_exp_hc_inject_norm( + hc, None, None, layers[0].ahc_norm, core.eps, hc_count + ) + for i, L in enumerate(layers): + if i == core.ple_layer: + # The n-gram path convolves along each request's rows: it runs on all rows. + whole = hc if gather is None else gather(hc) + eager_on_graph(whole, ple_hc, ple_normed, inner=self._ple_block) + hc, normed = ple_hc[lo:hi], ple_normed[lo:hi] + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + elif by_rstd: + mixed, inject = _hc_mixer_rstd( + core, hc, rstd, L.ahc_norm, L.ahc_first, L.ahc_up, True + ) + else: + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + if core.is_attn[i]: + block_out = self._attention_block( + mixed, i, L, proj_qkv, index_scratch, attn_out, pos3, pos + ) + else: + block_out = self._gdn_block(mixed, i, L, gdn_out) + if by_rstd: + hc, rstd = qwen4_exp_hc_inject_rstd(hc, block_out, inject, core.eps, hc_count) + mixed, inject = _hc_mixer_rstd( + core, hc, rstd, L.mhc_norm, L.mhc_first, L.mhc_up, True + ) + else: + hc, normed = qwen4_exp_hc_inject_norm( + hc, block_out, inject, L.mhc_norm, core.eps, hc_count + ) + mixed, inject = _hc_mixer(core, normed, L.mhc_first, L.mhc_up, True) + next_norm = layers[i + 1].ahc_norm if i + 1 < len(layers) else S.fhc_norm + # The injection forms routed + sigmoid(gate) * shared, rounded once. + if gather is None: + routed, gate_logits, shared = _moe_block( + core, mixed, L, pad_rows, pad_mask=pad_mask + ) + else: + routed, gate_logits, shared = _moe_block_sp( + core, + mixed, + L, + gather, + self._block_sum, + None if pad_mask is None else pad_mask[lo:hi], + ) + if by_rstd: + hc, rstd = qwen4_exp_hc_inject_rstd( + hc, routed, inject, core.eps, hc_count, shared, gate_logits + ) + else: + hc, normed = qwen4_exp_hc_inject_norm( + hc, routed, inject, next_norm, core.eps, hc_count, shared, gate_logits + ) + del routed, gate_logits, shared + # The stack-final mixer with no injection weight is the final norm. + if by_rstd: + final, _ = _hc_mixer_rstd(core, hc, rstd, S.fhc_norm, S.fhc_first, S.fhc_up, False) + else: + final, _ = _hc_mixer(core, normed, S.fhc_first, S.fhc_up, False) + if not core.mtp_enabled: + return final if gather is None else gather(final) + # `lm_head` reads the `[T, hidden]` mix, the MTP module the `[T, stream]` stream; packed + # because a breakable graph body returns one tensor (`split_spec_output`). + packed = torch.cat([final, hc], -1) + return packed if gather is None else gather(packed) + + # ------------------------------------------------------------- the draft --- + + def _capture_selection(self, md, topk_indices, visible_blocks) -> None: + """IndexShare: store each request's selection at the row the draft loop continues from + (a context request's last chunk row, a generation request's last accepted row), with its + complete-group count, in the draft layer's share slot.""" + batch = md.num_seqs + assert md.qsa_mtp_num_accepted is not None, ( + "IndexShare needs the worker's accepted counts to pick each request's capture row; " + "the draft loop did not supply them" + ) + seq_lens = md.seq_lens_cuda[:batch].to(torch.int64) + starts = torch.cumsum(seq_lens, 0) - seq_lens + accepted = md.qsa_mtp_num_accepted[:batch].to(torch.int64) + offsets = torch.minimum(torch.clamp(accepted - 1, min=0), seq_lens - 1) + offsets[: md.num_contexts] = seq_lens[: md.num_contexts] - 1 + rows = starts + offsets + slot = self.core._draft_bound.share_slot + md.qsa_shared_topk_indices[slot, :batch].copy_(torch.index_select(topk_indices, 0, rows)) + md.qsa_shared_topk_visible_blocks[slot, :batch].copy_( + torch.index_select(visible_blocks, 0, rows) + ) + # Host-side readiness, recorded in program order. + md.qsa_shared_topk_captured_slots.add(slot) + + def _draft_extend_query_rows(self, md, draft, qkv, q_index, num_ctx: int, rows: int): + """Draft-layer sparse attention on a draft-extend step with context rows, over only the + rows the draft loop continues from (each context request's last row, every generation + row); the worker discards the rest. Query rows score row by row, since a chunk's last row + sees every key the chunk stored. Returns `(out, query)`: `[n, q_width]`, int64 rows.""" + core = self.core + num_contexts = md.num_contexts + ends = torch.cumsum(md.seq_lens_cuda[:num_contexts].to(torch.int64), 0) - 1 + query = torch.cat( + [ends, torch.arange(num_ctx, rows, dtype=torch.int64, device=ends.device)] + ) + n = num_contexts + rows - num_ctx + logical = md.qsa_logical_positions[:rows].index_select(0, query) + req_idx = md.qsa_req_idx_per_token[:rows].index_select(0, query) + scores = qsa_paged_index_scores( + q=q_index.index_select(0, query), + index_cache=draft.index_cache, + block_table=md.qsa_block_table, + query_positions=logical, + request_indices=req_idx, + tokens_per_block=md.tokens_per_block, + compress_ratio=core.compress_ratio, + only_visible_blocks=True, + context_rows=False, + wide_split=True, + ) + selected = torch.empty( + [n, core.token_topk + core.compress_ratio - 1], dtype=torch.int32, device=qkv.device + ) + _selection( + core, + scores, + md.qsa_visible_blocks[:rows].index_select(0, query), + logical, + md.qsa_sequence_lengths[:rows].index_select(0, query), + md.qsa_topk_indices[:rows].index_select(0, query), + md.qsa_topk_row_starts[:rows].index_select(0, query), + selected, + ) + # q keeps the packed rows' strides (gathered whole), so the call shares the draft decode + # step's split-K geometry and the kernels its warmup compiled. + out = _sparse_gqa( + core, + draft.kv, + _packed_q(core, qkv.index_select(0, query)), + md.qsa_block_table, + selected, + req_idx, + md.tokens_per_block, + None, + ) + return torch.reshape(out, [n, core.q_width]), query + + def mtp_forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + """One eager draft step with context rows (the draft loop's first). Sparse attention runs + only on the rows the worker gathers (`_draft_extend_query_rows`) unless IndexShare + captures every row's selection.""" + core = self.core + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert core._bound is not None, "the trunk's cache-manager bind has not run" + md = attn_metadata + rows = hidden_states.shape[0] + assert input_ids.shape[0] == rows and hidden_states.shape[1] == core.stream, ( + f"the draft step got {input_ids.shape[0]} token ids and {tuple(hidden_states.shape)} " + f"hidden states; the MTP module reads one {core.stream}-wide stream row per token" + ) + # Per call: the worker rewrites `attn_metadata` in place between draft steps. + step = self.step_args(md) + num_ctx = md.num_ctx_tokens + gen_seqs = md.num_seqs - md.num_contexts + p, _ = _verify_rows(core, None, gen_seqs, rows - num_ctx) + step["predicted_tokens_per_seq"] = p + draft, layer_idx = core._draft_step_bind(md) + assert rows == md.num_tokens, ( + f"the draft step carries {rows} rows for a batch its metadata describes as " + f"{md.num_tokens} tokens; the QSA row mapping covers the second" + ) + pad_rows = _draft_pad_rows(core, all_rank_num_tokens, rows) + # Same host buffer as the trunk, so every draft step takes the trunk's branch. + batch = md.num_seqs + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + # IndexShare: the loop's first step runs the indexer and captures what later steps reuse. + share = core.mtp_index_share and md.qsa_in_mtp_draft_loop + assert not (share and md.qsa_indexer_skip_topk), ( + "the worker asked a draft step with context rows to reuse a selection" + ) + pos3, pos, coords = _draft_positions(position_ids) + M = core._mtp + req_idx = md.qsa_req_idx_per_token[:rows] + logical = md.qsa_logical_positions[:rows] + block_table = md.qsa_block_table + tokens_per_block = md.tokens_per_block + + x, mixed, inject = _mtp_front(core, M, embed_tokens, input_ids, hidden_states) + proj, qkv, gate, k_dst, v_dst, index_scratch = _mtp_attention_in( + core, M, mixed, pos3, pos, True + ) + # The rows the tail runs on (`_draft_extend_query_rows`), or None for every row. + tail_rows = None + q_index = _mtp_index_branch( + core, + M, + proj, + index_scratch, + draft, + (req_idx, logical, block_table, tokens_per_block, coords), + ) + if sparse and num_ctx and not share: + kv = draft.kv + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + out, query = self._draft_extend_query_rows(md, draft, qkv, q_index, num_ctx, rows) + if core.dp_size == 1: + # Only the query rows are read: the tail runs on them, the rest stay zero. + attn, tail_rows = out, query + gate, x, inject = (t.index_select(0, query) for t in (gate, x, inject)) + else: + attn = torch.zeros([rows, core.q_width], dtype=qkv.dtype, device=qkv.device) + attn.index_copy_(0, query, out) + elif sparse: + kv = draft.kv + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + topk_indices = md.qsa_topk_indices[:rows] + visible_blocks = md.qsa_visible_blocks[:rows] + seq_lens_q = md.qsa_sequence_lengths[:rows] + row_starts = md.qsa_topk_row_starts[:rows] + parts = [(0, num_ctx, True)] if num_ctx else [] + if num_ctx < rows: + parts.append((num_ctx, rows, False)) + selected = torch.empty( + [rows, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=qkv.device, + ) + # Every context row is scored, so they go in the trunk prefill's chunks. + for lo, hi, context_rows in parts: + _score_and_select( + core, + draft.index_cache, + q_index, + block_table, + logical, + req_idx, + tokens_per_block, + visible_blocks, + seq_lens_q, + topk_indices, + row_starts, + lo, + hi, + context_rows, + selected[lo:hi], + ) + if share: + self._capture_selection(md, topk_indices, visible_blocks) + attn = torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device) + for lo, hi, context_rows in parts: + # The context rows reach the trunk step's pages: its host count serves here. + out = _sparse_gqa( + core, + kv, + _packed_q(core, qkv)[lo:hi], + block_table, + selected[lo:hi], + req_idx[lo:hi], + tokens_per_block, + logical[lo:hi] if context_rows else None, + self._step.fp8_pages if context_rows else None, + ) + attn[lo:hi].copy_(torch.reshape(out, [hi - lo, core.q_width])) + else: + attn = thop_attention( + q=qkv, + output=torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device), + local_layer_idx=layer_idx, + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(attn, gate, direct_launch=True) + out = _mtp_tail(core, M, x, attn, inject, pad_rows) + if tail_rows is None: + return out + full = out.new_zeros([rows, out.shape[1]]) + full.index_copy_(0, tail_rows, out) + return full + + +# ------------------------------------------------------------------- decode --- + + +#: Generation-only steps up to this many rows run on a highest-priority stream with the MoE +#: shared expert on a default-priority side stream, so the routed (critical) path gets the SMs +#: first; on larger steps the starved shared expert would gate the join. +_DECODE_HIGH_PRIORITY_MAX_ROWS = 8 # capacity: fixed: a dispatch threshold, not a buffer size + + +def _on_decode_stream(core, rows, step, *args, **kwargs): + """Run a generation-only step of `rows` rows on the core's high-priority decode stream, + ordered after the caller's stream, which then waits for it (graph edges under capture; PDL + chains stay on one stream). Above the row limit or without the stream, run in line.""" + hi = getattr(core, "_decode_stream", None) if rows <= _DECODE_HIGH_PRIORITY_MAX_ROWS else None + if hi is None: + return step(*args, **kwargs) + caller = torch.cuda.current_stream() + hi.wait_stream(caller) + with torch.cuda.stream(hi): + out = step(*args, **kwargs) + caller.wait_stream(hi) + return out + + +class DecodeTarget: + """Generation-only batches (no context rows), every op inside the captured decode graph.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2) -> None: + self.core = core + self._contract_pending = step_contract_enabled() + # Dropped once it has reported. + self._kv_probe = _KvRangeProbe.for_target(core, self._contract_pending) + self._layers = _layer_views(core) + self._stack = _stack_views(core) + self._index_split = _index_split_group(core) + + def _check_step_contract(self, md) -> None: + if not self._contract_pending: + return + try: + self.step_args(md) + except AttributeError as exc: + raise AssertionError(f"attention metadata surface drifted: {exc}") from exc + self._contract_pending = False + + def step_args(self, md: TrtllmAttentionMetadata) -> dict: + """See PrefillTarget.step_args. Context counts are stated, not read: routing + guarantees them, and a decode-only CUDA graph would freeze whatever it read.""" + return _step_args( + md, + num_contexts=0, + num_ctx_tokens=0, + paged_context=self.core.quant_mode == QM_NVFP4_KV_CACHE, + ) + + def _verify_qsl(self, seq_lens, batch: int, p: int, device) -> torch.Tensor: + """PLE segmentation `[0, p, ..., batch * p]` of a generation-only verify step, cached + when built outside capture (warmup) so a captured step launches nothing for it.""" + key = (batch, p, device) + cache = self.__dict__.setdefault("_verify_qsl_cache", {}) + hit = cache.get(key) + if hit is not None: + return hit + if torch.cuda.is_current_stream_capturing(): + return cu_seqlens_from_seq_lens(seq_lens, torch.int64) + qsl = torch.arange(batch + 1, dtype=torch.int64, device=device) * p + cache[key] = qsl + return qsl + + # --------------------------------------------------------------- blocks --- + + def _attention_block(self, x, i, L, qkv, index_scratch, attn_out, qsa, step, stage): + """One gated GQA layer plus its QSA index branch for a generation-only batch. The index + plane is kept on every step; above the dense threshold the sparse core replaces the dense + op, and over a bf16 or e4m3 pool all work up to the index scores is one launch. Verify + rows (`stage`) have their index slots staged for `SpeculativeRollback` before the store.""" + core = self.core + ( + req_idx, + logical, + seq_lens_q, + visible_blocks, + topk_indices, + topk_row_starts, + block_table, + tokens_per_block, + pos3, + pos, + coords, + sparse, + ) = qsa + num_tokens = x.shape[0] + mm = _step_projection(core) + proj = mm(x, L.attn_in) + qkv_view, gate, _ = torch.split( + proj, [core.qkv_width, core.q_width, core.index_scratch_width], 1 + ) + pools = core._bound.pools + prologue = _decode_prologue(pools.kv[i], sparse) + if stage: + core.spec_rollback.stage_index(pools.position_cache) + plan = getattr(self, "_index_plan", None) if sparse else None + if prologue: + _decode_prologue_call( + core, + proj, + qkv, + index_scratch, + i, + L, + (req_idx, logical, block_table, tokens_per_block, pos3, coords), + step["predicted_tokens_per_seq"], + None if plan is None else plan.iq_rows, + ) + else: + _, k_dst, v_dst = _attn_in_heads(core, proj) + _attention_rope(core, proj, index_scratch, L, pos3, pos) + _index_plane( + core, proj, i, L, (req_idx, logical, block_table, tokens_per_block, coords) + ) + # One compact `[q | k | v]` copy: read flat by the dense op, per head by sparse GQA. + qkv.copy_(qkv_view) + if self._kv_probe is not None: + # Both branches leave the normalized, rotated K/V in the packed copy. + k_rows, v_rows = torch.split( + torch.reshape(qkv, [qkv.shape[0], -1, core.head_dim])[:, core.heads_q :], + core.heads_kv, + 1, + ) + if not self._kv_probe.observe(i, k_rows, v_rows): + self._kv_probe = None + + if sparse: + kv = pools.kv[i] + if not prologue: + _store_kv(core, kv, k_dst, v_dst, req_idx, logical, block_table, tokens_per_block) + selected = _decode_index_selection( + core, + getattr(self, "_index_split", None), + _index_query(core, index_scratch), + qsa[:6] + (block_table, tokens_per_block), + num_tokens, + step["predicted_tokens_per_seq"], + proj.device, + core._bound.pools.index_cache[i], + plan, + q_permuted=prologue, + order=getattr(self, "_row_order", None), + ) + attn = _sparse_gqa( + core, + kv, + _packed_q(core, qkv), + block_table, + selected, + req_idx, + tokens_per_block, + None, + output_gate=gate, + rows_per_request=step["predicted_tokens_per_seq"], + ) + out = torch.reshape(attn, [num_tokens, core.q_width]) + else: + out = thop_attention( + q=qkv, + output=attn_out, + local_layer_idx=core._bound.pools.pool_idx[i], + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(out, gate, direct_launch=True) + return _tp_partial(core, mm(out, L.o)) + + def _gdn_block(self, x, i, L, si_all, verify, gen_p) -> torch.Tensor: + """One Gated-DeltaNet layer over a generation-only batch: one GEMM for the four input + projections, the conv step in place, then recurrence and gated norm in one launch per + (row, v-head), so over a bf16 pool a row's output does not depend on its batch. A verify + step (`verify`, `gen_p` rows a sequence) takes `_gdn_verify` instead.""" + core = self.core + pools = core._bound.pools + mm = _step_projection(core) + # The fused launch is FlashInfer's warp split-K kernel: graphed steps only, as `mm`. + tactic = ( + _gdn_in_proj_conv_tactic(core, x, L, si_all, verify) + if verify is not None and mm is _decode_projection + else None + ) + mq_out = None + if tactic is not None: + # The projection and the conv of its q|k|v columns in one launch. + full, mq_out = gdn_in_proj_conv( + x, + L.in_proj_dec, + L.conv_w, + pools.conv[i], + si_all, + core._bound.int_conv[i], + verify[0], + _PAD_SLOT_ID, + gen_p, + tactic, + _DECODE_PROJ_PDL, + ) + proj = full[:, : sum(core.in_proj_split)] + else: + # Its alignment columns, past `sum(in_proj_split)`, are dropped by the view. + proj = mm(x, L.in_proj_dec)[:, : sum(core.in_proj_split)] + if verify is not None: + mixed_qkv, z, a, b = torch.split(proj, core.in_proj_split, 1) + num_tokens = x.shape[0] + fold_norm = _gdn_verify_folds_norm(core, num_tokens // gen_p, gen_p) + out = _gdn_verify( + core, + mixed_qkv, + a, + b, + i, + L, + si_all, + num_tokens // gen_p, + gen_p, + verify, + mq_out=mq_out, + z=z if fold_norm else None, + ) + if fold_norm: + # The replay returned the normed, gated rows. + o = torch.reshape(out, [num_tokens, core.value_dim]) + return _tp_partial(core, mm(o, L.out)) + return self._gdn_epilogue(out, z, L, num_tokens) + mixed_qkv = torch.split(proj, core.in_proj_split, 1)[0] + causal_conv1d_update( + x=mixed_qkv, + conv_state=pools.conv[i], + weight=L.conv_w, + bias=None, + silu_activation=True, + cache_seqlens=None, + conv_state_indices=si_all, + pad_slot_id=_PAD_SLOT_ID, + ) + # Computes both gates and the q/k L2 norm itself (softplus (1, 20), eps 1e-6). + o = gdn_decode_rows_gated_norm( + proj, + pools.ssm[i], + si_all, + L.a_log, + L.dt_bias, + L.gnorm, + core.eps, + core.num_k_heads, + core.head_k_dim, + ) + return _tp_partial(core, mm(o, L.out)) + + def _gdn_epilogue(self, out, z, L, num_tokens) -> torch.Tensor: + """A verify step's gated RMS norm (sigmoid gate, plain weight) and output projection.""" + core = self.core + o = rms_norm_gated_token_major( + torch.reshape(out, [-1, core.head_v_dim]), + torch.reshape(z, [num_tokens, core.num_v_heads, core.head_v_dim]), + weight=L.gnorm, + eps=core.eps, + fp8_scale=None, + gate_activation="sigmoid", + direct_launch=True, + ) + o = torch.reshape(o, [num_tokens, core.value_dim]) + return _tp_partial(core, _step_projection(core)(o, L.out)) + + def _ple_front(self, input_ids, ple_state, stage) -> tuple: + """The PLE part that reads no residual: verify staging of both pools (before the n-gram + history is written), then the n-gram embedding's `(key, value)` projections.""" + core = self.core + S = self._stack + conv_states, ngram_pool, si_all, qsl, rows = ple_state + if stage: + core.spec_rollback.stage_ple_verify(ngram_pool, conv_states, input_ids) + ngram = ple_ngram_embed( + input_ids, + ngram_pool, + si_all, + None, + qsl, + core.w["ple_table"], + S.ple_tscale, + S.ple_mult, + S.ple_vocab, + S.ple_offs, + core.eos_token_id, + rows, + row_window=core._ple_table_host.window, + ) + if core._ple_table_host.needs_sum: + ngram = core.tp_sum(ngram) + return cublas_mm(ngram, S.ple_key), cublas_mm(ngram, S.ple_val) + + def _ple_block( + self, hc, input_ids, ple_state, next_norm, stage, front=None, inject_block=None + ) -> tuple: + """The n-gram side path added into the stream and normed for the attention-side site; + returns `(hc, normed)`. No carry flags: every generation sequence carries. A verify step + (`stage`) stages both pools' pre-step rows and the conv input the second launch writes.""" + core = self.core + S = self._stack + conv_states, ngram_pool, si_all, qsl, rows = ple_state + key, val = front if front is not None else self._ple_front(input_ids, ple_state, stage) + conv_input = torch.empty_like(hc) if stage else None + from_pool = self._ple_from_pool(stage) + out = ple_gate_conv_inject_norm( + hc, + key, + val, + S.ple_nk_w, + S.ple_nq_w, + S.ple_nc_w, + S.ple_conv_w, + conv_states, + si_all, + qsl, + next_norm, + core.eps, + core.hc_count, + _PLE_CLAMP_MIN, + core.ple_dilation, + _PAD_SLOT_ID, + rows, + conv_input=conv_input, + inject_block=inject_block, + # The three launches chain through PDL (off with TRTLLM_ENABLE_PDL=0); the norm + # weights are read before the wait. + pdl=get_env_enable_pdl(), + # every sequence here is a generation request the commit rewrites + publish_end=0 if from_pool else None, + ) + if stage: + core.spec_rollback.stage_ple_conv_input(conv_input) + return out + + def _ple_from_pool(self, stage) -> bool: + """Whether this verify step's PLE launch leaves the conv pool rows unpublished, for the + fused rollback commit to rebuild from the pool and the staged conv input.""" + return bool(stage) and self.core.spec_rollback.fused_pending() + + # -------------------------------------------------------------- forward --- + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + **kwargs, + ): + core = self.core + md = attn_metadata + self._check_step_contract(md) + step = self.step_args(md) + spec_metadata = kwargs.get("spec_metadata") + mm = md.mamba_metadata + pos3 = take_step_positions(md, position_ids) + if pos3 is None: + pos3 = _copy_positions(position_ids) + pos = pos3[0] + coords = torch.transpose(torch.reshape(position_ids, [_QSA_POSITION_AXES, -1]), 0, 1) + + if input_ids.dim() == 1 and input_ids.is_contiguous(): + # The embedding and its replication over the streams in one launch. + hc = embed_rows(input_ids, core.w["embed"], core.hc_count, False) + else: + hc = torch.cat( + [nn.functional.embedding(input_ids, core.w["embed"])] * core.hc_count, -1 + ) + num_tokens = hc.shape[0] + dt = hc.dtype + dev = hc.device + batch = md.num_seqs + # Decode CUDA graphs pad with real dummy requests, so they are in the metadata. + assert num_tokens == md.num_tokens, ( + f"the engine handed {num_tokens} activation rows for a generation batch its " + f"metadata describes as {md.num_tokens} tokens" + ) + # None under a decode CUDA graph, which pads every attention-DP rank to one batch. + pad_rows = core._dp_pad_rows(md, num_tokens) + gen_p, is_verify = _verify_rows(core, spec_metadata, batch, num_tokens) + step["predicted_tokens_per_seq"] = gen_p + si_all = mm.state_indices[:batch] + verify = _verify_state(core, mm, batch) if is_verify else None + pools = core._bound.pools + # As in PrefillTarget.prepare: rebuild the PLE segmentation when `gen_p > 1`. + ple_qsl = ( + mm.query_start_loc_long + if gen_p == 1 + else self._verify_qsl(md.seq_lens_cuda[:batch], batch, gen_p, dev) + ) + ple_state = (pools.ple_state[0], pools.ple_state[1], si_all, ple_qsl, gen_p) + + # Rollback staging as in PrefillTarget.prepare (`fused` under a decode CUDA graph). + rollback = core.spec_rollback + if rollback is not None: + rollback.discard() + if is_verify: + rollback.begin( + num_contexts=0, + prefix_tokens=0, + num_gens=batch, + rows_per_gen=gen_p, + slots=si_all, + request_indices=md.qsa_req_idx_per_token[:num_tokens], + logical_positions=md.qsa_logical_positions[:num_tokens], + block_table=md.qsa_block_table, + tokens_per_block=md.tokens_per_block, + fused=md.is_cuda_graph, + position_cache=pools.position_cache, + ) + + # Host-read dense/sparse decision: per capture, as the short/long decode graph families. + sparse = batch > 0 and int(md.kv_lens_runtime[:batch].max()) > core.dense_threshold + qsa = ( + md.qsa_req_idx_per_token[:num_tokens], + md.qsa_logical_positions[:num_tokens], + md.qsa_sequence_lengths[:num_tokens], + md.qsa_visible_blocks[:num_tokens], + md.qsa_topk_indices[:num_tokens], + md.qsa_topk_row_starts[:num_tokens], + md.qsa_block_table, + md.tokens_per_block, + pos3, + pos, + coords, + sparse, + ) + split = getattr(self, "_index_split", None) + self._index_plan = None + if sparse and split is not None: + self._index_plan = _index_split_plan(split, qsa, num_tokens, gen_p) + self._row_order = _RowOrder() if sparse else None + + # Scratch shared by all attention layers; each layer rewrites what it reads. + qkv = torch.empty([num_tokens, core.qkv_width], dtype=dt, device=dev) + index_scratch = torch.empty([num_tokens, core.index_scratch_width], dtype=dt, device=dev) + attn_out = torch.empty([num_tokens, core.q_width], dtype=dt, device=dev) + + hc_count = core.hc_count + layers, S = self._layers, self._stack + # The PLE front reads no residual, so it runs on the PLE stream beside the first layers; + # a row-sharded table's sum is a TP collective, which stays on the main stream. + ple_stream = getattr(core, "_ple_stream", None) + ple_front = None + if ple_stream is not None and core.ple_layer > 0 and not core._ple_table_host.needs_sum: + caller = torch.cuda.current_stream() + ple_stream.wait_stream(caller) + with torch.cuda.stream(ple_stream): + ple_front = self._ple_front(input_ids, ple_state, is_verify) + # Each site injects the previous MoE block's `(routed, inject, shared, gate_logits)`. + moe = (None, None, None, None) + for i, L in enumerate(layers): + if i == core.ple_layer: + # The PLE path adds into the injected stream and normalizes it itself. + routed, inject, shared, gate_logits = moe + if isinstance(routed, _TpPartial): + routed = core.tp_sum(routed.partial) + # Fold the previous MoE injection into the PLE launches (bitwise equal to + # `qwen4_exp_hc_inject_norm` followed by the PLE op). + fold = None + if routed is not None and _ple_fold_fits(hc, routed, inject, shared): + fold = (routed, inject, shared, gate_logits) + elif routed is not None or inject is not None: + hc, _ = qwen4_exp_hc_inject_norm( + hc, routed, inject, L.ahc_norm, core.eps, hc_count, shared, gate_logits + ) + if ple_front is not None: + torch.cuda.current_stream().wait_stream(ple_stream) + hc, normed = self._ple_block( + hc, + input_ids, + ple_state, + L.ahc_norm, + is_verify, + front=ple_front, + inject_block=fold, + ) + ple_front = None + mixed, inject = _hc_mixer(core, normed, L.ahc_first, L.ahc_up, True) + else: + hc, mixed, inject = _hc_site( + core, + hc, + moe[0], + moe[1], + L.ahc_norm, + L.ahc_first, + L.ahc_up, + True, + *moe[2:], + ) + moe = None + if core.is_attn[i]: + block_out = self._attention_block( + mixed, i, L, qkv, index_scratch, attn_out, qsa, step, is_verify + ) + else: + block_out = self._gdn_block(mixed, i, L, si_all, verify, gen_p) + hc, mixed, inject = _hc_site( + core, hc, block_out, inject, L.mhc_norm, L.mhc_first, L.mhc_up, True + ) + routed, gate_logits, shared = _moe_block( + core, mixed, L, pad_rows, mm=_step_projection(core), defer=True + ) + moe = (routed, inject, shared, gate_logits) + del routed, gate_logits, shared + hc, final, _ = _hc_site( + core, hc, moe[0], moe[1], S.fhc_norm, S.fhc_first, S.fhc_up, False, *moe[2:] + ) + del moe + if spec_metadata is None: + return final + # `lm_head` reads the `[T, hidden]` mix, the MTP module the `[T, stream]` stream. + return final, hc + + # ------------------------------------------------------------- the draft --- + + def _capture_selection(self, md, topk_indices, visible_blocks, batch, p): + """IndexShare: store each request's selection at its last accepted row (row + `num_accepted - 1` of its `p`) in the draft layer's share slot. Runs on the capture + stream and returns it (None if in line); the caller joins it before `topk_indices` / + `visible_blocks` are rewritten or the shared rows are read.""" + assert md.qsa_mtp_num_accepted is not None, ( + "IndexShare needs the worker's accepted counts to pick each request's capture row; " + "the draft loop did not supply them" + ) + stream = getattr(self.core, "_capture_stream", None) + if stream is not None: + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream) if stream is not None else contextlib.nullcontext(): + slot = self.core._draft_bound.share_slot + accepted = md.qsa_mtp_num_accepted[:batch].to(torch.int64) + rows = torch.arange(batch, dtype=torch.int64, device=topk_indices.device) * p + rows = rows + torch.clamp(accepted - 1, min=0, max=p - 1) + md.qsa_shared_topk_indices[slot, :batch].copy_( + torch.index_select(topk_indices, 0, rows) + ) + md.qsa_shared_topk_visible_blocks[slot, :batch].copy_( + torch.index_select(visible_blocks, 0, rows) + ) + # Host-side readiness, recorded in program order: under capture this runs while the + # graph is recorded, so a graph holds the reuse only after the capture it reads. + md.qsa_shared_topk_captured_slots.add(slot) + return stream + + def mtp_forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + """One draft step over generation rows only (`p` rows a request on the first step of a + batch without context, one otherwise), inside the decode graph when the trunk is. + + Its projections take `_step_projection`: the steps after the first of a step with + context rows, and every step of an eager trunk step, run eagerly at any batch size. + + Under IndexShare the draft-extend step captures each request's selection and later steps + skip the index branch, rebuilding it with `qsa_mtp_reuse_topk`; the index keys they skip + lie past the committed length and the next draft-extend step rewrites them.""" + core = self.core + assert isinstance(attn_metadata, TrtllmAttentionMetadata) + assert core._bound is not None, "the trunk's cache-manager bind has not run" + md = attn_metadata + rows = hidden_states.shape[0] + assert input_ids.shape[0] == rows and hidden_states.shape[1] == core.stream, ( + f"the draft step got {input_ids.shape[0]} token ids and {tuple(hidden_states.shape)} " + f"hidden states; the MTP module reads one {core.stream}-wide stream row per token" + ) + step = self.step_args(md) + gen_seqs = md.num_seqs + p, _ = _verify_rows(core, None, gen_seqs, rows) + step["predicted_tokens_per_seq"] = p + draft, layer_idx = core._draft_step_bind(md) + assert rows == md.num_tokens, ( + f"the draft step carries {rows} rows for a batch its metadata describes as " + f"{md.num_tokens} tokens; the QSA row mapping covers the second" + ) + pad_rows = _draft_pad_rows(core, all_rank_num_tokens, rows) + # The worker advances only device KV lengths, so this matches the trunk's host decision. + sparse = gen_seqs > 0 and int(md.kv_lens_runtime[:gen_seqs].max()) > core.dense_threshold + share = core.mtp_index_share and md.qsa_in_mtp_draft_loop + reuse = share and md.qsa_indexer_skip_topk + assert not share or reuse == (p == 1), ( + f"the worker's reuse flag ({md.qsa_indexer_skip_topk}) disagrees with the step's " + f"shape ({p} rows per request): only the draft decode steps after the draft-extend " + "step reuse its selection" + ) + pos3, pos, coords = _draft_positions(position_ids) + M = core._mtp + req_idx = md.qsa_req_idx_per_token[:rows] + logical = md.qsa_logical_positions[:rows] + block_table = md.qsa_block_table + tokens_per_block = md.tokens_per_block + mm = _step_projection(core) + + x, mixed, inject = _mtp_front(core, M, embed_tokens, input_ids, hidden_states, mm=mm) + qkv, gate, q_index = _mtp_attention_planes( + core, + M, + mixed, + draft, + (req_idx, logical, block_table, tokens_per_block, pos3, pos, coords), + p, + not reuse, + sparse, + mm, + ) + # Rows whose outputs are read: each request's continuing row on draft-extend, else all. + cont, pre = None, {} + if _draft_continue_ok(core, md, p, share): + meta_rows = ( + md.qsa_visible_blocks, + md.qsa_sequence_lengths, + md.qsa_logical_positions, + md.qsa_req_idx_per_token, + ) + if all(t.is_contiguous() for t in (md.qsa_mtp_num_accepted, *meta_rows)): + # The rows, the metadata and the step's operands at them in two launches. + cont, meta = draft_continue_select(md.qsa_mtp_num_accepted, gen_seqs, p, meta_rows) + pre.update(zip(("visible", "seq_lens", "logical", "req_idx"), meta)) + named = {"x": x, "inject": inject} + if sparse: + named.update(qkv=qkv, gate=gate) + if q_index is not None: + named["q_index"] = q_index.reshape(rows, -1) + named = {k: t for k, t in named.items() if t.dim() == 2 and t.stride(1) == 1} + pre.update(zip(named, draft_rows_gather(cont, tuple(named.values())))) + if "q_index" in pre: + pre["q_index"] = pre["q_index"].view(gen_seqs, *q_index.shape[1:]) + else: + # The fused select needs contiguous vectors; else `sel` gathers each operand. + cont = _draft_continue_rows(core, md, gen_seqs, p, share) + n = rows if cont is None else gen_seqs + + def sel(t, name=None): + if cont is None: + return t + got = pre.get(name) + return got if got is not None else t.index_select(0, cont) + + capture = None # the stream the IndexShare capture runs on, joined below + if sparse: + kv = draft.kv + topk_indices = md.qsa_topk_indices[:n] + visible_blocks = sel(md.qsa_visible_blocks[:rows], "visible") + seq_lens_q = sel(md.qsa_sequence_lengths[:rows], "seq_lens") + q_logical, q_req_idx = sel(logical, "logical"), sel(req_idx, "req_idx") + selected = torch.empty( + [n, core.token_topk + core.compress_ratio - 1], + dtype=torch.int32, + device=qkv.device, + ) + if reuse: + slot = draft.share_slot + assert slot in md.qsa_shared_topk_captured_slots, ( + "a draft decode step would reuse a selection that no draft-extend step of " + "this loop captured" + ) + qsa_mtp_reuse_topk( + shared_indices=md.qsa_shared_topk_indices[slot], + shared_visible_blocks=md.qsa_shared_topk_visible_blocks[slot], + request_indices=q_req_idx, + visible_blocks=visible_blocks, + output=topk_indices, + ) + # Use the expansion's output as the selection when it matches (no copy). + expanded = qsa_expand_block_indices( + topk_indices, + q_logical, + seq_lens_q, + compress_ratio=core.compress_ratio, + token_topk=core.token_topk, + ) + if expanded.shape == selected.shape and expanded.dtype == selected.dtype: + selected = expanded + else: + selected.copy_(expanded) + else: + # IndexShare's capture keeps every row's block top-k: no split there. + selected = _decode_index_selection( + core, + None if share else getattr(self, "_index_split", None), + sel(q_index, "q_index"), + (q_req_idx, q_logical, seq_lens_q, visible_blocks, topk_indices, + md.qsa_topk_row_starts[:n], block_table, tokens_per_block), + n, + n // gen_seqs, + qkv.device, + draft.index_cache, + ) # fmt: skip + if share: + capture = self._capture_selection(md, topk_indices, visible_blocks, gen_seqs, p) + attn = torch.reshape( + _sparse_gqa( + core, + kv, + _packed_q(core, sel(qkv, "qkv")), + block_table, + selected, + q_req_idx, + tokens_per_block, + None, + output_gate=sel(gate, "gate"), + rows_per_request=n // gen_seqs if n % gen_seqs == 0 else 1, + ), + [n, core.q_width], + ) + else: + attn = thop_attention( + q=qkv, + output=torch.empty([rows, core.q_width], dtype=qkv.dtype, device=qkv.device), + local_layer_idx=layer_idx, + num_heads=core.heads_q, + num_kv_heads=core.heads_kv, + head_size=core.head_dim, + quant_mode=core.quant_mode, + kv_scale_orig_quant=core.kv_scale_orig_quant, + kv_scale_quant_orig=core.kv_scale_quant_orig, + **step, + **_CALL_CONSTANTS, + ) + fused_sigmoid_mul_inplace(attn, gate, direct_launch=True) + attn = sel(attn) + out = _mtp_tail(core, M, sel(x, "x"), attn, sel(inject, "inject"), pad_rows, mm=mm) + if capture is not None: + torch.cuda.current_stream().wait_stream(capture) + if cont is None: + return out + # The worker reads only the continuing rows; the rest are left unwritten. + full = out.new_empty([rows, out.shape[1]]) + full.index_copy_(0, cont, out) + return full + + +# ------------------------------------------------------------ the MTP module --- + + +class MTPLayer: + """The checkpoint's MTP module (one decoder layer behind `fc_embedding` / `fc_hidden`), run + once per draft step and routed to a target by each call's metadata phase. Returns the + residual un-collapsed (`shared_head` owns the mixer). Not an `nn.Module`: its weights live + in the core's `ParameterDict`, and a second registration would break load coverage.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2, logits_processor) -> None: + self.core = core + self.logits_processor = logits_processor + # The spec worker when it samples with the TP key exchange (`_sharded_head`). + self.worker = None + # The engine step's `spec_metadata`, set by the shell before the worker runs. + self.spec_metadata = None + + def __call__(self, *args, **kwargs) -> torch.Tensor: + return self.forward(*args, **kwargs) + + def _sharded_head(self, lm_head, rows: int) -> bool: + """Whether the draft logits may stay this rank's vocabulary shard (greedy tokens via the + key exchange); not under rejection sampling, which needs the gathered fp32 logits.""" + worker, md = self.worker, self.spec_metadata + if worker is None or md is None: + return False + exchange = worker.key_exchange + return bool( + exchange is not None + and not md.wants_advanced_draft_sampling + and worker.guided_decoder is None + and 1 <= rows <= exchange.MAX_ROWS + and lm_head.tp_mode == TensorParallelMode.COLUMN + and lm_head.gather_output + and lm_head.padding_size == 0 + ) + + def shared_head(self, hidden_states, lm_head, attn_metadata, return_context_logits): + """The module's own stack-final mixer, then `lm_head`; with the key exchange + (`_sharded_head`) this rank's bf16 vocabulary shard.""" + core = self.core + M = core._mtp + _, final, _ = _hc_site( + core, hidden_states, None, None, M.fhc_norm, M.fhc_first, M.fhc_up, False + ) + if self._sharded_head(lm_head, final.shape[0]): + return lm_head(final, gather_output=False) + return self.logits_processor.forward(final, lm_head, attn_metadata, return_context_logits) + + def forward( + self, + embed_tokens: torch.Tensor, + all_rank_num_tokens, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + **kwargs, + ) -> torch.Tensor: + target = self.core._select_target(attn_metadata) + args = (embed_tokens, all_rank_num_tokens, input_ids, position_ids, hidden_states) + if isinstance(target, DecodeTarget): + return _on_decode_stream( + self.core, input_ids.shape[0], target.mtp_forward, *args, attn_metadata, **kwargs + ) + return target.mtp_forward(*args, attn_metadata, **kwargs) + + +class DraftModel: + """The spec worker's handle on the drafter: `mtp_layers`, `embed_tokens` and `lm_head`. + `embed_tokens` reads through the core, since materialization replaces the meta tensors.""" + + def __init__(self, core: Qwen4ExpUnifiedModelingV2, lm_head, logits_processor) -> None: + self.core = core + self.mtp_layers = [MTPLayer(core, logits_processor)] + self.lm_head = lm_head + + @property + def embed_tokens(self) -> torch.Tensor: + return self.core.w["embed"] + + +def _spec_rollback_begin_ok( + request_indices, logical_positions, block_table, position_cache +) -> bool: + """Whether `spec_rollback_begin` accepts these operands (mirrors its `is_valid`).""" + return ( + logical_positions.dtype == torch.int64 + and request_indices.dtype in (torch.int32, torch.int64) + and block_table.dtype == torch.int32 + and block_table.dim() == 2 + and block_table.stride(1) == 1 + and position_cache.dtype == torch.int32 + and position_cache.dim() == 3 + and position_cache.shape[2] == 3 + and position_cache.stride(2) == 1 + ) + + +class SpeculativeRollback: + """Undoes what a verify step's rejected draft rows wrote to side caches the runtime does not + promote (it promotes the GDN state): the PLE n-gram history and conv state, each attention + layer's QSA index-K plane and the shared QSA position plane. + + For a request that accepted `a` rows (golden token included, `1 <= a <= p`) the commit sets + the n-gram row to row `a - 1`'s id window minus its oldest id, the conv row to columns + `[a, a + state_len)` of `[pre-step state | the p rows' conv input]`, and every index-K and + position slot written by an ordinal `>= a` back to its pre-verify value. + + The targets stage during the forward and the spec worker calls `commit_speculative_states` + after its draft loop in the same forward, so a captured graph holds both ends. A `fused` step + (decode graphs) stages and commits the PLE pools in one launch each.""" + + def __init__(self) -> None: + self._pending: dict | None = None + # (base addresses, page strides, plane count) of the index-K planes, once bound. + self._planes: tuple | None = None + + def set_index_planes(self, planes: list) -> None: + """Bind the attention layers' index-K planes (static for the manager's lifetime).""" + ptrs, strides = _plane_table(planes) + self._planes = (ptrs, strides, len(planes)) + + # ------------------------------------------------------------- staging --- + + def discard(self) -> None: + self._pending = None + + def begin( + self, + *, + num_contexts: int, + prefix_tokens: int, + num_gens: int, + rows_per_gen: int, + slots: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + fused: bool = False, + position_cache: torch.Tensor | None = None, + ) -> None: + """Open a verify step's staging, before the trunk writes any index plane. Generation rows + follow the `prefix_tokens` context rows as `num_gens` runs of `rows_per_gen` (row + `s * p + t` is ordinal `t` of request `s`); each row's QSA slot is resolved as + `qsa_paged_index_store` does and every index plane's slots are gathered. A `fused` step + given `position_cache` also gathers the position plane, in one launch.""" + n, p = num_gens, rows_per_gen + assert n > 0 and p > 1, ( + f"a verify step stages generation requests carrying drafts; got {n} requests of " + f"{p} rows" + ) + assert logical_positions.shape[0] == n * p, ( + f"{n} generation requests of {p} rows are {n * p} rows; got " + f"{logical_positions.shape[0]} positions" + ) + assert self._planes is not None, "the attention layers' index planes are not bound" + ptrs, strides, count = self._planes + if ( + fused + and position_cache is not None + and _spec_rollback_begin_ok( + request_indices, logical_positions, block_table, position_cache + ) + ): + pages, within, index_all, position_old = spec_rollback_begin( + request_indices, + logical_positions, + block_table, + tokens_per_block, + ptrs, + strides, + position_cache, + ) + self._pending = dict( + num_contexts=num_contexts, + prefix=prefix_tokens, + n=n, + p=p, + slots=slots.to(torch.long), + pages=pages, + within=within, + staged_layers=0, + position=(position_cache, position_old), + index_all=index_all, + fused=fused, + ngram_stream=None, + ) + return + positions = logical_positions.to(torch.long) + pages = block_table[request_indices.to(torch.long), positions // tokens_per_block] + pages = pages.to(torch.long) + within = positions % tokens_per_block + self._pending = dict( + num_contexts=num_contexts, + prefix=prefix_tokens, + n=n, + p=p, + slots=slots.to(torch.long), + pages=pages, + within=within, + staged_layers=0, + position=None, + index_all=qsa_rollback_gather(ptrs, strides, pages, within, count), + fused=fused, + ngram_stream=None, + ) + + def fused_pending(self) -> bool: + return self._pending is not None and bool(self._pending["fused"]) + + def stage_ple_verify( + self, ngram_pool: torch.Tensor, conv_pool: torch.Tensor, input_ids: torch.Tensor + ) -> None: + """Stage the PLE pools before either is written: the slots' pre-step conv rows (not on a + fused step, `DecodeTarget._ple_from_pool`) and the n-gram candidates, row `t`'s being the + last `context` ids of `[history | ids of rows 0..t]`.""" + pending = self._pending + prefix, n, p = pending["prefix"], pending["n"], pending["p"] + slots = pending["slots"] + pending["ngram_pool"] = ngram_pool + pending["conv_pool"] = conv_pool + if pending["fused"]: + pending["ngram_stream"], pending["conv_prev"] = spec_rollback_stage( + ngram_pool, + conv_pool, + slots, + input_ids[prefix : prefix + n * p], + p, + conv=False, + ) + return + pending["conv_prev"] = conv_pool[slots] + context = ngram_pool.shape[1] + ids = torch.reshape(input_ids[prefix : prefix + n * p], [n, p]).to(ngram_pool.dtype) + stream = torch.cat([ngram_pool[slots], ids], 1) + # Window k of the unfold is the history after k rows. + pending["ngram_candidates"] = stream.unfold(1, context, 1)[:, 1:] + + def stage_ple_conv_input(self, conv_input: torch.Tensor) -> None: + """Stage the generation rows' conv input (`[T, C]`, token-major); a mixed batch's rows + are copied so its context prefix is not kept alive through the draft loop.""" + pending = self._pending + prefix, n, p = pending["prefix"], pending["n"], pending["p"] + conv_x = torch.reshape(conv_input[prefix:], [n, p, -1]) + pending["conv_x"] = conv_x.clone() if prefix else conv_x + + def stage_index(self, position_cache: torch.Tensor) -> None: + """Before an attention layer's index store: on the first, stage the position slots.""" + pending = self._pending + if pending["position"] is None: + pages, within = pending["pages"], pending["within"] + pending["position"] = (position_cache, position_cache[pages, within]) + pending["staged_layers"] += 1 + + # -------------------------------------------------------------- commit --- + + def commit_speculative_states( + self, + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor, + num_contexts: int, + ) -> None: + """Rewrite the staged planes to the accepted prefix. Called by the spec worker.""" + pending, self._pending = self._pending, None + if pending is None: + return + n, p = pending["n"], pending["p"] + assert num_contexts == pending["num_contexts"], ( + f"the worker commits with {num_contexts} context requests and the forward staged " + f"{pending['num_contexts']}" + ) + assert state_indices.shape[0] == num_contexts + n, ( + f"the worker hands {state_indices.shape[0]} slots for a batch the forward staged " + f"as {num_contexts} + {n} requests" + ) + assert pending.get("conv_x") is not None and pending["staged_layers"] > 0, ( + "a verify step staged neither its PLE rows nor any attention layer; the forward " + "did not run the staging it opened" + ) + ptrs, strides, _ = self._planes + position, position_old = pending["position"] + pages, within = pending["pages"], pending["within"] + if pending["fused"]: + spec_rollback_commit( + num_accepted_tokens[num_contexts : num_contexts + n], + state_indices[num_contexts:], + pending["ngram_pool"], + pending["ngram_stream"], + pending["conv_pool"], + pending["conv_prev"], + pending["conv_x"], + position, + position_old, + pages, + within, + ptrs, + strides, + pending["index_all"], + ) + return + accepted = num_accepted_tokens[num_contexts : num_contexts + n].to(torch.long) + slots = state_indices[num_contexts:].to(torch.long) + rows = torch.arange(n, device=accepted.device) + ngram_pool, conv_pool = pending["ngram_pool"], pending["conv_pool"] + state_len = conv_pool.shape[-1] + ngram_pool[slots] = pending["ngram_candidates"][rows, accepted - 1] + # [n, C, state_len + p]; window k of the unfold is the state after k tokens. + history = torch.cat([pending["conv_prev"], torch.transpose(pending["conv_x"], 1, 2)], 2) + conv_pool[slots] = history.unfold(2, state_len, 1)[rows, :, accepted] + # [n * p]: True at every ordinal the sampler rejected. + rejected = torch.reshape( + torch.arange(p, device=accepted.device).unsqueeze(0) >= accepted.unsqueeze(1), [-1] + ) + qsa_rollback_restore(ptrs, strides, pages, within, rejected, pending["index_all"]) + position[pages, within] = torch.where( + rejected[:, None], position_old, position[pages, within] + ) + + +# -------------------------------------------------------------------- shell --- + + +class _WidenedHead: + """`lm_head` widening bf16 logits with the vectorized `bf16_to_fp32` (bit-identical to + `.float()`), so the logits processor's own `.float()` is a no-op.""" + + def __init__(self, lm_head) -> None: + self.lm_head = lm_head + + def __call__(self, hidden_states: torch.Tensor) -> torch.Tensor: + logits = self.lm_head(hidden_states) + if logits.dtype == torch.bfloat16 and logits.is_contiguous(): + return bf16_to_fp32(logits) + return logits + + +class _WideningLogitsProcessor(LogitsProcessor): + """The shell's logits processor over `_WidenedHead`, for trunk and draft logits.""" + + def forward( + self, hidden_states, lm_head, attn_metadata, return_context_logits: bool = False + ) -> torch.Tensor: + return super().forward( + hidden_states, _WidenedHead(lm_head), attn_metadata, return_context_logits + ) + + +def _field(block, name: str, default=None): + """`block`'s `name` -- a mapping's entry or an object's attribute -- or `default` where it + is absent or null.""" + value = block.get(name) if isinstance(block, dict) else getattr(block, name, None) + return default if value is None else value + + +def _text_settings(text) -> dict: + """`_TEXT_SETTINGS`' entries as `text` declares them. The rope fields are the target's + reading: the `rope_parameters` dict, or the flattened `rope_scaling` with the theta and + the partial factor at the top level.""" + rope = getattr(text, "rope_parameters", None) or getattr(text, "rope_scaling", None) or {} + settings = {name: getattr(text, name, None) for name in _TEXT_SETTINGS} + settings.update( + rope_theta=rope.get("rope_theta", getattr(text, "rope_theta", None)), + partial_rotary_factor=rope.get( + "partial_rotary_factor", getattr(text, "partial_rotary_factor", None) + ), + mrope_section=tuple(rope.get("mrope_section") or ()), + mrope_interleaved=rope.get("mrope_interleaved"), + rope_factor=rope.get("factor"), + attn_output_gate=_field(text, "attn_output_gate", True), + mtp_num_hidden_layers=_field(text, "mtp_num_hidden_layers", 1), + mtp_use_dedicated_embeddings=_field(text, "mtp_use_dedicated_embeddings", False), + ) + mtp = getattr(text, "mtp", None) + settings.update( + { + "mtp.num_hidden_layers": _field(mtp, "num_hidden_layers", 1), + "mtp.layer_types": tuple(_field(mtp, "layer_types", ("full_attention",))), + "mtp.mtp_use_hidden_state_from_layer": _field(mtp, "mtp_use_hidden_state_from_layer"), + } + ) + return settings + + +class ModelingV2Qwen38FlashNextFp8Sm103Unified( + DecoderModelForCausalLM[Qwen4ExpUnifiedModelingV2, PretrainedConfig] +): + """Qwen3.8-Flash-Next-FP8 at tp1, tp2, tep2, dep2, tp4, tep4 or dep4 on sm_103.""" + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: + """V2: the only KV cache manager with PLE state and QSA's index and position planes.""" + return "V2" + + @staticmethod + def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: str) -> None: + """Raise `NoTarget` for a checkpoint or engine choice this target does not run. + + `route` calls it once the GPU, `layout` (routing's `_FP8_LAYOUTS`) and draft path match, + so `auto` falls back to the built-in implementation instead of failing later.""" + cfg = model_config.pretrained_config + text = getattr(cfg, "text_config", None) + vision = getattr(cfg, "vision_config", None) + if text is None or vision is None: + raise NoTarget( + f"{type(cfg).__name__} carries no text_config/vision_config; the target reads " + "both halves of the normalized Qwen4-Exp composite" + ) + shape = ( + text.num_hidden_layers, + text.hidden_size, + text.num_experts, + text.num_attention_heads, + text.linear_num_value_heads, + text.vocab_size, + ) + vshape = ( + vision.depth, + vision.hidden_size, + vision.num_heads, + vision.spatial_merge_size, + vision.out_hidden_size, + vision.num_position_embeddings, + tuple(getattr(vision, "deepstack_visual_indexes", None) or ()), + vision.hidden_act, + ) + if (shape, vshape) != (_TEXT_SHAPE, _VISION_SHAPE): + raise NoTarget( + f"text shape {shape}, vision shape {vshape}; the target is Qwen3.8-Flash-Next's " + f"{_TEXT_SHAPE}, vision {_VISION_SHAPE}" + ) + differs = {k: v for k, v in _text_settings(text).items() if v != _TEXT_SETTINGS[k]} + if differs: + raise NoTarget( + f"the checkpoint has Qwen3.8-Flash-Next's shape but declares {differs}; the " + f"target implements { ({k: _TEXT_SETTINGS[k] for k in differs}) }" + ) + # The opt-in switches, read here so a misspelled value fails before the build; one that + # would do nothing at this deployment declines rather than being ignored. + mapping = model_config.mapping + _ple_host.host_table_mode() + if _weights.moe_tpb_requested() and not _weights.moe_tpb_enabled(mapping): + raise NoTarget( + f"{_weights.MOE_TPB_ENV}=rr at {layout}: the block-major MoE decode copy runs at " + "tp4 and tep4 only; unset it" + ) + rowtile_groups = _weights.switch(ROWTILE_SCRATCH_GROUPS_ENV) + strategy = AllReduceStrategy(int(model_config.allreduce_strategy)) + if _pool.attention_tp_size(mapping) > 1 and strategy in ( + AllReduceStrategy.UB, + AllReduceStrategy.LOWPRECISION, + ): + raise NoTarget( + f"allreduce_strategy={strategy.name} at {layout}: the target calls the all-reduce " + "op directly, without the buffers the stock AllReduce module sets up for it" + ) + # As the stock factory: only an attention-DP group has a MoE round trip to override. + if mapping.enable_attention_dp and mapping.dp_size > 1: + comm_method = _moe_comm_method() + if comm_method not in ("ALLGATHER", "NVLINK_ONE_SIDED"): + raise NoTarget( + f"{MOE_COMM_METHOD_ENV}={comm_method} at {layout}: the target runs the " + "attention-DP MoE round trip as ALLGATHER (the default) or NVLINK_ONE_SIDED" + ) + if comm_method == "NVLINK_ONE_SIDED": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + NVLinkOneSided, + ) + + if mpi_disabled() or not NVLinkOneSided.is_platform_supported(): + raise NoTarget( + f"{MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at {layout}: the all-to-all MoE " + "round trip needs MPI and MNNVL on this node" + ) + spec = model_config.spec_config + if spec is not None: + # One linear MTP chain, verified at `max_draft_len + 1` rows on the GDN cached replay. + if ( + getattr(spec, "num_nextn_predict_layers", None) not in (None, 1) + or getattr(spec, "use_dynamic_tree", False) + or getattr(spec, "eagle_choices", None) is not None + or spec.max_total_draft_tokens != spec.max_draft_len + ): + raise NoTarget( + "an MTP draft other than one linear chain over the checkpoint's single MTP " + "module" + ) + if not 1 <= spec.max_draft_len < _MAX_VERIFY_WIDTH: + raise NoTarget( + f"max_draft_len={spec.max_draft_len}; the verify step is certified at " + f"max_draft_len 1..{_MAX_VERIFY_WIDTH - 1}" + ) + if ( + getattr(spec, "draft_len_schedule", None) is not None + or getattr(spec, "acceptance_rate_threshold", None) is not None + ): + raise NoTarget( + "the draft length changes at runtime (draft_len_schedule, max_concurrency or " + "acceptance_rate_threshold), but the verify step runs at the one width " + "max_draft_len + 1" + ) + if not is_gdn_replay_enabled(): + raise NoTarget( + "MTP with TRTLLM_USE_GDN_REPLAY=0: the verify step runs the GDN cached replay, " + "not the legacy per-step recurrent states; leave TRTLLM_USE_GDN_REPLAY unset" + ) + if is_gdn_flashinfer_replay_enabled() and spec.max_draft_len + 1 in (4, 8): + raise NoTarget( + f"TRTLLM_USE_GDN_FLASHINFER_REPLAY with max_draft_len={spec.max_draft_len}: " + "the manager lays the replay history out for the FlashInfer ring kernel, " + "which the verify step does not run" + ) + prefill_graph = str(llm_args.prefill_cuda_graph_backend.value) + if prefill_graph == "piecewise": + raise NoTarget( + "prefill_cuda_graph_backend='piecewise' makes the engine torch.compile the model, " + "which the target does not implement; 'breakable' runs the prefill step as a " + "breakable CUDA graph" + ) + if mapping.enable_attention_dp and prefill_graph == "breakable": + raise NoTarget( + f"prefill_cuda_graph_backend='breakable' with attention DP ({layout}): the engine " + "pads every rank to one prefill bucket, so a rank holding only generation " + "requests is handed more rows than its batch, which the decode step refuses" + ) + # Unset, max_seq_len is inferred after the build from the text tower's + # max_position_embeddings (the pinned rope declares no scaling factor). + windows = llm_args.kv_cache_config.max_attention_window + full = llm_args.max_seq_len or model_config.max_seq_len or text.max_position_embeddings + if windows is not None and any(w < full for w in windows): + raise NoTarget( + f"kv_cache_config.max_attention_window={windows}; the target implements no " + "sliding window: every attention layer's window is max_seq_len" + ) + if getattr(llm_args.multimodal_config, "vision_gpu_normalize", False): + raise NoTarget( + "multimodal_config.vision_gpu_normalize: the input processor then ships uint8 " + "pixels, and the vision tower runs at the bf16 activation dtype" + ) + kv_cache_quant_algo = engine_kv_cache_quant_algo(model_config, llm_args) + if kv_cache_quant_algo not in (None, QuantAlgo.FP8, QuantAlgo.NVFP4): + raise NoTarget( + f"a {kv_cache_quant_algo} KV cache; the target stores K/V as the checkpoint's " + "bf16, as FP8 or as NVFP4" + ) + if rowtile_groups and kv_cache_quant_algo != QuantAlgo.FP8: + stored = "bf16" if kv_cache_quant_algo is None else kv_cache_quant_algo + raise NoTarget( + f"{ROWTILE_SCRATCH_GROUPS_ENV} with a {stored} KV cache: the grouped row-tile " + "context launches run over an FP8 KV cache only; unset it" + ) + if model_config.torch_dtype != torch.bfloat16: + raise NoTarget( + f"the engine runs the model in {model_config.torch_dtype}; the target is bf16" + ) + if str(model_config.moe_backend_requested).upper() not in _TRUNK_MOE_BACKENDS: + raise NoTarget( + f"moe_config.backend={model_config.moe_backend_requested!r}; the target runs the " + f"routed experts on {', '.join(_TRUNK_MOE_BACKENDS)}" + ) + if llm_args.moe_config.load_balancer is not None and mapping.moe_ep_size > 1: + raise NoTarget( + f"moe_config.load_balancer (EPLB) at {layout}: the target runs fixed contiguous " + "expert windows and registers no layer with the balancer, so the placement " + "would do nothing" + ) + state_dtype = effective_mamba_ssm_cache_dtype( + model_config, llm_args.kv_cache_config.mamba_ssm_cache_dtype + ) + if state_dtype not in _SSM_STATE_DTYPES: + served = " or ".join(str(dtype).removeprefix("torch.") for dtype in _SSM_STATE_DTYPES) + raise NoTarget( + f"a {state_dtype} recurrent-state cache; the target serves {served} " + "(kv_cache_config.mamba_ssm_cache_dtype)" + ) + + def __init__(self, model_config: ModelConfig): + cfg = model_config.pretrained_config + assert cfg is not None + # Adopt the normalized text tower as the config in place: the pools are sized from it, + # and the engine keeps its own reference to this `ModelConfig`. + text = getattr(cfg, "text_config", None) + assert text is not None, ( + "this target is the language tower of the composite Qwen4-Exp export; " + "pretrained_config carries no text_config" + ) + assert getattr(text, "architectures", None) == ["Qwen4ExpForCausalLM"], ( + f"the text tower must be the normalized Qwen4-Exp text config; got architectures " + f"{getattr(text, 'architectures', None)!r}" + ) + composite = cfg + model_config.pretrained_config = text + super().__init__( + Qwen4ExpUnifiedModelingV2(model_config, composite), + config=model_config, + hidden_size=text.hidden_size, + vocab_size=text.vocab_size, + ) + # `lm_head` is outside the quantization scheme: a dense bf16 `LMHead`. + assert self.lm_head.weight.dtype == torch.bfloat16, ( + f"expected a bf16 lm_head; the shell built {self.lm_head.weight.dtype}" + ) + self.logits_processor = _WideningLogitsProcessor() + # No separate draft checkpoint: the MTP module's weights are the target's own. + self.draft_config = None + self.spec_config = getattr(model_config, "spec_config", None) + self.draft_model = None + self.spec_worker = None + # Read by the engine (`model_engine._spec_decode_fast_paths`, `runtime_hooks_of`). + self.spec_decode_fast_paths = False + self.runtime_hooks = None + if self.spec_config is not None: + self.draft_model = DraftModel(self.model, self.lm_head, self.logits_processor) + # Must match the resource manager, or the draft step gets no draft layer mapped. + self.use_separate_draft_kv_cache = should_use_separate_draft_kv_cache(self.spec_config) + from tensorrt_llm._torch.distributed import ll_exchange + + # The engine's one-engine fast paths, written for linear one-model MTP-Eagle: the + # only speculation routing and check_deployment admit. + self.spec_decode_fast_paths = True + self.runtime_hooks = _runtime_hooks() + # Collective over the TP group (every rank constructs the model in step); + # None without attention TP or peer memory (the NCCL gathers stay). + key_exchange = ll_exchange.create_tp_key_exchange( + model_config.mapping, model_config.max_num_tokens + ) + self.spec_worker = get_spec_worker( + self.spec_config, + model_config, + model_config.mapping, + use_separate_draft_kv_cache=self.use_separate_draft_kv_cache, + key_exchange=key_exchange, + spec_decode_fast_paths=True, + ) + assert self.spec_worker is not None, ( + f"the engine resolved no spec worker for spec_dec_mode " + f"{self.spec_config.spec_dec_mode!r}" + ) + if getattr(self.spec_worker, "key_exchange", None) is not None: + self.draft_model.mtp_layers[0].worker = self.spec_worker + self.spec_worker.set_draft_model(self.draft_model) + # Nothing the runtime owns rolls back the PLE pools and QSA side caches. + self.spec_worker.register_auxiliary_state_handler(self.model.spec_rollback) + # Media placeholder ids, built lazily (`__init__` runs under meta init). + self._mm_token_ids_value = (self.model.image_token_id, self.model.video_token_id) + self._mm_token_ids_cache: torch.Tensor | None = None + # Replicated on every rank; built once the weights are real (`post_load_weights`). + self.vision_tower: _vision.VisionTower | None = None + + @property + def multimodal_token_ids(self) -> torch.Tensor: + """Placeholder ids whose rows vision embeddings replace (in vocabulary here, so the + executor's out-of-vocabulary fallback would select nothing).""" + if self._mm_token_ids_cache is None: + self._mm_token_ids_cache = torch.tensor(self._mm_token_ids_value, dtype=torch.int32) + return self._mm_token_ids_cache + + @property + def multimodal_data_device_paths(self) -> list[str]: + """`multimodal_data` tensors moved to the device; the `(t, h, w)` grids stay on the host.""" + return list(_vision.DEVICE_PATHS) + + def validate_request(self, request) -> None: + """Reject at admission a media request the vision tower would assert on.""" + _vision.validate_request(self.vision_tower, request) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.IntTensor | None = None, + position_ids: torch.IntTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + return_context_logits: bool = False, + spec_metadata=None, + lora_params: dict | None = None, + **kwargs, + ): + """One engine step: `begin_step`, the vision tower (the inherited forward drops + `multimodal_params`), then the trunk; with a spec worker the shell gathers the verified + logits rows (`spec_metadata.gather_ids`) and hands the worker the ungathered stream.""" + self.model.begin_step( + attn_metadata, + input_ids, + position_ids, + inputs_embeds=inputs_embeds, + lora_params=lora_params, + spec_metadata=spec_metadata, + vision=_vision.multimodal_embeds(self.vision_tower, kwargs), + ) + if self.spec_worker is None: + assert spec_metadata is None, ( + "spec_metadata arrived without a spec worker; this engine was built without a " + "speculative_config" + ) + return super().forward( + attn_metadata, + input_ids, + position_ids, + inputs_embeds, + return_context_logits, + spec_metadata, + lora_params, + **kwargs, + ) + assert spec_metadata is not None, "the spec worker is built but no spec_metadata came" + final, hc = self.model.split_spec_output( + self.model( + attn_metadata=attn_metadata, + input_ids=input_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + lora_params=lora_params, + spec_metadata=spec_metadata, + **kwargs, + ) + ) + rows = nn.functional.embedding(spec_metadata.gather_ids, final) + if self.spec_worker.key_exchange is not None: + # This rank's vocabulary shard for a greedy/temperature-only batch, else gathered fp32. + logits = self.spec_worker.target_logits( + rows, + self.lm_head, + self.logits_processor, + attn_metadata, + spec_metadata, + self.draft_model, + ) + else: + logits = self.logits_processor.forward(rows, self.lm_head, attn_metadata, True) + # The draft head decides from the same batch whether its logits may stay sharded. + self.draft_model.mtp_layers[0].spec_metadata = spec_metadata + # A prefill CUDA graph pads the rows; the worker reshapes per request, so trim them. + if attn_metadata.padded_num_tokens is not None: + n = attn_metadata.num_tokens + input_ids, hc = input_ids[:n], hc[:n] + position_ids = position_ids[..., :n] + return self.spec_worker( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hc, + logits=logits, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_model=self.draft_model, + resource_manager=kwargs.get("resource_manager"), + ) + + def load_weights(self, weights, *args, **kwargs): + # A reload re-derives everything `post_load_weights` builds from the weights. + self.model._prefill = self.model._decode = None + _weights.MODEL_WEIGHTS.load(self, weights) + + def post_load_weights(self): + super().post_load_weights() + self.model.post_load_weights() + self.vision_tower = _vision.VisionTower(self.model.w, self.model.vision_config) + # The CPU-only load checks build the shell over host weights; they launch nothing. + if self.lm_head.weight.is_cuda: + self.vision_tower.warm_up() + self._load_eager_gemm_kernels() + + def _load_eager_gemm_kernels(self) -> None: + """Load, once per rank, every cuBLASLt kernel an eager bf16 GEMM can pick (the core's + projections and `lm_head`, at `_projection_warm_rows`) so no serving step loads one.""" + core, head = self.model, self.lm_head + device = head.weight.device + projections = _eager_projections( + core, core._prefill._layers, core._prefill._stack, core._mtp + ) + _load_projection_kernels(projections, device) + rows = min(core.max_num_tokens, _LM_HEAD_WARM_ROWS) + x = torch.zeros([rows, head.in_features], dtype=head.weight.dtype, device=device) + for m in _projection_warm_rows(1, rows): + head.apply_linear(x[:m], head.bias) + torch.cuda.synchronize(device) + # Load the grouped routed-experts path before any prefill CUDA graph is captured. + dims = getattr(core, "dims", None) + if ( + _MOE_PREFILL_GROUPED_MIN_ROWS + and dims is not None + and getattr(core, "dp_size", 1) == 1 + and core.max_num_tokens >= _MOE_PREFILL_GROUPED_MIN_ROWS + ): + warm_prefill_grouped(device, dims.local_experts) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py new file mode 100644 index 000000000000..777b6b725c00 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ops used only by this target; the family's shared ops are in `models/qwen4_exp/ops`.""" diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py new file mode 100644 index 000000000000..99d7d950867c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/draft_continue_rows__decode.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A generation-only draft-extend step's continuing rows and their operands, in two launches. + +The draft layer's draft-extend step (`p` rows a request) runs its attention core and tail only on +each request's continuing row, `r * p + clamp(num_accepted[r] - 1, 0, p - 1)`: +`draft_continue_select` forms those rows and gathers the QSA row metadata at them; +`draft_rows_gather` gathers the step's 2-D operands at them. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Requests a program of `draft_continue_select` covers. +SEQ_BLOCK = 256 +#: Columns a program of `draft_rows_gather` copies. +COL_BLOCK = 1024 +#: Operands one `draft_rows_gather` launch takes. +MAX_OPERANDS = 6 + + +@triton.jit(do_not_specialize=["num_seqs", "rows_per_request"]) +def _continue_select_kernel( + accepted, + v0, + v1, + v2, + v3, + rows_out, + o0, + o1, + o2, + o3, + num_seqs, + rows_per_request, + BLOCK: tl.constexpr, +): + seqs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + valid = seqs < num_seqs + last = tl.load(accepted + seqs, mask=valid, other=1).to(tl.int64) - 1 + last = tl.minimum(tl.maximum(last, 0), rows_per_request - 1) + rows = seqs.to(tl.int64) * rows_per_request + last + tl.store(rows_out + seqs, rows, mask=valid) + tl.store(o0 + seqs, tl.load(v0 + rows, mask=valid), mask=valid) + tl.store(o1 + seqs, tl.load(v1 + rows, mask=valid), mask=valid) + tl.store(o2 + seqs, tl.load(v2 + rows, mask=valid), mask=valid) + tl.store(o3 + seqs, tl.load(v3 + rows, mask=valid), mask=valid) + + +@triton.jit +def _copy_row(src, dst, s_src, row, idx, width, cols): + mask = cols < width + tl.store(dst + row * width + cols, tl.load(src + idx * s_src + cols, mask=mask), mask=mask) + + +@triton.jit( + do_not_specialize=[ + "w0", "w1", "w2", "w3", "w4", "w5", "s0", "s1", "s2", "s3", "s4", "s5", + ], + do_not_specialize_on_alignment=["x0", "x1", "x2", "x3", "x4", "x5"], +) # fmt: skip +def _rows_gather_kernel( + index, + x0, x1, x2, x3, x4, x5, + y0, y1, y2, y3, y4, y5, + w0, w1, w2, w3, w4, w5, + s0, s1, s2, s3, s4, s5, + COUNT: tl.constexpr, + BLOCK: tl.constexpr, +): # fmt: skip + row = tl.program_id(0).to(tl.int64) + which = tl.program_id(1) + cols = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + idx = tl.load(index + row).to(tl.int64) + if which == 0: + _copy_row(x0, y0, s0, row, idx, w0, cols) + if COUNT > 1: + if which == 1: + _copy_row(x1, y1, s1, row, idx, w1, cols) + if COUNT > 2: + if which == 2: + _copy_row(x2, y2, s2, row, idx, w2, cols) + if COUNT > 3: + if which == 3: + _copy_row(x3, y3, s3, row, idx, w3, cols) + if COUNT > 4: + if which == 4: + _copy_row(x4, y4, s4, row, idx, w4, cols) + if COUNT > 5: + if which == 5: + _copy_row(x5, y5, s5, row, idx, w5, cols) + + +def _select_spec(seqs: int, p: int) -> dict: + return dict(seqs=seqs, rows_per_request=p) + + +class DraftContinueSelect(OpWrapper): + """`rows, (g0, g1, g2, g3) = draft_continue_select(accepted, num_seqs, p, (v0, v1, v2, v3))`: + `rows[r] = r * p + clamp(accepted[r] - 1, 0, p - 1)` (int64, `[num_seqs]`) and + `g_k[r] = v_k[rows[r]]` (fresh, `v_k`'s dtype), for `r < num_seqs`. + + - `accepted`: the worker's accepted counts (`qsa_mtp_num_accepted`, int32, contiguous); + - `v_k`: 1-D contiguous row vectors of at least `num_seqs * p` entries (the QSA row metadata). + + Bitwise `_draft_continue_rows` followed by four `index_select`s.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep2 HT draft-extend step: 192 requests (the largest batch)", spec=_select_spec(192, 3)), + Cell(why="two programs: 300 requests, 4 rows each", spec=_select_spec(300, 4)), + ) # fmt: skip + + note = """ + Source: one Triton kernel (`_continue_select_kernel`), grid `ceil(num_seqs / 256)`; the + request count and rows a request are runtime arguments (one compile per dtype set). + """ + + def __call__(self, accepted, num_seqs: int, rows_per_request: int, vectors): + dev = accepted.device + rows = torch.empty([num_seqs], dtype=torch.int64, device=dev) + outs = tuple(torch.empty([num_seqs], dtype=v.dtype, device=dev) for v in vectors) + if num_seqs: + _continue_select_kernel[(triton.cdiv(num_seqs, SEQ_BLOCK),)]( + accepted, *vectors, rows, *outs, num_seqs, rows_per_request, BLOCK=SEQ_BLOCK, + num_warps=4, + ) # fmt: skip + return rows, outs + + def reference(self, accepted, num_seqs: int, rows_per_request: int, vectors): + last = torch.clamp(accepted[:num_seqs].to(torch.int64) - 1, min=0, max=rows_per_request - 1) + rows = torch.arange(num_seqs, dtype=torch.int64, device=accepted.device) * rows_per_request + rows = rows + last + return rows, tuple(torch.index_select(v, 0, rows) for v in vectors) + + def is_valid(self, accepted, num_seqs: int, rows_per_request: int, vectors) -> None: + assert len(vectors) == 4 and rows_per_request >= 1 + assert accepted.is_contiguous() and accepted.numel() >= num_seqs + for v in vectors: + assert v.dim() == 1 and v.is_contiguous() and v.numel() >= num_seqs * rows_per_request + + +def _gather_spec(rows: int, widths: tuple) -> dict: + return dict(rows=rows, widths=widths) + + +class DraftRowsGather(OpWrapper): + """`outs = draft_rows_gather(index, operands)`: `outs[k] = operands[k][index]` for up to six + 2-D operands, each with unit column stride (any row stride), as fresh contiguous tensors of + `[index.numel(), operands[k].shape[1]]`. + + Bitwise `index_select(operands[k], 0, index)`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell( + why="tep2 HT draft-extend step at 64 requests: index query, QKV, gate, residual, inject", + spec=_gather_spec(64, (512, 2048, 1536, 10240, 4)), + ), + Cell(why="one operand, 7 rows", spec=_gather_spec(7, (300,))), + ) + + note = """ + Source: one Triton kernel (`_rows_gather_kernel`), grid `(rows, operands, ceil(max width / + 1024))`; widths and row strides are runtime arguments, the operand count a constexpr. + """ + + def __call__(self, index, operands): + count = len(operands) + assert 1 <= count <= MAX_OPERANDS + rows = index.numel() + outs = [torch.empty([rows, t.shape[1]], dtype=t.dtype, device=t.device) for t in operands] + if rows: + pad = MAX_OPERANDS - count + xs = list(operands) + [operands[0]] * pad + ys = outs + [outs[0]] * pad + ws = [t.shape[1] for t in operands] + [0] * pad + ss = [t.stride(0) for t in operands] + [0] * pad + width = max(t.shape[1] for t in operands) + _rows_gather_kernel[(rows, count, triton.cdiv(width, COL_BLOCK))]( + index, *xs, *ys, *ws, *ss, COUNT=count, BLOCK=COL_BLOCK, num_warps=4 + ) + return tuple(outs) + + def reference(self, index, operands): + return tuple(torch.index_select(t, 0, index) for t in operands) + + def is_valid(self, index, operands) -> None: + assert 1 <= len(operands) <= MAX_OPERANDS and index.dim() == 1 + for t in operands: + assert t.dim() == 2 and (t.stride(1) == 1 or t.shape[1] == 1) + assert t.device == index.device + + +draft_continue_select = DraftContinueSelect() +draft_rows_gather = DraftRowsGather() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py new file mode 100644 index 000000000000..f57416bd72bd --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_fp8_pages_to_bf16__prefill.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A batch's pages of an e4m3 paged K/V pool, widened into a bf16 scratch pool, and the block +table that addresses them there: the context launches over a kv_fp8 pool read their requests' +pages through this scratch with the bf16 row-tile kernel. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Elements a program widens per loop step. +_CHUNK = 4096 + +#: A dep4 rank's K/V geometry and QSA table at max_seq_len 262144 (8196 columns); `slot_pages` +#: role pages a slot (24 for the 12 trunk layers, 26 with the MTP draft layer's planes last). +_CHECKPOINT = dict( + kv_heads=2, + head_dim=256, + tokens_per_block=32, + width=8196, + slots=32, + slot_pages=24, + layer=11, + holes=False, + straddle=False, +) + + +def _spec(kv_lens, **overrides) -> dict: + """`kv_lens` is one KV length per listed request, in order.""" + return {**_CHECKPOINT, "kv_lens": tuple(kv_lens), **overrides} + + +# Per-step and per-engine sizes are not specialized on (none feeds a vector access). +@triton.jit(do_not_specialize=["num_pool_pages", "listed", "bt_stride", "table_stride"]) +def _fp8_pages_to_bf16_kernel( + k_cache, + v_cache, + block_table, + slots, + counts, + offsets, + table, + k_out, + v_out, + num_pool_pages, + listed, + stride_page, + bt_stride, + table_stride, + PAGE_ELEMS: tl.constexpr, + CHUNK: tl.constexpr, +): + request = tl.program_id(0) + count = tl.load(counts + request) + base = tl.load(offsets + request).to(tl.int64) + slot = tl.load(slots + request).to(tl.int64) + for page in range(tl.program_id(1), count, tl.num_programs(1)): + dst = base + page + if dst < listed: + src = tl.load(block_table + slot * bt_stride + page).to(tl.int64) + resident = (src >= 0) & (src < num_pool_pages) + tl.store(table + slot * table_stride + page, tl.where(resident, dst, -1).to(tl.int32)) + safe = tl.where(resident, src, 0) + for start in tl.static_range(0, PAGE_ELEMS, CHUNK): + elems = start + tl.arange(0, CHUNK) + k = tl.load(k_cache + safe * stride_page + elems, mask=resident, other=0.0) + v = tl.load(v_cache + safe * stride_page + elems, mask=resident, other=0.0) + tl.store(k_out + dst * PAGE_ELEMS + elems, k.to(tl.bfloat16)) + tl.store(v_out + dst * PAGE_ELEMS + elems, v.to(tl.bfloat16)) + + +#: Programs per request; each strides the request's pages. +_PAGE_PROGRAMS = 1024 # capacity: fixed: a launch width; a request may list any page count + + +class QsaFp8PagesToBf16(OpWrapper): + """K and V of listed requests' pages of an e4m3 HND paged pool, as bf16 scratch pools, + and the block table that addresses them there. + + Request `r` lists its first `counts[r]` table pages, `block_table[slots[r], j]` for `j < + counts[r]`, at scratch pages `offsets[r] + j` (those below `listed`, the scratch's page + count):: + + k_out[offsets[r] + j] = bf16(k_cache[block_table[slots[r], j]]) + + and `v_out` likewise. Each e4m3 code widens exactly to bf16 (no scale: the pool holds + K/V at unit scale, as `qsa_paged_sparse_gqa` reads it), so a bf16 kernel over the scratch + reads the values the e4m3 kernels read. `table[slots[r], j]` becomes `offsets[r] + j`, + or -1 where the pool's entry is unmapped or out of the pool (whose scratch page is then + zeros); the call writes no other entry of `table`, which the caller fills with -1. + `offsets` is the exclusive prefix sum of `counts`. Returns `(k_out, v_out, table)`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="one request, one page", spec=_spec((17,))), + Cell( + why="dep4 deep chunk: one request at 198160 tokens (6193 pages) over a pool " + "straddling 2^31 elements of the trunk plane", + spec=_spec((198160,), straddle=True), + ), + Cell( + why="three requests, a chunked continuation and two short ones", + spec=_spec((40000 + 3192, 3000, 900)), + ), + Cell(why="holes: unmapped and out-of-pool entries", spec=_spec((4000, 2000), holes=True)), + Cell( + why="a tp4 / tep4 rank's single KV head, [128, 8196] table", + spec=_spec((9000, 2100), kv_heads=1, slots=128), + ), + Cell( + why="ht_dep4 (dep4 + MTP): the trunk's 26 role pages a slot, the draft layer's " + "planes (layer 12), one request at 196576 tokens straddling 2^31", + spec=_spec((196576,), slot_pages=26, layer=12, straddle=True), + ), + ) + + note = """ + Source: one Triton kernel (`_fp8_pages_to_bf16_kernel`), `(requests, min(width, 1024))` + programs, each striding its request's pages. Page ids are widened to int64 before the + stride product: the engine's trunk planes pass 2^31 elements. + + Preconditions: e4m3 role planes `[pages, Hkv, tpb, D]` with a page's heads, tokens and + dims contiguous (`stride(1) == tpb * D`, `stride(2) == D`, unit last stride; any page + stride), K's and V's at equal strides; `k_out` / `v_out` contiguous bf16 `[listed, Hkv, + tpb, D]`; `table` int32 of `block_table`'s shape. Bitwise deterministic (a copy). + """ + + def __call__( + self, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + slots: torch.Tensor, + counts: torch.Tensor, + offsets: torch.Tensor, + table: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + requests, listed = slots.shape[0], k_out.shape[0] + _, heads, tpb, dim = k_cache.shape + if k_cache.dtype != torch.float8_e4m3fn or v_cache.dtype != k_cache.dtype: + raise ValueError("the widening copy reads e4m3 K/V planes") + if k_cache.stride() != v_cache.stride() or k_cache.stride()[1:] != (tpb * dim, dim, 1): + raise ValueError("K/V planes must share strides with contiguous head pages") + if not (k_out.is_contiguous() and v_out.is_contiguous()): + raise ValueError("the bf16 scratch pools must be contiguous") + if k_out.dtype != torch.bfloat16 or k_out.shape[1:] != k_cache.shape[1:]: + raise ValueError("the scratch pools must be bf16 pages of the pool's geometry") + page_elems = heads * tpb * dim + if requests and listed: + _fp8_pages_to_bf16_kernel[(requests, min(block_table.shape[1], _PAGE_PROGRAMS))]( + k_cache, + v_cache, + block_table, + slots, + counts, + offsets, + table, + k_out, + v_out, + k_cache.shape[0], + listed, + k_cache.stride(0), + block_table.stride(0), + table.stride(0), + PAGE_ELEMS=page_elems, + CHUNK=min(_CHUNK, page_elems), + num_warps=8, + ) + return k_out, v_out, table + + def reference( + self, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + slots: torch.Tensor, + counts: torch.Tensor, + offsets: torch.Tensor, + table: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + k_ref, v_ref, table_ref = k_out.clone(), v_out.clone(), table.clone() + listed, pool_pages = k_out.shape[0], k_cache.shape[0] + for r in range(slots.shape[0]): + slot, base = int(slots[r]), int(offsets[r]) + count = max(0, min(int(counts[r]), block_table.shape[1], listed - base)) + if count == 0: + continue + src = block_table[slot, :count].long() + resident = (src >= 0) & (src < pool_pages) + dst = torch.arange(base, base + count, device=src.device) + table_ref[slot, :count] = torch.where(resident, dst, -1).to(table.dtype) + safe = torch.where(resident, src, 0) + keep = resident[:, None, None, None] + k_ref[base : base + count] = torch.where(keep, k_cache[safe].float(), 0.0).to( + k_out.dtype + ) + v_ref[base : base + count] = torch.where(keep, v_cache[safe].float(), 0.0).to( + v_out.dtype + ) + return k_ref, v_ref, table_ref + + +qsa_fp8_pages_to_bf16 = QsaFp8PagesToBf16() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py new file mode 100644 index 000000000000..a78b664f0edb --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_cute__prefill.py @@ -0,0 +1,1018 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index scores of a context chunk's rows on SM103 tensor cores (CuTe DSL, tcgen05). + +Bitwise equal to the family's `_qsa_paged_index_scores_qtile_kernel` (`only_visible_blocks`): +columns at and past a row's visible block count are unspecified. A persistent CTA takes +(64-row tile, `CHUNK_TILES` x 128-block chunk) items; per tile one tcgen05 MMA chain +`D[block, 4 r + h]` (M 128, N 256, K 128, bf16 in, fp32 in tensor memory, two accumulators). +Warps 9-10 gather q rows and keys with `cp.async` into a `STAGES`-slot SW128 ring, warp 8 issues +the MMAs, warps 0-7 store the scores. A tile spanning several requests runs once per request. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import cutlass.cute.nvgpu.tcgen05 as tcgen05 +import cutlass.pipeline as pipeline +import cutlass.utils.blackwell_helpers as sm100_utils +import torch +from cutlass._mlir.dialects import nvvm as _nvvm +from cutlass.cute.nvgpu import OperandMajorMode, cpasync +from cutlass.cute.runtime import from_dlpack, make_ptr +from cutlass.cute.typing import Float32, Int32, Int64 +from cutlass.cutlass_dsl import dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + qsa_paged_index_scores as _family_index_scores, +) + +try: + from cutlass.memory import SmemAllocator +except ImportError: # DSL releases before the memory namespace split + from cutlass.utils import SmemAllocator + +BF16 = cutlass.BFloat16 +HEADS = 4 +DIM = 128 +#: Query rows a tile (the MMA's N = rows x heads). +QROWS = 64 +BN = QROWS * HEADS +#: Compressed blocks a tile (the MMA's M). +BM = 128 +#: Key ring slots (32 KB each); with the 64 KB query operand, ~192 KB of shared memory. +STAGES = 4 +CHUNK_TILES = 8 +#: Epilogue warps: two per tensor-memory lane quadrant, each taking half the accumulator. +EPI_WARPS = 8 +MMA_WARP = 8 +LOAD_WARP0 = 9 +LOAD_WARPS = 2 +#: Accumulator columns an epilogue thread loads at a time (8 rows x 4 heads). +EPI_COLS = 32 +#: Items a CTA's metadata table holds (more: each role derives an item's metadata itself). +MAX_ITEMS = 64 +THREADS = 32 * (EPI_WARPS + 1 + LOAD_WARPS) +LOAD_THREADS = 32 * LOAD_WARPS +#: Keys a page holds: one load thread's two token rows (`__init__`'s assert). +KEYS_PER_PAGE = 2 * (LOAD_THREADS // 16) + + +@dsl_user_op +def _fence_before(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.BEFORE_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def _fence_after(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.AFTER_THREAD_SYNC, loc=loc, ip=ip) + + +class _IndexScoresKernel: + def __init__(self, *, tokens_per_block, compress_ratio): + assert tokens_per_block % compress_ratio == 0 + self.tpb = tokens_per_block + self.cr = compress_ratio + self.kpp = tokens_per_block // compress_ratio # keys a page + assert BM % self.kpp == 0 and BM // self.kpp <= 32 + assert self.kpp == KEYS_PER_PAGE, "a thread's keys: two token rows of each page" + self.pages_per_tile = BM // self.kpp + + @cute.jit + def __call__( + self, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + pos: cute.Pointer, + req: cute.Pointer, + out_t: cute.Tensor, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + out_stride: Int32, + scale: Float32, + num_ctas: Int32, + stream, + ): + # The output arrives as a tensor, as the TVM-FFI entry takes its stream from a GPU + # tensor; the kernel addresses it through its pointer and row stride. + out = out_t.iterator + + def make_mma(): + return sm100_utils.make_trivial_tiled_mma( + BF16, + BF16, + OperandMajorMode.K, + OperandMajorMode.K, + Float32, + tcgen05.CtaGroup.ONE, + (BM, BN), + tcgen05.OperandSource.SMEM, + ) + + mma, mma_acc = make_mma(), make_mma() + a_layout = sm100_utils.make_smem_layout_a(mma, (BM, BN, DIM), BF16, STAGES) + b_layout = sm100_utils.make_smem_layout_b(mma, (BM, BN, DIM), BF16, 1) + + @cute.struct + class SharedStorage: + k_full: cute.struct.MemRange[Int64, STAGES] + k_empty: cute.struct.MemRange[Int64, STAGES] + q_full: cute.struct.MemRange[Int64, 1] + q_empty: cute.struct.MemRange[Int64, 1] + acc_full: cute.struct.MemRange[Int64, 2] + acc_empty: cute.struct.MemRange[Int64, 2] + tmem_holding: Int32 + vis: cute.struct.MemRange[Int32, QROWS] + meta_starts: cute.struct.MemRange[Int64, MAX_ITEMS] + meta_tot: cute.struct.MemRange[Int32, MAX_ITEMS] + meta_hi: cute.struct.MemRange[Int32, MAX_ITEMS] + meta_req: cute.struct.MemRange[Int32, MAX_ITEMS] + sA: cute.struct.Align[cute.struct.MemRange[BF16, cute.cosize(a_layout)], 1024] + sB: cute.struct.Align[cute.struct.MemRange[BF16, cute.cosize(b_layout)], 1024] + + self.shared_storage = SharedStorage + self.kernel( + mma, + mma_acc, + a_layout, + b_layout, + q, + cache, + table, + pos, + req, + out, + rows, + width, + q_stride_row, + cache_stride_page, + cache_stride_token, + table_stride, + out_stride, + scale, + ).launch( + grid=[num_ctas, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + min_blocks_per_mp=1, + ) + + @cute.jit + def item_geometry(self, item: Int32, nrt: Int32, rows: Int32, width: Int32): + """Items chunk-major, so a wave's CTAs share a chunk's keys in L2.""" + ch = item // nrt + rt = item - ch * nrt + row0 = rt * QROWS + rows_in = cutlass.min(rows - row0, Int32(QROWS)) + c0 = ch * (BM * CHUNK_TILES) + c1 = cutlass.min(c0 + BM * CHUNK_TILES, width) + return row0, rows_in, c0, c1 + + @cute.jit + def item_meta(self, meta, li: Int32, nmine: Int32, mReq, mPos, row0, rows_in, c0, c1, lane): + """(run starts, tiles, first run's window bound, first run's request) of the CTA's + item `li`: from the table, or derived here past it.""" + mS = cute.make_tensor(meta[0], cute.make_layout((MAX_ITEMS,))) + mT = cute.make_tensor(meta[1], cute.make_layout((MAX_ITEMS,))) + mH = cute.make_tensor(meta[2], cute.make_layout((MAX_ITEMS,))) + mR = cute.make_tensor(meta[3], cute.make_layout((MAX_ITEMS,))) + starts = Int64(0) + total = Int32(0) + hi1 = Int32(0) + req1 = Int32(0) + if nmine <= MAX_ITEMS: + starts = mS[li] + total = mT[li] + hi1 = mH[li] + req1 = mR[li] + else: + starts = self.group_starts(mReq, row0, rows_in, lane) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + if s == 0: + hi1 = cutlass.min(vis, c1) + total = total + nt + s = e + req1 = Int32(mReq[row0]) + return starts, total, hi1, req1 + + @cute.jit + def group_starts(self, mReq, row0: Int32, rows_in: Int32, lane: Int32): + """Bit j set where tile row j starts a run of one request (warp-uniform).""" + starts = Int64(0) + for half in cutlass.range_constexpr(2): + j = lane + 32 * half + flag = cutlass.Boolean(False) + if j < rows_in: + if j == 0: + flag = True + else: + flag = Int64(mReq[row0 + j]) != Int64(mReq[row0 + j - 1]) + bal = cute.arch.vote_ballot_sync(flag) + starts = starts | ((Int64(bal) & Int64(0xFFFFFFFF)) << Int64(32 * half)) + return starts + + @cute.jit + def group_end(self, starts: Int64, s: Int32, rows_in: Int32): + """The first row after `s` that starts another run, or `rows_in`.""" + rest = starts & ~((Int64(2) << Int64(s)) - Int64(1)) + e = rows_in + if rest != Int64(0): + low = rest & (Int64(0) - rest) + m = low - Int64(1) + e = Int32(cute.arch.popc(Int32(m & Int64(0xFFFFFFFF)))) + Int32( + cute.arch.popc(Int32(m >> Int64(32))) + ) + return e + + @cute.jit + def group_tiles(self, mPos, row0: Int32, e: Int32, c0: Int32, c1: Int32): + """Tiles a run ending before row `e` walks in the chunk, and its widest window.""" + vis = (Int32(mPos[row0 + e - 1]) + 1) // self.cr + hi = cutlass.min(vis, c1) + ntiles = Int32(0) + if hi > c0: + ntiles = (hi - c0 + (BM - 1)) // BM + return ntiles, vis + + # ------------------------------------------------------------------ kernel + @cute.kernel + def kernel( + self, + mma: cute.TiledMma, + mma_acc: cute.TiledMma, + a_layout: cute.ComposedLayout, + b_layout: cute.ComposedLayout, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + pos: cute.Pointer, + req: cute.Pointer, + out: cute.Pointer, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + out_stride: Int32, + scale: Float32, + ): + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + tidx, _, _ = cute.arch.thread_idx() + lane = cute.arch.lane_idx() + bid, _, _ = cute.arch.block_idx() + nctas, _, _ = cute.arch.grid_dim() + mma.set(tcgen05.Field.ACCUMULATE, False) + mma_acc.set(tcgen05.Field.ACCUMULATE, True) + + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + k_full = storage.k_full.data_ptr() + k_empty = storage.k_empty.data_ptr() + q_full = storage.q_full.data_ptr() + q_empty = storage.q_empty.data_ptr() + acc_full = storage.acc_full.data_ptr() + acc_empty = storage.acc_empty.data_ptr() + if tidx == 0: + for i in cutlass.range_constexpr(STAGES): + cute.arch.mbarrier_init(k_full + i, LOAD_THREADS) + cute.arch.mbarrier_init(k_empty + i, 1) + cute.arch.mbarrier_init(q_full, LOAD_THREADS) + cute.arch.mbarrier_init(q_empty, 1) + for i in cutlass.range_constexpr(2): + cute.arch.mbarrier_init(acc_full + i, 1) + cute.arch.mbarrier_init(acc_empty + i, 32 * EPI_WARPS) + cute.arch.mbarrier_init_fence() + + tmem_users = pipeline.NamedBarrier(barrier_id=1, num_threads=32 * (EPI_WARPS + 1)) + if warp_idx == MMA_WARP: + cute.arch.alloc_tmem(512, storage.tmem_holding.ptr) + cute.arch.relinquish_tmem_alloc_permit() + cute.arch.sync_threads() + + thr = mma.get_slice(0) + sA = storage.sA.get_tensor(a_layout.outer, swizzle=a_layout.inner) + sB = storage.sB.get_tensor(b_layout.outer, swizzle=b_layout.inner) + per_row = cute.make_layout((1 << 30,)) + mReq = cute.make_tensor(req, per_row) + mPos = cute.make_tensor(pos, per_row) + # The widest visible window of the call bounds the chunks (the table is max_seq_len wide). + sMax = cute.make_tensor(storage.vis.data_ptr(), cute.make_layout((QROWS,))) + vmax = Int32(0) + for i in cutlass.range(tidx, rows, THREADS, unroll=4): + vmax = cutlass.max(vmax, (Int32(mPos[i]) + 1) // self.cr) + vmax = cute.arch.warp_redux_sync(vmax, "max") + if lane == 0: + sMax[warp_idx] = vmax + cute.arch.sync_threads() + vmax = Int32(0) + if lane < THREADS // 32: + vmax = sMax[lane] + vmax = cute.arch.warp_redux_sync(vmax, "max") + table_width = width + width = cutlass.min(width, vmax) + cute.arch.sync_threads() + nrt = (rows + (QROWS - 1)) // QROWS + nch = (width + (BM * CHUNK_TILES - 1)) // (BM * CHUNK_TILES) + items = nrt * nch + # The CTA's items' metadata (see `item_meta`), once, a warp an item. + meta = ( + storage.meta_starts.data_ptr(), + storage.meta_tot.data_ptr(), + storage.meta_hi.data_ptr(), + storage.meta_req.data_ptr(), + ) + mS = cute.make_tensor(meta[0], cute.make_layout((MAX_ITEMS,))) + mT = cute.make_tensor(meta[1], cute.make_layout((MAX_ITEMS,))) + mH = cute.make_tensor(meta[2], cute.make_layout((MAX_ITEMS,))) + mR = cute.make_tensor(meta[3], cute.make_layout((MAX_ITEMS,))) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + if nmine <= MAX_ITEMS: + for li in cutlass.range(warp_idx, nmine, THREADS // 32, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts = self.group_starts(mReq, row0, rows_in, lane) + total = Int32(0) + hi1 = Int32(0) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + if s == 0: + hi1 = cutlass.min(vis, c1) + total = total + nt + s = e + if lane == 0: + mS[li] = starts + mT[li] = total + mH[li] = hi1 + mR[li] = Int32(mReq[row0]) + cute.arch.sync_threads() + + if warp_idx >= LOAD_WARP0: + self.load_role( + tidx - 32 * LOAD_WARP0, + lane, + bid, + nctas, + items, + nrt, + meta, + sA, + storage.sA.data_ptr(), + sB, + q, + cache, + table, + mReq, + mPos, + rows, + width, + q_stride_row, + cache_stride_page, + cache_stride_token, + table_stride, + k_full, + k_empty, + q_full, + q_empty, + ) + elif warp_idx == MMA_WARP: + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + self.mma_role( + mma, + mma_acc, + thr, + sA, + sB, + tmem_ptr, + lane, + bid, + nctas, + items, + nrt, + meta, + mReq, + mPos, + rows, + width, + k_full, + k_empty, + q_full, + q_empty, + acc_full, + acc_empty, + ) + tmem_users.arrive_and_wait() + cute.arch.dealloc_tmem(tmem_ptr, 512) + else: + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + self.epilogue_role( + thr, + tmem_ptr, + tidx, + lane, + bid, + nctas, + items, + nrt, + meta, + mReq, + mPos, + storage.vis.data_ptr(), + out, + rows, + width, + table_width, + out_stride, + scale, + acc_full, + acc_empty, + ) + tmem_users.arrive_and_wait() + + # ------------------------------------------------------------------ loaders (warps 9-10) + @cute.jit + def load_role( + self, + lt: Int32, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + sA, + sA_raw: cute.Pointer, + sB, + q: cute.Pointer, + cache: cute.Pointer, + table: cute.Pointer, + mReq, + mPos, + rows: Int32, + width: Int32, + q_stride_row: Int32, + cache_stride_page: Int32, + cache_stride_token: Int32, + table_stride: Int32, + k_full, + k_empty, + q_full, + q_empty, + ): + atom = cute.make_copy_atom( + cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL), BF16, num_bits_per_copy=128 + ) + # The query operand's element-coordinate view: (8 elements, (chunk, 64-dim block), row). + sB_c = cute.make_tensor( + sB.iterator, + cute.make_layout((8, (8, DIM // 64), BN), stride=(1, (8, BN * 64), 64)), + ) + chunk = lt % 16 + kq = lt // 16 + kstep: cutlass.Constexpr = LOAD_THREADS // 16 + toff = [] + dbase = [] + for v in cutlass.range_constexpr(self.kpp // kstep): + toff.append(((kq + kstep * v) * self.cr + self.cr - 1) * cache_stride_token + chunk * 8) + # SW128 K-major: 16-byte chunk c of row m at chunk c ^ (m % 8) of its 128-byte row, + # each 64-dim half a BM x 128-byte block. + kr = kq + kstep * v + dbase.append((chunk // 8) * (BM * 64) + kr * 64 + ((chunk % 8) ^ kr) * 8) + ring = Int32(0) + ring_phase = Int32(1) + q_phase = Int32(1) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + # The item's query rows: B row n = 4 r + h, 16 chunks of 16 bytes a row. + cute.arch.mbarrier_wait(q_empty, q_phase) + q_phase = q_phase ^ 1 + for it in cutlass.range_constexpr(BN * 16 // LOAD_THREADS): + x = lt + LOAD_THREADS * it + n = x // 16 + c = x % 16 + r = n // HEADS + h = n % HEADS + if r < rows_in: + src = cute.make_tensor( + q + + cute.assume( + Int64(row0 + r) * q_stride_row + h * DIM + c * 8, divby=8 + ), + cute.make_layout((8,)), + ) + cute.copy(atom, src, sB_c[(None, (c % 8, c // 8), n)]) + cute.arch.cp_async_mbarrier_arrive_noinc(q_full) + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt = total + hi = hi1 + r_id = Int64(req1) + if starts != Int64(1): + # Several runs: each its own tiles, window and request. + nt, vis = self.group_tiles(mPos, row0, e, c0, c1) + hi = cutlass.min(vis, c1) + r_id = Int64(mReq[row0 + s]) + mTable = cute.make_tensor(table, cute.make_layout((1 << 30,))) + # Lane p holds the tile's page p; the next tile's are read during its copies. + page_next = self.tile_page(mTable, r_id, table_stride, c0, hi, lane) + for j in cutlass.range(0, nt, 1, unroll=1): + b0 = c0 + j * BM + page = page_next + if j + 1 < nt: + page_next = self.tile_page( + mTable, r_id, table_stride, b0 + BM, hi, lane + ) + cute.arch.mbarrier_wait(k_empty + ring, ring_phase) + full = b0 + BM <= hi + # Key m = kq + kstep * it: page it // 2 (a broadcast shuffle), token row + # kq + kstep * (it % 2); only a partial tile tests the bound per key. + sbase = ring * (BM * DIM) + if full: + self.copy_keys( + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + False, + ) + else: + self.copy_keys( + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + True, + ) + cute.arch.cp_async_mbarrier_arrive_noinc(k_full + ring) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + s = e + # Tail: every slot released, so no load is in flight when the CTA retires. + for i in cutlass.range_constexpr(STAGES): + cute.arch.mbarrier_wait(k_empty + ring, ring_phase) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + + @cute.jit + def copy_keys( + self, + atom, + cache, + sA_raw, + page, + cache_stride_page, + toff, + dbase, + sbase, + kq, + b0, + hi, + bounded: cutlass.Constexpr, + ): + kstep: cutlass.Constexpr = LOAD_THREADS // 16 + for pp in cutlass.range_constexpr(BM // self.kpp): + pg = cute.arch.shuffle_sync(page, offset=pp) + pbase = Int64(pg) * cache_stride_page + for par in cutlass.range_constexpr(2): + it = 2 * pp + par + src = cute.make_tensor( + cache + cute.assume(pbase + toff[par], divby=8), cute.make_layout((8,)) + ) + dst = cute.make_tensor( + sA_raw + cute.assume(dbase[par] + sbase + pp * (self.kpp * 64), divby=8), + cute.make_layout((8,)), + ) + if cutlass.const_expr(bounded): + if b0 + kq + kstep * it < hi: + cute.copy(atom, src, dst) + else: + cute.copy(atom, src, dst) + + @cute.jit + def tile_page(self, mTable, r_id, table_stride: Int32, b0: Int32, hi: Int32, lane: Int32): + lp0 = (b0 * self.cr + self.cr - 1) // self.tpb + page = Int32(0) + if lane < self.pages_per_tile: + if b0 + lane * self.kpp < hi: + page = mTable[r_id * table_stride + lp0 + lane] + return page + + # ------------------------------------------------------------------ MMA (warp 8) + @cute.jit + def mma_role( + self, + mma: cute.TiledMma, + mma_acc: cute.TiledMma, + thr, + sA, + sB, + tmem_ptr, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + mReq, + mPos, + rows: Int32, + width: Int32, + k_full, + k_empty, + q_full, + q_empty, + acc_full, + acc_empty, + ): + tCrA = thr.make_fragment_A(sA) + tCrB = thr.make_fragment_B(sB) + tAcc_f = thr.make_fragment_C(thr.partition_shape_C((BM, BN))) + ring = Int32(0) + ring_phase = Int32(0) + q_phase = Int32(0) + t = Int32(0) # tiles so far: accumulator t % 2, its phase (t // 2) % 2 + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + cute.arch.mbarrier_wait(q_full, q_phase) + q_phase = q_phase ^ 1 + for j in cutlass.range(0, total, 1, unroll=1): + ab = t % 2 + cute.arch.mbarrier_wait(k_full + ring, ring_phase) + cute.arch.mbarrier_wait(acc_empty + ab, ((t // 2) % 2) ^ 1) + _fence_after() + tAcc = cute.make_tensor(tmem_ptr + ab * BN, tAcc_f.layout) + for kb in cutlass.range_constexpr(cute.size(tCrA, mode=[2])): + if cutlass.const_expr(kb == 0): + cute.gemm( + mma, tAcc, tCrA[None, None, kb, ring], tCrB[None, None, kb, 0], tAcc + ) + else: + cute.gemm( + mma_acc, + tAcc, + tCrA[None, None, kb, ring], + tCrB[None, None, kb, 0], + tAcc, + ) + with cute.arch.elect_one(): + tcgen05.commit(k_empty + ring) + tcgen05.commit(acc_full + ab) + ring = ring + 1 + if ring == STAGES: + ring = Int32(0) + ring_phase = ring_phase ^ 1 + t = t + 1 + with cute.arch.elect_one(): + tcgen05.commit(q_empty) + + # ------------------------------------------------------------------ epilogue (warps 0-7) + @cute.jit + def head_sum(self, v0, v1, v2, v3): + """The family's head reduction order, bit for bit.""" + z = Float32(0.0) + r0 = cute.arch.fmax(v0, z) + r1 = cute.arch.fmax(v1, z) + r2 = cute.arch.fmax(v2, z) + r3 = cute.arch.fmax(v3, z) + return (r0 + r2) + (r1 + r3) + + @cute.jit + def epilogue_role( + self, + thr, + tmem_ptr, + tidx: Int32, + lane: Int32, + bid: Int32, + nctas: Int32, + items: Int32, + nrt: Int32, + meta, + mReq, + mPos, + vis_ptr: cute.Pointer, + out: cute.Pointer, + rows: Int32, + width: Int32, + table_width: Int32, + out_stride: Int32, + scale: Float32, + acc_full, + acc_empty, + ): + epi_bar = pipeline.NamedBarrier(barrier_id=2, num_threads=32 * EPI_WARPS) + sVis = cute.make_tensor(vis_ptr, cute.make_layout((QROWS,))) + tAcc_f = thr.make_fragment_C(thr.partition_shape_C((BM, BN))) + cols = EPI_COLS + rpc = cols // HEADS # rows a load + half = tidx // 128 # the accumulator half (rows 0-31 / 32-63) this warp reads + tAcc0 = cute.make_tensor(tmem_ptr, tAcc_f.layout) + tAcc_c = cute.make_tensor( + tAcc0.iterator, cute.composition(tAcc0.layout, cute.make_layout((BM, cols))) + ) + cAcc = cute.make_identity_tensor((BM, BN)) + tCcAcc = thr.partition_C(cAcc) + tCcAcc_c = cute.make_tensor( + tCcAcc.iterator, cute.composition(tCcAcc.layout, cute.make_layout((BM, cols))) + ) + load = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(cols)), Float32), + tAcc_c, + ) + thr_load = load.get_slice(tidx % 128) + tL = thr_load.partition_S(tAcc_c) + tLc = thr_load.partition_D(tCcAcc_c) + mOut = cute.make_tensor(out, cute.make_layout((1 << 30,))) + m = tLc[0][0] + t = Int32(0) + nmine = Int32(0) + if bid < items: + nmine = (items - bid + nctas - 1) // nctas + for li in cutlass.range(0, nmine, 1, unroll=1): + row0, rows_in, c0, c1 = self.item_geometry(bid + li * nctas, nrt, rows, width) + starts, total, hi1, req1 = self.item_meta( + meta, li, nmine, mReq, mPos, row0, rows_in, c0, c1, lane + ) + if total > 0: + # One request's 64 rows, every tile inside the table: stores unmasked. + whole = (rows_in == QROWS) & (starts == Int64(1)) & (c0 + total * BM <= table_width) + if not whole: + # The rows' visible block counts (bounded by the table width). + epi_bar.arrive_and_wait() + if tidx < QROWS: + v = Int32(0) + if tidx < rows_in: + v = cutlass.min((Int32(mPos[row0 + tidx]) + 1) // self.cr, table_width) + sVis[tidx] = v + epi_bar.arrive_and_wait() + s = Int32(0) + while s < rows_in: + e = self.group_end(starts, s, rows_in) + nt = total + if starts != Int64(1): + nt, _ = self.group_tiles(mPos, row0, e, c0, c1) + for j in cutlass.range(0, nt, 1, unroll=1): + ab = t % 2 + b = c0 + j * BM + m + base = Int64(row0) * out_stride + b + cute.arch.mbarrier_wait(acc_full + ab, (t // 2) % 2) + _fence_after() + for cc in cutlass.range_constexpr(BN // 2 // cols): + rv = cute.make_rmem_tensor(tLc.shape, Float32) + col0 = half * (BN // 2) + cc * cols + cute.copy( + load, + cute.make_tensor(tL.iterator + ab * BN + col0, tL.layout), + rv, + ) + cute.arch.fence_view_async_tmem_load() + if cutlass.const_expr(cc == BN // 2 // cols - 1): + _fence_before() + cute.arch.mbarrier_arrive(acc_empty + ab) + r0 = half * (QROWS // 2) + cc * rpc + for rr in cutlass.range_constexpr(rpc): + val = ( + self.head_sum( + rv[HEADS * rr], + rv[HEADS * rr + 1], + rv[HEADS * rr + 2], + rv[HEADS * rr + 3], + ) + * scale + ) + r = r0 + rr + if whole: + mOut[base + Int64(r) * out_stride] = val + else: + if r >= s: + if r < e: + if b < sVis[r]: + mOut[base + Int64(r) * out_stride] = val + t = t + 1 + s = e + + +_COMPILED: dict = {} +_SMS: dict = {} +_INDEX = {torch.int32: Int32, torch.int64: Int64} + + +def _compiled(tpb, cr, pos_dtype, req_dtype, out): + key = (tpb, cr, pos_dtype, req_dtype) + fn = _COMPILED.get(key) + if fn is None: + kernel = _IndexScoresKernel(tokens_per_block=tpb, compress_ratio=cr) + gmem = cute.AddressSpace.gmem + + def fake(dtype, align): + return make_ptr(dtype, 16 * 1024, gmem, assumed_align=align) + + fn = _COMPILED[key] = cute.compile( + kernel, + fake(BF16, 16), + fake(BF16, 16), + fake(Int32, 4), + fake(_INDEX[pos_dtype], 4), + fake(_INDEX[req_dtype], 4), + from_dlpack(out, assumed_align=16).mark_layout_dynamic(leading_dim=1), + 1, + 1, + 8, + 8, + 8, + 8, + 8, + 1.0, + 1, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + return fn + + +def supports(q, index_cache, tokens_per_block, compress_ratio) -> bool: + """A context-row call this kernel serves: bf16 4 x 128 index heads (unit dim stride, heads + 128 apart, 16-byte rows), a `[pages, tpb, 1, 128]` bf16 pool with 256-byte token rows.""" + return ( + q.dtype == torch.bfloat16 + and index_cache.dtype == torch.bfloat16 + and q.ndim == 3 + and q.shape[1] == HEADS + and q.shape[2] == DIM + and q.stride(2) == 1 + and q.stride(1) == DIM + and q.stride(0) % 8 == 0 + and q.data_ptr() % 16 == 0 + and index_cache.ndim == 4 + and index_cache.shape[2] == 1 + and index_cache.shape[3] == DIM + and index_cache.stride(3) == 1 + and index_cache.stride(1) % 8 == 0 + and index_cache.stride(0) % 8 == 0 + and index_cache.data_ptr() % 16 == 0 + and tokens_per_block % compress_ratio == 0 + and tokens_per_block // compress_ratio == KEYS_PER_PAGE + and torch.cuda.get_device_capability(q.device) == (10, 3) + ) + + +def _qsa_index_scores_prefill( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, +): + """The family's context-row index scores (`only_visible_blocks`): `[rows, max_blocks]` fp32, + columns at and past each row's visible count unspecified.""" + rows = q.shape[0] + max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + out = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + if rows == 0: + return out + fn = _compiled( + tokens_per_block, + compress_ratio, + query_positions.dtype, + request_indices.dtype, + out, + ) + dev = q.device.index if q.device.index is not None else 0 + sms = _SMS.get(dev) + if sms is None: + sms = _SMS[dev] = torch.cuda.get_device_properties(dev).multi_processor_count + nch = -(-max_blocks // (BM * CHUNK_TILES)) + items = -(-rows // QROWS) * nch + fn( + q.data_ptr(), + index_cache.data_ptr(), + block_table.data_ptr(), + query_positions.data_ptr(), + request_indices.data_ptr(), + out, + rows, + max_blocks, + q.stride(0), + index_cache.stride(0), + index_cache.stride(1), + block_table.stride(0), + out.stride(0), + float(q.shape[2] ** -0.5), + max(1, min(items, sms)), + ) + return out + + +class QsaIndexScoresPrefill(OpWrapper): + """The family's `qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=True)` + on context rows `supports` admits: `[rows, max_blocks]` fp32, the same bits over each row's + visible columns, the columns at and past them unspecified.""" + + def __call__( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> torch.Tensor: + return _qsa_index_scores_prefill( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + ) + + def reference( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> torch.Tensor: + """The family entry's reference over context rows: NaN past each row's visible columns.""" + return _family_index_scores.reference( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + only_visible_blocks=True, + context_rows=True, + ) + + def is_valid( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + ) -> None: + # The kernel is built for this geometry and checks none of it. + assert supports(q, index_cache, tokens_per_block, compress_ratio), ( + "a call outside `supports`" + ) + + +qsa_index_scores_prefill = QsaIndexScoresPrefill() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py new file mode 100644 index 000000000000..fd76c8732f6f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_scores_groups_cute__decode.py @@ -0,0 +1,512 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index scores of a verify step's request groups: a CuTe DSL kernel with a cp.async ring. + +A CTA (4 warps) walks `TILES` tiles (`BN` keys each) of one request group with `STAGES` tiles in +flight: the program's page numbers are looked up once into shared memory, each tile's keys go to +an XOR-swizzled shared ring with `cp.async` (four threads a key), and the scores come from +`mma.sync` m16n8k16 with the group's query columns as the A fragments (rows 4 t + h: verify row t, +index head h) and the keys as B. A key's score per row is the family's +(`qsa_paged_index_scores` with `rows_per_request`): ReLU of each head's dot, summed +(h0 + h1) + (h2 + h3), times 128^-0.5. The dot's accumulation order differs from the family +kernel's, so the scores differ in the last bits. + +Precondition (`serves`): verify rows, `rows_per_request` (2 or 3) consecutive rows of one request +each, 4 bf16 index heads of 128; the index pool a `[pages, tokens, 1, 128]` view with token rows +256 B apart. Columns at and past a row's window are left unwritten. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_index_scores import ( + qsa_paged_index_scores as _family_index_scores, +) + +from .qsa_sparse_gqa_splitk_cute__decode import ( + _asm, + _cp16z, + _f32_bits, + _ir, + _ldg1, + _ldg4, + _lds1, + _lds4, + _mma_bf16, + _ptr, + _stg1, + _sts1, +) + +HEADS = 4 +HEAD_DIM = 128 +#: Keys a tile: 64 keeps a 2-stage ring small enough for several CTAs an SM. +BN = 64 +WARPS = 4 +THREADS = 32 * WARPS +#: Bytes of a key row in the ring: unpadded, with 16-byte chunk `ch` of key `k` stored at chunk +#: `ch ^ 4 (k & 1)`, so the two keys a quarter-warp's 16-byte loads and cp.async writes touch fall +#: in disjoint bank halves without row padding. +K_LD = 256 +#: Tiles a CTA walks, and tiles in flight (the fastest of the configurations measured on GB300). +TILES = 8 +STAGES = 2 + + +@dsl_user_op +def _stg2f(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(_f32_bits(a), cutlass.Int32, loc, ip), + _ir(_f32_bits(b), cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _shfl_xor_f(x, m, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + f"shfl.sync.bfly.b32 $0, $1, {m}, 31, 0xffffffff;", + "=f,f", + loc, + ip, + side=True, + ) + ) + + +class _IdxGroupsKernel: + def __init__(self, *, rpr, tpb, cr, tiles, stages, bt_width, num_requests, pdl, bn=BN): + assert 2 <= rpr <= 3 and tpb % cr == 0 and bn in (64, 128) + self.T, self.tpb, self.cr = rpr, tpb, cr + self.bn = bn + self.tiles, self.stages = tiles, max(2, min(stages, tiles)) + self.bt_width, self.num_requests, self.pdl = bt_width, num_requests, pdl + self.k_ld = K_LD + self.o_pg = 0 + self.pg_bytes = tiles * bn * 4 + self.o_ring = -(-self.pg_bytes // 128) * 128 + self.smem = self.o_ring + self.stages * bn * self.k_ld + 128 + + @cute.jit + def __call__( + self, + q_ptr: cute.Pointer, + cache_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_t: cute.Tensor, + rows: cutlass.Int32, + q_stride_row: cutlass.Int32, + page_stride: cutlass.Int64, + out_stride: cutlass.Int32, + max_blocks: cutlass.Int32, + groups: cutlass.Int32, + col_progs: cutlass.Int32, + stream, + ): + self.kernel( + q_ptr.toint(), + cache_ptr.toint(), + bt_ptr.toint(), + pos_ptr.toint(), + req_ptr.toint(), + # the output arrives as a tensor: the TVM-FFI entry takes its stream from it + out_t.iterator.toint(), + rows, + q_stride_row, + page_stride, + out_stride, + max_blocks, + col_progs, + ).launch( + grid=[groups * col_progs, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + ) + + @cute.jit + def _issue(self, j, buf, tidx, s_pg, s_ring, cache_base, page_stride): + """cp.async of tile `j`'s `bn` keys into ring slot `buf`, one commit group: thread t serves + chunks q4, q4 + 4, q4 + 8, q4 + 12 of key t // 4 + 32 i (64 contiguous bytes for the four + threads of a key); a key past the window (page -1) gets zeros.""" + tpb: cutlass.Constexpr = self.tpb + cr: cutlass.Constexpr = self.cr + bn: cutlass.Constexpr = self.bn + k_ld: cutlass.Constexpr = self.k_ld + q4 = tidx % 4 + dst0 = s_ring + buf * (bn * k_ld) + for i in cutlass.range_constexpr(bn // (THREADS // 4)): + k = tidx // 4 + (THREADS // 4) * i + page = _lds1(s_pg + (j * bn + k) * 4) + ok = page >= 0 + n = cutlass.Int32(cutlass.select_(ok, cutlass.Int32(16), cutlass.Int32(0))) + tok = (k * cr + cr - 1) % tpb + src = ( + cache_base + + ( + cutlass.Int64(cutlass.max(page, 0)) * page_stride + + cutlass.Int64(tok * HEAD_DIM) + ) + * 2 + ) + for cc in cutlass.range_constexpr(4): + ch = q4 + 4 * cc + _cp16z(dst0 + k * k_ld + 16 * (ch ^ ((k & 1) << 2)), src + 16 * ch, n) + cute.arch.cp_async_commit_group() + + @cute.kernel + def kernel( + self, + q_base: cutlass.Int64, + cache_base: cutlass.Int64, + bt_base: cutlass.Int64, + pos_base: cutlass.Int64, + req_base: cutlass.Int64, + out_base: cutlass.Int64, + rows: cutlass.Int32, + q_stride_row: cutlass.Int32, + page_stride: cutlass.Int64, + out_stride: cutlass.Int32, + max_blocks: cutlass.Int32, + col_progs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + g = lane // 4 + c = lane % 4 + rpr: cutlass.Constexpr = self.T + tpb: cutlass.Constexpr = self.tpb + cr: cutlass.Constexpr = self.cr + tiles: cutlass.Constexpr = self.tiles + stages: cutlass.Constexpr = self.stages + bn: cutlass.Constexpr = self.bn + ntw: cutlass.Constexpr = self.bn // (8 * WARPS) # n8 tiles a warp + group = bidx // col_progs + prog = bidx % col_progs + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_pg = sbase + self.o_pg + s_ring = sbase + self.o_ring + + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + # ---- the group's rows: request, windows ------------------------------------------- + r0 = group * rpr + lead = cutlass.Int64(_ldg1(req_base + cutlass.Int64(cutlass.min(r0, rows - 1)) * 4)) + safe_req = cutlass.min( + cutlass.max(lead, cutlass.Int64(0)), cutlass.Int64(self.num_requests - 1) + ) + vis = [] + max_vis = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + r = r0 + t + p64 = cutlass.Int64(_ldg1(pos_base + cutlass.Int64(cutlass.min(r, rows - 1)) * 8)) + v = cutlass.Int32( + cutlass.select_(r < rows, cutlass.Int32((p64 + 1) // cr), cutlass.Int32(0)) + ) + vis.append(v) + max_vis = cutlass.max(max_vis, v) + max_vis = cutlass.min(max_vis, max_blocks) + first = prog * tiles + n_t = cutlass.max(cutlass.min(cutlass.Int32(tiles), (max_vis + bn - 1) // bn - first), 0) + + # ---- the program's page numbers (page of each tile key, -1 past the window) ------ + for i in cutlass.range_constexpr(-(-(tiles * bn) // THREADS)): + kk = tidx + THREADS * i + blk = first * bn + kk + lp = (blk * cr + cr - 1) // tpb + ok = (blk < max_vis) & (lp < self.bt_width) + pg = cutlass.Int32(-1) + if ok: + pg = _ldg1(bt_base + (safe_req * self.bt_width + cutlass.Int64(lp)) * 4) + if kk < tiles * bn: + _sts1(s_pg + kk * 4, pg) + + # ---- q of the group as A fragments: row m = 4 t + h; thread (g, c) holds dims + # 32 m4 + 8 c .. + 7 of rows g and g + 8 (m4 = 0..3), the keys read the same dims. ----- + qa = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Int32) + qb = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Int32) + ta = g // 4 + tb = (g + 8) // 4 + ha = g % 4 + ra = cutlass.min(r0 + ta, rows - 1) + rb = cutlass.min(r0 + tb, rows - 1) + oka = (ta < rpr) & (r0 + ta < rows) + okb = (tb < rpr) & (r0 + tb < rows) + for m4 in cutlass.range_constexpr(4): + off = cutlass.Int64((32 * m4 + 8 * c) * 2) + wa = _ldg4(q_base + (cutlass.Int64(ra) * q_stride_row + ha * HEAD_DIM) * 2 + off) + wb = _ldg4(q_base + (cutlass.Int64(rb) * q_stride_row + ha * HEAD_DIM) * 2 + off) + for e in cutlass.range_constexpr(4): + qa[m4, e] = cutlass.Int32(cutlass.select_(oka, wa[e], cutlass.Int32(0))) + qb[m4, e] = cutlass.Int32(cutlass.select_(okb, wb[e], cutlass.Int32(0))) + cute.arch.sync_threads() + + # one commit group a ring slot, empty past the program's tiles: the loop's wait_group(stages - 2) + # then always leaves exactly the newer tiles in flight + for s in cutlass.range_constexpr(stages - 1): + if s < n_t: + self._issue(s, s, tidx, s_pg, s_ring, cache_base, page_stride) + else: + cute.arch.cp_async_commit_group() + + scale = cutlass.Float32(HEAD_DIM**-0.5) + out_a = out_base + cutlass.Int64(ra) * out_stride * 4 + out_b = out_base + cutlass.Int64(rb) * out_stride * 4 + vis_a = vis[0] + vis_b = vis[0] + for t in cutlass.range_constexpr(rpr): + vis_a = cutlass.Int32(cutlass.select_(ta == t, vis[t], vis_a)) + vis_b = cutlass.Int32(cutlass.select_(tb == t, vis[t], vis_b)) + for j in cutlass.range(n_t): + buf = j % stages + cute.arch.cp_async_wait_group(stages - 2) + cute.arch.sync_threads() + if j + stages - 1 < n_t: + self._issue( + j + stages - 1, + (j + stages - 1) % stages, + tidx, + s_pg, + s_ring, + cache_base, + page_stride, + ) + else: + cute.arch.cp_async_commit_group() + rk = s_ring + buf * (bn * self.k_ld) + for nt in cutlass.range_constexpr(ntw): + key = warp * (8 * ntw) + nt * 8 + g + zero = cutlass.Float32(0.0) + d = [zero, zero, zero, zero] + for m4 in cutlass.range_constexpr(4): + # chunk 4 m4 + c of key `key` (parity g & 1) sits at chunk (4 m4 + c) ^ 4 (g & 1) + b = _lds4(rk + key * self.k_ld + 16 * ((4 * m4 + c) ^ ((g & 1) << 2))) + a_lo = (qa[m4, 0], qb[m4, 0], qa[m4, 1], qb[m4, 1]) + a_hi = (qa[m4, 2], qb[m4, 2], qa[m4, 3], qb[m4, 3]) + d = _mma_bf16(a_lo, (b[0], b[1]), d) + d = _mma_bf16(a_hi, (b[2], b[3]), d) + # d0, d1: row g (t = g // 4, head g % 4), keys 2c, 2c + 1 of the n8 tile; + # d2, d3: row g + 8. A row's heads are lanes 4 apart: (h0 + h1) + (h2 + h3). + rr = [cutlass.max(x, zero) for x in d] + pr = [rr[e] + _shfl_xor_f(rr[e], 4) for e in range(4)] + sc = [pr[e] + _shfl_xor_f(pr[e], 8) for e in range(4)] + col = (first + j) * bn + warp * (8 * ntw) + nt * 8 + 2 * c + if ha == 0: + # both keys inside the row's window: one 8-byte store; only the first: 4 bytes + if oka & (col + 1 < vis_a): + _stg2f(out_a + cutlass.Int64(col) * 4, sc[0] * scale, sc[1] * scale) + if oka & (col + 1 == vis_a): + _stg1(out_a + cutlass.Int64(col) * 4, _f32_bits(sc[0] * scale)) + if okb & (col + 1 < vis_b): + _stg2f(out_b + cutlass.Int64(col) * 4, sc[2] * scale, sc[3] * scale) + if okb & (col + 1 == vis_b): + _stg1(out_b + cutlass.Int64(col) * 4, _f32_bits(sc[2] * scale)) + + +_COMPILED: dict = {} + + +def serves(q: torch.Tensor, index_cache: torch.Tensor, rows_per_request: int) -> bool: + """Verify rows this kernel serves: 2 or 3 rows a request, 4 bf16 index heads of 128 with unit + dim stride and 16-byte aligned rows, a `[pages, tokens, 1, 128]` bf16 pool view with token rows + 256 B apart and 16-byte aligned pages.""" + return ( + rows_per_request in (2, 3) + and q.ndim == 3 + and q.shape[1] == HEADS + and q.shape[2] == HEAD_DIM + and q.dtype == torch.bfloat16 + and q.stride(2) == 1 + and q.stride(1) == HEAD_DIM + and (q.stride(0) * 2) % 16 == 0 + and q.data_ptr() % 16 == 0 + and q.shape[0] % rows_per_request == 0 + and index_cache.ndim == 4 + and index_cache.shape[2] == 1 + and index_cache.shape[3] == HEAD_DIM + and index_cache.dtype == torch.bfloat16 + and index_cache.stride(3) == 1 + and index_cache.stride(1) == HEAD_DIM + and (index_cache.stride(0) * 2) % 16 == 0 + and index_cache.data_ptr() % 16 == 0 + ) + + +def _index_scores_groups( + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, +) -> torch.Tensor: + """`qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=False, + rows_per_request=...)`'s scores to the last bits (`[rows, max_blocks]` fp32, columns at and + past a row's window unspecified).""" + rows = q.shape[0] + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1] + assert query_positions.dtype == torch.int64 and request_indices.dtype == torch.int32 + max_blocks = block_table.shape[1] * tokens_per_block // compress_ratio + out = torch.empty((rows, max_blocks), dtype=torch.float32, device=q.device) + if rows == 0: + return out + groups = rows // rows_per_request + col_progs = -(-max_blocks // (BN * TILES)) + pdl = bool(use_pdl and get_env_enable_pdl()) + key = (rows_per_request, tokens_per_block, compress_ratio, tuple(block_table.shape), pdl) + args = ( + _ptr(q, cutlass.BFloat16), + _ptr(index_cache, cutlass.BFloat16), + _ptr(block_table, cutlass.Int32, 4), + _ptr(query_positions, cutlass.Int64, 8), + _ptr(request_indices, cutlass.Int32, 4), + from_dlpack(out, assumed_align=16).mark_layout_dynamic(leading_dim=1), + cutlass.Int32(rows), + cutlass.Int32(q.stride(0)), + cutlass.Int64(index_cache.stride(0)), + cutlass.Int32(out.stride(0)), + cutlass.Int32(max_blocks), + cutlass.Int32(groups), + cutlass.Int32(col_progs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + fn = _COMPILED.get(key) + if fn is None: + kern = _IdxGroupsKernel( + rpr=rows_per_request, + tpb=tokens_per_block, + cr=compress_ratio, + tiles=TILES, + stages=STAGES, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + pdl=pdl, + ) + fn = _COMPILED[key] = cute.compile(kern, *args, options="--opt-level 2 --enable-tvm-ffi") + fn( + q.data_ptr(), + index_cache.data_ptr(), + block_table.data_ptr(), + query_positions.data_ptr(), + request_indices.data_ptr(), + out, + rows, + q.stride(0), + index_cache.stride(0), + out.stride(0), + max_blocks, + groups, + col_progs, + ) + return out + + +class QsaIndexScoresGroups(OpWrapper): + """The family's `qsa_paged_index_scores(..., only_visible_blocks=True, context_rows=False, + rows_per_request=...)` on verify rows `serves` admits: `[rows, max_blocks]` fp32 to the last + bits over each row's visible columns, the columns at and past them unspecified.""" + + def __call__( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> torch.Tensor: + return _index_scores_groups( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + use_pdl, + rows_per_request, + ) + + def reference( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> torch.Tensor: + """The family entry's reference over verify rows: NaN past each row's visible columns. + `use_pdl` picks a launch, not a result.""" + return _family_index_scores.reference( + q, + index_cache, + block_table, + query_positions, + request_indices, + tokens_per_block, + compress_ratio, + only_visible_blocks=True, + context_rows=False, + use_pdl=use_pdl, + rows_per_request=rows_per_request, + ) + + def is_valid( + self, + q: torch.Tensor, + index_cache: torch.Tensor, + block_table: torch.Tensor, + query_positions: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + use_pdl: bool, + rows_per_request: int, + ) -> None: + # The kernel is built for this geometry and checks none of it. + assert serves(q, index_cache, rows_per_request), "a call outside `serves`" + + +index_scores_groups = QsaIndexScoresGroups() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py new file mode 100644 index 000000000000..3f0b46e974fc --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_index_split_plan__decode.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Which requests each TP rank scores in a decode step's split QSA index selection, by work. + +A tep decode step splits its QSA index scoring and block top-k by request over the TP group +(`_decode_index_selection`). A split by position would put the CUDA graph padding (always at the +tail) on one rank, so one launch assigns the requests by work, on the device (a graph replay's +batch is not known to the host at capture): + +- each request's cost is its first row's visible-block count; +- the requests in descending cost (ties to the lower request index) are dealt to the ranks in + snake order (0, 1, .., t - 1, t - 1, .., 0, ...): every rank gets exactly `n / t` requests, so + the kernels' shapes stay the position split's; +- in the permuted order rank `r`'s requests are positions `[r * n / t, (r + 1) * n / t)`: the + request at sorted position `s = lap * t + seat` goes to permuted position + `owner * n / t + local(lap)`, where `local` interleaves the laps from both ends + (0, last, 1, last - 1, ...) so long and short requests alternate in a rank's share. + +Outputs, for `rows = n * p` rows of `p` a request: + +- `iq_rows` `[rows]` int32: each row's position in the permuted order (where the attention + prologue stores the row's index query, so each rank's queries are one contiguous slice); +- `perm_rows` `[rows]` int32: the row at each permuted position; +- this rank's rows of the four QSA row vectors (request index, logical position, sequence length, + visible blocks) and the top-k row starts, in permuted order, `[rows / t]` each, fresh. + +Every rank computes the same permutation from the same replicated metadata, so the gathered +selection, scattered back by `perm_rows`, is bitwise the position split's. +""" + +from __future__ import annotations + +from typing import NamedTuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Requests compared at once in the ranking (a `[BLOCK, CHUNK]` tile). +_CHUNK = 32 + + +class SplitPlan(NamedTuple): + """`index_split_plan`'s outputs (see the module docstring).""" + + iq_rows: torch.Tensor + perm_rows: torch.Tensor + req_idx: torch.Tensor + logical: torch.Tensor + seq_lens: torch.Tensor + visible: torch.Tensor + row_starts: torch.Tensor + + +@triton.jit(do_not_specialize=["num_requests", "rows_per_request", "share", "rank"]) +def _split_plan_kernel( + visible, + req_idx, + logical, + seq_lens, + row_starts, + iq_rows, + perm_rows, + o_req, + o_logical, + o_seq, + o_visible, + o_starts, + num_requests, + rows_per_request, + share, + rank, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, + CHUNK: tl.constexpr, +): + i = tl.arange(0, BLOCK) + live = i < num_requests + first = i.to(tl.int64) * rows_per_request + cost = tl.load(visible + first, mask=live, other=0) + # Stable descending rank: requests strictly costlier, or as costly with a lower index. + pos = tl.zeros([BLOCK], dtype=tl.int32) + for j0 in range(0, num_requests, CHUNK): + j = j0 + tl.arange(0, CHUNK) + cj = tl.load(visible + j.to(tl.int64) * rows_per_request, mask=j < num_requests, other=0) + ahead = (cj[None, :] > cost[:, None]) | ( + (cj[None, :] == cost[:, None]) & (j[None, :] < i[:, None]) + ) + ahead = ahead & (j[None, :] < num_requests) + pos += tl.sum(ahead.to(tl.int32), axis=1) + lap = pos // WORLD + seat = pos % WORLD + owner = tl.where(lap % 2 == 0, seat, WORLD - 1 - seat) + half = (share + 1) // 2 + local_lap = tl.where(lap < half, 2 * lap, 2 * (share - 1 - lap) + 1) + slot = owner * share + local_lap + mine = live & (owner == rank) + for t in range(0, rows_per_request): + g = first + t + k = slot.to(tl.int64) * rows_per_request + t + tl.store(iq_rows + g, k.to(tl.int32), mask=live) + tl.store(perm_rows + k, g.to(tl.int32), mask=live) + local = local_lap.to(tl.int64) * rows_per_request + t + tl.store(o_req + local, tl.load(req_idx + g, mask=mine), mask=mine) + tl.store(o_logical + local, tl.load(logical + g, mask=mine), mask=mine) + tl.store(o_seq + local, tl.load(seq_lens + g, mask=mine), mask=mine) + tl.store(o_visible + local, tl.load(visible + g, mask=mine), mask=mine) + tl.store(o_starts + local, tl.load(row_starts + g, mask=mine), mask=mine) + + +def _spec(requests: int, p: int, world: int, rank: int, kind: str) -> dict: + return dict(requests=requests, rows_per_request=p, world=world, rank=rank, kind=kind) + + +class IndexSplitPlan(OpWrapper): + """`plan = index_split_plan(visible, req_idx, logical, seq_lens, row_starts, num_requests, + rows_per_request, world, rank)`: the work-balanced request assignment of a decode step's split + index selection (module docstring), a `SplitPlan`. + + - `visible`, `req_idx`, `seq_lens`, `row_starts`: int32, `logical`: int64, each a contiguous + `[>= rows]` QSA row vector (`rows = num_requests * rows_per_request`, `p` rows a request, + consecutive); + - `num_requests % world == 0`, `num_requests <= 1024`. + + Bitwise `reference`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep2 HT verify step: 64 requests x 3 rows, rank 1, 3 padding requests at the tail", + spec=_spec(64, 3, 2, 1, "padded")), + Cell(why="the largest batch (192), ties everywhere (equal windows)", spec=_spec(192, 3, 2, 1, "equal")), + Cell(why="tep4: 128 requests over 4 ranks, rank 2", spec=_spec(128, 3, 4, 2, "served")), + ) # fmt: skip + + note = """ + Source: one Triton program (`_split_plan_kernel`, 4 warps), `BLOCK` the next power of two of + the request count (a compile per block size and world); counts are runtime arguments. The + ranking compares every pair of requests in `[BLOCK, CHUNK]` tiles. + """ + + def __call__( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> SplitPlan: + rows = num_requests * rows_per_request + local = rows // world + dev = visible.device + plan = SplitPlan( + iq_rows=torch.empty([rows], dtype=torch.int32, device=dev), + perm_rows=torch.empty([rows], dtype=torch.int32, device=dev), + req_idx=torch.empty([local], dtype=req_idx.dtype, device=dev), + logical=torch.empty([local], dtype=logical.dtype, device=dev), + seq_lens=torch.empty([local], dtype=seq_lens.dtype, device=dev), + visible=torch.empty([local], dtype=visible.dtype, device=dev), + row_starts=torch.empty([local], dtype=row_starts.dtype, device=dev), + ) + if num_requests: + block = max(_CHUNK, triton.next_power_of_2(num_requests)) + _split_plan_kernel[(1,)]( + visible, req_idx, logical, seq_lens, row_starts, *plan, + num_requests, rows_per_request, num_requests // world, rank, + WORLD=world, BLOCK=block, CHUNK=_CHUNK, num_warps=4, + ) # fmt: skip + return plan + + def reference( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> SplitPlan: + p, n, t = rows_per_request, num_requests, world + share = n // t + cost = [int(c) for c in visible[: n * p : p].tolist()] + order = sorted(range(n), key=lambda r: (-cost[r], r)) + slot = [0] * n + half = (share + 1) // 2 + for s, r in enumerate(order): + lap, seat = divmod(s, t) + owner = seat if lap % 2 == 0 else t - 1 - seat + local = 2 * lap if lap < half else 2 * (share - 1 - lap) + 1 + slot[r] = owner * share + local + dev = visible.device + iq = torch.tensor( + [slot[g // p] * p + g % p for g in range(n * p)], dtype=torch.int32, device=dev + ) + perm = torch.empty_like(iq) + perm[iq.long()] = torch.arange(n * p, dtype=torch.int32, device=dev) + mine = perm[rank * share * p : (rank + 1) * share * p].long() + return SplitPlan( + iq_rows=iq, + perm_rows=perm, + req_idx=req_idx.index_select(0, mine), + logical=logical.index_select(0, mine), + seq_lens=seq_lens.index_select(0, mine), + visible=visible.index_select(0, mine), + row_starts=row_starts.index_select(0, mine), + ) + + def is_valid( + self, + visible, + req_idx, + logical, + seq_lens, + row_starts, + num_requests: int, + rows_per_request: int, + world: int, + rank: int, + ) -> None: + rows = num_requests * rows_per_request + assert world >= 1 and 0 <= rank < world and num_requests % world == 0 + assert num_requests <= 1024 and rows_per_request >= 1 + for v in (visible, req_idx, logical, seq_lens, row_starts): + assert v.dim() == 1 and v.is_contiguous() and v.numel() >= rows + + +index_split_plan = IndexSplitPlan() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py new file mode 100644 index 000000000000..8870b8e9aaec --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_index_store.py @@ -0,0 +1,395 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA index side-cache store: raw index keys and position coordinates into their pages. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_index_store`.""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +_INTEGER_DTYPES = (torch.int32, torch.int64) + + +# Row count, pool/table extents and the strides that vary per engine are not specialized on, so +# a new batch or pool does not recompile; none of them is a vector access worth a variant. +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_requests", + "table_width", + "num_pages", + "stride_c_axis", + "stride_t_request", + ], + do_not_specialize_on_alignment=[ + "coords", + "request_indices", + "logical_positions", + "block_table", + ], +) +def _qsa_paged_index_store_kernel( + token_k, + coords, + index_cache, + position_cache, + request_indices, + logical_positions, + block_table, + num_rows, + num_requests, + table_width, + num_pages, + stride_k_row, + stride_k_head, + stride_k_dim, + stride_c_row, + stride_c_axis, + stride_i_page, + stride_i_slot, + stride_i_head, + stride_i_dim, + stride_p_page, + stride_p_slot, + stride_p_axis, + stride_request, + stride_position, + stride_t_request, + stride_t_column, + HKV: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + AXES: tl.constexpr, + BLOCK_A: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + # Indices are widened to int64 before any arithmetic: plane offsets pass 2^31 elements. + rows = (tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)).to(tl.int64) + live = rows < num_rows + request = tl.load(request_indices + rows * stride_request, mask=live, other=-1).to(tl.int64) + position = tl.load(logical_positions + rows * stride_position, mask=live, other=-1).to(tl.int64) + column = position // TOKENS_PER_BLOCK + slot = position % TOKENS_PER_BLOCK + live &= (request >= 0) & (request < num_requests) & (position >= 0) & (column < table_width) + page = tl.load( + block_table + request * stride_t_request + column * stride_t_column, mask=live, other=-1 + ).to(tl.int64) + live &= (page >= 0) & (page < num_pages) + + dims = tl.arange(0, BLOCK_D) + key_mask = live[:, None] & (dims < HEAD_DIM)[None, :] + for head in tl.static_range(HKV): + key = tl.load( + token_k + + rows[:, None] * stride_k_row + + head * stride_k_head + + dims[None, :] * stride_k_dim, + mask=key_mask, + other=0.0, + ) + tl.store( + index_cache + + page[:, None] * stride_i_page + + slot[:, None] * stride_i_slot + + head * stride_i_head + + dims[None, :] * stride_i_dim, + key.to(index_cache.dtype.element_ty), + mask=key_mask, + ) + + axes = tl.arange(0, BLOCK_A) + axis_mask = live[:, None] & (axes < AXES)[None, :] + coordinate = tl.load( + coords + rows[:, None] * stride_c_row + axes[None, :] * stride_c_axis, + mask=axis_mask, + other=0, + ) + tl.store( + position_cache + + page[:, None] * stride_p_page + + slot[:, None] * stride_p_slot + + axes[None, :] * stride_p_axis, + coordinate.to(position_cache.dtype.element_ty), + mask=axis_mask, + ) + + +def _launch_rows(rows: int) -> Tuple[int, int]: + """Rows per program and warps: one row per program up to 256 rows, then tiles of 16.""" + if rows <= 256: + return 1, 1 + return 16, 4 + + +#: Index geometry: `token_k` is the k head of the packed `[T, 4 + 1, 128]` index projection; the +#: coordinates are the `[T, 3]` transpose of the engine's `[3, max_num_tokens]` position buffer. +_CKPT = dict( + hkv=1, + head_dim=128, + index_heads=4, + tokens_per_block=32, + axes=3, + slots=32, + width=260, + key_dtype=torch.bfloat16, + request_dtype=torch.int32, + position_dtype=torch.int64, + key_layout="packed", + cache_layout="plain", + hole=None, +) + +#: ISL 8192 + OSL 1024 (positions to 9215), 128 requests, 16384 rows in one step. +_WIDE = dict(slots=160, width=300) +_PREFILL_16K = ((0, 8192), (0, 8192)) + +#: 32 MTP7 verify blocks (8 rows each), some straddling a page edge. +_MTP7_32 = tuple((29 + 61 * i, 8) for i in range(32)) + + +def _spec(segments, pages, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CKPT, "segments": segments, "pages": pages, **overrides} + + +#: One rank of a member. `token_k` is the index key head of the fused input projection row +#: (`weights.py`'s `attn_in_rows`): 7424 wide under attention TP2, 4352 under TP4, 14080 under +#: DP and tp1. The bf16 index pool coalesces the 12 sparse layers' planes: `[pages, 32, 1, 128]` +#: at page stride 12 x 4096; the position plane is a contiguous `[pages, 32, 3]` int32 pool. +#: `pages` is the NVFP4 tep4 engine's pool (245835 pages of 32); the table is 128 x 292. +#: `straddle` puts the last row on the pool's last page, past 2^31 elements. +_DEFAULT = dict( + _CKPT, + key_layout="fused_row", + cache_layout="coalesced", + layers=12, + layer=11, + page_stride=49152, + slots=128, + width=292, + pages=245835, + straddle=True, +) + +#: 128 decode rows on both sides of page edges; the last at 9215. +_DECODE_128 = tuple((31 + 72 * i, 1) for i in range(127)) + ((9215, 1),) +#: 128 MTP3 verify blocks of 4 rows, some across a page edge, the last ending at 9215. +_MTP3_128 = tuple((29 + 71 * i, 4) for i in range(127)) + ((9212, 4),) + + +class QsaPagedIndexStore(OpWrapper): + """Write each row's raw index key and position coordinates into its paged slot. + + For row t, with `page = block_table[request_indices[t], logical_positions[t] + // tokens_per_block]` and `slot = logical_positions[t] % tokens_per_block`:: + + index_cache[page, slot] = token_k[t] # cast to the cache dtype + position_cache[page, slot, :A] = position_coordinates[t] # A = coordinates' width + + A row that resolves to no slot (negative or out-of-table request, position or page column, + or a table entry that is `-1` or past the pool) is skipped. The key is stored raw (un-normed, + un-rotated): `qsa_prefill_compress` applies the norm and rotation itself. Returns + `(index_cache, position_cache)`, written in place. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="decode MTP7: 32 requests x 8 rows (256), the last one-row-per-program " + "launch, blocks straddling pages", + spec=_spec(_MTP7_32, pages=1200, slots=40), + ), + Cell( + why="prefill 16384 rows (max_num_tokens), 16 rows per program, token_k off the " + "fused projection row", + spec=_spec(_PREFILL_16K, pages=600, key_layout="fused_row", **_WIDE), + ), + Cell( + why="NVFP4-KV side planes: index and position planes as strided views of one " + "per-slot buffer holding 12 index-K pages and the position page; one row " + "on an unallocated (-1) table entry is skipped", + spec=_spec( + ((0, 1), (31, 1), (32, 1), (33, 40)), + pages=8, + cache_layout="shared_page", + layers=12, + page_stride=49344, + position_strides=(24672, 3, 1), + hole="unallocated_page", + ), + ), + Cell( + why="default tep4 decode: one row at position 9215 on the pool's last page, token_k " + "off the 4352-wide fused row, the coalesced bf16 index plane", + spec=dict(_DEFAULT, member="tep4", segments=((9215, 1),), row_width=4352), + ), + Cell( + why="tp2 decode: 128 requests one row each (the largest captured batch), 7424-wide row", + spec=dict(_DEFAULT, member="tp2", segments=_DECODE_128, row_width=7424), + ), + Cell( + why="dep4 MTP3 verify: 128 requests x 4 rows (512), 16 rows per program, " + "14080-wide row", + spec=dict(_DEFAULT, member="dep4", segments=_MTP3_128, row_width=14080), + ), + ) + + note = """ + Source: this target's Triton kernel `_qsa_paged_index_store_kernel` (Triton JIT). + + The target passes `qsa_req_idx_per_token` (int32) and `qsa_logical_positions` (int64) sliced + to the step's rows, `qsa_block_table` unsliced, the layer's `get_index_k_buffer(i)` and the + shared `get_qsa_position_buffer()` (rewritten idempotently by every sparse layer). Every + operand is read through its own strides. Under fp8 and NVFP4 KV both planes are strided views + of one per-slot buffer (index page stride 12 x 4096 + 192, position strides (24672, 3, 1)); + the cells check that a page's other index-K roles and layer planes survive the call. + + Preconditions: + - destination slots are unique across the batch; two rows naming one (page, slot) both + store, unordered. The engine's mapping cannot collide. + - coordinates fit the position plane's dtype: wider values wrap, floats truncate. + Shape, rank and dtype mismatches raise `ValueError` before launching; `rows == 0` returns + before launching. CUDA-graph capturable; a replay follows buffers rewritten in place. + + Not certified: an fp8 index plane, keys other than bf16, index geometry other than 1 head x + 128 at 32-token pages, batches other than the cells', strided metadata or tables, stride-0 + coordinates, multiple devices, concurrent streams onto one pool, duplicate destinations, + skip conditions other than a `-1` table entry, tep2/dep2/tp1 at their own cells (certified + through tp2 and dep4 at the same row widths), each member's own pool size (the cells use the + NVFP4 tep4 engine's 245835 pages), and positions past 9215. + """ + + def __call__( + self, + token_k: torch.Tensor, + position_coordinates: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + rows = request_indices.shape[0] + if request_indices.dim() != 1 or logical_positions.shape != request_indices.shape: + raise ValueError( + f"request_indices {tuple(request_indices.shape)} and logical_positions " + f"{tuple(logical_positions.shape)} must be one-dimensional and equally long" + ) + if request_indices.dtype not in _INTEGER_DTYPES or ( + logical_positions.dtype not in _INTEGER_DTYPES + or block_table.dtype not in _INTEGER_DTYPES + ): + raise ValueError("QSA index store metadata and block table must be int32 or int64") + if token_k.dim() != 3 or token_k.shape[0] != rows: + raise ValueError(f"token_k {tuple(token_k.shape)} must be [{rows}, heads, dim]") + if position_coordinates.dim() != 2 or position_coordinates.shape[0] != rows: + raise ValueError( + f"position_coordinates {tuple(position_coordinates.shape)} must be [{rows}, axes]" + ) + if index_cache.dim() != 4 or index_cache.shape[1:] != ( + tokens_per_block, + *token_k.shape[1:], + ): + raise ValueError( + f"index_cache {tuple(index_cache.shape)} must be [pages, {tokens_per_block}, " + f"{token_k.shape[1]}, {token_k.shape[2]}]" + ) + if ( + position_cache.dim() != 3 + or position_cache.shape[1] != tokens_per_block + or position_cache.shape[2] < position_coordinates.shape[1] + ): + raise ValueError( + f"position_cache {tuple(position_cache.shape)} must be [pages, " + f"{tokens_per_block}, >= {position_coordinates.shape[1]}]" + ) + if block_table.dim() != 2: + raise ValueError(f"block_table {tuple(block_table.shape)} must be two-dimensional") + if rows == 0: + return index_cache, position_cache + block_rows, warps = _launch_rows(rows) + axes = position_coordinates.shape[1] + _qsa_paged_index_store_kernel[(triton.cdiv(rows, block_rows),)]( + token_k, + position_coordinates, + index_cache, + position_cache, + request_indices, + logical_positions, + block_table, + rows, + block_table.shape[0], + block_table.shape[1], + index_cache.shape[0], + *token_k.stride(), + *position_coordinates.stride(), + *index_cache.stride(), + *position_cache.stride(), + request_indices.stride(0), + logical_positions.stride(0), + *block_table.stride(), + HKV=token_k.shape[1], + HEAD_DIM=token_k.shape[2], + BLOCK_D=triton.next_power_of_2(token_k.shape[2]), + AXES=axes, + BLOCK_A=triton.next_power_of_2(axes), + TOKENS_PER_BLOCK=tokens_per_block, + BLOCK_ROWS=block_rows, + num_warps=warps, + ) + return index_cache, position_cache + + def reference( + self, + token_k: torch.Tensor, + position_coordinates: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Both planes after the store, as fresh tensors; the inputs are not touched. + + Destinations are decided in host Python integers, independent of the kernel's arithmetic. + """ + expected_index, expected_position = index_cache.clone(), position_cache.clone() + table = block_table.tolist() + pool = index_cache.shape[0] + kept, pages, slots = [], [], [] + rows = zip(request_indices.tolist(), logical_positions.tolist()) + for row, (request, position) in enumerate(rows): + if not 0 <= request < len(table) or position < 0: + continue + column, slot = divmod(position, tokens_per_block) + if column >= len(table[request]) or not 0 <= table[request][column] < pool: + continue + kept.append(row) + pages.append(table[request][column]) + slots.append(slot) + if not kept: + return expected_index, expected_position + device = index_cache.device + kept_t, page_t, slot_t = ( + torch.tensor(v, dtype=torch.long, device=device) for v in (kept, pages, slots) + ) + axes = position_coordinates.shape[-1] + expected_index[page_t, slot_t] = token_k[kept_t].to(index_cache.dtype) + expected_position[page_t, slot_t, :axes] = position_coordinates[kept_t].to( + position_cache.dtype + ) + return expected_index, expected_position + + +qsa_paged_index_store = QsaPagedIndexStore() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py new file mode 100644 index 000000000000..b9f10ccdc806 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_kv_store.py @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse path's paged K/V append: this step's rows into the HND main pool. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_kv_store`.""" + +from __future__ import annotations + +from typing import Tuple + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from . import qsa_triton_kernels__stable_jit as _qsa_kernels + +#: Main K/V geometry: k and v are head slices of the packed `[T, 24 + 2 + 2, 256]` QKV row; +#: `slots` x `width` is `qsa_block_table` (max_num_sequences x max_blocks_per_seq). +_CKPT = dict( + hkv=2, + head_dim=256, + heads_q=24, + tokens_per_block=32, + slots=32, + width=260, + row_dtype=torch.bfloat16, + cache_dtype=torch.bfloat16, + request_dtype=torch.int32, + position_dtype=torch.int64, + hole=None, + # Heads after v in each row: 27 for the fused projection row [q | k | v | gate | index]. + row_pad_heads=0, +) + +_F8 = torch.float8_e4m3fn + +_DECODE = ((0, 1), (31, 1), (32, 1), (33, 1), (100, 1), (255, 1), (7, 1), (64, 1)) + +#: 32 MTP7 verify blocks (8 rows each), some straddling a page edge. +_MTP7_32 = tuple((29 + 61 * i, 8) for i in range(32)) + +#: One rank's pool as `Qwen4ExpHybridCacheManager` lays it out: a page holds the K and V planes +#: of all 12 attention layers (24 role pages of `[hkv, 32, 256]`), so a layer's +#: `get_buffers(i, kv_layout="HND")` view steps 24 role pages a page; the cells drive layer 47 +#: (`layer_slot` 11). The table is 128 x 292 at `max_seq_len` 9216. +_ENGINE = dict(_CKPT, pool="engine", role_pages=24, layer_slot=11, slots=128, width=292) + +#: tp4 / tep4 rank: fused row `[q 6 | k 1 | v 1 | gate 6 | index 3]` (17 heads, stride 4352). +_TP4 = dict(_ENGINE, hkv=1, heads_q=6, row_pad_heads=9, row_stride=4352) + +#: dep4 rank: tp1 row `[q 24 | k 2 | v 2 | gate 24 | index 3]` (55 heads, stride 14080), e4m3 pool. +_DEP4_FP8 = dict(_ENGINE, row_pad_heads=27, row_stride=14080, cache_dtype=torch.float8_e4m3fn) + +#: tp2 / tep2 rank: fused row `[q 12 | k 1 | v 1 | gate 12 | index 3]` (29 heads, stride 7424). +_TP2 = dict(_ENGINE, hkv=1, heads_q=12, row_pad_heads=15, row_stride=7424) + +#: dep4 rank over a bf16 pool. +_DEP4 = dict(_ENGINE, row_pad_heads=27, row_stride=14080) + +#: dep4 with one-model MTP: the draft layer joins the trunk's manager (13 layers, 26 role pages). +_DEP4_MTP = dict(_DEP4, role_pages=26) + +#: Attention pool pages per rank (`maxNumBlocks` less the recurrent slots) of the NVFP4 +#: checkpoint's engines at KV fraction 0.6, `max_batch_size` 128, `max_seq_len` 9216; the FP8 +#: cells reuse them at the same per-page shape. +PAGES_TEP4 = 257109 +PAGES_DEP4_FP8 = 171362 +PAGES_DEP4 = 119680 + +#: 128 decode rows: page-aligned, mid-page, the last position (9215), then a stride-577 walk. +_DECODE_128 = ((0, 1), (31, 1), (32, 1), (9215, 1)) + tuple( + ((i * 577 + 45) % 9216, 1) for i in range(124) +) + +#: An 8192-row context step: a chunk from mid-page to 9215, and a request's first 1023 tokens. +_PREFILL_8192 = ((2047, 7169), (0, 1023)) + +#: 128 MTP3 verify blocks (512 rows) ending at position 9215, at every offset in a page. +_MTP3_128 = tuple(((29 + 71 * i) % 9212, 4) for i in range(127)) + ((9212, 4),) + + +def _spec(segments, pages, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CKPT, "segments": segments, "pages": pages, **overrides} + + +def _engine(base: dict, member: str, segments, pages: int, **overrides) -> dict: + """An engine-pool cell: `rows` is the step's row count, `member` whose pool `pages` is.""" + rows = sum(n for _, n in segments) + return { + **base, + "member": member, + "segments": segments, + "pages": pages, + "rows": rows, + **overrides, + } + + +class QsaPagedKvStore(OpWrapper): + """Scatter every row's K and V into its request's page of the HND pool. + + One Triton launch over `(rows, Hkv)`. For row t:: + + page = block_table[request_indices[t], logical_positions[t] // tokens_per_block] + slot = logical_positions[t] % tokens_per_block + k_cache[page, :, slot, :] = k[t] v_cache[page, :, slot, :] = v[t] + + `logical_positions` is absolute, so prefill, decode and MTP rows are the same call. Values + are converted once to the pool's dtype (an fp8-e4m3 pool takes no scale: readers use kv + scale 1.0). A row that resolves to no slot (negative or out-of-table request, position or + page column, or a table entry that is `-1` or past the pool) is skipped. Returns + `(k_cache, v_cache)`, written in place. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="decode: 8 rows off the fused projection row (row stride 14080), " + "at aligned, mid and page-crossing positions, plus one row on an " + "unallocated (-1) table entry that is skipped", + spec=_spec(_DECODE, pages=64, row_pad_heads=27, hole="unallocated_page"), + ), + Cell( + why="fp8 decode MTP7: 32 requests x 8 rows (256), blocks straddling pages", + spec=_spec(_MTP7_32, pages=1200, slots=40, cache_dtype=_F8), + ), + # Engine pools at one rank's page count. + Cell( + why="tep4 decode, 1 row: hkv 1 off the 4352-wide row, position 9215, " + "257109-page bf16 pool", + spec=_engine(_TP4, "tep4", ((9215, 1),), PAGES_TEP4), + ), + Cell( + why="tp4 context, 8192 rows (max_num_tokens): a mid-page chunk to 9215 and a " + "request's first 1023", + spec=_engine(_TP4, "tp4", _PREFILL_8192, PAGES_TEP4), + ), + Cell( + why="tp2 decode, 128 rows (max_batch_size) off the 7424-wide row, aligned, mid " + "and page-crossing", + spec=_engine(_TP2, "tp2", _DECODE_128, PAGES_TEP4), + ), + Cell( + why="dep4 MTP3 verify, 128 requests x 4 rows (512) to position 9215: hkv 2 off " + "the 14080-wide row, 26 role pages a page (the draft layer's planes beside the " + "trunk's)", + spec=_engine(_DEP4_MTP, "dep4", _MTP3_128, PAGES_DEP4), + ), + Cell( + why="dep4 kv_fp8 decode, 128 rows (max_batch_size): hkv 2 off the 14080-wide row " + "into a 171362-page e4m3 pool", + spec=_engine(_DEP4_FP8, "dep4", _DECODE_128, PAGES_DEP4_FP8), + ), + Cell( + why="dep4 context, 8192 rows (max_num_tokens): hkv 2 off the 14080-wide row, a " + "mid-page chunk to 9215 and a request's first 1023", + spec=_engine(_DEP4, "dep4", _PREFILL_8192, PAGES_DEP4), + ), + ) + + note = """ + Source: `qsa_triton_kernels__stable_jit._qsa_kv_store_stable_kernel` (Triton JIT). + + Preconditions: + - destination slots are unique across the batch; two rows naming one (page, slot) both + store, unordered. The engine's mapping cannot collide. + - `request_indices` and `logical_positions` are contiguous (`is_valid`): the kernel gets + no stride for them. + - both are cast to int32 in the kernel, so positions must stay below 2^31 (the engine's + stay below 2^14). + Pool/table extents, the table's row stride and the row count are runtime ints, so one + compiled kernel serves every pool and batch; CUDA-graph capturable, and a replay follows + positions rewritten in place. Page ids are widened to int64 before the stride product. + + Not certified: duplicate destinations, multiple devices, a scaled fp8 pool, fp8 values past + 448, the packed-QKV-row layout over a bf16 pool or at one KV head, fp16 rows, geometry other + than D 256 / Hkv 1-2 / 32-token pages, int32 positions or int64 request indices, head-major + rows, sliced or transposed tables, `rows == 0`, unresolvable rows other than a `-1` entry + or a negative position, positions past 9215, tables wider than 292 columns, and the FP8 + checkpoint's engines at their own page counts. + """ + + def __call__( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + _qsa_kernels.qsa_paged_kv_store( + k=k, + v=v, + k_cache=k_cache, + v_cache=v_cache, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + return k_cache, v_cache + + def reference( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Both caches after the scatter, as fresh tensors; the inputs are not touched.""" + expected_k, expected_v = k_cache.clone(), v_cache.clone() + requests = request_indices.long() + positions = logical_positions.long() + columns = positions.div(tokens_per_block, rounding_mode="floor") + table_rows, table_width = block_table.shape + addressable = ( + (requests >= 0) & (requests < table_rows) & (positions >= 0) & (columns < table_width) + ) + pages = block_table[ + requests.clamp(0, table_rows - 1), columns.clamp(0, table_width - 1) + ].long() + addressable &= (pages >= 0) & (pages < k_cache.shape[0]) + page, slot = pages[addressable], (positions % tokens_per_block)[addressable] + expected_k[page, :, slot, :] = k[addressable].to(expected_k.dtype) + expected_v[page, :, slot, :] = v[addressable].to(expected_v.dtype) + return expected_k, expected_v + + def is_valid( + self, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> None: + # A strided view lands rows at other rows' slots, with nothing raised. + assert request_indices.is_contiguous(), "request_indices must be a contiguous 1-D tensor" + assert logical_positions.is_contiguous(), ( + "logical_positions must be a contiguous 1-D tensor" + ) + + +qsa_paged_kv_store = QsaPagedKvStore() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py new file mode 100644 index 000000000000..6114b460c996 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_paged_sparse_gqa.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA: attention over an explicitly selected set of paged K/V tokens. + +This target's copy of the family entry `models/qwen4_exp/ops/attention/qsa_paged_sparse_gqa.py`, +launching this target's stable-JIT kernels.""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from .qsa_triton_kernels__stable_jit import ( + can_fuse_splitk_output_gate as _can_fuse_qsa_splitk_output_gate, +) +from .qsa_triton_kernels__stable_jit import qsa_paged_sparse_gqa as _triton_qsa_paged_sparse_gqa + +_CHECKPOINT = dict( + q_heads=24, + kv_heads=2, + head_dim=256, + tokens_per_block=32, + compress_ratio=4, + token_topk=2048, + cache_dtype=torch.bfloat16, + q_dtype=torch.bfloat16, + # q is the first 24 heads of the packed [T, 24 + 2 + 2, 256] QKV buffer, or ("fused_row") + # of the fused input projection row. + q_layout="qkv_slice", + request_dtype=torch.int32, + pages=260, +) + +_SPLIT_K = ("splitk", "merge") +_PREFILL = ("prefill",) +_FP8 = dict(cache_dtype=torch.float8_e4m3fn) + + +def _spec(**overrides) -> dict: + """A cell spec: this checkpoint's geometry unless the cell says otherwise. + + `segments` is `(first position, rows)` per request, in row order; + `prefill_pair` says whether `query_positions` / `compress_ratio` are + passed, which the target does exactly when the step has a context request. + """ + return {**_CHECKPOINT, **overrides} + + +def _decode(segments, launches=_SPLIT_K, **overrides) -> dict: + base = dict(segments=segments, slots=32, prefill_pair=False, launches=launches) + return _spec(**{**base, **overrides}) + + +#: The engine's trunk layout (`Qwen4ExpHybridCacheManager.get_buffers`): a pool page holds the +#: K and V planes of the 12 attention layers (24 role pages); layer 11's planes are the last. +_TRUNK = dict(slot_pages=24, layer=11) + +#: The element offset the trunk cells' pages straddle in the K and V planes. +_OFFSET_EDGE = 2**31 + +#: One tp4 / tep4 rank: 6 q heads and one KV head; a 128-row table of 292 columns +#: (max_seq_len 9216), 516 at MTP's 16384. +_TP4 = dict(q_heads=6, kv_heads=1, slots=128, pages=292, **_TRUNK) +_TP4_MTP = dict(_TP4, pages=516) +#: One dep4 rank: 24 / 2 heads and a 32-row table. +_DEP4 = dict(q_heads=24, kv_heads=2, slots=32, pages=292, **_TRUNK) +_DEP4_FP8 = dict(_DEP4, **_FP8) + +#: q and the output gate off one rank's fused input projection row `[q | k | v | gate | index]` +#: in heads of 256: 17 heads at a tp4 / tep4 rank, 55 at dep4. +_TP4_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=17) +_DEP4_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=55) + +#: One tp2 / tep2 rank: 12 q heads and one KV head, a 128-row table of 292 columns. +_TP2 = dict(q_heads=12, kv_heads=1, slots=128, pages=292, **_TRUNK) +#: A tp2 / tep2 rank's fused row: 29 heads. +_TP2_ROW = dict(q_layout="fused_row", gate="fused_row", row_heads=29) +#: One tp1 or dep2 rank: 24 / 2 heads and a 128-row table. +_TP1 = dict(q_heads=24, kv_heads=2, slots=128, pages=292, **_TRUNK) +_TP1_FP8 = dict(_TP1, **_FP8) + + +def _straddling(segments, kv_heads: int) -> int: + """The pool page count that puts a cell's requests (the pool's last pages) on both sides of + `_OFFSET_EDGE` elements of the K plane.""" + tpb, d, slot_pages, layer = 32, 256, _TRUNK["slot_pages"], _TRUNK["layer"] + needed = sum(math.ceil((s + n) / tpb) for s, n in segments) + 7 + role_page = kv_heads * tpb * d + edge_page = math.ceil((_OFFSET_EDGE - 2 * layer * role_page) / (slot_pages * role_page)) + return max(edge_page + needed // 2, needed) + + +def _default(segments, member: dict, launches, **overrides) -> dict: + """A cell at `member`'s heads, table and trunk layout, its pool straddling `_OFFSET_EDGE`.""" + return _decode( + segments, + launches=launches, + rows=sum(n for _, n in segments), + pool_pages=_straddling(segments, member["kv_heads"]), + offset_edge=_OFFSET_EDGE, + **{**member, **overrides}, + ) + + +def _spread(count: int, rows: int, last_start: int) -> tuple: + """`count` requests of `rows` rows: one starting at `last_start` (its last row at the + envelope's last position), the others spread from position 0 up.""" + span = last_start + 1 + return tuple(((5 + 1129 * g) % span, rows) for g in range(count - 1)) + ((last_start, rows),) + + +class QsaPagedSparseGqa(OpWrapper): + """Sparse GQA over the selected tokens of an HND paged K/V pool. + + For row `r`, query head `h`, KV head `h // (Hq // Hkv)` (contiguous groups), over the + columns `j` of `selected_tokens[r]` that name a resident token:: + + t = selected_tokens[r, j] # logical position + page = block_table[request_indices[r], t // tokens_per_block] + s_j = (q[r, h] . k_cache[page, kv, t % tpb]) * softmax_scale + out[r, h] = sum_j softmax(s)_j * v_cache[page, kv, t % tpb] + + Non-resident columns (negative index, page past the table, negative or out-of-pool table + entry, request outside the table) contribute nothing; a row left with nothing returns zeros. + Indices are not deduplicated. Returns a fresh contiguous `[rows, Hq, D]` tensor of `q.dtype`. + + Two kernels, chosen inside the call: with `query_positions` and `compress_ratio` given and + `rows * Hkv > 512`, a causal-prefix prefill kernel; otherwise split-K, plus a merge kernel + when it used more than one split. + + **The prefill pair is a bound, not a mask.** The prefill kernel reads only the + `min(visible // cr, block_topk) * cr + visible % cr` leading columns of a row + (`visible = query_positions[r] + 1`); a selection with valid entries past the bound is + silently truncated. QSA's top-k + expansion emit exactly that prefix. + + **The output gate rides the merge when `output_gate` is passed:** each merged head row is + rounded to `q.dtype`, multiplied in fp32 by `sigmoid(output_gate)` and rounded again (the + ungated call followed by `fused_sigmoid_mul_inplace`). `output_gate` is `[rows, Hq * D]` or + `[rows, Hq, D]` in `q.dtype` with unit last stride. Only a multi-split launch has a merge to + carry it; `fuses_output_gate` says whether a call does, and the op raises `ValueError` if + handed a gate it cannot carry. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + # ---- tp4 / tep4 / dep4 ranks: 6 / 1 and 24 / 2 heads, trunk-layout pools ---------- + Cell( + why="tp4/tep4 decode, 128 requests x 1 row (the largest captured batch): " + "6 q / 1 kv heads, 8-way split-K + merge, [128, 292] table", + spec=_default(_spread(128, 1, 9215), _TP4, _SPLIT_K), + ), + Cell( + why="tep4 MTP7 verify, 128 requests x 8 rows = 1024: 6 q / 1 kv heads, one split " + "and no merge, [128, 516] table (max_seq_len 16384), rows to position 16383", + spec=_default(_spread(128, 8, 16376), _TP4_MTP, ("splitk",), rows_per_request=8), + ), + # ---- the gated decode / verify site, one cell per 6 / 1 launch bucket ------------- + Cell( + why="tep4 gated decode 1 row in a CUDA graph: q and the gate off the 17-head " + "(4352-wide) fused row, 64-way split-K + gated merge with PDL, captured, refilled " + "and replayed", + spec=_default( + _spread(1, 1, 9215), _TP4, _SPLIT_K, splits=64, pdl=True, capture=True, **_TP4_ROW + ), + ), + Cell( + why="tep4 gated decode 3 rows (PDL's 2..4-row branch): 64-way split-K + gated merge " + "with PDL, 17-head fused row", + spec=_default(_spread(3, 1, 9215), _TP4, _SPLIT_K, splits=64, pdl=True, **_TP4_ROW), + ), + Cell( + why="tep4 gated decode 16 rows (the 5..31-row branch): 32-way split-K + gated merge, " + "17-head fused row", + spec=_default(_spread(16, 1, 9215), _TP4, _SPLIT_K, splits=32, **_TP4_ROW), + ), + Cell( + why="tep4 gated decode 128 x 1 row in a CUDA graph (the largest captured batch): " + "8-way split-K + gated merge, 17-head fused row, captured, refilled and replayed", + spec=_default( + _spread(128, 1, 9215), _TP4, _SPLIT_K, splits=8, capture=True, **_TP4_ROW + ), + ), + Cell( + why="tep4 MTP3 gated verify 128 x 4 rows = 512 (the 257..512-row branch): 4-way " + "split-K + gated merge, [128, 516] table, rows to position 16383, 17-head fused row", + spec=_default( + _spread(128, 4, 16380), + _TP4_MTP, + _SPLIT_K, + splits=4, + rows_per_request=4, + **_TP4_ROW, + ), + ), + Cell( + why="dep4 kv_fp8 context 8192 rows from position 0 with the prefill pair: 24 q / 2 " + "kv heads over an e4m3 pool on the prefill kernel, [32, 292] table", + spec=_default(((0, 8192),), _DEP4_FP8, _PREFILL, prefill_pair=True), + ), + Cell( + why="tep4 8192-row context step (max_num_tokens) with the prefill pair, q off the " + "17-head fused row: a context request 0..8064 and 127 generation rows to position " + "9215, all on the prefill kernel", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP4, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=17, + ), + ), + # ---- this target's tp2 / tep2 ranks: 12 / 1 heads off the 29-head fused row -------- + Cell( + why="tp2/tep2 gated decode 1 row at position 9215 in a CUDA graph: 12 q / 1 kv heads, " + "q and the gate off the 29-head (7424-wide) fused row, 64-way split-K + gated merge " + "with PDL, [128, 292] table, captured, refilled and replayed", + spec=_default( + _spread(1, 1, 9215), _TP2, _SPLIT_K, splits=64, pdl=True, capture=True, **_TP2_ROW + ), + ), + Cell( + why="tp2/tep2 8192-row step (max_num_tokens) with the prefill pair, q off the 29-head " + "fused row: a context request 0..8064 and 127 generation rows to position 9215, all " + "on the prefill kernel, 12 / 1 heads", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP2, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=29, + ), + ), + # ---- this target's tp1 / dep2 ranks: 24 / 2 heads off the 55-head fused row -------- + Cell( + why="tp1/dep2 gated decode 128 x 1 row in a CUDA graph: 24 q / 2 kv heads, q and the " + "gate off the 55-head (14080-wide) fused row, 8-way split-K + gated merge, [128, 292] " + "table", + spec=_default( + _spread(128, 1, 9215), _TP1, _SPLIT_K, splits=8, capture=True, **_DEP4_ROW + ), + ), + Cell( + why="tp1/dep2 MTP3 verify 128 x 4 = 512 rows in a CUDA graph: at 24 / 2 heads that is " + "1024 programs, so one split and no merge, and the gate off the 55-head row cannot " + "ride it (fuses_output_gate False); q off that row, rows to position 9215", + spec=_default( + _spread(128, 4, 9212), + _TP1, + ("splitk",), + splits=1, + capture=True, + rows_per_request=4, + gate_fused=False, + **_DEP4_ROW, + ), + ), + Cell( + why="kv_fp8 gated decode 128 x 1 row in a CUDA graph: 24 / 2 heads over an e4m3 pool, " + "8-way split-K + gated merge, 55-head fused row, [128, 292] table", + spec=_default( + _spread(128, 1, 9215), _TP1_FP8, _SPLIT_K, splits=8, capture=True, **_DEP4_ROW + ), + ), + Cell( + why="tp1/dep2 8192-row step (max_num_tokens) with the prefill pair, q off the 55-head " + "fused row: a context request 0..8064 and 127 generation rows to position 9215, all " + "on the prefill kernel, 24 / 2 heads, [128, 292] table", + spec=_default( + ((0, 8065),) + _spread(127, 1, 9215), + _TP1, + _PREFILL, + prefill_pair=True, + q_layout="fused_row", + row_heads=55, + ), + ), + ) + + note = """ + Source: `_qsa_prefill_pair_stable_kernel`, `_qsa_splitk_stable_kernel` and + `_qsa_merge_stable_kernel` in `qsa_triton_kernels__stable_jit` (Triton JIT), copies of the + shared QSA kernels with pool and table extents, row count and metadata alignment + unspecialized: one compile per call site layout, pool dtype and launch bucket + (`splitk_launch_config`), whatever the engine's pool or batch. + + The target passes `qsa_block_table` (physical page ids), `qsa_expand_block_indices`' + output as `selected_tokens` (logical positions, `-1`-padded) and the layer's role planes. + + Preconditions: + - whenever the prefill kernel can run, each selection row is packed at the front with + exactly the bound's count of valid entries (`qsa_topk_select` + expansion satisfy it); + passed because some rows are context, the pair puts the generation rows on the + prefill kernel too. + - an fp8 pool is read at unit scale (there is no KV scale argument). + - `D` a power of two; `Hq % Hkv == 0`. + Inputs are validated loudly (`ValueError`), so `is_valid` adds nothing. Page ids are + widened to int64 before the stride product (the trunk cells straddle 2^31 elements). + The 1-row and 128-row decodes and the one-split verify are captured in a CUDA graph and + replayed over inputs refilled in place. + + Not certified: head geometries other than 24/2, 12/1 and 6/1 x 256; `tokens_per_block` + other than 32; more than 8192 rows; `rows == 0`; capture of the prefill kernel and of the + 1024-row one-split verify at 6/1; gated decode at 24/2 below 128 rows; a trunk with the MTP + draft layer appended (26 role pages); an fp8 pool at 6/1 or 12/1; positions past 16383; + rows whose selection resolves no token; an fp16 pool or q; int64 request indices; the + engines' own pool page counts (the cells' pools hold 5592 to 33167 pages). + """ + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: Optional[torch.Tensor] = None, + compress_ratio: Optional[int] = None, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return _triton_qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + @staticmethod + def fuses_output_gate( + q: torch.Tensor, + k_cache: torch.Tensor, + selected_tokens: torch.Tensor, + output_gate: torch.Tensor, + ) -> bool: + """Whether a call with these operands and no prefill pair carries `output_gate`.""" + return _can_fuse_qsa_splitk_output_gate(q, k_cache, selected_tokens, output_gate) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: Optional[torch.Tensor] = None, + compress_ratio: Optional[int] = None, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """fp32 sparse GQA over every resident selected token, rounded once; with + `output_gate`, then gated in fp32 and rounded again. `query_positions` and + `compress_ratio` are ignored (on a prefix-packed selection the bound reads exactly the + valid entries).""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads = k_cache.shape[0], k_cache.shape[1] + groups = num_q_heads // num_kv_heads + num_requests, table_width = block_table.shape + requests = request_indices.long() + out = torch.empty(rows, num_q_heads, head_dim, dtype=torch.float32, device=q.device) + for start in range(0, rows, 32): + stop = min(start + 32, rows) + chunk = selected_tokens[start:stop].long() + logical_page = chunk.clamp_min(0) // tokens_per_block + in_page = chunk.clamp_min(0) % tokens_per_block + request = requests[start:stop] + owned = (request >= 0) & (request < num_requests) + physical = block_table[ + request.clamp(0, num_requests - 1)[:, None].expand_as(logical_page), + logical_page.clamp(max=table_width - 1), + ].long() + resident = ( + (chunk >= 0) + & owned[:, None] + & (logical_page < table_width) + & (physical >= 0) + & (physical < num_pages) + ) + physical = physical.clamp(0, num_pages - 1) + # What a non-resident column gathers (a clamped page) is not part of the answer: + # zeroed, so a pool holding NaN outside the mapped pages cannot reach the sum. + absent = ~resident[:, :, None, None] + keys = k_cache[physical, :, in_page, :].float().masked_fill(absent, 0.0) + values = v_cache[physical, :, in_page, :].float().masked_fill(absent, 0.0) + grouped = q[start:stop].reshape(-1, num_kv_heads, groups, head_dim).float() + scores = torch.einsum("bhgd,bkhd->bhgk", grouped, keys) * softmax_scale + scores = scores.masked_fill(~resident[:, None, None, :], float("-inf")) + # A row with nothing resident softmaxes over all -inf: NaN, i.e. zeros. + weights = torch.nan_to_num(torch.softmax(scores, dim=-1), nan=0.0) + pooled = torch.einsum("bhgk,bkhd->bhgd", weights, values) + out[start:stop] = pooled.reshape(stop - start, num_q_heads, head_dim) + if output_gate is None: + return out.to(q.dtype) + gate = output_gate.reshape(rows, num_q_heads, head_dim).float() + return (out.to(q.dtype).float() * torch.sigmoid(gate)).to(q.dtype) + + +qsa_paged_sparse_gqa = QsaPagedSparseGqa() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py new file mode 100644 index 000000000000..5a1385019612 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_grouped_cute__decode.py @@ -0,0 +1,2817 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA over an e4m3 paged K/V pool for MTP verify rows, request-grouped: CuTe DSL +(tcgen05) split-K over the union of a request's rows' selections, then a row merge. + +Derived from `qsa_paged_sparse_gqa_nvfp4_verify` (same warp roles, tcgen05 pipeline, softmax, +epilogue and row merge) with an exact e4m3 -> f16 widening, and the union built in a shared hash +table instead of a token bitmap, so any block-table width is served. + +One CTA serves one (group of T = `rows_per_request` rows, KV head, split): 128 MMA rows, row +m = t * G + h (verify row t, q head h of the KV head's G; rows past T * G idle), over the group's +union selection, so each selected K/V token is loaded and widened once for the group's rows. The +union lists the first row's four-token groups in column order, then each later row's new ones +(deterministic whatever order the hash atomics ran in); per (tile, verify row) a 64-bit selection +word masks positions the row did not select to -inf, so the softmax offset is the row's own max +and causality is the selection's. Splits take interleaved tiles of the union. + +Roles (11 warps): 0-3 softmax and epilogue (one thread an MMA row), 4 the tcgen05 MMAs, 5 / 6 the +K / V bulk copies into the raw ring, 7-10 the e4m3 -> f16 widening. Tensor memory: O in columns +0-255, S/P buffers at 256 and 320, Q at 384. The hash lives in raw slot 0 and the staged q rows +and the union list in raw slot 1 until the tiles start: one CTA an SM. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple, Optional + +import cutlass +import cutlass.cute as cute +import cutlass.cute.nvgpu.tcgen05 as tcgen05 +import cutlass.utils.blackwell_helpers as sm100_utils +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass._mlir.dialects import nvvm as _nvvm +from cutlass.cute.nvgpu import OperandMajorMode +from cutlass.cute.runtime import from_dlpack +from cutlass.cute.typing import Float32, Int32, Int64 +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from .qsa_sparse_gqa_splitk_cute__decode import ( + MAX_SPLITS, + _asm, + _bf2_hi, + _bf2_lo, + _bf2_pack, + _bf2_to_f2, + _bits_f32, + _cp16z, + _e4m3x4, + _f32_bits, + _ir, + _ldg1, + _ldg4, + _lds1, + _lds4, + _ptr, + _round_bf16, + _stg1, + _sts1, + _sts4, + qsa_sparse_gqa_splitk_cute, +) + +try: + from cutlass.memory import SmemAllocator +except ImportError: # DSL releases before the memory namespace split + from cutlass.utils import SmemAllocator + +F16 = cutlass.Float16 +LOG2E = 1.4426950408889634 +HEAD_DIM = 256 +#: MMA rows a CTA (T verify rows x the KV head's q heads) and positions a tile. +BM = 128 +BN = 64 +#: Four-token groups a tile. +GPT = BN // 4 +#: Bytes of a raw e4m3 K or V row in shared memory: unpadded (a four-token group lands as one +#: 1 KB bulk copy; the widening rotates its reads instead). +RAW_LD = 256 +RAW_K = BN * RAW_LD +#: One raw slot: K rows, V rows. +RAW_BUF = 2 * RAW_K +#: Bytes of one f16 K or V slot (64 positions x 256 dims). +F16_SLOT = BN * HEAD_DIM * 2 +#: The rings: raw e4m3 tiles in flight, f16 K slots, f16 V slots (32 KB each, 192 KB in all). +#: The gather is bound by the bytes in flight an SM, so the raw ring takes the room; one f16 slot +#: each suffices once the widening runs a tile ahead of the MMA (fastest split measured on GB300). +RAW_SLOTS = 4 +K_SLOTS = 1 +V_SLOTS = 1 +#: Shared memory a CTA may claim (227 KB), less barriers, scan scratch and alignment. +SMEM_BUDGET = 232448 - 4096 +#: A row's reference max moves only past this many log2 units (P <= 2^8, exact in f16 range). +RESCALE_TH = 8.0 +SOFT_WARPS = 4 +MMA_WARP = 4 +LOAD_WARP0 = 5 +#: Warps issuing the bulk copies: warp 5 the K groups, warp 6 the V groups (lanes 0-15, one group +#: a lane); a warp's bulk copies issue one lane after another, so two warps halve the issue time. +LOAD_WARPS = 2 +DQ_WARP0 = 7 +DQ_WARPS = 4 +THREADS = 32 * (DQ_WARP0 + DQ_WARPS) +TMEM_O = 0 +TMEM_S0 = 256 +TMEM_Q = 384 +TMEM_COLS = 512 +CHUNK_COLS = 32 +#: Bytes of a staged q row (bf16, 512 + 16 of padding). +Q_LD = 528 +#: Hash slots of the union (keys, then masks, in raw slot 0): twice the most groups T = 4 rows +#: name, so a probe chain stays short. +HASH = 4096 +#: Most rows a group (the hash mask holds four bits a row). +MAX_RPR = 4 +#: Most splits of one (group, KV head) union. +MAX_PLAN_SPLITS = 32 +#: Most splits of a last-wave unit (tail mode). +MAX_TAIL_SPLITS = 4 +#: Fewer request groups than this take the per-row kernel. +MIN_GROUPS = 8 +#: Scan scratch words: (row, column round) x warp counts. +SCAN_WORDS = 256 +# --- inline PTX beyond the per-row kernel's --- + + +@dsl_user_op +def _popc(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(T.i32(), [_ir(x, cutlass.Int32, loc, ip)], "popc.b32 $0, $1;", "=r,r", loc, ip) + ) + + +@dsl_user_op +def _ctz(x, *, loc=None, ip=None): + """Trailing zeros of a nonzero word.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b32 t; brev.b32 t, $1; clz.b32 $0, t; }", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _ballot(x, *, loc=None, ip=None): + """`vote.sync.ballot` of `x != 0` over the full warp.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .pred p; setp.ne.u32 p, $1, 0; vote.sync.ballot.b32 $0, p, 0xffffffff; }", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _sts_u16(saddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; cvt.u16.u32 h, $2; st.shared.u16 [$1], h; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds_u16(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; ld.shared.u16 h, [$1]; cvt.u32.u16 $0, h; }", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _cas_shared(saddr, cmp, val, *, loc=None, ip=None): + """`atom.shared.cas.b32`: the old word at `saddr`.""" + return cutlass.Int32( + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(cmp, cutlass.Int32, loc, ip), + _ir(val, cutlass.Int32, loc, ip), + ], + "atom.shared.cas.b32 $0, [$1], $2, $3;", + "=r,r,r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _red_or_shared(saddr, v, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "{ red.shared.or.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s(saddr, gaddr, nbytes, mbar, *, loc=None, ip=None): + """`cp.async.bulk` global -> this CTA's shared memory, completing `nbytes` on `mbar`.""" + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + ], + "{ cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], $3, [$4]; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _arrive_expect_tx(mbar, nbytes, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(mbar, cutlass.Int32, loc, ip), _ir(nbytes, cutlass.Int32, loc, ip)], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg4(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(v, cutlass.Int32, loc, ip) for v in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg4f(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(v, cutlass.Float32, loc, ip) for v in (a, b, c, d)], + "{ st.global.v4.f32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,f,f,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg2(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v2.b32 {$0, $1}, [$2];", + "=r,=r,l", + loc, + ip, + side=True, + ) + return [cutlass.Int32(llvm.extractvalue(T.i32(), res, [i], loc=loc, ip=ip)) for i in range(2)] + + +@dsl_user_op +def _stg2(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Int32, loc, ip), + _ir(b, cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _red_shared(op, saddr, v, *, loc=None, ip=None): + """red.shared..s32 (op: add, min, max).""" + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "{ red.shared.%s.s32 [$1], $2; mov.u32 $0, 0; }" % op, + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg1cg(gaddr, *, loc=None, ip=None): + """4-byte global load at L2 (`.cg`): data another CTA of this launch wrote.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.cg.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg4cg(gaddr, *, loc=None, ip=None): + """16-byte global load at L2 (`.cg`): data another CTA of this launch wrote.""" + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.cg.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return [cutlass.Int32(llvm.extractvalue(T.i32(), res, [i], loc=loc, ip=ip)) for i in range(4)] + + +@dsl_user_op +def _atom_add_acqrel_g(gaddr, v, *, loc=None, ip=None): + """atomicAdd on a global int32, acq_rel at GPU scope (after a CTA barrier it releases every + thread's prior stores, CUTLASS's generic-barrier pattern); returns the old value.""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(v, cutlass.Int32, loc, ip)], + "atom.acq_rel.gpu.global.add.u32 $0, [$1], $2;", + "=r,l,r", + loc, + ip, + side=True, + ) + ) + + +def _spread4(nib): + """Bits 0..3 of `nib` to bits 0, 8, 16, 24.""" + return (nib & 1) | ((nib & 2) << 7) | ((nib & 4) << 14) | ((nib & 8) << 21) + + +@dsl_user_op +def _tc_fence_before(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.BEFORE_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def _tc_fence_after(*, loc=None, ip=None): + _nvvm.tcgen05_fence(_nvvm.Tcgen05FenceKind.AFTER_THREAD_SYNC, loc=loc, ip=ip) + + +#: Swizzle: B -> the descriptor's layout type (none, 32B, 64B, 128B). +_SW_TYPES = {0: 0, 1: 6, 2: 4, 3: 2} +#: Layout type -> swizzle atom size along MN (16-byte units). +_SW_ATOM_MN = {0: 1, 6: 2, 4: 4, 2: 8} + + +def _instr_desc(op) -> int: + """The 32-bit instruction descriptor of an f16/bf16 x f16/bf16 -> f32 tcgen05 MMA op.""" + fmt = {cutlass.Float16: 0, cutlass.BFloat16: 1} + a_major = 0 if op.a_major_mode == tcgen05.OperandMajorMode.K else 1 + b_major = 0 if op.b_major_mode == tcgen05.OperandMajorMode.K else 1 + m, n = op.shape_mnk[0], op.shape_mnk[1] + assert op.acc_dtype is cutlass.Float32 and m in (64, 128) and n % 8 == 0 and 8 <= n <= 256 + d = 0 + d |= 1 << 4 # c_format f32 + d |= fmt[op.a_dtype] << 7 + d |= fmt[op.b_dtype] << 10 + d |= a_major << 15 + d |= b_major << 16 + d |= (n >> 3) << 17 + d |= (m >> 4) << 24 + return d + + +def _smem_desc_base(layout_u128, swizzle, k_major: bool) -> int: + """The 64-bit shared-memory descriptor without its start address, for an operand whose + (MN, K) layout `layout_u128` is in 16-byte units (CUTLASS `mma_sm100_desc.hpp`).""" + assert swizzle.num_base == 4 and swizzle.num_shift == 3 + ltype = _SW_TYPES[swizzle.num_bits] + atom_mn = _SW_ATOM_MN[ltype] + if not k_major: + canon = cute.logical_divide(layout_u128, (atom_mn, 8)) + s00 = canon.stride[0][0] + s10 = canon.stride[1][0] + assert (ltype == 0 or s00 == 1) and s10 == atom_mn, "not a canonical UMMA MN layout" + s01, s11 = canon.stride[0][1], canon.stride[1][1] + sbo, lbo = (s01, s11) if ltype == 0 else (s11, s01) + else: + canon = cute.logical_divide(layout_u128, (8, 2)) + s00 = canon.stride[0][0] + s10 = canon.stride[1][0] + assert s00 == atom_mn and (ltype == 0 or s10 == 1), "not a canonical UMMA K layout" + sbo, lbo = canon.stride[0][1], s10 + d = 0 + d |= (lbo & 0x3FFF) << 16 + d |= (sbo & 0x3FFF) << 32 + d |= 1 << 46 # descriptor version + d |= (ltype & 0x7) << 61 + return d + + +def _gemm_ts(op, acc_tmem, a_tmem, tCrA, tCrB, sB, zero_init: bool): + """acc (+)= A B over every K step: A in tensor memory at `a_tmem` (fragment `tCrA`), B the + shared-memory operand `sB` (fragment `tCrB`, one stage). Issued by one elected thread of + the calling warp; the K loop unrolls into one PTX block (precomputed descriptors).""" + idesc = _instr_desc(op) + k_major_b = op.b_major_mode == tcgen05.OperandMajorMode.K + base_b = _smem_desc_base( + cute.recast_layout(128, op.b_dtype.width, sB.layout[0]), + sB.iterator.type.swizzle_type, + k_major_b, + ) + b_lo = base_b & 0xFFFFFFFF + b_hi = (base_b >> 32) & 0xFFFFFFFF + a_layout = cute.recast_layout(32, tCrA.element_type.width, tCrA.layout) + n_k = cute.size(tCrA.shape[2]) + off_a = [cute.crd2idx((0, 0, k), a_layout) for k in range(n_k)] + off_b = [cute.crd2idx((0, 0, k), tCrB.layout) for k in range(n_k)] + start_b = Int32(b_lo) | Int32((sB[None, None, 0].iterator.toint() & 0x3FFFF) >> 4) + body = "{\n\t.reg .pred leader;\n\t.reg .b32 db_lo;\n\t.reg .b64 db;\n\telect.sync _|leader, -1;\n\t" + for k in range(n_k): + acc = "0" if (zero_init and k == 0) else "1" + body += ( + f"add.u32 db_lo, $2, {hex(off_b[k])};\n\t" + f"mov.b64 db, {{db_lo, {hex(b_hi)}}};\n\t" + f"@leader tcgen05.mma.cta_group::1.kind::f16 [$0], [$1 + {hex(off_a[k])}], db, " + f"{hex(idesc)}, {acc};\n\t" + ) + body += "}\n" + llvm.inline_asm( + None, + [ + Int32(cute.arch.make_warp_uniform(acc_tmem)).ir_value(), + Int32(cute.arch.make_warp_uniform(a_tmem)).ir_value(), + Int32(cute.arch.make_warp_uniform(start_b)).ir_value(), + ], + body, + "r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + ) + + +# --- the grouped kernel --- + + +class _Layout(NamedTuple): + """The shared-memory plan of one compile: the union's group capacity, local tiles a CTA may + own in one pass, the union-list offset in raw slot 1 and the table size in int32 words.""" + + ug_max: int + tmax: int + u_ug: int + tab_words: int + + +def _layout(rpr: int, group: int, width: int, splits: int, rings=None): + """The kernel's shared-memory plan for `rpr` rows a group over a `width`-column expanded + selection, or None where it does not fit: the hash (keys and masks) fills raw slot 0, the + staged q rows and the union list share raw slot 1.""" + raw_n, ks_n, vs_n = rings or (RAW_SLOTS, K_SLOTS, V_SLOTS) + if rpr < 1 or rpr > MAX_RPR or rpr * group > BM: + return None + # an expanded selection names at most ceil(width / 4) four-token groups a row + ug_max = rpr * -(-width // 4) + tmax = -(-(-(-ug_max // GPT)) // splits) + # the union list: raw slot 2 where the ring has one, else after the q rows in raw slot 1 + u_ug = RAW_BUF if raw_n >= 3 else -(-(rpr * group * Q_LD) // 16) * 16 + tab_words = 2 * tmax * GPT + 2 * tmax * rpr + fits = ( + 2 * HASH * 4 <= RAW_BUF + and 5 * ug_max <= 3 * HASH + and rpr * group * Q_LD <= RAW_BUF + and u_ug + ug_max * 2 <= RAW_BUF * (2 if raw_n >= 3 else 1) + and rpr * -(-(-(-width // 4)) // THREADS) <= SCAN_WORDS // (THREADS // 32) + and 4 * tab_words + 4 * SCAN_WORDS + raw_n * RAW_BUF + (ks_n + vs_n) * F16_SLOT + <= SMEM_BUDGET + and raw_n >= 2 + and rpr * group * Q_LD <= ks_n * F16_SLOT + ) + return _Layout(ug_max, tmax, u_ug, tab_words) if fits else None + + +def _sigmoid_fast(x): + """1 / (1 + 2^(-x log2 e)) on ex2.approx and rcp.approx (relative error ~1e-6).""" + return cute.arch.rcp_approx(1.0 + cute.math.exp2(-x * LOG2E, fastmath=True)) + + +def _tree_fmax(vals): + """Max of a short list, pairwise (traced as Python).""" + while len(vals) > 1: + vals = [ + cute.arch.fmax(vals[j], vals[j + 1]) if j + 1 < len(vals) else vals[j] + for j in range(0, len(vals), 2) + ] + return vals[0] + + +def _tree_sum(vals): + """Sum of a short list, pairwise (traced as Python).""" + while len(vals) > 1: + vals = [ + vals[j] + vals[j + 1] if j + 1 < len(vals) else vals[j] for j in range(0, len(vals), 2) + ] + return vals[0] + + +def _phase(g, n): + """Parity of use `g` of a ring of `n` slots.""" + return (g // n) % 2 + + +class _GroupedKernel: + def __init__( + self, + *, + rpr, + group, + kv_heads, + width, + tpb, + splits, + tail, + scale, + num_pages, + bt_width, + num_requests, + k_strides, + q_strides, + req64, + gated, + pdl, + bal=False, + bal_ctas=0, + ): + self.RS, self.KS, self.VS = RAW_SLOTS, K_SLOTS, V_SLOTS + lay = _layout( + rpr, group, width, 1 if (tail or bal) else splits, (self.RS, self.KS, self.VS) + ) + assert lay is not None, "no shared-memory plan for this geometry (see `_layout`)" + assert tpb & (tpb - 1) == 0 and tpb % 4 == 0, "a power-of-two page of whole groups" + self.T = rpr + self.group, self.kvh, self.width, self.tpb = group, kv_heads, width, tpb + self.splits = splits + #: tail mode: CTAs below `n_whole` serve whole units, the rest `splits`-way splits + self.tail = tail + #: balanced mode: each CTA's unit and split from `_bal_schedule`; split units merge in + #: `fixup` + self.bal = bal + self.bal_ctas = bal_ctas + #: histogram bins of the schedule: a unit's union tiles are at most rpr * ceil(width / 4) + #: groups / GPT + self.hmax = -(-(-(-(rpr * -(-width // 4)) // GPT) + 1) // 32) * 32 + self.scale_log2 = scale * LOG2E + self.num_pages, self.bt_width, self.num_requests = num_pages, bt_width, num_requests + self.ks, self.qs = k_strides, q_strides + self.req64, self.gated, self.pdl = req64, gated, pdl + self.heads = group * kv_heads + self.rows_m = rpr * group + #: softmax warps holding live MMA rows (the rest only keep the barriers' counts) + self.live_warps = -(-self.rows_m // 32) + self.ug_max, self.tmax = lay.ug_max, lay.tmax + self.raw_bytes = self.RS * RAW_BUF + # union scratch: the hash in raw slot 0, the q rows then the union list in raw slot 1 + self.u_hk = 0 + self.u_hm = HASH * 4 + self.u_q = RAW_BUF + self.u_ug = RAW_BUF + lay.u_ug + # tables (int32 words). Per group: its first token's pool row (page * tpb + slot, -1 + # when it does not resolve) and its (token, row) selection bytes; per (tile, row): the + # 64-bit selection word. + self.t_tok = 0 + self.t_gmask = self.tmax * GPT + self.t_bits = 2 * self.tmax * GPT + self.tab_words = lay.tab_words + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + q_ptr: cute.Pointer, + k_ptr: cute.Pointer, + v_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_ptr: cute.Pointer, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + gate_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + n_whole: cutlass.Int32, + tail_row0: cutlass.Int32, + n_ctas: cutlass.Int32, + tab_ptr: cute.Pointer, + cnt_ptr: cute.Pointer, + est_ptr: cute.Pointer, + stream, + ): + def qk(): + return sm100_utils.make_trivial_tiled_mma( + F16, + F16, + OperandMajorMode.K, + OperandMajorMode.K, + Float32, + tcgen05.CtaGroup.ONE, + (BM, BN), + tcgen05.OperandSource.TMEM, + ) + + def pv(): + return sm100_utils.make_trivial_tiled_mma( + F16, + F16, + OperandMajorMode.K, + OperandMajorMode.MN, + Float32, + tcgen05.CtaGroup.ONE, + (BM, HEAD_DIM), + tcgen05.OperandSource.TMEM, + ) + + qk_mma, qk_mma_acc, pv_mma_acc = qk(), qk(), pv() + q_layout = sm100_utils.make_smem_layout_a(qk_mma, (BM, BN, HEAD_DIM), F16, 1) + k_layout = sm100_utils.make_smem_layout_b(qk_mma, (BM, BN, HEAD_DIM), F16, self.KS) + vk_layout = sm100_utils.make_smem_layout_b(qk_mma, (BM, BN, HEAD_DIM), F16, self.VS) + v_layout = sm100_utils.make_smem_layout_b(pv_mma_acc, (BM, HEAD_DIM, BN), F16, self.VS) + p_layout = sm100_utils.make_smem_layout_a(pv_mma_acc, (BM, HEAD_DIM, BN), F16, 1) + + @cute.struct + class SharedStorage: + q_full: cute.struct.MemRange[Int64, 1] + raw_full: cute.struct.MemRange[Int64, self.RS] + raw_empty: cute.struct.MemRange[Int64, self.RS] + k_full: cute.struct.MemRange[Int64, self.KS] + k_empty: cute.struct.MemRange[Int64, self.KS] + v_full: cute.struct.MemRange[Int64, self.VS] + v_empty: cute.struct.MemRange[Int64, self.VS] + s_full: cute.struct.MemRange[Int64, 2] + p_full: cute.struct.MemRange[Int64, 2] + o_full: cute.struct.MemRange[Int64, 1] + o_final: cute.struct.MemRange[Int64, 1] + tmem_holding: Int32 + scan: cute.struct.MemRange[Int32, SCAN_WORDS] + tab: cute.struct.Align[cute.struct.MemRange[Int32, self.tab_words], 16] + raw: cute.struct.Align[cute.struct.MemRange[cutlass.Int8, self.raw_bytes], 128] + sK: cute.struct.Align[cute.struct.MemRange[F16, cute.cosize(k_layout)], 1024] + sV: cute.struct.Align[cute.struct.MemRange[F16, cute.cosize(vk_layout)], 1024] + + self.shared_storage = SharedStorage + self.vk_layout = vk_layout + bt = cute.make_tensor( + bt_ptr, cute.make_layout((self.num_requests, self.bt_width), stride=(self.bt_width, 1)) + ) + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel( + qk_mma, + qk_mma_acc, + pv_mma_acc, + q_layout, + k_layout, + vk_layout, + v_layout, + p_layout, + sel_t.iterator.toint(), + q_ptr.toint(), + k_ptr.toint(), + v_ptr.toint(), + bt, + req, + out_ptr.toint(), + part_ptr.toint(), + lse_ptr.toint(), + gate_ptr.toint(), + rows, + sel_stride, + gate_stride, + gate_hs, + n_whole, + tail_row0, + tab_ptr.toint(), + cnt_ptr.toint(), + est_ptr.toint(), + ).launch( + grid=[n_ctas, 1, 1], + block=[THREADS, 1, 1], + stream=stream, + min_blocks_per_mp=1, + use_pdl=self.pdl, + ) + + @cute.jit + def _bal_schedule(self, tidx, warp, lane, bidx, rows, est_base, tab_base, s_w): + """Balanced mode, every thread: this CTA's (unit, split, unit's CTAs, unit's first CTA): + unit u takes ceil(e_u / L) CTAs, L the least load in [ceil(max e / MAX_BAL_SPLITS), max e] + whose counts fit `bal_ctas`, units on consecutive CTAs; unit -1 past the plan. Scratch + `s_w` is raw slot 0 (free until the first union).""" + rpr: cutlass.Constexpr = self.T + kvh_n: cutlass.Constexpr = self.kvh + n_ctas: cutlass.Constexpr = self.bal_ctas + hmax: cutlass.Constexpr = self.hmax + nw: cutlass.Constexpr = -(-n_ctas // 32) + s_res = s_w # 16-byte aligned (raw slot 0) + s_emax = s_w + 16 + s_l = s_w + 20 + s_tot = s_w + 32 + s_h = s_w + 32 + 32 * 4 + units = ((rows + rpr - 1) // rpr) * kvh_n + for i in cutlass.range_constexpr(-(-hmax // THREADS)): + k = tidx + i * THREADS + if k < hmax: + _sts1(s_h + k * 4, cutlass.Int32(0)) + if tidx == 0: + _sts1(s_emax, cutlass.Int32(1)) + _sts1(s_l, cutlass.Int32(hmax)) + _sts4(s_res, cutlass.Int32(-1), cutlass.Int32(0), cutlass.Int32(1), cutlass.Int32(0)) + live_u = tidx < units + e = cutlass.Int32(0) + if live_u: + e = cutlass.min( + cutlass.max(_ldg1(est_base + cutlass.Int64(tidx // kvh_n) * 4), cutlass.Int32(1)), + cutlass.Int32(hmax - 1), + ) + cute.arch.sync_threads() + if live_u: + _red_shared("add", s_h + e * 4, cutlass.Int32(1)) + _red_shared("max", s_emax, e) + cute.arch.sync_threads() + emax = _lds1(s_emax) + lo = cutlass.max((emax + MAX_BAL_SPLITS - 1) // MAX_BAL_SPLITS, cutlass.Int32(1)) + # thread t tests L = t + 1: sum over the histogram of h[x] * ceil(x / L), ceil by a + # 16-bit reciprocal (exact: numerators below 2^9, L below 2^8) + cand = tidx + 1 + recip = cutlass.Int32(65536) // cand + 1 + cnt = cutlass.Int32(0) + for x4 in cutlass.range_constexpr(hmax // 4): + h4 = _lds4(s_h + x4 * 16) + for k in cutlass.range_constexpr(4): + x = 4 * x4 + k + if const_expr(x > 0): + cnt = cnt + h4[k] * (((x + cand - 1) * recip) >> 16) + if (cand >= lo) & (cand <= emax) & (cnt <= n_ctas): + _red_shared("min", s_l, cand) + cute.arch.sync_threads() + load = _lds1(s_l) + n = cutlass.Int32(0) + if live_u: + n = (e + load - 1) // load + # exclusive scan of n over units (threads 0 .. n_ctas - 1, warp scans + warp totals) + inc = n + for i in cutlass.range_constexpr(5): + up = cute.arch.shuffle_sync_up(inc, offset=1 << i) + if lane >= (1 << i): + inc = inc + up + if lane == 31: + _sts1(s_tot + warp * 4, inc) + cute.arch.sync_threads() + before = cutlass.Int32(0) + for w in cutlass.range_constexpr(nw): + if warp > w: + before = before + _lds1(s_tot + w * 4) + base = before + inc - n + if live_u & (bidx >= base) & (bidx < base + n): + _sts4(s_res, tidx, bidx - base, n, base) + cute.arch.sync_threads() + res = _lds4(s_res) + if tidx == 0: + _stg4(tab_base + cutlass.Int64(bidx) * 16, res[0], res[1], res[2], res[3]) + cute.arch.sync_threads() + return res + + # ---- pieces every role runs (every thread of the CTA, barrier 0) ----------------------- + + @cute.jit + def _pass_rows(self, rq: cute.Tensor, handled): + lead = cutlass.Int32(-1) + for t in tuple(range(self.T - 1, -1, -1)): + lead = cutlass.Int32(cutlass.select_(((handled >> t) & 1) == 0, rq[t], lead)) + act = cutlass.Int32(0) + for t in cutlass.range_constexpr(self.T): + act = act | cutlass.Int32( + cutlass.select_( + (((handled >> t) & 1) == 0) & (rq[t] == lead), + cutlass.Int32(1 << t), + cutlass.Int32(0), + ) + ) + return lead, act + + @cute.jit + def _hash_insert(self, s_hk, s_hm, gid, bits): + """Insert group `gid` (linear probing over `HASH` slots; a key never moves once + placed) and OR `bits` into its mask; returns the slot.""" + slot = ((gid * cutlass.Int32(-1640531535)) >> 20) & (HASH - 1) + probing = cutlass.Int32(1) + while probing != 0: + old = _cas_shared(s_hk + slot * 4, cutlass.Int32(-1), gid) + if (old == -1) | (old == gid): + probing = cutlass.Int32(0) + else: + slot = (slot + 1) & (HASH - 1) + _red_or_shared(s_hm + slot * 4, bits) + return slot + + @cute.jit + def _union( + self, act, r0, rows, sel_base, sel_stride, tidx, warp, lane, s_hk, s_hm, s_scan, s_ug + ): + """The union of the pass's rows' groups: hash slots of the owned groups, in (row, + column) order, at `s_ug`; returns their count.""" + rpr: cutlass.Constexpr = self.T + width: cutlass.Constexpr = self.width + n_q: cutlass.Constexpr = -(-width // 4) # four-column quads a row + n_i: cutlass.Constexpr = -(-n_q // THREADS) + nwarps: cutlass.Constexpr = THREADS // 32 + # clear the hash: keys -1, masks 0 + for i in cutlass.range_constexpr(-(-(HASH // 4) // THREADS)): + idx = tidx + i * THREADS + if idx < HASH // 4: + _sts4(s_hk + idx * 16, -1, -1, -1, -1) + _sts4(s_hm + idx * 16, 0, 0, 0, 0) + # thread: columns 4q .. 4q + 3 of each row, every load first + toks = [] + for t in cutlass.range_constexpr(rpr): + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + for i in cutlass.range_constexpr(n_i): + for u in cutlass.range_constexpr(4): + colc = cutlass.min(4 * (tidx + i * THREADS) + u, width - 1) + toks.append(_ldg1(row_sel + cutlass.Int64(colc * 4))) + cute.arch.sync_threads() + gids = [] + slots = [] + for t in cutlass.range_constexpr(rpr): + live_t = ((act >> t) & 1) != 0 + for i in cutlass.range_constexpr(n_i): + q4 = tidx + i * THREADS + gid = cutlass.Int32(-1) + bits = cutlass.Int32(0) + for u in cutlass.range_constexpr(4): + tok = toks[(t * n_i + i) * 4 + u] + ok = live_t & (4 * q4 + u < width) & (tok >= 0) + gid = cutlass.Int32(cutlass.select_(ok & (gid < 0), tok >> 2, gid)) + bits = bits | cutlass.Int32( + cutlass.select_(ok, cutlass.Int32(1) << (tok & 3), cutlass.Int32(0)) + ) + slot = cutlass.Int32(-1) + if gid >= 0: + slot = self._hash_insert(s_hk, s_hm, gid, bits << (4 * t)) + gids.append(gid) + slots.append(slot) + cute.arch.sync_threads() + # owner: the lowest row naming the group; counts per (row, round) and warp + owns = [] + for t in cutlass.range_constexpr(rpr): + for i in cutlass.range_constexpr(n_i): + k = t * n_i + i + own = cutlass.Int32(0) + if slots[k] >= 0: + msk = _lds1(s_hm + slots[k] * 4) + own = cutlass.Int32( + cutlass.select_((_ctz(msk) >> 2) == t, cutlass.Int32(1), cutlass.Int32(0)) + ) + owns.append(own) + ball = _ballot(own) + if lane == 0: + _sts1(s_scan + (k * nwarps + warp) * 4, _popc(ball)) + cute.arch.sync_threads() + # place: rounds in (row, round) order, warps within a round, lanes within a warp + lt_mask = (cutlass.Int32(1) << lane) - 1 + total = cutlass.Int32(0) + for k in cutlass.range_constexpr(rpr * n_i): + cnt_lane = cutlass.Int32(0) + if lane < nwarps: + cnt_lane = _lds1(s_scan + (k * nwarps + lane) * 4) + before_w = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, cnt_lane, cutlass.Int32(0))) + ) + round_tot = cute.arch.warp_reduction_sum(cnt_lane) + ball = _ballot(owns[k]) + pos = total + before_w + _popc(ball & lt_mask) + if owns[k] != 0: + if pos < self.ug_max: + _sts_u16(s_ug + pos * 2, slots[k]) + total = total + round_tot + cute.arch.sync_threads() + return cutlass.min(total, self.ug_max) + + @cute.jit + def _resolve(self, lead, split, nsplit, n_loc, n_ug, tidx, s_hk, s_hm, s_ug, s_tab, mBT): + """Every local tile's groups (pool row of the group's first token, (token, row) + selection bytes), then each (tile, verify row)'s 64-bit selection word.""" + rpr: cutlass.Constexpr = self.T + tmax: cutlass.Constexpr = self.tmax + tpb: cutlass.Constexpr = self.tpb + n_rs: cutlass.Constexpr = -(-(tmax * GPT) // THREADS) + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + s_bits = s_tab + self.t_bits * 4 + pages = [] + gids = [] + msks = [] + for i in cutlass.range_constexpr(n_rs): + slot = tidx + i * THREADS + kl = slot // GPT + ugi = (split + kl * nsplit) * GPT + slot % GPT + ok = (kl < n_loc) & (ugi < n_ug) + hs = cutlass.Int32(0) + if ok: + hs = _lds_u16(s_ug + cutlass.min(ugi, self.ug_max - 1) * 2) + gid = cutlass.Int32(cutlass.select_(ok, _lds1(s_hk + hs * 4), cutlass.Int32(0))) + msk = cutlass.Int32(cutlass.select_(ok, _lds1(s_hm + hs * 4), cutlass.Int32(0))) + col = cutlass.min(cutlass.max((gid * 4) // tpb, 0), self.bt_width - 1) + pg = mBT[cutlass.max(lead, 0), col] + pages.append(cutlass.Int32(cutlass.select_(ok, pg, cutlass.Int32(-1)))) + gids.append(gid) + msks.append(msk) + for i in cutlass.range_constexpr(n_rs): + slot = tidx + i * THREADS + page = pages[i] + gid = gids[i] + msk = msks[i] + mw = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + mw = mw | (_spread4((msk >> (4 * t)) & 0xF) << t) + okp = (page >= 0) & (page < self.num_pages) & (msk != 0) + tok = page * tpb + (gid * 4) % tpb + tok = cutlass.Int32(cutlass.select_(okp, tok, cutlass.Int32(-1))) + mw = cutlass.Int32(cutlass.select_(okp, mw, cutlass.Int32(0))) + if slot < tmax * GPT: + _sts1(s_tok + slot * 4, tok) + _sts1(s_gmask + slot * 4, mw) + cute.arch.sync_threads() + for i in cutlass.range_constexpr(-(-(tmax * rpr) // THREADS)): + w = tidx + i * THREADS + if w < tmax * rpr: + kl = w // rpr + t = w % rpr + lo = cutlass.Int32(0) + hi = cutlass.Int32(0) + for gq in cutlass.range_constexpr(GPT): + nb = (_lds1(s_gmask + (kl * GPT + gq) * 4) >> t) & 0x01010101 + nb = (nb | (nb >> 7) | (nb >> 14) | (nb >> 21)) & 0xF + if const_expr(gq < 8): + lo = lo | (nb << (4 * gq)) + else: + hi = hi | (nb << (4 * (gq - 8))) + _sts1(s_bits + w * 8, lo) + _sts1(s_bits + w * 8 + 4, hi) + cute.arch.sync_threads() + + @cute.jit + def _pass_tiles( + self, + act, + lead, + r0, + rows, + sel_base, + sel_stride, + split, + nsplit, + tidx, + warp, + lane, + s_raw, + s_scan, + s_tab, + mBT, + ): + """Union + resolve of one pass; returns the CTA's local tile count.""" + s_hk = s_raw + self.u_hk + s_hm = s_raw + self.u_hm + s_ug = s_raw + self.u_ug + n_ug = self._union( + act, r0, rows, sel_base, sel_stride, tidx, warp, lane, s_hk, s_hm, s_scan, s_ug + ) + n_tiles = (n_ug + GPT - 1) // GPT + n_loc = cutlass.Int32( + cutlass.select_( + split < n_tiles, + (n_tiles - split + nsplit - 1) // nsplit, + cutlass.Int32(0), + ) + ) + self._resolve(lead, split, nsplit, n_loc, n_ug, tidx, s_hk, s_hm, s_ug, s_tab, mBT) + return n_loc + + @cute.kernel + def kernel( + self, + qk_mma: cute.TiledMma, + qk_mma_acc: cute.TiledMma, + pv_mma_acc: cute.TiledMma, + q_layout: cute.ComposedLayout, + k_layout: cute.ComposedLayout, + vk_layout: cute.ComposedLayout, + v_layout: cute.ComposedLayout, + p_layout: cute.ComposedLayout, + sel_base: cutlass.Int64, + q_base: cutlass.Int64, + k_base: cutlass.Int64, + v_base: cutlass.Int64, + mBT: cute.Tensor, + mReq: cute.Tensor, + out_base: cutlass.Int64, + part_base: cutlass.Int64, + lse_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + n_whole: cutlass.Int32, + tail_row0: cutlass.Int32, + tab_base: cutlass.Int64, + cnt_base: cutlass.Int64, + est_base: cutlass.Int64, + ): + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + qk_mma.set(tcgen05.Field.ACCUMULATE, False) + qk_mma_acc.set(tcgen05.Field.ACCUMULATE, True) + pv_mma_acc.set(tcgen05.Field.ACCUMULATE, True) + rpr: cutlass.Constexpr = self.T + group: cutlass.Constexpr = self.group + kvh_n: cutlass.Constexpr = self.kvh + splits: cutlass.Constexpr = self.splits + if const_expr(self.tail): + # units (group, KV head) below n_whole: one CTA each; the rest: `splits` CTAs each + is_whole = bidx < n_whole + t_cta = bidx - n_whole + unit = cutlass.Int32(cutlass.select_(is_whole, bidx, n_whole + t_cta // splits)) + split = cutlass.Int32(cutlass.select_(is_whole, cutlass.Int32(0), t_cta % splits)) + nsplit = cutlass.Int32( + cutlass.select_(is_whole, cutlass.Int32(1), cutlass.Int32(splits)) + ) + elif const_expr(self.bal): + # read from the plan table after the dependency wait + unit = cutlass.Int32(0) + split = cutlass.Int32(0) + nsplit = cutlass.Int32(1) + else: + unit = bidx // splits + split = bidx % splits + nsplit = cutlass.Int32(splits) + gi = unit // kvh_n + kvh = unit % kvh_n + r0 = gi * rpr + + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + q_full = storage.q_full.data_ptr() + raw_full = storage.raw_full.data_ptr() + raw_empty = storage.raw_empty.data_ptr() + k_full = storage.k_full.data_ptr() + k_empty = storage.k_empty.data_ptr() + v_full = storage.v_full.data_ptr() + v_empty = storage.v_empty.data_ptr() + s_full = storage.s_full.data_ptr() + p_full = storage.p_full.data_ptr() + o_full = storage.o_full.data_ptr() + o_final = storage.o_final.data_ptr() + s_scan = cutlass.Int32(storage.scan.data_ptr().toint()) + s_tab = cutlass.Int32(storage.tab.data_ptr().toint()) + s_raw = cutlass.Int32(storage.raw.data_ptr().toint()) + if tidx == 0: + cute.arch.mbarrier_init(q_full, 32 * SOFT_WARPS) + for i in cutlass.range_constexpr(self.RS): + cute.arch.mbarrier_init(raw_full + i, LOAD_WARPS) + cute.arch.mbarrier_init(raw_empty + i, 32 * DQ_WARPS) + for i in cutlass.range_constexpr(self.KS): + cute.arch.mbarrier_init(k_full + i, BN) + cute.arch.mbarrier_init(k_empty + i, 1) + for i in cutlass.range_constexpr(self.VS): + cute.arch.mbarrier_init(v_full + i, BN) + cute.arch.mbarrier_init(v_empty + i, 1) + for i in cutlass.range_constexpr(2): + cute.arch.mbarrier_init(s_full + i, 1) + cute.arch.mbarrier_init(p_full + i, 32 * SOFT_WARPS) + cute.arch.mbarrier_init(o_full, 1) + cute.arch.mbarrier_init(o_final, 1) + cute.arch.mbarrier_init_fence() + if warp == MMA_WARP: + cute.arch.alloc_tmem(TMEM_COLS, storage.tmem_holding.ptr) + cute.arch.relinquish_tmem_alloc_permit() + cute.arch.sync_threads() + tmem_ptr = cute.arch.retrieve_tmem_ptr(Float32, 16, storage.tmem_holding.ptr) + + qk_thr = qk_mma.get_slice(0) + pv_thr = pv_mma_acc.get_slice(0) + sK = storage.sK.get_tensor(k_layout.outer, swizzle=k_layout.inner) + sVr = storage.sV.get_tensor(vk_layout.outer, swizzle=vk_layout.inner) + sV = cute.make_tensor(cute.recast_ptr(sVr.iterator, v_layout.inner), v_layout.outer) + s_k16 = cutlass.Int32(storage.sK.data_ptr().toint()) + s_v16 = cutlass.Int32(storage.sV.data_ptr().toint()) + + # ---- softmax-side tensor-memory views (warps 0-3: one thread an MMA row) ------------- + st = tidx + tStS = qk_thr.make_fragment_C(qk_thr.partition_shape_C((BM, BN))) + tStS0 = cute.make_tensor(tmem_ptr + TMEM_S0, tStS.layout) + tOtO_f = pv_thr.make_fragment_C(pv_thr.partition_shape_C((BM, HEAD_DIM))) + tOtO = cute.make_tensor(tmem_ptr + TMEM_O, tOtO_f.layout) + tScS = qk_thr.partition_C(cute.make_identity_tensor((BM, BN))) + load_s = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), Float32), + tStS0, + ) + thr_load_s = load_s.get_slice(st) + tLtS0 = thr_load_s.partition_S(tStS0) + tLcS = thr_load_s.partition_D(tScS) + p_cols = BN // 2 + tStP0 = cute.make_tensor( + tStS0.iterator, cute.composition(tStS0.layout, cute.make_layout((BM, p_cols))) + ) + tScP = cute.make_tensor( + tScS.iterator, cute.composition(tScS.layout, cute.make_layout((BM, p_cols))) + ) + store_p = tcgen05.make_tmem_copy( + cute.make_copy_atom(tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(p_cols)), Float32), + tStP0, + ) + thr_store_p = store_p.get_slice(st) + tPtP0 = thr_store_p.partition_D(tStP0) + tPcP = thr_store_p.partition_S(tScP) + tOcO = pv_thr.partition_C(cute.make_identity_tensor((BM, HEAD_DIM))) + tOtO_c = cute.make_tensor( + tOtO.iterator, cute.composition(tOtO.layout, cute.make_layout((BM, CHUNK_COLS))) + ) + tOcO_c = cute.make_tensor( + tOcO.iterator, cute.composition(tOcO.layout, cute.make_layout((BM, CHUNK_COLS))) + ) + load_o = tcgen05.make_tmem_copy( + cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(CHUNK_COLS)), Float32 + ), + tOtO_c, + ) + store_o = tcgen05.make_tmem_copy( + cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(CHUNK_COLS)), Float32 + ), + tOtO_c, + ) + thr_load_o = load_o.get_slice(st) + thr_store_o = store_o.get_slice(st) + tLtO = thr_load_o.partition_S(tOtO_c) + tLcO = thr_load_o.partition_D(tOcO_c) + tStO = thr_store_o.partition_D(tOtO_c) + tview = (load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO) + + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + + idle = cutlass.Int32(0) + bal_base = cutlass.Int32(0) + if const_expr(self.bal): + # this CTA's (unit, split, unit's CTAs, unit's first CTA); unit -1: past the plan + ent = self._bal_schedule(tidx, warp, lane, bidx, rows, est_base, tab_base, s_raw) + idle = cutlass.Int32(cutlass.select_(ent[0] < 0, cutlass.Int32(1), cutlass.Int32(0))) + unit = cutlass.max(ent[0], cutlass.Int32(0)) + split = ent[1] + nsplit = ent[2] + bal_base = ent[3] + gi = unit // kvh_n + kvh = unit % kvh_n + r0 = gi * rpr + m = tidx + t_row = m // group + h = m % group + live = (m < self.rows_m) & (r0 + t_row < rows) + if const_expr(self.bal): + live = live & (idle == 0) + safe_row = cutlass.min(r0 + t_row, rows - 1) + qs0, qs1 = self.qs + q_row = q_base + (cutlass.Int64(safe_row) * qs0 + cutlass.Int64(kvh * group + h) * qs1) * 2 + q_s = s_raw + self.u_q + cutlass.min(m, self.rows_m - 1) * Q_LD + + if warp < SOFT_WARPS: + # O = 0; this row's q copied into raw slot 1 now, into tensor memory (f16) after the + # first pass's resolve. Dead rows stage zeros. + rZ = cute.make_rmem_tensor(tLcO.shape, Float32) + rZ.fill(0.0) + for c in cutlass.range_constexpr(HEAD_DIM // CHUNK_COLS): + cute.copy( + store_o, rZ, cute.make_tensor(tStO.iterator + c * CHUNK_COLS, tStO.layout) + ) + cute.arch.fence_view_async_tmem_store() + if m < self.rows_m: + qn = cutlass.Int32(cutlass.select_(live, cutlass.Int32(16), cutlass.Int32(0))) + for i in cutlass.range_constexpr(HEAD_DIM // 8): + _cp16z(q_s + 16 * i, q_row + cutlass.Int64(16 * i), qn) + cute.arch.cp_async_commit_group() + + rq = cute.make_rmem_tensor(cute.make_layout((rpr,), stride=(1,)), cutlass.Int32) + handled = cutlass.Int32(0) + for t in cutlass.range_constexpr(rpr): + row = r0 + t + xr = cutlass.Int64(mReq[cutlass.min(row, rows - 1)]) + okr = (row < rows) & (xr >= 0) & (xr < self.num_requests) + rq[t] = cutlass.Int32(cutlass.select_(okr, cutlass.Int32(xr), cutlass.Int32(-1))) + handled = handled | cutlass.Int32( + cutlass.select_(okr, cutlass.Int32(0), cutlass.Int32(1 << t)) + ) + full = (1 << rpr) - 1 + if const_expr(self.bal): + handled = cutlass.Int32(cutlass.select_(idle != 0, cutlass.Int32(full), handled)) + scale_log2 = Float32(self.scale_log2) + neg_inf = -Float32.inf + row_max = neg_inf + row_sum = Float32(0.0) + q_pending = cutlass.Int32(1) + g = cutlass.Int32(0) + for _pass in cutlass.range(rpr): + if handled != full: + lead, act = self._pass_rows(rq, handled) + n_loc = self._pass_tiles( + act, + lead, + r0, + rows, + sel_base, + sel_stride, + split, + nsplit, + tidx, + warp, + lane, + s_raw, + s_scan, + s_tab, + mBT, + ) + if warp >= DQ_WARP0: + self.widen_tiles( + tidx - 32 * DQ_WARP0, + g, + n_loc, + s_raw, + s_tab, + s_k16, + s_v16, + raw_full, + raw_empty, + k_full, + k_empty, + v_full, + v_empty, + ) + elif warp >= LOAD_WARP0: + self.load_tiles( + tidx - 32 * LOAD_WARP0, + g, + n_loc, + kvh, + s_raw, + s_tab, + k_base, + v_base, + raw_full, + raw_empty, + q_full, + ) + elif warp == MMA_WARP: + self.mma_tiles( + g, + n_loc, + qk_mma, + pv_mma_acc, + qk_thr, + pv_thr, + q_layout, + p_layout, + sK, + sV, + tmem_ptr, + q_full, + k_full, + k_empty, + v_full, + v_empty, + s_full, + p_full, + o_full, + ) + else: + if q_pending != 0: + self.stage_q(q_s, live, tview, q_full) + q_pending = cutlass.Int32(0) + row_max, row_sum = self.softmax_tiles( + g, + n_loc, + row_max, + row_sum, + live, + t_row, + warp, + s_tab, + scale_log2, + tview, + s_full, + p_full, + o_full, + ) + g = g + n_loc + cute.arch.sync_threads() # [P] end of the pass + handled = handled | act + + if warp == MMA_WARP: + with cute.arch.elect_one(): + tcgen05.commit(o_final) + if warp < SOFT_WARPS: + cute.arch.cp_async_wait_group(0) + epi = ( + row_max, + row_sum, + live, + warp, + m, + r0, + rows, + kvh, + bidx if self.bal else split, # balanced: the partial slot is the CTA + nsplit, + tail_row0, + scale_log2, + out_base, + part_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + tview, + o_final, + tidx, + s_k16, + ) + if const_expr(self.bal): + if idle == 0: + self.epilogue(*epi) + else: + self.epilogue(*epi) + # every role done with tensor memory (the softmax warps read O last) + cute.arch.sync_threads() + if warp == MMA_WARP: + cute.arch.dealloc_tmem(tmem_ptr, TMEM_COLS) + if const_expr(self.bal): + if (idle == 0) & (nsplit > 1): + self.fixup( + tidx, + unit, + kvh, + r0, + rows, + nsplit, + bal_base, + cnt_base, + part_base, + lse_base, + out_base, + gate_base, + gate_stride, + gate_hs, + s_scan, + s_raw, + ) + + # ------------------------------------------------------------------ loaders (warps 5-6) + @cute.jit + def load_tiles( + self, lt, g, n_loc, kvh, s_raw, s_tab, k_base, v_base, raw_full, raw_empty, q_full + ): + """Raw tile copies: lane l < 16 of warp 5 (K) and of warp 6 (V) bulk-copies group l's four + rows (1 KB, contiguous in the pool); each warp's lane 0 first arms the slot's mbarrier with + its half of the tile's bytes. A group that does not resolve is not copied (the widening + zeroes it).""" + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + tpb: cutlass.Constexpr = self.tpb + ks0, ks1, ks2 = self.ks + if lt < 32 * LOAD_WARPS: + gi = lt % 16 + is_v = lt >= 32 + base = cutlass.select_(is_v, v_base, k_base) + cutlass.Int64(kvh * ks1) + dst0 = ( + s_raw + + cutlass.Int32(cutlass.select_(is_v, cutlass.Int32(RAW_K), cutlass.Int32(0))) + + gi * (4 * RAW_LD) + ) + rf0 = cutlass.Int32(raw_full.toint()) + for kl in cutlass.range(n_loc): + gg = g + kl + r = gg % self.RS + cute.arch.mbarrier_wait(raw_empty + r, _phase(gg, self.RS) ^ 1) + if r == 1: + cute.arch.mbarrier_wait(q_full, 0) + slot = kl * GPT + gi + tok0 = _lds1(s_tok + slot * 4) + mw = _lds1(s_gmask + slot * 4) + ok = (tok0 >= 0) & (mw != 0) & ((lt % 32) < 16) + nb = cutlass.Int32( + cutlass.select_(ok, cutlass.Int32(4 * HEAD_DIM), cutlass.Int32(0)) + ) + total = cute.arch.warp_reduction_sum(nb) + if (lt % 32) == 0: + _arrive_expect_tx(rf0 + r * 8, total) + cute.arch.sync_warp() + if ok: + off = cutlass.Int64(tok0 // tpb) * ks0 + cutlass.Int64((tok0 % tpb) * ks2) + _bulk_g2s(dst0 + r * RAW_BUF, base + off, nb, rf0 + r * 8) + + # ------------------------------------------------------------------ widen (warps 7-10) + @cute.jit + def widen_tiles( + self, + dt, + g, + n_loc, + s_raw, + s_tab, + s_k16, + s_v16, + raw_full, + raw_empty, + k_full, + k_empty, + v_full, + v_empty, + ): + """Thread `dt` < 64 widens K row `dt` of each tile, the others V row `dt - 64`: the raw + row into registers (the raw slot is then free), then into the f16 SW128 slot once the + MMA has released it.""" + is_v = dt >= BN + n = dt % BN + s_tok = s_tab + self.t_tok * 4 + s_gmask = s_tab + self.t_gmask * 4 + # bulk copies land unpadded rows: chunk c of the read order is the row's chunk (c + n) % 16 + rot = n % 16 + for kl in cutlass.range(n_loc): + gg = g + kl + r = gg % self.RS + slot = kl * GPT + n // 4 + tok0 = _lds1(s_tok + slot * 4) + mw = _lds1(s_gmask + slot * 4) + live_row = (tok0 >= 0) & (((mw >> (8 * (n % 4))) & 0xFF) != 0) + cute.arch.mbarrier_wait(raw_full + r, _phase(gg, self.RS)) + row_off = ( + s_raw + + r * RAW_BUF + + cutlass.Int32(cutlass.select_(is_v, cutlass.Int32(RAW_K), cutlass.Int32(0))) + + n * RAW_LD + ) + w = [_lds4(row_off + 16 * ((c + rot) % 16)) for c in range(16)] + cute.arch.mbarrier_arrive(raw_empty + r) + # a row no verify row selected (not copied) widens to zeros: P = 0 times a stale NaN + # would poison P V + w = [ + [cutlass.Int32(cutlass.select_(live_row, x, cutlass.Int32(0))) for x in wc] + for wc in w + ] + if is_v: + b = gg % self.VS + cute.arch.mbarrier_wait(v_empty + b, _phase(gg, self.VS) ^ 1) + self._widen_row(w, s_v16 + b * F16_SLOT, n, rot) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(v_full + b) + else: + b = gg % self.KS + cute.arch.mbarrier_wait(k_empty + b, _phase(gg, self.KS) ^ 1) + self._widen_row(w, s_k16 + b * F16_SLOT, n, rot) + cute.arch.fence_view_async_shared() + cute.arch.mbarrier_arrive(k_full + b) + + @cute.jit + def _widen_row(self, w, dst, n, rot): + """Raw e4m3 row words `w` (16 x 4) -> f16 SW128 row `n` at `dst`: 16-byte chunk ci (dims + 8 ci .. 8 ci + 7) of the row's 64-dim block ci // 8 at 8192 (ci // 8) + 128 n + 16 ((ci % + 8) ^ (n % 8)).""" + for c in cutlass.range_constexpr(16): + for half in cutlass.range_constexpr(2): + a = _e4m3x4(w[c][2 * half]) + b = _e4m3x4(w[c][2 * half + 1]) + ci = 2 * ((c + rot) % 16) + half + _sts4( + dst + 8192 * (ci // 8) + 128 * n + 16 * ((ci % 8) ^ (n % 8)), + a[0], + a[1], + b[0], + b[1], + ) + + # ------------------------------------------------------------------ MMA (warp 4) + @cute.jit + def mma_tiles( + self, + g, + n_loc, + qk_mma, + pv_mma_acc, + qk_thr, + pv_thr, + q_layout, + p_layout, + sK, + sV, + tmem_ptr, + q_full, + k_full, + k_empty, + v_full, + v_empty, + s_full, + p_full, + o_full, + ): + """QK(g), then QK(j) / PV(j - 1) pairs, then the last PV: S/P buffer j % 2 at TMEM column + TMEM_S0 + 64 (j % 2); QK(j) overwrites it only after PV(j - 2) was issued (tcgen05 MMAs of + one thread execute in order).""" + tQ = cute.make_tensor(tmem_ptr + TMEM_Q, q_layout.outer) + tSrQ = qk_thr.make_fragment_A(tQ)[None, None, None, 0] + tSrQ = cute.make_tensor(cute.recast_ptr(tmem_ptr + TMEM_Q, dtype=F16), tSrQ.layout) + tSrK = qk_thr.make_fragment_B(sK) + tOrV = pv_thr.make_fragment_B(sV) + tP = cute.make_tensor(tmem_ptr + TMEM_S0, p_layout.outer) + tOrP = pv_thr.make_fragment_A(tP)[None, None, None, 0] + tOrP = cute.make_tensor(cute.recast_ptr(tmem_ptr + TMEM_S0, dtype=F16), tOrP.layout) + tbase = cutlass.Int32(tmem_ptr.toint()) + qk_op = qk_mma.op + pv_op = pv_mma_acc.op + if n_loc > 0: + cute.arch.mbarrier_wait(q_full, 0) + cute.arch.mbarrier_wait(k_full + g % self.KS, _phase(g, self.KS)) + _tc_fence_after() + self.issue_qk(qk_op, tbase, tSrQ, tSrK, sK, g % 2, g % self.KS) + with cute.arch.elect_one(): + tcgen05.commit(s_full + g % 2) + tcgen05.commit(k_empty + g % self.KS) + for j in cutlass.range(1, n_loc + 1, 1, unroll=1): + gj = g + j + if j < n_loc: + cute.arch.mbarrier_wait(k_full + gj % self.KS, _phase(gj, self.KS)) + _tc_fence_after() + self.issue_qk(qk_op, tbase, tSrQ, tSrK, sK, gj % 2, gj % self.KS) + with cute.arch.elect_one(): + tcgen05.commit(s_full + gj % 2) + tcgen05.commit(k_empty + gj % self.KS) + gp = gj - 1 + cute.arch.mbarrier_wait(v_full + gp % self.VS, _phase(gp, self.VS)) + cute.arch.mbarrier_wait(p_full + gp % 2, _phase(gp, 2)) + _tc_fence_after() + self._issue_pv(pv_op, tbase, tOrP, tOrV, sV, gp) + with cute.arch.elect_one(): + tcgen05.commit(v_empty + gp % self.VS) + tcgen05.commit(o_full) + + @cute.jit + def _issue_pv(self, pv_op, tbase, tOrP, tOrV, sV, gp): + """O += P(gp) V(gp): P from tensor memory over the S it came from, V from its slot.""" + _gemm_ts( + pv_op, + tbase + TMEM_O, + tbase + TMEM_S0 + (gp % 2) * BN, + tOrP, + tOrV[None, None, None, gp % self.VS], + sV[None, None, None, gp % self.VS], + False, + ) + + @cute.jit + def issue_qk(self, qk_op, tbase, tSrQ, tSrK, sK, sbuf, kslot): + """S[sbuf] = Q K(kslot)^T: Q from tensor memory, K from SW128 slot `kslot`.""" + _gemm_ts( + qk_op, + tbase + TMEM_S0 + sbuf * BN, + tbase + TMEM_Q, + tSrQ, + tSrK[None, None, None, kslot], + sK[None, None, None, kslot], + True, + ) + + # ------------------------------------------------------------- softmax + epilogue (warps 0-3) + @cute.jit + def stage_q(self, q_s, live, tview, q_full): + """This MMA row's q (bf16 in raw slot 1) -> f16 into tensor memory (the QK A operand).""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + p_cols: cutlass.Constexpr = BN // 2 + cute.arch.cp_async_wait_group(0) + for c in cutlass.range_constexpr(HEAD_DIM // (2 * p_cols)): + rQ = cute.make_rmem_tensor(tPcP.shape, Float32) + for i in cutlass.range_constexpr(p_cols // 4): + w4 = _lds4(q_s + 16 * (c * (p_cols // 4) + i)) + for e in cutlass.range_constexpr(4): + rQ[4 * i + e] = _bits_f32( + cutlass.Int32(cutlass.select_(live, _bf2_to_f2(w4[e]), cutlass.Int32(0))) + ) + cute.copy( + store_p, + rQ, + cute.make_tensor(tPtP0.iterator + (TMEM_Q - TMEM_S0 + c * p_cols), tPtP0.layout), + ) + cute.arch.fence_view_async_tmem_store() + _tc_fence_before() + cute.arch.mbarrier_arrive(q_full) + + @cute.jit + def softmax_tiles( + self, + g, + n_loc, + row_max, + row_sum, + live, + t_row, + warp, + s_tab, + scale_log2, + tview, + s_full, + p_full, + o_full, + ): + """Per tile: S from tensor memory, the max over the row's own entries, the reference max + moved (and O rescaled) only past RESCALE_TH, P = 2^(s - max) as f16 over the S.""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + rpr: cutlass.Constexpr = self.T + s_bits = s_tab + self.t_bits * 4 + tau = Float32(RESCALE_TH) + neg_inf = -Float32.inf + live_warp = warp < self.live_warps + for kl in cutlass.range(n_loc, unroll=1): + gg = g + kl + buf = gg % 2 + cute.arch.mbarrier_wait(s_full + buf, _phase(gg, 2)) + _tc_fence_after() + if live_warp: + lo = cutlass.Int32(0) + hi = cutlass.Int32(0) + if live: + lo = _lds1(s_bits + (kl * rpr + t_row) * 8) + hi = _lds1(s_bits + (kl * rpr + t_row) * 8 + 4) + rS = cute.make_rmem_tensor(tLcS.shape, Float32) + tLtS_b = cute.make_tensor(tLtS0.iterator + buf * BN, tLtS0.layout) + for c in cutlass.range_constexpr(cute.size(rS, mode=[1])): + cute.copy(load_s, tLtS_b[None, c, 0, 0], rS[None, c, 0, 0]) + cute.arch.fence_view_async_tmem_load() + tile_max = neg_inf + for i in cutlass.range_constexpr(BN): + word = lo if i < 32 else hi + x = rS[i] + x = x if (word & (1 << (i % 32))) != 0 else neg_inf + rS[i] = x + tile_max = cute.arch.fmax(tile_max, x) + new_max = cute.arch.fmax(row_max, tile_max) + use_max = row_max + factor = Float32(1.0) + if row_max == neg_inf: + use_max = new_max + elif (new_max - row_max) * scale_log2 > tau: + use_max = new_max + factor = cute.math.exp2((row_max - new_max) * scale_log2, fastmath=True) + need = factor != Float32(1.0) + if cute.arch.vote_any_sync(need): + # O holds PV(..gg - 1): wait for it, then rescale this warp's rows + cute.arch.mbarrier_wait(o_full, (gg - 1) % 2) + _tc_fence_after() + for c in cutlass.range_constexpr(HEAD_DIM // CHUNK_COLS): + rO = cute.make_rmem_tensor(tLcO.shape, Float32) + cute.copy( + load_o, + cute.make_tensor(tLtO.iterator + c * CHUNK_COLS, tLtO.layout), + rO, + ) + for i in cutlass.range_constexpr(cute.size(rO)): + rO[i] = rO[i] * factor + cute.copy( + store_o, + rO, + cute.make_tensor(tStO.iterator + c * CHUNK_COLS, tStO.layout), + ) + cute.arch.fence_view_async_tmem_store() + row_max = use_max + safe = use_max + if use_max == neg_inf: + safe = Float32(0.0) + neg = -safe * scale_log2 + rP = cute.make_rmem_tensor(tPcP.shape, Float32) + rP16 = cute.make_tensor(cute.recast_ptr(rP.iterator, dtype=F16), rS.layout) + acc0 = (Float32(0.0), Float32(0.0)) + acc1 = (Float32(0.0), Float32(0.0)) + for i in cutlass.range_constexpr(0, BN, 4): + e = cute.make_rmem_tensor((4,), Float32) + e[0], e[1] = cute.arch.fma_packed_f32x2( + (rS[i], rS[i + 1]), (scale_log2, scale_log2), (neg, neg) + ) + e[2], e[3] = cute.arch.fma_packed_f32x2( + (rS[i + 2], rS[i + 3]), (scale_log2, scale_log2), (neg, neg) + ) + for u in cutlass.range_constexpr(4): + rS[i + u] = cute.math.exp2(e[u], fastmath=True) + acc0 = cute.arch.add_packed_f32x2(acc0, (rS[i], rS[i + 1])) + acc1 = cute.arch.add_packed_f32x2(acc1, (rS[i + 2], rS[i + 3])) + row_sum = row_sum * factor + ((acc0[0] + acc0[1]) + (acc1[0] + acc1[1])) + rP16.store(rS.load().to(F16)) + cute.copy(store_p, rP, cute.make_tensor(tPtP0.iterator + buf * BN, tPtP0.layout)) + cute.arch.fence_view_async_tmem_store() + _tc_fence_before() + cute.arch.mbarrier_arrive(p_full + buf) + return row_max, row_sum + + @cute.jit + def epilogue( + self, + row_max, + row_sum, + live, + warp, + m, + r0, + rows, + kvh, + split, + nsplit, + tail_row0, + scale_log2, + out_base, + part_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + tview, + o_final, + st, + s_stage, + ): + """O / l: a whole unit's rows as bf16, a split's as an fp32 partial and its log2-sum-exp for + the merge (rows from `tail_row0` on). Gated whole units read the gate before the last P V + completes, stage bf16(O / l) in the free K slots, then apply the gate in coalesced rows.""" + load_s, tLtS0, tLcS, store_p, tPtP0, tPcP, load_o, store_o, tLtO, tLcO, tStO = tview + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + splits: cutlass.Constexpr = self.splits + tailm: cutlass.Constexpr = self.tail or self.bal + has_whole: cutlass.Constexpr = tailm or splits == 1 + has_split: cutlass.Constexpr = tailm or splits > 1 + gated_whole: cutlass.Constexpr = has_whole and self.gated + rows_m: cutlass.Constexpr = self.rows_m + cpr: cutlass.Constexpr = HEAD_DIM // 8 # 16-byte chunks a row + n_gc: cutlass.Constexpr = -(-(rows_m * cpr) // (32 * SOFT_WARPS)) + n_chunks: cutlass.Constexpr = HEAD_DIM // CHUNK_COLS + neg_inf = -Float32.inf + row = r0 + m // group + h = m % group + head = kvh * group + h + prow = cutlass.Int64(row - tail_row0) + prows = cutlass.Int64(rows - tail_row0) + if const_expr(self.bal): + # a CTA's partial slot holds its unit's T rows + prow = cutlass.Int64(m // group) + prows = cutlass.Int64(self.T) + whole = nsplit == 1 + gregs = [] + if const_expr(gated_whole): + for j in cutlass.range_constexpr(n_gc): + k = st + 32 * SOFT_WARPS * j + gm = cutlass.min(k // cpr, rows_m - 1) + grow = cutlass.min(r0 + gm // group, rows - 1) + ghead = kvh * group + gm % group + gregs.append( + _ldg4( + gate_base + + ( + cutlass.Int64(grow) * gate_stride + + cutlass.Int64(ghead) * gate_hs + + (k % cpr) * 8 + ) + * 2 + ) + ) + cute.arch.mbarrier_wait(o_final, 0) + _tc_fence_after() + if warp < self.live_warps: + inv = Float32(0.0) + if row_sum > Float32(0.0): + inv = Float32(1.0) / row_sum + for c in cutlass.range_constexpr(n_chunks): + rO = cute.make_rmem_tensor(tLcO.shape, Float32) + cute.copy(load_o, cute.make_tensor(tLtO.iterator + c * CHUNK_COLS, tLtO.layout), rO) + for v8 in cutlass.range_constexpr(CHUNK_COLS // 8): + vals = [rO[8 * v8 + i] * inv for i in range(8)] + d0 = c * CHUNK_COLS + 8 * v8 + oaddr = out_base + ((cutlass.Int64(row) * heads + head) * HEAD_DIM + d0) * 2 + paddr = ( + part_base + + (((cutlass.Int64(split) * prows + prow) * heads + head) * HEAD_DIM + d0) + * 4 + ) + packed = [_bf2_pack(vals[2 * e], vals[2 * e + 1]) for e in range(4)] + saddr = s_stage + cutlass.min(m, rows_m - 1) * Q_LD + d0 * 2 + if const_expr(has_whole): + if whole: + if const_expr(gated_whole): + if m < rows_m: + _sts4(saddr, packed[0], packed[1], packed[2], packed[3]) + else: + if live: + _stg4(oaddr, packed[0], packed[1], packed[2], packed[3]) + if const_expr(has_split): + if live & (nsplit > 1): + _stg4f(paddr, vals[0], vals[1], vals[2], vals[3]) + _stg4f(paddr + 16, vals[4], vals[5], vals[6], vals[7]) + if const_expr(has_split): + if live & (nsplit > 1): + lse = neg_inf + if row_sum > Float32(0.0): + lse = row_max * scale_log2 + cute.math.log2(row_sum) + _stg1( + lse_base + ((cutlass.Int64(split) * prows + prow) * heads + head) * 4, + _f32_bits(lse), + ) + if const_expr(gated_whole): + if whole: + cute.arch.barrier(barrier_id=1, number_of_threads=32 * SOFT_WARPS) + for j in cutlass.range_constexpr(n_gc): + k = st + 32 * SOFT_WARPS * j + gm = cutlass.min(k // cpr, rows_m - 1) + grow = r0 + gm // group + ghead = kvh * group + gm % group + a4 = _lds4(s_stage + gm * Q_LD + (k % cpr) * 16) + g4 = gregs[j] + res = [ + _bf2_pack( + _bf2_lo(a4[e]) * _sigmoid_fast(_bf2_lo(g4[e])), + _bf2_hi(a4[e]) * _sigmoid_fast(_bf2_hi(g4[e])), + ) + for e in range(4) + ] + if (k < rows_m * cpr) & (grow < rows): + _stg4( + out_base + + ((cutlass.Int64(grow) * heads + ghead) * HEAD_DIM + (k % cpr) * 8) + * 2, + res[0], + res[1], + res[2], + res[3], + ) + + @cute.jit + def fixup( + self, + tidx, + unit, + kvh, + r0, + rows, + nsplit, + bal_base, + cnt_base, + part_base, + lse_base, + out_base, + gate_base, + gate_stride, + gate_hs, + s_flag, + s_w, + ): + """Balanced mode, a unit of several CTAs: the unit's last CTA to finish (a ticket a unit) + merges its partials in slot order as `_RowMergeKernel` does, applies the gate, stores the + rows and resets the ticket; the result does not depend on which CTA merges.""" + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + rows_m: cutlass.Constexpr = self.rows_m + rpr: cutlass.Constexpr = self.T + d4: cutlass.Constexpr = HEAD_DIM // 4 + n_c: cutlass.Constexpr = rows_m * d4 + per: cutlass.Constexpr = -(-n_c // THREADS) + nsp: cutlass.Constexpr = MAX_BAL_SPLITS + neg_inf = -Float32.inf + # the CTA's partials and log-sum-exps (stored before the barrier) released by thread 0's + # acq_rel ticket; the last CTA's thread 0 acquires the others', the barrier passes them on + cute.arch.sync_threads() + if tidx == 0: + _sts1(s_flag, _atom_add_acqrel_g(cnt_base + cutlass.Int64(unit) * 4, cutlass.Int32(1))) + cute.arch.sync_threads() + if _lds1(s_flag) == nsplit - 1: + if tidx == 0: + _stg1(cnt_base + cutlass.Int64(unit) * 4, cutlass.Int32(0)) + # weights of each (verify row, head) of the unit: slots past `nsplit` weigh 0 + for i in cutlass.range_constexpr(-(-rows_m // THREADS)): + k = tidx + i * THREADS + if k < rows_m: + head = kvh * group + k % group + ls = [] + for s_ in cutlass.range_constexpr(nsp): + slot = bal_base + cutlass.min(cutlass.Int32(s_), nsplit - 1) + lv = _bits_f32( + _ldg1cg( + lse_base + + ((cutlass.Int64(slot) * rpr + k // group) * heads + head) * 4 + ) + ) + ls.append(Float32(cutlass.select_(nsplit > s_, lv, neg_inf))) + mx = _tree_fmax(ls) + ws = [ + Float32( + cutlass.select_(mx > neg_inf, cute.math.exp2(lv - mx), Float32(0.0)) + ) + for lv in ls + ] + for s_ in cutlass.range_constexpr(nsp): + _sts1(s_w + (s_ * rows_m + k) * 4, _f32_bits(ws[s_])) + _sts1(s_w + (nsp * rows_m + k) * 4, _f32_bits(_tree_sum(ws))) + cute.arch.sync_threads() + for j in cutlass.range_constexpr(per): + c = tidx + THREADS * j + k = cutlass.min(c // d4, rows_m - 1) + d = (c % d4) * 4 + t = k // group + head = kvh * group + k % group + row = r0 + t + ps = [ + _ldg4cg( + part_base + + ( + ( + ( + cutlass.Int64( + bal_base + cutlass.min(cutlass.Int32(s_), nsplit - 1) + ) + * rpr + + t + ) + * heads + + head + ) + * HEAD_DIM + + d + ) + * 4 + ) + for s_ in range(nsp) + ] + gw = None + if const_expr(self.gated): + gw = _ldg2( + gate_base + + ( + cutlass.Int64(cutlass.min(row, rows - 1)) * gate_stride + + cutlass.Int64(head) * gate_hs + + d + ) + * 2 + ) + ws = [_bits_f32(_lds1(s_w + (s_ * rows_m + k) * 4)) for s_ in range(nsp)] + den = _bits_f32(_lds1(s_w + (nsp * rows_m + k) * 4)) + res = [] + for e in cutlass.range_constexpr(4): + acc = _tree_sum([ws[s_] * _bits_f32(ps[s_][e]) for s_ in range(nsp)]) + merged = Float32(cutlass.select_(den > Float32(0.0), acc / den, Float32(0.0))) + if const_expr(self.gated): + if const_expr(e % 2 == 0): + gx = _bf2_lo(gw[e // 2]) + else: + gx = _bf2_hi(gw[e // 2]) + merged = _round_bf16(merged) * _sigmoid_fast(gx) + res.append(merged) + if (c < n_c) & (row < rows): + _stg2( + out_base + ((cutlass.Int64(row) * heads + head) * HEAD_DIM + d) * 2, + _bf2_pack(res[0], res[1]), + _bf2_pack(res[2], res[3]), + ) + + +#: Threads of a merge CTA (four values a thread at a time). +MERGE_THREADS = 256 +#: Merge CTAs a row, each a contiguous share of the row's heads x head_dim values. +MERGE_PARTS = 8 + + +class _RowMergeKernel: + """The split partials of a row range merged as the per-row kernel's merge does + (log2-sum-exp weights, fp32), the gate applied to the bf16-rounded result: `parts` CTAs a + row, every load of a thread issued before the first use.""" + + def __init__(self, *, heads, splits, gated, pdl, parts): + self.heads, self.splits, self.gated, self.pdl = heads, splits, gated, pdl + self.parts = parts + + @cute.jit + def __call__( + self, + gate_t: cute.Tensor, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + out_ptr: cute.Pointer, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + self.kernel( + part_ptr.toint(), + lse_ptr.toint(), + out_ptr.toint(), + gate_t.iterator.toint(), + rows, + gate_stride, + gate_hs, + ).launch( + grid=[rows * self.parts, 1, 1], + block=[MERGE_THREADS, 1, 1], + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + part_base: cutlass.Int64, + lse_base: cutlass.Int64, + out_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + heads: cutlass.Constexpr = self.heads + sp: cutlass.Constexpr = self.splits + parts: cutlass.Constexpr = self.parts + n_all: cutlass.Constexpr = heads * HEAD_DIM // 4 + n_c: cutlass.Constexpr = -(-n_all // parts) + per: cutlass.Constexpr = -(-n_c // MERGE_THREADS) + neg_inf = -Float32.inf + row = bidx // parts + c0 = (bidx % parts) * n_c + r64 = cutlass.Int64(row) + cs = [ + cutlass.min(c0 + cutlass.min(tidx + MERGE_THREADS * j, n_c - 1), n_all - 1) + for j in range(per) + ] + gws = [] + if const_expr(self.gated): + # the gate is an input of the attention call, ready before the split kernel ran + for c in cs: + head = c // (HEAD_DIM // 4) + d = (c % (HEAD_DIM // 4)) * 4 + gws.append( + _ldg2(gate_base + (r64 * gate_stride + cutlass.Int64(head) * gate_hs + d) * 2) + ) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + lss = [] + pss = [] + for j in cutlass.range_constexpr(per): + head = cs[j] // (HEAD_DIM // 4) + lss.append( + [ + _bits_f32( + _ldg1(lse_base + ((cutlass.Int64(s_) * rows + r64) * heads + head) * 4) + ) + for s_ in range(sp) + ] + ) + pss.append( + [ + _ldg4( + part_base + + ((cutlass.Int64(s_) * rows + r64) * (heads * HEAD_DIM) + 4 * cs[j]) * 4 + ) + for s_ in range(sp) + ] + ) + for j in cutlass.range_constexpr(per): + ls = lss[j] + mx = _tree_fmax(ls) + ws = [ + Float32(cutlass.select_(mx > neg_inf, cute.math.exp2(lv - mx), Float32(0.0))) + for lv in ls + ] + den = _tree_sum(ws) + res = [] + for e in cutlass.range_constexpr(4): + acc = _tree_sum([ws[s_] * _bits_f32(pss[j][s_][e]) for s_ in range(sp)]) + merged = Float32(0.0) + if den > Float32(0.0): + merged = acc / den + if const_expr(self.gated): + if const_expr(e % 2 == 0): + gx = _bf2_lo(gws[j][e // 2]) + else: + gx = _bf2_hi(gws[j][e // 2]) + merged = _round_bf16(merged) * _sigmoid_fast(gx) + res.append(merged) + if (tidx + MERGE_THREADS * j < n_c) & (c0 + tidx + MERGE_THREADS * j < n_all): + _stg2( + out_base + (r64 * (heads * HEAD_DIM) + 4 * cs[j]) * 2, + _bf2_pack(res[0], res[1]), + _bf2_pack(res[2], res[3]), + ) + + +#: Balanced launch: most CTAs of one unit (the last CTA's merge reads this many partials). +MAX_BAL_SPLITS = 4 +#: The plan kernel's threads a CTA, and tokens read a four-column quad (1: the quad's first token, +#: which names the group in the expansion layout, where a quad's valid tokens are a prefix). +PLAN_THREADS, PLAN_QUAD = 512, 1 +#: Most bitmap words (32 four-token groups each) of a plan CTA: a 262144-token block table. +MAX_PLAN_WORDS = 4096 + + +class _BalPlanKernel: + """Balanced launch, step 1, one CTA a request group: the group's union tiles (distinct + four-token groups its valid rows select, / `GPT`, at least 1) to `est`, for + `_GroupedKernel._bal_schedule`.""" + + def __init__(self, *, rpr, width, words, num_requests, pdl): + self.T, self.width, self.words = rpr, width, words + self.num_requests, self.pdl = num_requests, pdl + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + req_ptr: cute.Pointer, + est_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + n_groups: cutlass.Int32, + stream, + ): + @cute.struct + class SharedStorage: + red: cute.struct.MemRange[Int32, 32] + bm: cute.struct.Align[cute.struct.MemRange[Int32, self.words], 16] + + self.shared_storage = SharedStorage + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel(sel_t.iterator.toint(), req, est_ptr.toint(), rows, sel_stride).launch( + grid=[n_groups, 1, 1], + block=[PLAN_THREADS, 1, 1], + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + sel_base: cutlass.Int64, + mReq: cute.Tensor, + est_base: cutlass.Int64, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = cute.arch.lane_idx() + rpr: cutlass.Constexpr = self.T + width: cutlass.Constexpr = self.width + words: cutlass.Constexpr = self.words + n_q: cutlass.Constexpr = -(-width // 4) + n_i: cutlass.Constexpr = -(-n_q // PLAN_THREADS) + nwarps: cutlass.Constexpr = PLAN_THREADS // 32 + smem = SmemAllocator() + storage = smem.allocate(self.shared_storage) + s_red = cutlass.Int32(storage.red.data_ptr().toint()) + s_bm = cutlass.Int32(storage.bm.data_ptr().toint()) + r0 = bidx * rpr + for i in cutlass.range_constexpr(-(-words // PLAN_THREADS)): + idx = tidx + i * PLAN_THREADS + if idx < words: + _sts1(s_bm + idx * 4, cutlass.Int32(0)) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + # the first valid token of a quad names its four-token group + nu: cutlass.Constexpr = PLAN_QUAD + toks = [] + oks = [] + for t in cutlass.range_constexpr(rpr): + row = r0 + t + xr = cutlass.Int64(mReq[cutlass.min(row, rows - 1)]) + oks.append((row < rows) & (xr >= 0) & (xr < self.num_requests)) + row_sel = sel_base + cutlass.Int64(cutlass.min(row, rows - 1)) * sel_stride * 4 + for i in cutlass.range_constexpr(n_i): + for u in cutlass.range_constexpr(nu): + colc = cutlass.min(4 * (tidx + i * PLAN_THREADS) + u, width - 1) + toks.append(_ldg1(row_sel + cutlass.Int64(colc * 4))) + cute.arch.sync_threads() + for t in cutlass.range_constexpr(rpr): + for i in cutlass.range_constexpr(n_i): + q4 = tidx + i * PLAN_THREADS + gid = cutlass.Int32(-1) + for u in cutlass.range_constexpr(nu): + tok = toks[(t * n_i + i) * nu + u] + ok = oks[t] & (4 * q4 + u < width) & (tok >= 0) + gid = cutlass.Int32(cutlass.select_(ok & (gid < 0), tok >> 2, gid)) + if (gid >= 0) & (gid < words * 32): + _red_or_shared(s_bm + (gid >> 5) * 4, cutlass.Int32(1) << (gid & 31)) + cute.arch.sync_threads() + cnt = cutlass.Int32(0) + for i in cutlass.range_constexpr(-(-words // PLAN_THREADS)): + idx = tidx + i * PLAN_THREADS + if idx < words: + cnt = cnt + _popc(_lds1(s_bm + idx * 4)) + cnt = cute.arch.warp_reduction_sum(cnt) + if lane == 0: + _sts1(s_red + warp * 4, cnt) + cute.arch.sync_threads() + if tidx == 0: + n = cutlass.Int32(0) + for w in cutlass.range_constexpr(nwarps): + n = n + _lds1(s_red + w * 4) + _stg1( + est_base + cutlass.Int64(bidx) * 4, + cutlass.max((n + GPT - 1) // GPT, cutlass.Int32(1)), + ) + + +# --- launch --- + +_COMPILED: dict = {} +#: Geometries whose every reachable launch plan is compiled. +_PLANNED: set = set() + + +@functools.lru_cache(maxsize=None) +def _num_sms(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def plan_splits(n_groups: int, kv_heads: int, width: int, num_sms: int) -> int: + """Splits so the grid fills the GPU once (one CTA an SM): `num_sms // units` CTAs a unit, and + three splits from half to three quarters of a wave while they fit in two waves + (3 * units <= 2 * num_sms). Outputs of different splits differ in the last bits.""" + est_tiles = -(-int(width * 1.25) // BN) + units = n_groups * kv_heads + band = num_sms // 2 < units <= (num_sms * 3) // 4 + if band and 3 * units <= 2 * num_sms: + return min(3, est_tiles) + return max(1, min(MAX_PLAN_SPLITS, est_tiles, num_sms // units)) + + +def plan_launch(n_groups: int, kv_heads: int, width: int, num_sms: int): + """(splits, n_whole) of a launch: n_whole None = every unit (group, KV head) takes `splits` + CTAs (one wave: `plan_splits`); else units below n_whole take one CTA and the units of the + last, partial wave (whole groups) `splits` CTAs each, merged afterwards.""" + units = n_groups * kv_heads + if units <= num_sms: + return plan_splits(n_groups, kv_heads, width, num_sms), None + waves = -(-units // num_sms) + tail = units - (waves - 1) * num_sms + tail = -(-tail // kv_heads) * kv_heads + tail_splits = min(MAX_TAIL_SPLITS, num_sms // tail) + if tail_splits < 2: + return 1, None + return tail_splits, units - tail + + +def reachable_plans(max_groups: int, kv_heads: int, width: int, num_sms: int) -> list: + """Every `(splits, tail)` pair `plan_launch` returns for `MIN_GROUPS..max_groups` groups.""" + plans = set() + for n_groups in range(MIN_GROUPS, max(MIN_GROUPS, max_groups) + 1): + splits, n_whole = plan_launch(n_groups, kv_heads, width, num_sms) + plans.add((max(1, min(MAX_SPLITS, int(splits))), n_whole is not None)) + return sorted(plans) + + +#: Launches from this many units (request groups x KV heads) to one wave take the balanced plan +#: (`_BalPlanKernel`, then the grouped kernel's balanced mode) instead of `plan_launch`'s. A unit's +#: split count depends on the batch, so a row's last bits depend on the other requests' unions. +BALANCED_MIN_UNITS = 88 + + +def takes_balanced(units: int, num_sms: int) -> bool: + """Whether a launch of `units` (group, KV head) units takes the balanced plan: from + `BALANCED_MIN_UNITS` units to one wave.""" + return BALANCED_MIN_UNITS <= units <= num_sms + + +def _plan_words(bt_width: int, tpb: int) -> int: + """Bitmap words (32 four-token groups each) of a plan CTA over a block table.""" + return -(-(bt_width * tpb // 4) // 32) + + +#: The balanced launch's persistent int32 words a device (zeros once; the kernels leave every +#: ticket at zero): one launch at a time on a device. +_BAL_BUFS: dict = {} + + +def _bal_buffers(device: torch.device, n: int) -> torch.Tensor: + """`[8 n + 16]` int32 on `device`: each CTA's plan entry (4 words a CTA, words 0..4n), the + units' merge tickets (4n..5n), the groups' union tiles (5n + 4 .. 6n + 4).""" + key = (device.index if device.index is not None else torch.cuda.current_device(), n) + buf = _BAL_BUFS.get(key) + if buf is None: + buf = torch.zeros(8 * n + 16, dtype=torch.int32, device=device) + _BAL_BUFS[key] = buf + return buf + + +def _compile_plan(pkey: tuple, plan_kw: dict, plan_args: tuple): + """The balanced launch's plan kernel of `pkey`, compiled once.""" + fn = _COMPILED.get(pkey) + if fn is None: + fn = cute.compile( + _BalPlanKernel(**plan_kw), *plan_args, options="--opt-level 2 --enable-tvm-ffi" + ) + _COMPILED[pkey] = fn + return fn + + +def takes_grouped(q, k_cache, selected_tokens, rows_per_request) -> bool: + """Whether `sparse_gqa_grouped` serves this call (on shapes only, so a per-capture constant): + an e4m3 pool on SM103, 2..4 rows a request, `MIN_GROUPS` groups, a shared-memory plan.""" + if q.dim() != 3 or k_cache.dim() != 4 or k_cache.dtype != torch.float8_e4m3fn: + return False + rows, num_q_heads, head_dim = q.shape + rpr = int(rows_per_request) + if rpr < 2 or rpr > MAX_RPR or -(-rows // rpr) < MIN_GROUPS or head_dim != HEAD_DIM: + return False + if k_cache.shape[3] != HEAD_DIM or num_q_heads % k_cache.shape[1]: + return False + if k_cache.stride(3) != 1 or k_cache.stride(2) != HEAD_DIM: + return False + if torch.cuda.get_device_capability(q.device) != (10, 3): + return False + group = num_q_heads // k_cache.shape[1] + return _layout(rpr, group, selected_tokens.shape[1], 1) is not None + + +def _compile_grouped(key: tuple, kern_kw: dict, compile_args: tuple): + """The grouped kernel of `key` (its `splits` and launch mode -- False, True for the tail + mode, "bal" for the balanced one -- at positions 5 and 6), compiled once.""" + fn = _COMPILED.get(key) + if fn is None: + kern = _GroupedKernel(splits=key[5], tail=key[6] is True, bal=key[6] == "bal", **kern_kw) + fn = cute.compile(kern, *compile_args, options="--opt-level 2 --enable-tvm-ffi") + _COMPILED[key] = fn + return fn + + +def _compile_merge(heads: int, splits: int, gated: bool, pdl: bool, merge_args: tuple, gdim: int): + """The row merge of `splits` partials, compiled once per gate tensor rank `gdim` (the + trunk's `[rows, Hq * D]` gate, the draft layer's `[rows, Hq, D]` view, q when ungated).""" + mkey = ("rowmerge", heads, splits, gated, pdl, MERGE_PARTS, gdim) + mfn = _COMPILED.get(mkey) + if mfn is None: + mfn = cute.compile( + _RowMergeKernel(heads=heads, splits=splits, gated=gated, pdl=pdl, parts=MERGE_PARTS), + *merge_args, + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[mkey] = mfn + return mfn + + +def sparse_gqa_grouped( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """The grouped kernel (and the row merge with more than one split) over an e4m3 pool; see + `QsaSparseGqaGroupedCute` for the contract.""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads, tpb, dim = k_cache.shape + rpr = int(rows_per_request) + assert head_dim == HEAD_DIM and dim == HEAD_DIM and tpb == tokens_per_block + assert k_cache.dtype == torch.float8_e4m3fn and v_cache.dtype == k_cache.dtype + assert q.dtype == torch.bfloat16 and q.stride(2) == 1 + assert (q.stride(0) * 2) % 16 == 0 and (q.stride(1) * 2) % 16 == 0 and q.data_ptr() % 16 == 0 + assert k_cache.stride() == v_cache.stride() and k_cache.stride(3) == 1 + assert k_cache.stride(2) == HEAD_DIM, "a page's token rows contiguous (a group: one bulk copy)" + assert v_cache.shape == k_cache.shape + assert k_cache.data_ptr() % 16 == 0 and v_cache.data_ptr() % 16 == 0 + assert all(s % 16 == 0 for s in k_cache.stride()[:3]), "16-byte aligned K/V rows" + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1], "row-contiguous block table" + assert selected_tokens.dtype == torch.int32 and selected_tokens.stride(1) == 1 + assert request_indices.is_contiguous() and request_indices.dtype in (torch.int32, torch.int64) + group = num_q_heads // num_kv_heads + width = selected_tokens.shape[1] + out = torch.empty((rows, num_q_heads, head_dim), dtype=torch.bfloat16, device=q.device) + if rows == 0: + return out + n_groups = -(-rows // rpr) + units = n_groups * num_kv_heads + n_whole = None + n_sms = _num_sms(q.device.index or 0) + words = _plan_words(block_table.shape[1], tpb) + bal_fits = ( + words <= MAX_PLAN_WORDS and units <= n_sms <= THREADS and _layout(rpr, group, width, 1) + ) + bal = bool(bal_fits and takes_balanced(units, n_sms)) + if bal: + splits = 1 + else: + splits, n_whole = plan_launch(n_groups, num_kv_heads, width, n_sms) + n_splits = max(1, min(MAX_SPLITS, int(splits))) + tail = n_whole is not None + assert _layout(rpr, group, width, 1 if (tail or bal) else n_splits) is not None, ( + "no shared-memory plan for this geometry: see `takes_grouped`" + ) + use_pdl = bool(get_env_enable_pdl()) + gated = output_gate is not None + gate_hs = head_dim + if gated: + assert output_gate.dtype == torch.bfloat16 and output_gate.stride(-1) == 1 + if output_gate.dim() == 3: + assert output_gate.shape == q.shape + gate_hs = output_gate.stride(1) + else: + assert output_gate.shape == (rows, num_q_heads * head_dim) + assert (output_gate.stride(0) * 2) % 16 == 0 and (gate_hs * 2) % 16 == 0 + assert output_gate.data_ptr() % 16 == 0, "16-byte aligned gate rows" + req64 = request_indices.dtype == torch.int64 + bbuf = _bal_buffers(q.device, n_sms) + if bal: + n_whole, tail_row0, n_ctas = 0, rows, n_sms + elif tail: + tail_row0 = (n_whole // num_kv_heads) * rpr + n_ctas = n_whole + (units - n_whole) * n_splits + elif n_splits > 1: + n_whole, tail_row0, n_ctas = 0, 0, units * n_splits + else: + n_whole, tail_row0, n_ctas = units, rows, units + merged = rows - tail_row0 + key = ( + rpr, + group, + num_kv_heads, + width, + tpb, + n_splits, + "bal" if bal else tail, + float(softmax_scale), + num_pages, + tuple(block_table.shape), + tuple(k_cache.stride()[:3]), + tuple(q.stride()[:2]), + req64, + gated, + use_pdl, + ) + if bal: + # a partial slot a CTA: its unit's T rows + part = torch.empty( + (n_ctas, rpr, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_ctas, rpr, num_q_heads), dtype=torch.float32, device=q.device) + elif merged > 0: + part = torch.empty( + (n_splits, merged, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_splits, merged, num_q_heads), dtype=torch.float32, device=q.device) + else: + part, lse = out, out + gate_t = output_gate if gated else q + gate_stride = output_gate.stride(0) if gated else 0 + compile_args = ( + # read with scalar loads: a row slice of the selection needs no copy + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(q, cutlass.BFloat16), + _ptr(k_cache, cutlass.Uint8), + _ptr(v_cache, cutlass.Uint8), + _ptr(block_table, cutlass.Int32, 4), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(out, cutlass.BFloat16), + _ptr(part, cutlass.Float32), + _ptr(lse, cutlass.Float32), + _ptr(gate_t, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cutlass.Int32(n_whole), + cutlass.Int32(tail_row0), + cutlass.Int32(n_ctas), + _ptr(bbuf, cutlass.Int32, 16), + _ptr(bbuf, cutlass.Int32, 4), + _ptr(bbuf, cutlass.Int32, 4), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + tab_addr = bbuf.data_ptr() + cnt_addr = tab_addr + 4 * (4 * n_sms) + est_addr = tab_addr + 4 * (5 * n_sms + 4) + plan_kw = dict( + rpr=rpr, width=width, words=words, num_requests=block_table.shape[0], pdl=use_pdl + ) + pkey = ("balplan", rpr, width, words, block_table.shape[0], req64, use_pdl) + plan_args = ( + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(bbuf, cutlass.Int32, 4), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(n_groups), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + merge_args = ( + from_dlpack(gate_t, assumed_align=16).mark_layout_dynamic(leading_dim=gate_t.dim() - 1), + _ptr(part, cutlass.Float32), + _ptr(lse, cutlass.Float32, 4), + _ptr(out, cutlass.BFloat16, 8), + cutlass.Int32(merged), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + kern_kw = dict( + rpr=rpr, + group=group, + kv_heads=num_kv_heads, + width=width, + tpb=tpb, + scale=float(softmax_scale), + num_pages=num_pages, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + k_strides=tuple(k_cache.stride()[:3]), + q_strides=tuple(q.stride()[:2]), + req64=req64, + gated=gated, + pdl=use_pdl, + bal_ctas=n_sms, + ) + # A launch plan depends on the call's row count: the first call of a geometry compiles every + # plan its row counts can reach, so no serving step compiles one. + if key[:5] + key[7:] not in _PLANNED: + _PLANNED.add(key[:5] + key[7:]) + max_groups = block_table.shape[0] + for plan_splits_, plan_tail in reachable_plans( + max_groups, num_kv_heads, width, _num_sms(q.device.index or 0) + ): + if _layout(rpr, group, width, 1 if plan_tail else plan_splits_) is None: + continue + _compile_grouped(key[:5] + (plan_splits_, plan_tail) + key[7:], kern_kw, compile_args) + if plan_tail or plan_splits_ > 1: + _compile_merge(num_q_heads, plan_splits_, gated, use_pdl, merge_args, gate_t.dim()) + if bal_fits: + _compile_grouped(key[:5] + (1, "bal") + key[7:], kern_kw, compile_args) + _compile_plan(pkey, plan_kw, plan_args) + fn = _compile_grouped(key, kern_kw, compile_args) + if bal: + pfn = _compile_plan(pkey, plan_kw, plan_args) + pfn( + selected_tokens, + request_indices.data_ptr(), + est_addr, + rows, + selected_tokens.stride(0), + n_groups, + ) + fn( + selected_tokens, + q.data_ptr(), + k_cache.data_ptr(), + v_cache.data_ptr(), + block_table.data_ptr(), + request_indices.data_ptr(), + out.data_ptr(), + part.data_ptr(), + lse.data_ptr(), + gate_t.data_ptr(), + rows, + selected_tokens.stride(0), + gate_stride, + gate_hs, + n_whole, + tail_row0, + n_ctas, + tab_addr, + cnt_addr, + est_addr, + ) + if bal or merged <= 0: + return out + mfn = _compile_merge(num_q_heads, n_splits, gated, use_pdl, merge_args, gate_t.dim()) + gate_tail = gate_t[tail_row0:] if gated else gate_t + mfn( + gate_tail, + part.data_ptr(), + lse.data_ptr(), + out[tail_row0:].data_ptr(), + merged, + gate_stride, + gate_hs, + ) + return out + + +# --- catalog entry --- + +#: The checkpoint's QSA geometry: 32-token pages, 2051 selected tokens a row, 256 head dims, a +#: `max_seq_len` 262144 table. +_QSA = dict(tokens_per_block=32, compress_ratio=4, token_topk=2048, head_dim=256, pages=8196) +_TEP2 = dict( + _QSA, q_heads=12, kv_heads=1, slots=128, slot_pages=24, layer=11, kv=torch.float8_e4m3fn +) +_DEP4 = dict( + _QSA, q_heads=24, kv_heads=2, slots=128, slot_pages=26, layer=11, kv=torch.float8_e4m3fn +) +_TEP2_ROW = dict(gate="row", row_heads=29) +_DEP4_ROW = dict(gate="row", row_heads=55) + + +def _grouped(member: dict, requests: int, rows: int = 3, last=60000, step=900, **kw) -> dict: + """A verify step of `requests` requests of `rows` rows each, request r's rows ending at + `last - step * r`; `overlap` spreads a request's rows' block scores (None: independent rows).""" + segments = tuple((last + 1 - rows - step * r, rows) for r in range(requests)) + return dict(member, segments=segments, rows_per_request=rows, **{"overlap": 0.05, **kw}) + + +class QsaSparseGqaGroupedCute(OpWrapper): + """`qsa_paged_sparse_gqa` for decode rows that come `rows_per_request` consecutive rows a + request (an MTP verify step) over an e4m3 paged pool: `[rows, Hq, D]` bf16. With `output_gate` + (`[rows, Hq * D]`, or `[rows, Hq, D]` with any head stride; bf16, unit inner stride) the + result is `bf16(attn) * sigmoid(gate)` rounded once more to bf16. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ht_tep2 MTP-2 verify: 128 x 3 = 384 rows, one CTA a group (gate fused into its " + "store), captured", + spec=_grouped(_TEP2, 128, step=400, capture=True, **_TEP2_ROW), + ), + Cell( + why="ht_tep2 MTP-2 verify: 160 x 3 = 480 rows, one CTA a group past the first wave, " + "the last wave's groups split 4 ways and merged, gated", + spec=_grouped(dict(_TEP2, slots=160), 160, step=300, **_TEP2_ROW), + ), + Cell( + why="8 requests x 3 rows deep in long sequences (196k..167k tokens): the fewest " + "groups the kernel takes, 18 splits, gated", + spec=_grouped(_TEP2, 8, last=196607, step=4099, **_TEP2_ROW), + ), + Cell( + why="16 x 3 rows whose selections are independent (union ~3x a row's), ungated", + spec=_grouped(_TEP2, 16, overlap=None), + ), + Cell( + why="ht_dep4 MTP-3 verify: 8 x 4 = 32 rows, 24 q / 2 KV heads over the e4m3 trunk " + "(26 role pages a slot), gated, captured", + spec=_grouped(_DEP4, 8, rows=4, capture=True, **_DEP4_ROW), + ), + Cell( + why="draft layer: 16 x 3 rows, the gate half of the interleaved q projection (head " + "stride 512)", + spec=_grouped(_TEP2, 16, gate="interleaved"), + ), + Cell( + why="groups spanning requests (3, 3, 1, 2, 3 x 6, 2 rows: misaligned and partial " + "groups), short sequences, one foreign row, a row slice of a wider selection, " + "ungated", + spec=dict( + _TEP2, + segments=((77, 3), (2047, 3), (9000, 1), (30000, 2)) + + tuple((40000 + 1000 * r, 3) for r in range(6)) + + ((5, 2),), + rows_per_request=3, + overlap=0.05, + row_offset=1, + foreign=True, + ), + ), + ) + + note = """ + Source: this target's CuTe DSL kernels (`_GroupedKernel`, `_RowMergeKernel`, `_BalPlanKernel`), + `cute.compile`d per (T, splits, launch mode); every reachable plan of a geometry is compiled on + its first call. Called by `modeling._sparse_gqa` for verify / draft-extend rows over an e4m3 + pool with `rows_per_request` > 1 where `serves` (an e4m3 pool on SM103, 2..4 rows a request, + at least `MIN_GROUPS` groups, T x q heads a KV head <= 128); the per-row + `qsa_sparse_gqa_splitk_cute` otherwise. Graph-capturable (the row count is a runtime value). + + Numerics: K/V widened exactly to f16, q rounded to f16, f16 MMAs with fp32 accumulation, P + rounded to f16, the softmax offset moved only past 2^8; within the decode op's band of the + fp32 reference, not bitwise the per-row kernel. Deterministic for one batch. + + Preconditions: as `qsa_sparse_gqa_splitk_cute`, plus expanded selections: each four-column + quad of a row names one four-token group at its own offsets (column % 4 == token % 4), no + group twice in a row. A NaN in a token one row selects reaches the group's other rows of that + request (P = 0 times NaN). A group's pool row `page * tokens_per_block + slot` is 32-bit + (pools below 2^26 pages). + + Not certified: the three-split band of `plan_splits` (77..87 units on 152 SMs), T = 2. + """ + + serves = staticmethod(takes_grouped) + + def __call__( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return sparse_gqa_grouped( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + rows_per_request, + output_gate=output_gate, + ) + + def reference( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + rows_per_request: int, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """The per-row entry's reference (fp32 sparse GQA over every resident selected token, + rounded once; gated in fp32 over that rounding and rounded again). `rows_per_request` + is ignored: the grouping changes which loads serve a row, not what it attends.""" + return qsa_sparse_gqa_splitk_cute.reference( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + output_gate, + ) + + +qsa_sparse_gqa_grouped_cute = QsaSparseGqaGroupedCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py new file mode 100644 index 000000000000..acb07f85d796 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_rowtile__prefill.py @@ -0,0 +1,1725 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA for context batches: several query rows per program, K/V read once per tile. + +A plan launch lists, per group of consecutive rows of one request, the union of the rows' +resident selected tokens as cache slots and re-indexes each row's selection onto it; a persistent +row-tile pass walks each list in 64-slot tiles, loading each tile's K/V once for every row and head +of the group. Groups whose rows span requests, or whose union costs more than the rows' own +columns, go to a lone-row pass over each row's own columns. A group is sized to whole 8-row +swizzle atoms (10 rows of 12 heads, 20 of 6). +""" + +from __future__ import annotations + +import math + +import cutlass +import cutlass.cute as cute +import torch +import triton +import triton.language as tl +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( + qsa_paged_sparse_gqa as _per_row_entry, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + BLOCK_M, + BLOCK_N, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + eligible as _cute_eligible, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + num_ctas as _rowtile_ctas, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rowtile_attention_cute import ( + rowtile_attention_cute as _rowtile_attention_cute, +) + +_INTEGER_DTYPES = (torch.int32, torch.int64) +# The per-row kernel's scan tile: the prefill bound is rounded up to it, as that kernel does. +_BOUND_TILE = 64 +# Programs of the lone-row pass per KV head; each walks the lone list with this stride. +_LONE_PROGRAMS = 256 # capacity: fixed: a launch width; the lone list may be any length +# A one-request group takes the row-tile pass when its list's 64-slot tiles, each costing this +# many quarters of a per-row tile (measured: about half), cost no more than its rows' own +# columns; at 2 every one-request group qualifies, as a union never holds more tiles. +_SHARED_TILE_COST = 2 +#: The O staging's swizzle atom in MMA rows: a group's rows x heads come in whole atoms. +_ROW_ATOM = 8 +_LONE_WARPS = 2 +_LONE_STAGES = 2 + + +@dsl_user_op +def _red_or_shared(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.or.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,l", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +# -------------------------------------------------------------------------------------- +# The token plan (CuTe DSL): one CTA a group ORs its rows' readable selected tokens into a +# shared-memory bitmap over the table's tokens, keeping each row's highest token. A one-request +# group masks each word to resident pages, takes list positions as a block-wide prefix of the +# words' popcounts and decides shared vs lone (`_SHARED_TILE_COST`); a shared group writes each +# listed token's cache slot and each row's re-indexed bits (rank = prefix + popcount below it). +# The list is the union's resident tokens in token order. +# -------------------------------------------------------------------------------------- + +_PLAN_THREADS = 512 +#: CTAs an SM the plan's registers are held to, so three groups' latency chains overlap. +_PLAN_MIN_BLOCKS = 3 +#: Shared scratch words of the plan CTA: rows' highest tokens at 0, rows' columns read at +#: `_MISC_COLS`, the lone list base at `_MISC_LONE`, the scan's warp totals at `_MISC_SCAN`. +_MISC_COLS = 32 +_MISC_LONE = 63 +_MISC_SCAN = 64 + + +@dsl_user_op +def _cu_popc64(x, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(x).ir_value(loc=loc, ip=ip)], + "popc.b64 $0, $1;", + "=r,l", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _cu_ctz64(x, *, loc=None, ip=None): + """Trailing zeros of a nonzero 64-bit word.""" + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(x).ir_value(loc=loc, ip=ip)], + "{ .reg .b64 t; brev.b64 t, $1; clz.b64 $0, t; }", + "=r,l", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _max_shared(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.max.s32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ld_shared_b64(addr, *, loc=None, ip=None): + return cutlass.Int64( + llvm.inline_asm( + T.i64(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip)], + "ld.shared.b64 $0, [$1];", + "=l,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _st_shared_b64(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ st.shared.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,l", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ld_shared_b32(addr, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _st_shared_b32(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +class _GroupPlanKernel: + """The token plan pass, one CTA a group (see the section comment).""" + + def __init__(self, topk, block_topk, compress_ratio, tokens_per_block, table_width, + num_words, rows_per_tile, packed_words, req64, qp64): # fmt: skip + self.topk, self.block_topk, self.cr = topk, block_topk, compress_ratio + self.tpb, self.width, self.num_words = tokens_per_block, table_width, num_words + self.rpt, self.pw = rows_per_tile, packed_words + self.req64, self.qp64 = req64, qp64 + # shared memory: union words | word prefixes | rows' packed words | row scratch + self.o_pref = num_words * 8 + self.o_pack = self.o_pref + ((num_words * 4 + 15) // 16) * 16 + self.o_misc = self.o_pack + rows_per_tile * packed_words * 8 + self.smem = self.o_misc + 96 * 4 + 64 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + qp_ptr: cute.Pointer, + req_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + info_ptr: cute.Pointer, + slots_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + plan_ptr: cute.Pointer, + lone_ptr: cute.Pointer, + lone_count_ptr: cute.Pointer, + rows: cutlass.Int32, + num_groups: cutlass.Int32, + sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, + slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, + num_cache_pages: cutlass.Int32, + stream, + ): + self.kernel( + sel_t.iterator.toint(), + qp_ptr.toint(), + req_ptr.toint(), + bt_ptr.toint(), + info_ptr.toint(), + slots_ptr.toint(), + packed_ptr.toint(), + plan_ptr.toint(), + lone_ptr.toint(), + lone_count_ptr.toint(), + rows, + sel_stride, + bt_stride, + slots_stride, + num_requests, + num_cache_pages, + ).launch( + grid=[num_groups, 1, 1], + block=[_PLAN_THREADS, 1, 1], + smem=self.smem, + stream=stream, + min_blocks_per_mp=_PLAN_MIN_BLOCKS, + ) + + @cute.kernel + def kernel(self, sel_base: cutlass.Int64, qp_base: cutlass.Int64, req_base: cutlass.Int64, + bt_base: cutlass.Int64, info_base: cutlass.Int64, slots_base: cutlass.Int64, + packed_base: cutlass.Int64, plan_base: cutlass.Int64, lone_base: cutlass.Int64, + lone_count_base: cutlass.Int64, rows: cutlass.Int32, sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, num_cache_pages: cutlass.Int32): # fmt: skip + tidx, _, _ = cute.arch.thread_idx() + group, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + warp = tidx // 32 + nthreads: cutlass.Constexpr = _PLAN_THREADS + nwarps: cutlass.Constexpr = _PLAN_THREADS // 32 + rpt: cutlass.Constexpr = self.rpt + nw: cutlass.Constexpr = self.num_words + tpb: cutlass.Constexpr = self.tpb + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem,), stride=(1,)), byte_alignment=128 + ) + s_u = cutlass.Int32(buf.iterator.toint()) + s_pref = s_u + self.o_pref + s_pack = s_u + self.o_pack + s_misc = s_u + self.o_misc + r0 = group * rpt + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < nw: + _st_shared_b64(s_u + w * 8, cutlass.Int64(0)) + if tidx < rpt: + _st_shared_b32(s_misc + tidx * 4, cutlass.Int32(-1)) + # each row's request and columns read (every thread, rpt rows) + reqs = [] + cols = [] + own_tiles = cutlass.Int32(0) + n_live = cutlass.Int32(0) + lowest = cutlass.Int32(2147483647) + highest_req = cutlass.Int32(-1) + for t in cutlass.range_constexpr(rpt): + row = r0 + t + live = row < rows + srow = cutlass.min(row, rows - 1) + if const_expr(self.req64): + rq = cutlass.Int32(_ldg_i64(req_base + cutlass.Int64(srow) * 8)) + else: + rq = _ldg_i32(req_base + cutlass.Int64(srow) * 4) + if const_expr(self.qp64): + qp = cutlass.Int32(_ldg_i64(qp_base + cutlass.Int64(srow) * 8)) + else: + qp = _ldg_i32(qp_base + cutlass.Int64(srow) * 4) + visible = qp + 1 + sb = cutlass.min(visible // self.cr, cutlass.Int32(self.block_topk)) + valid = sb * self.cr + visible % self.cr + bounded = cutlass.min(valid, cutlass.Int32(self.topk)) + cr_ = cutlass.min((bounded + _BOUND_TILE - 1) // _BOUND_TILE * _BOUND_TILE, + cutlass.Int32(self.topk)) # fmt: skip + cr_ = cutlass.max(cr_, cutlass.Int32(0)) + cr_ = cutlass.Int32(cutlass.select_(live, cr_, cutlass.Int32(0))) + reqs.append(rq) + cols.append(cr_) + if tidx == t: + _st_shared_b32(s_misc + (_MISC_COLS + t) * 4, cr_) + own_tiles = own_tiles + (cr_ + BLOCK_N - 1) // BLOCK_N + n_live = n_live + cutlass.Int32( + cutlass.select_(live, cutlass.Int32(1), cutlass.Int32(0)) + ) + lowest = cutlass.Int32(cutlass.select_(live, cutlass.min(lowest, rq), lowest)) + highest_req = cutlass.Int32( + cutlass.select_(live, cutlass.max(highest_req, rq), highest_req) + ) + one_request = (lowest == highest_req) & (lowest >= 0) & (lowest < num_requests) + cute.arch.sync_threads() + # walk 1: the rows' readable tokens into the union bitmap, each row's highest token. + # Items are (row, four-column quad) pairs, item = i * nthreads + tidx, kept in registers + # for walk 2. + nq: cutlass.Constexpr = -(-self.topk // 4) + n_items: cutlass.Constexpr = rpt * nq + n_rounds: cutlass.Constexpr = -(-n_items // nthreads) + toks = [] + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + q = item % nq + ncol = _ld_shared_b32(s_misc + (_MISC_COLS + t) * 4) + ncol = cutlass.Int32(cutlass.select_(item < n_items, ncol, cutlass.Int32(0))) + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + for k in cutlass.range_constexpr(4): + c = q * 4 + k + tok = cutlass.Int32(-1) + if c < ncol: + tok = _ldg_i32(row_sel + cutlass.Int64(c) * 4) + ok = (tok >= 0) & (tok // tpb < self.width) + toks.append(cutlass.Int32(cutlass.select_(ok, tok, cutlass.Int32(-1)))) + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + cur_w = cutlass.Int32(-1) + cur_m = cutlass.Int64(0) + hi_q = cutlass.Int32(-1) + for k in cutlass.range_constexpr(4): + tok = toks[i * 4 + k] + if tok >= 0: + w = tok >> 6 + if w != cur_w: + if cur_w >= 0: + _red_or_shared(s_u + cur_w * 8, cur_m) + cur_w = w + cur_m = cutlass.Int64(0) + cur_m = cur_m | (cutlass.Int64(1) << cutlass.Int64(tok & 63)) + hi_q = cutlass.max(hi_q, tok) + if cur_w >= 0: + _red_or_shared(s_u + cur_w * 8, cur_m) + # the row's highest token: a warp's 32 items span at most two rows (nq > 32) + t_first = cute.arch.shuffle_sync(t, 0) + same = t == t_first + hi_a = cutlass.Int32(cutlass.select_(same, hi_q, cutlass.Int32(-1))) + hi_b = cutlass.Int32(cutlass.select_(same, cutlass.Int32(-1), hi_q)) + for o in (1, 2, 4, 8, 16): + hi_a = cutlass.max(hi_a, cute.arch.shuffle_sync_bfly(hi_a, offset=o)) + hi_b = cutlass.max(hi_b, cute.arch.shuffle_sync_bfly(hi_b, offset=o)) + if lane == 0: + if hi_a >= 0: + _max_shared(s_misc + t_first * 4, hi_a) + t_last = cute.arch.shuffle_sync(t, 31) + if lane == 31: + if (t_last != t_first) & (hi_b >= 0): + _max_shared(s_misc + t_last * 4, hi_b) + cute.arch.sync_threads() + # row_info, the group's highest token + grp_hi = cutlass.Int32(-1) + for t in cutlass.range_constexpr(rpt): + h = _ld_shared_b32(s_misc + t * 4) + grp_hi = cutlass.max(grp_hi, h) + if tidx == t: + if r0 + t < rows: + _stg_i32(info_base + cutlass.Int64(r0 + t) * 8, h) + _stg_i32(info_base + cutlass.Int64(r0 + t) * 8 + 4, cols[t]) + words_needed = cutlass.Int32( + cutlass.select_(one_request, (grp_hi + 64) // 64, cutlass.Int32(0)) + ) + words_needed = cutlass.min(words_needed, cutlass.Int32(nw)) + # resident tokens: a word's bits on a page outside the table or the pool are dropped + bt_row = bt_base + cutlass.Int64(cutlass.max(lowest, 0)) * bt_stride * 4 + cnts = [] + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + cnt = cutlass.Int32(0) + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + if u != 0: + first = w * 64 + for k in cutlass.range_constexpr(_word_pages(tpb)): + lp = first // tpb + k + lo_b = cutlass.max(lp * tpb - first, 0) + hi_b = cutlass.min((lp + 1) * tpb - first, 64) + if lo_b < hi_b: + ok = lp < self.width + pg = cutlass.Int32(-1) + if ok: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + ok = ok & (pg >= 0) & (pg < num_cache_pages) + span = _bits_range64(lo_b, hi_b) + u = cutlass.Int64(cutlass.select_(ok, u, u & ~span)) + _st_shared_b64(s_u + w * 8, u) + cnt = _cu_popc64(u) + cnts.append(cnt) + # list positions: block-wide exclusive prefix of the words' counts in word order; thread + # tidx owns words tidx + i * nthreads, so each round is scanned and the rounds chained. + base = cutlass.Int32(0) + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + incl = cnts[i] + for o in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=o) + if lane >= o: + incl = incl + up + if lane == 31: + _st_shared_b32(s_misc + (_MISC_SCAN + warp) * 4, incl) + cute.arch.sync_threads() + wv = cutlass.Int32(0) + tot = cutlass.Int32(0) + if lane < nwarps: + wv = _ld_shared_b32(s_misc + (_MISC_SCAN + lane) * 4) + tot = wv + before = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, wv, cutlass.Int32(0))) + ) + total = cute.arch.warp_reduction_sum(tot) + w = tidx + i * nthreads + if w < nw: + _st_shared_b32(s_pref + w * 4, base + before + incl - cnts[i]) + base = base + total + cute.arch.sync_threads() + listed = base + tiles = (listed + BLOCK_N - 1) // BLOCK_N + shared = one_request & (tiles * _SHARED_TILE_COST <= own_tiles * 4) + shared_i = cutlass.Int32(cutlass.select_(shared, cutlass.Int32(1), cutlass.Int32(0))) + if tidx == 0: + _stg_i32( + plan_base + cutlass.Int64(group) * 8, + cutlass.Int32(cutlass.select_(shared, tiles, cutlass.Int32(-1))), + ) + _stg_i32(plan_base + cutlass.Int64(group) * 8 + 4, listed) + if shared_i == 0: + lb = _atom_add_g32(lone_count_base, n_live) + _st_shared_b32(s_misc + _MISC_LONE * 4, lb) + if shared_i == 0: + cute.arch.sync_threads() + if tidx < rpt: + if r0 + tidx < rows: + lb = _ld_shared_b32(s_misc + _MISC_LONE * 4) + _stg_i32(lone_base + cutlass.Int64(lb + tidx) * 4, r0 + tidx) + if (shared_i != 0) & (listed > 0): + pw_used = (listed + 63) // 64 + # the rows' packed words, zeroed + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + _st_shared_b64(s_pack + i * 8, cutlass.Int64(0)) + # slots: one thread a word, its set bits in token order + slot_row = slots_base + cutlass.Int64(group) * slots_stride * 4 + for i in cutlass.range_constexpr((nw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + j = _ld_shared_b32(s_pref + w * 4) + pg_lp = cutlass.Int32(-1) + pg = cutlass.Int32(0) + while u != 0: + b = _cu_ctz64(u) + tok = w * 64 + b + lp = tok // tpb + if lp != pg_lp: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + pg_lp = lp + _stg_i32(slot_row + cutlass.Int64(j) * 4, pg * tpb + tok % tpb) + j = j + 1 + u = u & (u - 1) + cute.arch.sync_threads() + # walk 2 (walk 1's registers): each row's readable resident tokens -> its bit at the + # token's list position; a quad's ranks follow by popcount from its word's prefix. + for i in cutlass.range_constexpr(n_rounds): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tok0 = cutlass.max( + cutlass.max(toks[i * 4], toks[i * 4 + 1]), + cutlass.max(toks[i * 4 + 2], toks[i * 4 + 3]), + ) + if tok0 >= 0: + w = tok0 >> 6 + u = _ld_shared_b64(s_u + w * 8) + pre = _ld_shared_b32(s_pref + w * 4) + cur_w = cutlass.Int32(-1) + cur_m = cutlass.Int64(0) + for k in cutlass.range_constexpr(4): + tok = toks[i * 4 + k] + if tok >= 0: + uw = u + pw_pre = pre + if (tok >> 6) != w: + uw = _ld_shared_b64(s_u + (tok >> 6) * 8) + pw_pre = _ld_shared_b32(s_pref + (tok >> 6) * 4) + bit = cutlass.Int64(1) << cutlass.Int64(tok & 63) + if (uw & bit) != 0: + jj = pw_pre + _cu_popc64(uw & (bit - 1)) + pw_i = jj >> 6 + if pw_i != cur_w: + if cur_w >= 0: + _red_or_shared(s_pack + (t * pw_used + cur_w) * 8, cur_m) + cur_w = pw_i + cur_m = cutlass.Int64(0) + cur_m = cur_m | (cutlass.Int64(1) << cutlass.Int64(jj & 63)) + if cur_w >= 0: + _red_or_shared(s_pack + (t * pw_used + cur_w) * 8, cur_m) + cute.arch.sync_threads() + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + t = i // pw_used + k = i % pw_used + if r0 + t < rows: + _stg_i64(packed_base + (cutlass.Int64(r0 + t) * self.pw + k) * 8, + _ld_shared_b64(s_pack + i * 8)) # fmt: skip + + +def _word_pages(tpb: int) -> int: + """Pages an aligned 64-token word spans at most.""" + if 64 % tpb == 0: + return 64 // tpb + if tpb % 64 == 0: + return 1 + return -(-64 // tpb) + 1 + + +def _bits_range64(lo, hi): + """int64 with bits lo .. hi - 1 set (0 <= lo < hi <= 64).""" + ones = cutlass.Int64(-1) + below_hi = cutlass.Int64( + cutlass.select_(hi >= 64, ones, (cutlass.Int64(1) << cutlass.Int64(hi)) - 1) + ) + below_lo = (cutlass.Int64(1) << cutlass.Int64(lo)) - 1 + return below_hi & ~below_lo + + +@dsl_user_op +def _ldg_i32(addr, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b32 $0, [$1];", + "=r,l", has_side_effects=True, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +@dsl_user_op +def _ldg_i64(addr, *, loc=None, ip=None): + return cutlass.Int64( + llvm.inline_asm( + T.i64(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b64 $0, [$1];", + "=l,l", has_side_effects=True, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +@dsl_user_op +def _stg_i32(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ st.global.b32 [$1], $2; mov.u32 $0, 0; }", "=r,l,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _stg_i64(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int64(val).ir_value(loc=loc, ip=ip)], + "{ st.global.b64 [$1], $2; mov.u32 $0, 0; }", "=r,l,l", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _atom_add_g32(addr, val, *, loc=None, ip=None): + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "atom.global.add.u32 $0, [$1], $2;", "=r,l,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +_GROUP_PLAN: dict = {} + + +def _group_plan(selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, num_words, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages): # fmt: skip + """Launch the token plan pass (one CTA a group).""" + rows, final_topk = selected_tokens.shape + block_topk = (final_topk - compress_ratio + 1) // compress_ratio + req64 = request_indices.dtype == torch.int64 + qp64 = query_positions.dtype == torch.int64 + query_positions = query_positions.contiguous() + key = (final_topk, compress_ratio, tokens_per_block, table_width, rows_per_tile, + packed_words, req64, qp64) # fmt: skip + gm = cute.AddressSpace.gmem + ptrs = ( + query_positions.data_ptr(), request_indices.data_ptr(), block_table.data_ptr(), + row_info.data_ptr(), slots.data_ptr(), packed_bits.data_ptr(), group_plan.data_ptr(), + lone_rows.data_ptr(), lone_count.data_ptr(), + ) # fmt: skip + scalars = (rows, num_groups, selected_tokens.stride(0), block_table.stride(0), slots.stride(0), + block_table.shape[0], num_cache_pages) # fmt: skip + fn = _GROUP_PLAN.get(key) + if fn is None: + i32, i64 = cutlass.Int32, cutlass.Int64 + fn = cute.compile( + _GroupPlanKernel( + final_topk, + block_topk, + compress_ratio, + tokens_per_block, + table_width, + num_words, + rows_per_tile, + packed_words, + req64, + qp64, + ), # fmt: skip + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + make_ptr(i64 if qp64 else i32, ptrs[0], gm, assumed_align=4), + make_ptr(i64 if req64 else i32, ptrs[1], gm, assumed_align=4), + make_ptr(i32, ptrs[2], gm, assumed_align=4), + make_ptr(i32, ptrs[3], gm, assumed_align=8), + make_ptr(i32, ptrs[4], gm, assumed_align=4), + make_ptr(i64, ptrs[5], gm, assumed_align=8), + make_ptr(i32, ptrs[6], gm, assumed_align=8), + make_ptr(i32, ptrs[7], gm, assumed_align=4), + make_ptr(i32, ptrs[8], gm, assumed_align=4), + *(cutlass.Int32(x) for x in scalars), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GROUP_PLAN[key] = fn + fn(selected_tokens, *ptrs, *scalars) + + +# -------------------------------------------------------------------------------------- +# The block plan (CuTe DSL): the token plan's outputs, bit for bit, from the selection's 4-token +# blocks. Every token a row selects below `T0 = 4 * ((p + 1) // 4)` (`p` the group's lowest +# position) lies in a complete block: an aligned column quad below `T0` sets one bit of a block +# bitmap; tokens at or past `T0` (at most `rows_per_tile + 3`) go to one 64-token local word. A +# group naming a token below `T0` outside such a quad goes to the lone pass. Its shared-memory ORs +# are 32-bit (`red.shared.or.b64` is a compare-and-swap loop on SM100). +# -------------------------------------------------------------------------------------- + +_BP_THREADS = 512 +#: CTAs an SM the block plan's registers are held to. +_BP_MIN_BLOCKS = 4 +#: Shared scratch words of the block plan CTA (int32 slots of its misc area). +_BP_HI = 0 # rows' highest tokens (rows_per_tile) +_BP_COLS = 32 # rows' columns read +_BP_SCAN = 64 # the scan's warp totals (16) +_BP_LONE = 80 # the lone list base +_BP_FLAG = 81 # a token below T0 outside an aligned block quad +_BP_NSEG = 82 # the group's segments (runs of one request's rows), 0 for a lone group +_BP_SPLIT = 83 # the second segment's first row +_BP_NLIVE = 84 +_BP_OWN = 85 # the rows' own 64-column tiles +_BP_REQ0 = 86 # each segment's request +_BP_REQ1 = 87 +_BP_T00 = 88 # each segment's T0 +_BP_T01 = 89 +_BP_LOCN = 90 # the segment's resident local tokens +_BP_LOC = 92 # the segment's local word (int64 at words 92-93) +_BP_MISC_WORDS = 96 + + +def _span_pages(tokens: int, tpb: int, aligned: bool) -> int: + """Pages a run of `tokens` tokens spans at most (`aligned`: it starts on a multiple of + `tokens`).""" + if aligned and tokens % tpb == 0: + return tokens // tpb + if aligned and tpb % tokens == 0: + return 1 + return -(-tokens // tpb) + 1 + + +@dsl_user_op +def _stg_v4_i32(addr, a, b, c, d, *, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)] + + [cutlass.Int32(x).ir_value(loc=loc, ip=ip) for x in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", "=r,l,r,r,r,r", + has_side_effects=True, is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, ip=ip, + ) # fmt: skip + + +@dsl_user_op +def _ldg_nc_i32(addr, *, loc=None, ip=None): + """A global int32 load the compiler may schedule freely (no side effects).""" + return cutlass.Int32( + llvm.inline_asm( + T.i32(), [cutlass.Int64(addr).ir_value(loc=loc, ip=ip)], "ld.global.b32 $0, [$1];", + "=r,l", has_side_effects=False, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + ) # fmt: skip + + +def _word_page_ids(bt_row, lp0, tpb, width): + """The physical pages of a block word's logical pages `lp0 ..` (clamped into the table; the + caller drops pages past it), loaded together.""" + out = [] + for k in range(_span_pages(256, tpb, True)): # trace-time unrolled + lp = cutlass.min(lp0 + k, width - 1) + out.append(_ldg_nc_i32(bt_row + cutlass.Int64(lp) * 4)) + return out + + +@dsl_user_op +def _red_or_shared32(addr, val, *, loc=None, ip=None): + """A native shared-memory OR (`red.shared.or.b64` is a compare-and-swap loop on SM100).""" + llvm.inline_asm( + T.i32(), + [cutlass.Int32(addr).ir_value(loc=loc, ip=ip), cutlass.Int32(val).ir_value(loc=loc, ip=ip)], + "{ red.shared.or.b32 [$1], $2; mov.u32 $0, 0; }", "=r,r,r", has_side_effects=True, + is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) # fmt: skip + + +class _BlockPlanKernel: + """The block plan pass, one CTA a group (see the section comment).""" + + def __init__(self, topk, block_topk, tokens_per_block, table_width, block_words, + rows_per_tile, packed_words, req64, qp64): # fmt: skip + self.topk, self.block_topk = topk, block_topk + self.tpb, self.width, self.nbw = tokens_per_block, table_width, block_words + self.rpt, self.pw = rows_per_tile, packed_words + self.req64, self.qp64 = req64, qp64 + # shared memory: union block words | word prefixes | rows' packed words | misc + self.o_pref = block_words * 8 + self.o_pack = self.o_pref + ((block_words * 4 + 15) // 16) * 16 + self.o_misc = self.o_pack + rows_per_tile * packed_words * 8 + self.smem = self.o_misc + _BP_MISC_WORDS * 4 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + qp_ptr: cute.Pointer, + req_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + info_ptr: cute.Pointer, + slots_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + plan_ptr: cute.Pointer, + lone_ptr: cute.Pointer, + lone_count_ptr: cute.Pointer, + rows: cutlass.Int32, + num_groups: cutlass.Int32, + sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, + slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, + num_cache_pages: cutlass.Int32, + stream, + ): + self.kernel( + sel_t.iterator.toint(), qp_ptr.toint(), req_ptr.toint(), bt_ptr.toint(), + info_ptr.toint(), slots_ptr.toint(), packed_ptr.toint(), plan_ptr.toint(), + lone_ptr.toint(), lone_count_ptr.toint(), rows, sel_stride, bt_stride, slots_stride, + num_requests, num_cache_pages, + ).launch( + grid=[num_groups, 1, 1], + block=[_BP_THREADS, 1, 1], + smem=self.smem, + stream=stream, + min_blocks_per_mp=_BP_MIN_BLOCKS, + ) # fmt: skip + + def _quad(self, sel_base, r0, rows, sel_stride, s_misc, item, ra, rb): + """Walk item `item`'s four columns (a clamped column, masked after): each readable + token, -1 elsewhere (also for an item past the rows or outside rows [ra, rb)).""" + nq = -(-self.topk // 4) + t = cutlass.min(item // nq, self.rpt - 1) + q = item % nq + ncol = _ld_shared_b32(s_misc + (_BP_COLS + t) * 4) + in_seg = (item < self.rpt * nq) & (t >= ra) & (t < rb) + ncol = cutlass.Int32(cutlass.select_(in_seg, ncol, cutlass.Int32(0))) + row_sel = sel_base + cutlass.Int64(cutlass.min(r0 + t, rows - 1)) * sel_stride * 4 + tk = [] + for k in range(4): # trace-time unrolled + c = q * 4 + k + cc = cutlass.Int32(cutlass.select_(c < ncol, c, cutlass.Int32(0))) + tok = _ldg_nc_i32(row_sel + cutlass.Int64(cc) * 4) + ok = (c < ncol) & (tok >= 0) & (tok // self.tpb < self.width) + tk.append(cutlass.Int32(cutlass.select_(ok, tok, cutlass.Int32(-1)))) + return tk + + @cute.kernel + def kernel(self, sel_base: cutlass.Int64, qp_base: cutlass.Int64, req_base: cutlass.Int64, + bt_base: cutlass.Int64, info_base: cutlass.Int64, slots_base: cutlass.Int64, + packed_base: cutlass.Int64, plan_base: cutlass.Int64, lone_base: cutlass.Int64, + lone_count_base: cutlass.Int64, rows: cutlass.Int32, sel_stride: cutlass.Int32, + bt_stride: cutlass.Int32, slots_stride: cutlass.Int32, + num_requests: cutlass.Int32, num_cache_pages: cutlass.Int32): # fmt: skip + tidx, _, _ = cute.arch.thread_idx() + group, _, _ = cute.arch.block_idx() + lane = cute.arch.lane_idx() + warp = tidx // 32 + nthreads: cutlass.Constexpr = _BP_THREADS + nwarps: cutlass.Constexpr = _BP_THREADS // 32 + rpt: cutlass.Constexpr = self.rpt + nbw: cutlass.Constexpr = self.nbw + tpb: cutlass.Constexpr = self.tpb + pw: cutlass.Constexpr = self.pw + big = cutlass.Int32(2147483647) + smem = cutlass.utils.SmemAllocator() + buf = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem,), stride=(1,)), byte_alignment=128 + ) + s_u = cutlass.Int32(buf.iterator.toint()) + s_pref = s_u + self.o_pref + s_pack = s_u + self.o_pack + s_misc = s_u + self.o_misc + r0 = group * rpt + # the rows (warp 0, lane t holding row t): columns read, request and position; the + # group's segment (one request's rows; any other group is lone), live rows, own tiles, T0 + if warp == 0: + row = r0 + lane + live = (lane < rpt) & (row < rows) + srow = cutlass.min(row, rows - 1) + if const_expr(self.req64): + rq = cutlass.Int32(_ldg_i64(req_base + cutlass.Int64(srow) * 8)) + else: + rq = _ldg_i32(req_base + cutlass.Int64(srow) * 4) + if const_expr(self.qp64): + qp = cutlass.Int32(_ldg_i64(qp_base + cutlass.Int64(srow) * 8)) + else: + qp = _ldg_i32(qp_base + cutlass.Int64(srow) * 4) + visible = qp + 1 + sb = cutlass.min(visible // 4, cutlass.Int32(self.block_topk)) + valid = sb * 4 + visible % 4 + bounded = cutlass.min(valid, cutlass.Int32(self.topk)) + cr_ = cutlass.min((bounded + _BOUND_TILE - 1) // _BOUND_TILE * _BOUND_TILE, + cutlass.Int32(self.topk)) # fmt: skip + cr_ = cutlass.max(cr_, cutlass.Int32(0)) + cr_ = cutlass.Int32(cutlass.select_(live, cr_, cutlass.Int32(0))) + if lane < rpt: + _st_shared_b32(s_misc + (_BP_COLS + lane) * 4, cr_) + _st_shared_b32(s_misc + (_BP_HI + lane) * 4, cutlass.Int32(-1)) + q0 = cute.arch.shuffle_sync(rq, 0) # row r0 is always live + other = live & (rq != q0) + split = cutlass.Int32(cutlass.select_(other, lane, big)) + q1_lo = cutlass.Int32(cutlass.select_(other, rq, big)) + q1_hi = cutlass.Int32(cutlass.select_(other, rq, cutlass.Int32(-1))) + own = (cr_ + BLOCK_N - 1) // BLOCK_N + nl = cutlass.Int32(cutlass.select_(live, cutlass.Int32(1), cutlass.Int32(0))) + for o in (1, 2, 4, 8, 16): + split = cutlass.min(split, cute.arch.shuffle_sync_bfly(split, offset=o)) + q1_lo = cutlass.min(q1_lo, cute.arch.shuffle_sync_bfly(q1_lo, offset=o)) + q1_hi = cutlass.max(q1_hi, cute.arch.shuffle_sync_bfly(q1_hi, offset=o)) + own = own + cute.arch.shuffle_sync_bfly(own, offset=o) + nl = nl + cute.arch.shuffle_sync_bfly(nl, offset=o) + # rows of request q0 past the split break the run (three runs): lone + late0 = live & (rq == q0) & (lane > split) + seg1 = lane >= split + p0 = cutlass.Int32(cutlass.select_(live & (seg1 == False), qp, big)) # noqa: E712 + p1 = cutlass.Int32(cutlass.select_(live & seg1, qp, big)) + late = cutlass.Int32(cutlass.select_(late0, cutlass.Int32(1), cutlass.Int32(0))) + for o in (1, 2, 4, 8, 16): + p0 = cutlass.min(p0, cute.arch.shuffle_sync_bfly(p0, offset=o)) + p1 = cutlass.min(p1, cute.arch.shuffle_sync_bfly(p1, offset=o)) + late = late | cute.arch.shuffle_sync_bfly(late, offset=o) + if lane == 0: + ok0 = (q0 >= 0) & (q0 < num_requests) & (p0 >= 0) + ok1 = (q1_lo == q1_hi) & (q1_lo >= 0) & (q1_lo < num_requests) & (p1 >= 0) + ok1 = ok1 & (late == 0) + nseg = cutlass.Int32(0) + if split == big: + nseg = cutlass.Int32(cutlass.select_(ok0, cutlass.Int32(1), cutlass.Int32(0))) + _st_shared_b32(s_misc + _BP_NSEG * 4, nseg) + _st_shared_b32(s_misc + _BP_SPLIT * 4, cutlass.min(split, cutlass.Int32(rpt))) + _st_shared_b32(s_misc + _BP_REQ0 * 4, cutlass.max(q0, 0)) + _st_shared_b32(s_misc + _BP_REQ1 * 4, cutlass.max(cutlass.min(q1_lo, big - 1), 0)) + _st_shared_b32(s_misc + _BP_NLIVE * 4, nl) + _st_shared_b32(s_misc + _BP_OWN * 4, own) + _st_shared_b32(s_misc + _BP_T00 * 4, + cutlass.Int32(cutlass.select_(ok0, (p0 + 1) // 4 * 4, cutlass.Int32(0)))) # fmt: skip + _st_shared_b32(s_misc + _BP_T01 * 4, + cutlass.Int32(cutlass.select_(ok1, (p1 + 1) // 4 * 4, cutlass.Int32(0)))) # fmt: skip + _st_shared_b32(s_misc + _BP_FLAG * 4, cutlass.Int32(0)) + cute.arch.sync_threads() + nseg = _ld_shared_b32(s_misc + _BP_NSEG * 4) + split = _ld_shared_b32(s_misc + _BP_SPLIT * 4) + n_live = _ld_shared_b32(s_misc + _BP_NLIVE * 4) + own_tiles = _ld_shared_b32(s_misc + _BP_OWN * 4) + # the rows' packed words (a fixed `packed_words` stride a row), zeroed + for i in cutlass.range_constexpr((rpt * pw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < rpt * pw: + _st_shared_b64(s_pack + w * 8, cutlass.Int64(0)) + nq: cutlass.Constexpr = -(-self.topk // 4) + n_items: cutlass.Constexpr = rpt * nq + n_rounds: cutlass.Constexpr = -(-n_items // nthreads) + listed = cutlass.Int32(0) + for sg in cutlass.range_constexpr(1): + # a lone group (no segment) still walks its rows once: row_info's highest tokens + if sg < cutlass.max(nseg, 1): + # this segment: rows [ra, rb) of request `req`, list positions from `seg_base` + ra = cutlass.Int32(0) if sg == 0 else split + rb = cutlass.Int32(cutlass.select_(nseg == 2, split, cutlass.Int32(rpt))) + if sg == 1: + rb = cutlass.Int32(rpt) + req = _ld_shared_b32(s_misc + (_BP_REQ0 + sg) * 4) + t_lo = _ld_shared_b32(s_misc + (_BP_T00 + sg) * 4) + seg_base = listed + bt_row = bt_base + cutlass.Int64(req) * bt_stride * 4 + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < nbw: + _st_shared_b64(s_u + w * 8, cutlass.Int64(0)) + if tidx == 0: + _st_shared_b64(s_misc + _BP_LOC * 4, cutlass.Int64(0)) + cute.arch.sync_threads() + # walk 1: items are (row, four-column quad) pairs, item = i * nthreads + tidx; a + # block quad below T0 sets its block bit, any other quad its tokens' local bits + # (rolled: unrolled it spills). + for i in cutlass.range(0, n_rounds, unroll=1): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tk = self._quad(sel_base, r0, rows, sel_stride, s_misc, item, ra, rb) + b0 = tk[0] + is_blk = (b0 >= 0) & (b0 < t_lo) & ((b0 & 3) == 0) + is_blk = is_blk & (tk[1] == b0 + 1) & (tk[2] == b0 + 2) & (tk[3] == b0 + 3) + hi_q = cutlass.max(cutlass.max(tk[0], tk[1]), cutlass.max(tk[2], tk[3])) + if is_blk: + blk = b0 >> 2 + _red_or_shared32(s_u + (blk >> 5) * 4, cutlass.Int32(1) << (blk & 31)) + else: + if hi_q >= 0: + lm = cutlass.Int64(0) + bad = cutlass.Int32(0) + for k in cutlass.range_constexpr(4): + d = tk[k] - t_lo + inr = (d >= 0) & (d < 64) + live_k = tk[k] >= 0 + lm = lm | cutlass.Int64( + cutlass.select_( + live_k & inr, + cutlass.Int64(1) << cutlass.Int64(d & 63), + cutlass.Int64(0), + ) + ) + bad = bad | cutlass.Int32( + cutlass.select_( + live_k & ((d < 0) | (d >= 64)), + cutlass.Int32(1), + cutlass.Int32(0), + ) + ) + lm_lo = cutlass.Int32(lm & cutlass.Int64(0xFFFFFFFF)) + lm_hi = cutlass.Int32(lm >> cutlass.Int64(32)) + if lm_lo != 0: + _red_or_shared32(s_misc + _BP_LOC * 4, lm_lo) + if lm_hi != 0: + _red_or_shared32(s_misc + _BP_LOC * 4 + 4, lm_hi) + if bad != 0: + _st_shared_b32(s_misc + _BP_FLAG * 4, cutlass.Int32(1)) + # the row's highest token: a warp's 32 items span at most two rows (nq > 32) + t_first = cute.arch.shuffle_sync(t, 0) + same = t == t_first + hi_a = cutlass.Int32(cutlass.select_(same, hi_q, cutlass.Int32(-1))) + hi_b = cutlass.Int32(cutlass.select_(same, cutlass.Int32(-1), hi_q)) + for o in (1, 2, 4, 8, 16): + hi_a = cutlass.max(hi_a, cute.arch.shuffle_sync_bfly(hi_a, offset=o)) + hi_b = cutlass.max(hi_b, cute.arch.shuffle_sync_bfly(hi_b, offset=o)) + if lane == 0: + if hi_a >= 0: + _max_shared(s_misc + (_BP_HI + t_first) * 4, hi_a) + t_last = cute.arch.shuffle_sync(t, 31) + if lane == 31: + if (t_last != t_first) & (hi_b >= 0): + _max_shared(s_misc + (_BP_HI + t_last) * 4, hi_b) + cute.arch.sync_threads() + use = (_ld_shared_b32(s_misc + _BP_FLAG * 4) == 0) & (nseg > 0) + words_needed = cutlass.Int32( + cutlass.select_(use, (t_lo // 4 + 63) // 64, cutlass.Int32(0)) + ) + words_needed = cutlass.min(words_needed, cutlass.Int32(nbw)) + # resident blocks: a word's bits on a page outside the table or the pool dropped + cnts = [] + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + cnt = cutlass.Int32(0) + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + if u != 0: + first = w * 256 + pgs = _word_page_ids(bt_row, first // tpb, tpb, self.width) + for k in cutlass.range_constexpr(_span_pages(256, tpb, True)): + lp = first // tpb + k + lo_b = cutlass.max(lp * tpb - first, 0) // 4 + hi_b = cutlass.min((lp + 1) * tpb - first, 256) // 4 + if lo_b < hi_b: + span = _bits_range64(lo_b, hi_b) + pg = pgs[k] + ok = (lp < self.width) & (pg >= 0) & (pg < num_cache_pages) + u = cutlass.Int64(cutlass.select_(ok, u, u & ~span)) + _st_shared_b64(s_u + w * 8, u) + cnt = _cu_popc64(u) + cnts.append(cnt) + # the local word's resident tokens + if tidx == nthreads - 1: + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + if use & (lw != 0): + for k in cutlass.range_constexpr(_span_pages(64, tpb, False)): + lp = t_lo // tpb + k + lo_b = cutlass.max(lp * tpb - t_lo, 0) + hi_b = cutlass.min((lp + 1) * tpb - t_lo, 64) + if lo_b < hi_b: + span = _bits_range64(lo_b, hi_b) + if (lw & span) != 0: + ok = lp < self.width + pg = cutlass.Int32(-1) + if ok: + pg = _ldg_i32(bt_row + cutlass.Int64(lp) * 4) + ok = ok & (pg >= 0) & (pg < num_cache_pages) + lw = cutlass.Int64(cutlass.select_(ok, lw, lw & ~span)) + lw = cutlass.Int64(cutlass.select_(use, lw, cutlass.Int64(0))) + _st_shared_b64(s_misc + _BP_LOC * 4, lw) + _st_shared_b32(s_misc + _BP_LOCN * 4, _cu_popc64(lw)) + cute.arch.sync_threads() + # list positions of the blocks: exclusive prefix of the words' counts, round by round + base = cutlass.Int32(0) + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + if i * nthreads < words_needed: + incl = cnts[i] + for o in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=o) + if lane >= o: + incl = incl + up + if lane == 31: + _st_shared_b32(s_misc + (_BP_SCAN + warp) * 4, incl) + cute.arch.sync_threads() + wv = cutlass.Int32(0) + tot = cutlass.Int32(0) + if lane < nwarps: + wv = _ld_shared_b32(s_misc + (_BP_SCAN + lane) * 4) + tot = wv + before = cute.arch.warp_reduction_sum( + cutlass.Int32(cutlass.select_(lane < warp, wv, cutlass.Int32(0))) + ) + total = cute.arch.warp_reduction_sum(tot) + w = tidx + i * nthreads + if w < nbw: + _st_shared_b32(s_pref + w * 4, base + before + incl - cnts[i]) + base = base + total + cute.arch.sync_threads() + local_base = seg_base + base * 4 + listed = local_base + _ld_shared_b32(s_misc + _BP_LOCN * 4) + if use & (listed > seg_base): + # slots: a thread a block word, four (one 16-byte store) a listed block + slot_row = slots_base + cutlass.Int64(group) * slots_stride * 4 + for i in cutlass.range_constexpr((nbw + nthreads - 1) // nthreads): + w = tidx + i * nthreads + if w < words_needed: + u = _ld_shared_b64(s_u + w * 8) + j = seg_base + _ld_shared_b32(s_pref + w * 4) * 4 + lp0 = (w * 256) // tpb + pgs = _word_page_ids(bt_row, lp0, tpb, self.width) + while u != 0: + b = _cu_ctz64(u) + tok = (w * 64 + b) * 4 + which = tok // tpb - lp0 + pg = pgs[0] + for k in cutlass.range_constexpr(1, len(pgs)): + pg = cutlass.Int32(cutlass.select_(which == k, pgs[k], pg)) + s0 = pg * tpb + tok % tpb + if (seg_base & 3) == 0: + _stg_v4_i32(slot_row + cutlass.Int64(j) * 4, + s0, s0 + 1, s0 + 2, s0 + 3) # fmt: skip + else: + # a second segment's blocks may sit off a 16-byte boundary + _stg_i32(slot_row + cutlass.Int64(j) * 4, s0) + _stg_i32(slot_row + cutlass.Int64(j + 1) * 4, s0 + 1) + _stg_i32(slot_row + cutlass.Int64(j + 2) * 4, s0 + 2) + _stg_i32(slot_row + cutlass.Int64(j + 3) * 4, s0 + 3) + j = j + 4 + u = u & (u - 1) + if tidx == nthreads - 1: + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + j = local_base + while lw != 0: + b = _cu_ctz64(lw) + tok = t_lo + b + pg = _ldg_i32(bt_row + cutlass.Int64(tok // tpb) * 4) + _stg_i32(slot_row + cutlass.Int64(j) * 4, pg * tpb + tok % tpb) + j = j + 1 + lw = lw & (lw - 1) + # walk 2 (quads re-read): a listed block's four bits, a local token's one bit + lw = _ld_shared_b64(s_misc + _BP_LOC * 4) + for i in cutlass.range(0, n_rounds, unroll=1): + item = i * nthreads + tidx + t = cutlass.min(item // nq, rpt - 1) + tk = self._quad(sel_base, r0, rows, sel_stride, s_misc, item, ra, rb) + b0 = tk[0] + is_blk = (b0 >= 0) & (b0 < t_lo) & ((b0 & 3) == 0) + is_blk = is_blk & (tk[1] == b0 + 1) & (tk[2] == b0 + 2) & (tk[3] == b0 + 3) + row_pack = s_pack + t * (pw * 8) + if is_blk: + blk = b0 >> 2 + w = blk >> 6 + bit = cutlass.Int64(1) << cutlass.Int64(blk & 63) + uw = _ld_shared_b64(s_u + w * 8) + if (uw & bit) != 0: + k = _ld_shared_b32(s_pref + w * 4) + _cu_popc64(uw & (bit - 1)) + pos = seg_base + k * 4 + _red_or_shared32(row_pack + (pos >> 5) * 4, + cutlass.Int32(15) << (pos & 31)) # fmt: skip + else: + for k in cutlass.range_constexpr(4): + d = tk[k] - t_lo + if (tk[k] >= 0) & (d >= 0) & (d < 64): + bit = cutlass.Int64(1) << cutlass.Int64(d) + if (lw & bit) != 0: + jj = local_base + _cu_popc64(lw & (bit - 1)) + _red_or_shared32(row_pack + (jj >> 5) * 4, + cutlass.Int32(1) << (jj & 31)) # fmt: skip + cute.arch.sync_threads() + use = (nseg > 0) & (_ld_shared_b32(s_misc + _BP_FLAG * 4) == 0) + listed = cutlass.Int32(cutlass.select_(use, listed, cutlass.Int32(0))) + if tidx < rpt: + if r0 + tidx < rows: + _stg_i32(info_base + cutlass.Int64(r0 + tidx) * 8, + _ld_shared_b32(s_misc + (_BP_HI + tidx) * 4)) # fmt: skip + _stg_i32(info_base + cutlass.Int64(r0 + tidx) * 8 + 4, + _ld_shared_b32(s_misc + (_BP_COLS + tidx) * 4)) # fmt: skip + tiles = (listed + BLOCK_N - 1) // BLOCK_N + shared = use & (tiles * _SHARED_TILE_COST <= own_tiles * 4) + shared_i = cutlass.Int32(cutlass.select_(shared, cutlass.Int32(1), cutlass.Int32(0))) + if tidx == 0: + _stg_i32( + plan_base + cutlass.Int64(group) * 8, + cutlass.Int32(cutlass.select_(shared, tiles, cutlass.Int32(-1))), + ) + _stg_i32(plan_base + cutlass.Int64(group) * 8 + 4, listed) + if shared_i == 0: + lb = _atom_add_g32(lone_count_base, n_live) + _st_shared_b32(s_misc + _BP_LONE * 4, lb) + if shared_i == 0: + cute.arch.sync_threads() + if tidx < rpt: + if r0 + tidx < rows: + lb = _ld_shared_b32(s_misc + _BP_LONE * 4) + _stg_i32(lone_base + cutlass.Int64(lb + tidx) * 4, r0 + tidx) + if (shared_i != 0) & (listed > 0): + pw_used = (listed + 63) // 64 + for i in cutlass.range(tidx, rpt * pw_used, nthreads, unroll=1): + t = i // pw_used + k = i % pw_used + if r0 + t < rows: + _stg_i64(packed_base + (cutlass.Int64(r0 + t) * pw + k) * 8, + _ld_shared_b64(s_pack + (t * pw + k) * 8)) # fmt: skip + + +_BLOCK_PLAN: dict = {} + + +def block_plan_serves(compress_ratio: int, tokens_per_block: int, rows_per_tile: int) -> bool: + """Whether `_block_plan` serves a launch: blocks of 4 tokens, pages of whole blocks, a + group's rows in one warp.""" + return compress_ratio == 4 and tokens_per_block % 4 == 0 and rows_per_tile <= 32 + + +def _block_plan(selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages): # fmt: skip + """Launch the block plan pass (one CTA a group); the fused plan's outputs.""" + rows, final_topk = selected_tokens.shape + block_topk = (final_topk - compress_ratio + 1) // compress_ratio + block_words = triton.cdiv(triton.cdiv(table_width * tokens_per_block, 4), 64) + req64 = request_indices.dtype == torch.int64 + qp64 = query_positions.dtype == torch.int64 + query_positions = query_positions.contiguous() + key = (final_topk, tokens_per_block, table_width, rows_per_tile, packed_words, req64, qp64) + gm = cute.AddressSpace.gmem + ptrs = ( + query_positions.data_ptr(), request_indices.data_ptr(), block_table.data_ptr(), + row_info.data_ptr(), slots.data_ptr(), packed_bits.data_ptr(), group_plan.data_ptr(), + lone_rows.data_ptr(), lone_count.data_ptr(), + ) # fmt: skip + scalars = (rows, num_groups, selected_tokens.stride(0), block_table.stride(0), slots.stride(0), + block_table.shape[0], num_cache_pages) # fmt: skip + fn = _BLOCK_PLAN.get(key) + if fn is None: + i32, i64 = cutlass.Int32, cutlass.Int64 + fn = cute.compile( + _BlockPlanKernel( + final_topk, + block_topk, + tokens_per_block, + table_width, + block_words, + rows_per_tile, + packed_words, + req64, + qp64, + ), # fmt: skip + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + make_ptr(i64 if qp64 else i32, ptrs[0], gm, assumed_align=4), + make_ptr(i64 if req64 else i32, ptrs[1], gm, assumed_align=4), + make_ptr(i32, ptrs[2], gm, assumed_align=4), + make_ptr(i32, ptrs[3], gm, assumed_align=8), + make_ptr(i32, ptrs[4], gm, assumed_align=16), + make_ptr(i64, ptrs[5], gm, assumed_align=8), + make_ptr(i32, ptrs[6], gm, assumed_align=8), + make_ptr(i32, ptrs[7], gm, assumed_align=4), + make_ptr(i32, ptrs[8], gm, assumed_align=4), + *(cutlass.Int32(x) for x in scalars), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _BLOCK_PLAN[key] = fn + fn(selected_tokens, *ptrs, *scalars) + + +@triton.jit(do_not_specialize=["rows", "NUM_CACHE_PAGES", "NUM_REQUESTS"]) +def _rowtile_lone_kernel( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, + lone_rows, lone_count, output, softmax_scale, rows, + q_stride_row, q_stride_head: tl.constexpr, q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, v_stride_dim: tl.constexpr, + block_table_stride_request: tl.constexpr, block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + NUM_CACHE_PAGES, NUM_REQUESTS, LONE_PROGRAMS, + PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_H: tl.constexpr, BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, USE_PDL: tl.constexpr, + LONE_STAGES: tl.constexpr, +): # fmt: skip + """Rows of groups the row-tile pass does not serve: each over its own columns. q's row stride + is a runtime argument, so q in place on the projection row and a compact row share a compile.""" + if USE_PDL: + # Independent of the row-tile pass (disjoint rows): let it start now. + tl.extra.cuda.gdc_launch_dependents() + kv_head = tl.program_id(1) + for slot in range(tl.program_id(0), tl.load(lone_count), LONE_PROGRAMS): + _rowtile_lone_row( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, output, + softmax_scale, tl.load(lone_rows + slot).to(tl.int64), kv_head, + q_stride_row, q_stride_head, q_stride_dim, + k_stride_page, k_stride_head, k_stride_token, k_stride_dim, + v_stride_page, v_stride_head, v_stride_token, v_stride_dim, + block_table_stride_request, block_table_stride_page, + selected_stride_row, selected_stride_token, + output_stride_row, output_stride_head, output_stride_dim, + NUM_CACHE_PAGES, NUM_REQUESTS, PAGE_TABLE_WIDTH, GROUP_SIZE, TOKENS_PER_BLOCK, + BLOCK_H, BLOCK_N, HEAD_DIM, LONE_STAGES, + ) # fmt: skip + + +@triton.jit +def _rowtile_lone_row( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, output, + softmax_scale, row, kv_head, + q_stride_row, q_stride_head, q_stride_dim, + k_stride_page, k_stride_head, k_stride_token, k_stride_dim, + v_stride_page, v_stride_head, v_stride_token, v_stride_dim, + block_table_stride_request, block_table_stride_page, + selected_stride_row, selected_stride_token, + output_stride_row, output_stride_head, output_stride_dim, + NUM_CACHE_PAGES, NUM_REQUESTS, PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, TOKENS_PER_BLOCK: tl.constexpr, BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, HEAD_DIM: tl.constexpr, LONE_STAGES: tl.constexpr, +): # fmt: skip + request = tl.load(request_indices + row).to(tl.int64) + valid_request = (request >= 0) & (request < NUM_REQUESTS) + safe_request = tl.minimum(tl.maximum(request, 0), NUM_REQUESTS - 1) + page_base = block_table + safe_request * block_table_stride_request + columns_read = tl.load(row_info + row * 2 + 1) + head_offsets = tl.arange(0, BLOCK_H) + dim_offsets = tl.arange(0, HEAD_DIM) + query_heads = kv_head * GROUP_SIZE + head_offsets + head_live = head_offsets < GROUP_SIZE + q_tile = tl.load( + q + row * q_stride_row + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=head_live[:, None], + other=0.0, + ) # fmt: skip + accumulator = tl.zeros([BLOCK_H, HEAD_DIM], tl.float32) + running_max = tl.full([BLOCK_H], -1.0e30, tl.float32) + running_sum = tl.zeros([BLOCK_H], tl.float32) + token_offsets = tl.arange(0, BLOCK_N) + for tile in tl.range(0, tl.cdiv(columns_read, BLOCK_N), num_stages=LONE_STAGES): + columns = tile * BLOCK_N + token_offsets + tokens = tl.load( + selected_tokens + row * selected_stride_row + columns * selected_stride_token, + mask=columns < columns_read, + other=-1, + ).to(tl.int32) + safe_tokens = tl.maximum(tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + usable = valid_request & (tokens >= 0) & (logical_pages < PAGE_TABLE_WIDTH) + physical_pages = tl.load( + page_base + tl.minimum(logical_pages, PAGE_TABLE_WIDTH - 1) * block_table_stride_page, + mask=usable, + other=-1, + ).to(tl.int64) + usable = usable & (physical_pages >= 0) & (physical_pages < NUM_CACHE_PAGES) + safe_pages = tl.maximum(physical_pages, 0) + keys = tl.load( + k_cache + safe_pages[None, :] * k_stride_page + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + dim_offsets[:, None] * k_stride_dim, + mask=usable[None, :], + other=0.0, + ).to(q_tile.dtype) # fmt: skip + values = tl.load( + v_cache + safe_pages[:, None] * v_stride_page + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + dim_offsets[None, :] * v_stride_dim, + mask=usable[:, None], + other=0.0, + ).to(q_tile.dtype) # fmt: skip + scores = tl.dot(q_tile, keys) * (softmax_scale * 1.4426950408889634) + scores = tl.where(usable[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where(usable[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0) + accumulator = tl.dot( + probabilities.to(q_tile.dtype), values, accumulator * correction[:, None] + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + normalized = tl.where( + running_sum[:, None] > 0, accumulator / tl.maximum(running_sum[:, None], 1.0e-20), 0.0 + ) + tl.store( + output + row * output_stride_row + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized.to(output.dtype.element_ty), + mask=head_live[:, None], + ) # fmt: skip + + +def _check_inputs(q, k_cache, v_cache, block_table, selected_tokens, request_indices, + tokens_per_block, softmax_scale, query_positions, compress_ratio): # fmt: skip + if q.ndim != 3: + raise ValueError("QSA queries must have shape [rows, query_heads, head_dim]") + if k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA K/V caches must have matching HND paged shapes") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA sparse GQA requires a two-dimensional int32 block table") + if selected_tokens.ndim != 2 or selected_tokens.dtype != torch.int32: + raise ValueError("QSA selected tokens must be a two-dimensional int32 tensor") + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or selected_tokens.shape[1] <= 0: + raise ValueError("QSA selected-token rows must match non-empty query rows") + if request_indices.shape != (rows,) or request_indices.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA request indices must have one integer entry per row") + if not request_indices.is_contiguous(): + raise ValueError("QSA request indices must be contiguous") + if tokens_per_block <= 0 or k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA cache token dimension must match positive tokens_per_block") + if k_cache.shape[3] != head_dim or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA query and K/V cache head geometry is incompatible") + if q.dtype != torch.bfloat16: + raise ValueError("QSA queries must use BF16") + if k_cache.dtype != torch.bfloat16: + raise ValueError("QSA K/V cache must use BF16") + if selected_tokens.stride(1) != 1: + raise ValueError("QSA selected tokens must have unit column stride") + num_kv_heads = k_cache.shape[1] + if num_kv_heads <= 0 or num_q_heads <= 0 or num_q_heads % num_kv_heads: + raise ValueError("QSA query heads must be divisible by non-zero KV heads") + if k_cache.shape[0] <= 0 or block_table.shape[0] <= 0 or block_table.shape[1] <= 0: + raise ValueError("QSA sparse GQA requires non-empty cache and block-table geometry") + if k_cache.shape[0] * tokens_per_block >= 2**31: + raise ValueError("QSA row-tile cache slots must fit int32") + if not math.isfinite(softmax_scale) or softmax_scale <= 0: + raise ValueError("QSA sparse GQA softmax scale must be finite and positive") + if query_positions is None or compress_ratio is None: + raise ValueError("the row-tile kernel needs the prefill pair (positions, compress ratio)") + if query_positions.shape != (rows,) or query_positions.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA query positions must be one integer entry per row") + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + token_topk = selected_tokens.shape[1] - compress_ratio + 1 + if token_topk <= 0 or token_topk % compress_ratio: + raise ValueError("QSA selected-token width is incompatible with its compression ratio") + devices = {t.device for t in (q, k_cache, v_cache, block_table, selected_tokens, + request_indices, query_positions)} # fmt: skip + if len(devices) != 1 or next(iter(devices)).type != "cuda": + raise ValueError("QSA row-tile sparse GQA needs every tensor on one CUDA device") + if num_q_heads // num_kv_heads > BLOCK_M: + raise ValueError("a row's query heads per KV head must fit the 128-row tile") + if not _cute_eligible(q, k_cache, v_cache): + raise ValueError( + "the row-tile pass needs SM103, head_dim 256, unit-stride heads (q's heads packed) " + "and 16-byte-aligned rows" + ) + + +def rows_per_group(group_size: int) -> int: + """Rows of one request a row-tile item serves: the most whose `rows x group_size` MMA + rows fill whole 8-row swizzle atoms of the 128-row tile (10 at 12 heads, 20 at 6).""" + rows = BLOCK_M // group_size + while rows > 1 and rows * group_size % _ROW_ATOM: + rows -= 1 + return rows + + +def plan_launch(block_table, selected_tokens, request_indices, query_positions, + tokens_per_block, compress_ratio, rows_per_tile, num_cache_pages): # fmt: skip + """The row-tile plan of a context launch: `(row_info, slots, packed_bits, group_plan, + lone_rows, counters)` (`counters[0]` the lone list's length, `counters[1]` the row-tile + pass's ticket, zeroed), from one launch: on the selection's blocks wherever + `block_plan_serves`, else on its tokens. Both give the same values.""" + rows = selected_tokens.shape[0] + device = selected_tokens.device + table_width = block_table.shape[1] + num_words = triton.cdiv(table_width * tokens_per_block, 64) + num_groups = triton.cdiv(rows, rows_per_tile) + # A group lists at most its rows' columns: the bound on its slots and re-indexed bits. + listed_cap = min(num_words * 64, rows_per_tile * selected_tokens.shape[1]) + packed_words = triton.cdiv(listed_cap, 64) + slots = torch.empty((num_groups, packed_words * 64), dtype=torch.int32, device=device) + packed_bits = torch.zeros((rows, packed_words), dtype=torch.int64, device=device) + group_plan = torch.empty((num_groups, 2), dtype=torch.int32, device=device) + lone_rows = torch.empty((rows,), dtype=torch.int32, device=device) + counters = torch.zeros((2,), dtype=torch.int32, device=device) + lone_count = counters[:1] + row_info = torch.empty((rows, 2), dtype=torch.int32, device=device) + if block_plan_serves(compress_ratio, tokens_per_block, rows_per_tile): + _block_plan( + selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages, + ) # fmt: skip + else: + _group_plan( + selected_tokens, query_positions, request_indices, block_table, compress_ratio, + tokens_per_block, table_width, num_words, rows_per_tile, num_groups, packed_words, + row_info, slots, packed_bits, group_plan, lone_rows, lone_count, num_cache_pages, + ) # fmt: skip + return row_info, slots, packed_bits, group_plan, lone_rows, counters + + +def qsa_rowtile_sparse_gqa( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, +) -> torch.Tensor: + """Plan pass (`plan_launch`), lone-row pass, row-tile pass (`qsa_rowtile_attention_cute`); + returns `[rows, Hq, D]` contiguous.""" + _check_inputs(q, k_cache, v_cache, block_table, selected_tokens, request_indices, + tokens_per_block, softmax_scale, query_positions, compress_ratio) # fmt: skip + rows, num_q_heads, head_dim = q.shape + num_kv_heads = k_cache.shape[1] + group_size = num_q_heads // num_kv_heads + output = torch.empty((rows, num_q_heads, head_dim), dtype=q.dtype, device=q.device) + if rows == 0: + return output + table_width = block_table.shape[1] + rows_per_tile = rows_per_group(group_size) + kv_strides = ( + k_cache.stride(0), k_cache.stride(1), k_cache.stride(2), k_cache.stride(3), + v_cache.stride(0), v_cache.stride(1), v_cache.stride(2), v_cache.stride(3), + ) # fmt: skip + q_strides = (q.stride(0), q.stride(1), q.stride(2)) + common = (*q_strides, *kv_strides, block_table.stride(0), block_table.stride(1)) + geometry = dict( + NUM_CACHE_PAGES=k_cache.shape[0], NUM_REQUESTS=block_table.shape[0], + PAGE_TABLE_WIDTH=table_width, GROUP_SIZE=group_size, + TOKENS_PER_BLOCK=tokens_per_block, HEAD_DIM=head_dim, + ) # fmt: skip + out_strides = (output.stride(0), output.stride(1), output.stride(2)) + num_groups = triton.cdiv(rows, rows_per_tile) + row_info, slots, packed_bits, group_plan, lone_rows, counters = plan_launch( + block_table, selected_tokens, request_indices, query_positions, tokens_per_block, + compress_ratio, rows_per_tile, k_cache.shape[0], + ) # fmt: skip + lone_count = counters[:1] + use_pdl = get_env_enable_pdl() + _rowtile_lone_kernel[(min(rows, _LONE_PROGRAMS), num_kv_heads)]( + q, k_cache, v_cache, block_table, selected_tokens, request_indices, row_info, + lone_rows, lone_count, output, softmax_scale, rows, + *common, selected_tokens.stride(0), selected_tokens.stride(1), *out_strides, + BLOCK_H=max(16, triton.next_power_of_2(group_size)), BLOCK_N=64, + LONE_PROGRAMS=min(rows, _LONE_PROGRAMS), USE_PDL=use_pdl, LONE_STAGES=_LONE_STAGES, + num_warps=_LONE_WARPS, **geometry, + ) # fmt: skip + _rowtile_attention_cute[(_rowtile_ctas(num_groups * num_kv_heads, q.device),)]( + q, k_cache, v_cache, slots, packed_bits, group_plan, output, counters[1:], softmax_scale, + rows, + group_size=group_size, rows_per_tile=rows_per_tile, tokens_per_block=tokens_per_block, + use_pdl=use_pdl, + ) # fmt: skip + return output + + +_CHECKPOINT = dict( + q_heads=24, + kv_heads=2, + head_dim=256, + tokens_per_block=32, + compress_ratio=4, + token_topk=2048, + cache_dtype=torch.bfloat16, + q_dtype=torch.bfloat16, + # q is the first heads of the packed [T, Hq + 2 Hkv, 256] QKV buffer. + q_layout="qkv_slice", + request_dtype=torch.int32, + # The engine's table width at max_seq_len 262144, -1 past each sequence. + pages=8196, +) + +#: The trunk: 24 role pages a slot, layer 11's K and V read, pages straddling 2^31 elements. +_TRUNK = dict(cache_layout="trunk", slot_pages=24, layer=11, offset_edge=2**31) +#: Attention DP with one-model MTP: the draft layer's planes (layer 12) join the trunk's slot. +_TRUNK_MTP_DP = dict(cache_layout="trunk", slot_pages=26, layer=12, offset_edge=2**31) +#: tp4 / tep4 rank: 6 q heads and one KV head. +_TP4 = dict(q_heads=6, kv_heads=1, slots=128, **_TRUNK) +#: tp2 / tep2 rank: 12 q heads and one KV head. +_TP2 = dict(q_heads=12, kv_heads=1, slots=128, **_TRUNK) +#: dep4 rank over the bf16 trunk, or over the kv_fp8 pages widened into a bf16 scratch. +_DEP4 = dict(q_heads=24, kv_heads=2, slots=32, **_TRUNK) +_DEP4_SCRATCH = dict(q_heads=24, kv_heads=2, slots=32, cache_layout="scratch") + + +def _spec(segments, member, **overrides) -> dict: + """`segments` is `(first position, rows)` per request, in row order.""" + return {**_CHECKPOINT, **member, "segments": tuple(segments), **overrides} + + +class QsaSparseGqaRowtile(OpWrapper): + """Sparse GQA over the selected tokens of an HND paged K/V pool, for a + launch that carries the prefill pair (`query_positions`, `compress_ratio`). + + For row `r`, query head `h`, KV head `h // (Hq // Hkv)`, over the columns + `j < bound(r)` of `selected_tokens[r]` that name a resident token:: + + t = selected_tokens[r, j] # logical position + page = block_table[request_indices[r], t // tokens_per_block] + s_j = (q[r, h] . k_cache[page, kv, t % tpb]) * softmax_scale + out[r, h] = sum_j softmax(s)_j * v_cache[page, kv, t % tpb] + + `bound(r)` is the per-row kernel's prefill bound -- `min(visible // cr, + block_topk) * cr + visible % cr` columns with `visible = query_positions[r] + + 1`, rounded up to 64 -- so a row reads exactly the columns + `qsa_paged_sparse_gqa` reads on a context launch. Negative indices, pages + past the table, unmapped or out-of-pool pages and foreign requests + contribute nothing; a row left with nothing returns zeros. Returns a + fresh, contiguous `[rows, Hq, D]` tensor of `q.dtype`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="tp4/tep4 2049 rows from 9000: a last group of 9 rows, 6 q / 1 kv heads", + spec=_spec([(9000, 2049)], _TP4), + ), + Cell( + why="tp4/tep4 deep chunk: 8192 rows from 188416 ranking one score vector " + "(a sparse union), 6 q / 1 kv heads, trunk pool straddling 2^31", + spec=_spec([(188416, 8192)], _TP4, shared_scores=True), + ), + Cell( + why="dep4 kv_fp8 deep chunk: 8160 rows from 188416 over the widened bf16 " + "scratch, 24 q / 2 kv heads (10-row groups), [32, 8196] table", + spec=_spec([(188416, 8160)], _DEP4_SCRATCH, shared_scores=True), + ), + Cell( + why="dep4 core: three context requests (0+3000, 40000+4000, 120000+1160) in one " + "launch, 24 q / 2 kv heads, trunk pool straddling 2^31", + spec=_spec([(0, 3000), (40000, 4000), (120000, 1160)], _DEP4), + ), + Cell( + why="dep4 MTP draft layer: 2100 rows from 50000 over layer 12 of a 26-role-page " + "trunk (the draft layer in the trunk's manager), 24 q / 2 kv heads", + spec=_spec([(50000, 2100)], {**_DEP4, **_TRUNK_MTP_DP}, shared_scores=True), + ), + Cell( + why="tp2/tep2 deep chunk: 8192 rows from 120000, 12 q / 1 kv heads (10-row " + "groups), [128, 8196] table, trunk pool straddling 2^31", + spec=_spec([(120000, 8192)], _TP2, shared_scores=True), + ), + Cell( + why="tp4/tep4 engine position dtype: int64 query positions with int32 request " + "indices, 2049 rows from 9000, 6 q / 1 kv heads", + spec=_spec([(9000, 2049)], _TP4, position_dtype=torch.int64), + ), + ) + + note = """ + Source: this target's plan (CuTe DSL `_block_plan` / `_group_plan`), lone-row (Triton) and + row-tile (`qsa_rowtile_attention_cute`, tcgen05) passes: SM103, bf16 q and K/V, D 256. It + takes `qsa_paged_sparse_gqa`'s arguments with the prefill pair required. + + Preconditions (not checkable without reading the contents): + - within the bound, a row names each token at most once: the row-tile pass attends to + the set of tokens a row names, the per-row kernel to every column. QSA's top-k and + expansion never repeat one. Rows served by the lone-row pass keep per-column semantics. + - as for the per-row kernel: each row is prefix-packed with exactly the bound's count of + valid entries; entries past the 64-rounded bound are ignored. + + Work grows with the union of a group's selections; below a few hundred rows the per-row + kernel is faster. Compiled once per (group size, rows per tile, tokens per block, PDL). + Calls allocate their plan scratch, so CUDA-graph capture is not certified (the prefill path + runs eager). Deterministic. + + Not certified: int64 request indices; rows whose selection resolves no token + (all padding, a foreign request, unmapped pages); `rows == 0`. + """ + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, + ) -> torch.Tensor: + return qsa_rowtile_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + ) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor, + compress_ratio: int, + ) -> torch.Tensor: + """`qsa_paged_sparse_gqa.reference`: fp32 sparse GQA over every resident selected token, + rounded once (the prefill pair is ignored: the bound reads exactly the valid entries).""" + return _per_row_entry.reference( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + ) + + +qsa_sparse_gqa_rowtile = QsaSparseGqaRowtile() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py new file mode 100644 index 000000000000..d267f0d3aaee --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_sparse_gqa_splitk_cute__decode.py @@ -0,0 +1,1567 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA sparse GQA for decode rows over a bf16 or e4m3 paged pool: CuTe DSL split-K + merge. + +The split-K pipeline and merge of the family's `qsa_paged_sparse_gqa_nvfp4_decode`, reading K +and V as bf16 or exactly widened e4m3 -> f16 instead of NVFP4. + +One CTA (eight warps) serves one (row, KV head, split): it reads q before its PDL wait, +resolves the split's selected tokens to pool rows, streams 64-token K/V tiles through a +cp.async ring, and runs S = q K^T and O += P V on `mma.sync.m16n8k16` with an online +softmax in the log2 domain. The head dimension is permuted identically in q and K (lane +quad c of k-step pair m holds dims 32m + 8c .. 32m + 8c + 7). e4m3 V rows are widened into +an f16 tile over their own ring slot once every raw read of the slot is done, so three +slots fit two CTAs an SM. With one split the CTA stores (and gates) the output; otherwise +the merge kernel combines fp32 partials by their log2-sum-exp and applies the gate. +""" + +from __future__ import annotations + +import math +from typing import Optional + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_paged_sparse_gqa import ( + qsa_paged_sparse_gqa as _composition, +) + +TILE = 64 +WARPS = 8 +THREADS = WARPS * 32 +HEAD_DIM = 256 +#: Bytes of a raw K row in shared memory, padded so a fragment read is bank-conflict free. +K_LD = {2: 576, 1: 288} +#: Bytes of a raw e4m3 V row (unpadded: the widening pass rotates its reads instead). +V8_LD = 256 +#: Bytes of a V row of the MMA tile (256 + 8 elements: conflict-free ldmatrix). +SV_LD = 528 +#: Bytes of a P row (64 + 8 elements). +SP_LD = 144 +#: Tiles in flight at most (cp.async ring). +STAGES = 3 +LOG2E = 1.4426950408889634 +#: Output elements a merge CTA covers per head. +CHUNK = 128 +MAX_SPLITS = 64 + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _unpack(res, n, ty, loc, ip): + return [ + ty(llvm.extractvalue(T.i32() if ty is cutlass.Int32 else T.f32(), res, [i], loc=loc, ip=ip)) + for i in range(n) + ] + + +@dsl_user_op +def _cp16z(saddr, gaddr, nbytes, *, loc=None, ip=None): + """cp.async of 16 bytes, zero-filled past `nbytes` (0 or 16): no branch for a dead token.""" + _asm( + T.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + ], + "{ cp.async.cg.shared.global [$1], [$2], 16, $3; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts4(saddr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)] + + [_ir(v, cutlass.Int32, loc, ip) for v in (a, b, c, d)], + "{ st.shared.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts1(saddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds1(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds4(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldg4(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldsm4(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _ldsm4t(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (f16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([T.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _ex2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "ex2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_to_f2(w, *, loc=None, ip=None): + """bf16x2 -> f16x2 (through f32, round to nearest).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; .reg .f32 fl, fh; mov.b32 {lo, hi}, $1; " + "cvt.f32.bf16 fl, lo; cvt.f32.bf16 fh, hi; cvt.rn.f16x2.f32 $0, fh, fl; }", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _f2_pack(lo, hi, *, loc=None, ip=None): + """Two f32 -> f16x2 (lo in the low half).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.f16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_pack(lo, hi, *, loc=None, ip=None): + """Two f32 -> bf16x2 (lo in the low half).""" + return cutlass.Int32( + _asm( + T.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.bf16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg1(gaddr, a, *, loc=None, ip=None): + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.global.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg2f(gaddr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Float32, loc, ip), + _ir(b, cutlass.Float32, loc, ip), + ], + "{ st.global.v2.f32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg1(gaddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + T.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_lo(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_hi(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _f32_bits(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(T.i32(), [_ir(x, cutlass.Float32, loc, ip)], "mov.b32 $0, $1;", "=r,f", loc, ip) + ) + + +@dsl_user_op +def _bits_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(T.f32(), [_ir(x, cutlass.Int32, loc, ip)], "mov.b32 $0, $1;", "=f,r", loc, ip) + ) + + +@dsl_user_op +def _round_bf16(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + T.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $1; cvt.f32.bf16 $0, h; }", + "=f,f", + loc, + ip, + ) + ) + + +def _sigmoid(x): + """1 / (1 + exp(-x)) as the bf16 merge computes it.""" + return 1.0 / (1.0 + cute.math.exp(-cutlass.Float32(x))) + + +@dsl_user_op +def _e4m3x4(word, *, loc=None, ip=None): + """Four e4m3 bytes -> two f16x2 (exact), the lower byte of each pair in the low half.""" + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(word, cutlass.Int32, loc, ip)], + "{ .reg .b16 l, h; mov.b32 {l, h}, $2; " + "cvt.rn.f16x2.e4m3x2 $0, l; cvt.rn.f16x2.e4m3x2 $1, h; }", + "=r,=r,r", + loc, + ip, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds2(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([T.i32()] * 2), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v2.b32 {$0, $1}, [$2];", + "=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma_bf16(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (bf16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([T.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +class _SplitKernel: + def __init__( + self, + *, + elem, + group, + kv_heads, + width, + tpb, + splits, + tps, + scale, + num_pages, + bt_width, + num_requests, + kv_strides, + q_strides, + gated, + pdl, + ): + assert elem in (1, 2) and group <= 16 and tpb & (tpb - 1) == 0 + self.elem = elem # bytes an element of the pool: 2 (bf16) or 1 (e4m3) + self.group, self.kvh, self.width, self.tpb = group, kv_heads, width, tpb + self.splits, self.tps = splits, tps + self.n_tiles = -(-width // TILE) + self.scale = scale * LOG2E + self.num_pages, self.bt_width, self.num_requests = num_pages, bt_width, num_requests + self.kvs, self.qs = kv_strides, q_strides + # Two CTAs an SM (at most 128 registers a thread). + self.gated, self.pdl, self.min_blocks = gated, pdl, 2 + self.heads = group * kv_heads + self.k_ld = K_LD[elem] + self.stages = max(1, min(STAGES, tps)) + # shared memory (bytes): token table | ring of tile slots | P | cross-warp exchange. + # A slot holds the tile's K and V rows: bf16 V straight in its MMA tile; e4m3 V raw, + # widened into an MMA tile over the whole slot once its raw K and V have been read. + self.o_tok = 0 + self.tok_bytes = -(-tps * TILE * 4 // 128) * 128 + self.raw_k = TILE * self.k_ld + self.raw_v = TILE * (V8_LD if elem == 1 else SV_LD) + self.raw_buf = self.raw_k + self.raw_v + if elem == 1: + self.raw_buf = max(self.raw_buf, TILE * SV_LD) + self.o_raw = self.o_tok + self.tok_bytes + self.o_sp = self.o_raw + self.stages * self.raw_buf + self.o_red = self.o_sp + 16 * SP_LD + self.smem = self.o_red + 2 * WARPS * 16 * 4 + 128 + + @cute.jit + def __call__( + self, + sel_t: cute.Tensor, + q_ptr: cute.Pointer, + k_ptr: cute.Pointer, + v_ptr: cute.Pointer, + bt_ptr: cute.Pointer, + req_ptr: cute.Pointer, + out_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + gate_ptr: cute.Pointer, + rows: cutlass.Int32, + sel_stride: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + sel = cute.make_tensor( + sel_t.iterator, cute.make_layout((rows, self.width), stride=(sel_stride, 1)) + ) + bt = cute.make_tensor( + bt_ptr, cute.make_layout((self.num_requests, self.bt_width), stride=(self.bt_width, 1)) + ) + req = cute.make_tensor(req_ptr, cute.make_layout((rows,), stride=(1,))) + self.kernel( + sel, + q_ptr.toint(), + k_ptr.toint(), + v_ptr.toint(), + bt, + req, + out_ptr.toint(), + lse_ptr.toint(), + gate_ptr.toint(), + rows, + gate_stride, + gate_hs, + ).launch( + grid=[rows * self.kvh * self.splits, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=self.min_blocks, + ) + + @cute.jit + def _issue(self, j, buf, tidx, kvh, s_tok, s_raw, k_base, v_base): + """cp.async of tile `j`'s K and V rows into ring slot `buf`, committed as one group. + + The four threads of a token read 64 contiguous bytes an instruction; a token that does + not resolve gets zeros (src-size 0). A bf16 V row lands in its MMA tile swizzled: + 16-byte chunk 8 q + k at 8 q + (k ^ 2 q), the map `ldmatrix.trans` reads through.""" + tpb: cutlass.Constexpr = self.tpb + lt: cutlass.Constexpr = tpb.bit_length() - 1 + elem: cutlass.Constexpr = self.elem + k_ld: cutlass.Constexpr = self.k_ld + ks0, ks1, ks2 = self.kvs + rk = s_raw + buf * self.raw_buf + rv = rk + self.raw_k + tt = tidx // 4 + q4 = tidx % 4 + ent = _lds1(s_tok + (j * TILE + tt) * 4) + ok = ent >= 0 + e = cutlass.max(ent, 0) + pg = cutlass.Int64(e >> lt) + sl = e & (tpb - 1) + n = cutlass.Int32(cutlass.select_(ok, cutlass.Int32(16), cutlass.Int32(0))) + off = (pg * ks0 + cutlass.Int64(kvh * ks1 + sl * ks2)) * elem + if const_expr(elem == 1): + for i in cutlass.range_constexpr(4): + b = 64 * i + 16 * q4 + _cp16z(rk + tt * k_ld + b, k_base + off + b, n) + _cp16z(rv + tt * V8_LD + b, v_base + off + b, n) + else: + for i in cutlass.range_constexpr(8): + b = 64 * i + 16 * q4 + # chunk 4 i + q4 of the row, at its swizzled place + swz = 8 * (i // 2) + ((4 * (i % 2) + q4) ^ (2 * (i // 2))) + _cp16z(rk + tt * k_ld + b, k_base + off + b, n) + _cp16z(rv + tt * SV_LD + 16 * swz, v_base + off + b, n) + cute.arch.cp_async_commit_group() + + @cute.kernel + def kernel( + self, + mSel: cute.Tensor, + q_base: cutlass.Int64, + k_base: cutlass.Int64, + v_base: cutlass.Int64, + mBT: cute.Tensor, + mReq: cute.Tensor, + out_base: cutlass.Int64, + lse_base: cutlass.Int64, + gate_base: cutlass.Int64, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + g = lane // 4 + c = lane % 4 + elem: cutlass.Constexpr = self.elem + k_ld: cutlass.Constexpr = self.k_ld + group: cutlass.Constexpr = self.group + kvh_n: cutlass.Constexpr = self.kvh + splits: cutlass.Constexpr = self.splits + tps: cutlass.Constexpr = self.tps + n_tiles_all: cutlass.Constexpr = self.n_tiles + width: cutlass.Constexpr = self.width + tpb: cutlass.Constexpr = self.tpb + stages: cutlass.Constexpr = self.stages + row = bidx // (kvh_n * splits) + kvh = (bidx // splits) % kvh_n + split = bidx % splits + t0 = split * tps + n_t = cutlass.min(tps, n_tiles_all - t0) + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_tok = sbase + self.o_tok + s_raw = sbase + self.o_raw + s_p = sbase + self.o_sp + s_red = sbase + self.o_red + + if const_expr(self.pdl): + # The merge waits before it reads anything this grid writes. + cute.arch.griddepcontrol_launch_dependents() + + # ---- q of this KV head, before the wait: rows g and g + 8 of the m16 tile as + # permuted A fragments. Thread (g, c) holds dims 32m + 8c .. 32m + 8c + 7 of heads + # g and g + 8, m = 0..7; a head past the group holds zeros and stores nothing. + qa = cute.make_rmem_tensor(cute.make_layout((8, 4), stride=(4, 1)), cutlass.Int32) + qb = cute.make_rmem_tensor(cute.make_layout((8, 4), stride=(4, 1)), cutlass.Int32) + qs0, qs1 = self.qs + q_row = q_base + (cutlass.Int64(row) * qs0 + cutlass.Int64(kvh * group) * qs1) * 2 + for m in cutlass.range_constexpr(8): + off = cutlass.Int64((32 * m + 8 * c) * 2) + hac = cutlass.min(g, group - 1) + w4 = _ldg4(q_row + cutlass.Int64(hac) * (qs1 * 2) + off) + hb = g + 8 + hbc = cutlass.min(hb, group - 1) + w4b = _ldg4(q_row + cutlass.Int64(hbc) * (qs1 * 2) + off) + for e in cutlass.range_constexpr(4): + wa = w4[e] + wb = w4b[e] + if const_expr(elem == 1): + wa = _bf2_to_f2(wa) + wb = _bf2_to_f2(wb) + qa[m, e] = cutlass.Int32(cutlass.select_(g < group, wa, cutlass.Int32(0))) + qb[m, e] = cutlass.Int32(cutlass.select_(hb < group, wb, cutlass.Int32(0))) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + + # ---- the token table of this split: page * tpb + slot, or -1 --------------------- + rq = cutlass.Int64(mReq[row]) + req_ok = (rq >= 0) & (rq < self.num_requests) + safe_req = cutlass.Int32( + cutlass.min(cutlass.max(rq, cutlass.Int64(0)), cutlass.Int64(self.num_requests - 1)) + ) + for i in cutlass.range_constexpr(-(-(tps * TILE) // THREADS)): + cl = tidx + i * THREADS + col = t0 * TILE + cl + colc = cutlass.min(col, width - 1) + tok = mSel[row, colc] + ok = req_ok & (col < width) & (tok >= 0) + safe_tok = cutlass.max(tok, 0) + lp = safe_tok // tpb + ok = ok & (lp < self.bt_width) + page = mBT[safe_req, cutlass.min(lp, self.bt_width - 1)] + ok = ok & (page >= 0) & (page < self.num_pages) + ent = cutlass.Int32(cutlass.select_(ok, page * tpb + safe_tok % tpb, cutlass.Int32(-1))) + if cl < tps * TILE: + _sts1(s_tok + cl * 4, ent) + cute.arch.sync_threads() + + # Tiles 0 .. stages - 2 in flight (tile 0 alone when one tile is the whole split). + for s in cutlass.range_constexpr(max(stages - 1, 1)): + if s < n_t: + self._issue(s, s, tidx, kvh, s_tok, s_raw, k_base, v_base) + + neg_inf = cutlass.Float32(-math.inf) + st = cute.make_rmem_tensor(cute.make_layout((4,), stride=(1,)), cutlass.Float32) + st[0] = neg_inf # running max, head g + st[1] = neg_inf # running max, head g + 8 + st[2] = cutlass.Float32(0.0) # this thread's share of the running sum, head g + st[3] = cutlass.Float32(0.0) # ... head g + 8 (reduced over the CTA at the end) + acc = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), cutlass.Float32) + acc.fill(0.0) + score_scale = cutlass.Float32(self.scale) + + # loop-invariant per-thread offsets + vt = tidx // 4 + vq = tidx % 4 + v_dst = vt * SV_LD + 128 * vq + ktok = warp * 8 + g + k_src = ktok * k_ld + (8 * c if elem == 1 else 16 * c) + ta = warp * 8 + 2 * c + p_st_g = s_p + g * SP_LD + ta * 2 + p_st_h = p_st_g + 8 * SP_LD + p_ld = s_p + ((lane % 8) + 8 * ((lane // 8) % 2)) * SP_LD + 16 * (lane // 16) + v_ld_row = ((lane % 8) + 8 * ((lane // 8) % 2)) * SV_LD + ci0 = 4 * warp + lane // 16 + ci1 = ci0 + 2 + v_ld0 = v_ld_row + 16 * ((ci0 & ~7) | ((ci0 & 7) ^ (2 * ((ci0 >> 3) & 3)))) + v_ld1 = v_ld_row + 16 * ((ci1 & ~7) | ((ci1 & 7) ^ (2 * ((ci1 >> 3) & 3)))) + red_g = s_red + g * WARPS * 4 + red_h = s_red + (g + 8) * WARPS * 4 + + for j in cutlass.range(n_t): + buf = j % stages + if const_expr(stages > 2): + if j + 1 < n_t: + cute.arch.cp_async_wait_group(stages - 2) + else: + cute.arch.cp_async_wait_group(0) + else: + cute.arch.cp_async_wait_group(0) + # [A] tile j landed; every warp is past tile j - 1, so its ring slot is free + cute.arch.sync_threads() + if const_expr(stages > 1): + if j + stages - 1 < n_t: + self._issue( + j + stages - 1, + (j + stages - 1) % stages, + tidx, + kvh, + s_tok, + s_raw, + k_base, + v_base, + ) + rk = s_raw + buf * self.raw_buf + rv = rk + self.raw_k + # P V reads V from its MMA tile: where bf16 rows landed, or the e4m3 tile widened + # over the slot. + sv = rk if elem == 1 else rv + + # -- S = q K^T for this warp's eight tokens (two accumulator chains) + zero = cutlass.Float32(0.0) + sa = [zero, zero, zero, zero] + sb = [zero, zero, zero, zero] + kb = [] + for m in cutlass.range_constexpr(8): + if const_expr(elem == 1): + w2 = _lds2(rk + k_src + 32 * m) + lo0, hi0 = _e4m3x4(w2[0]) + lo1, hi1 = _e4m3x4(w2[1]) + kb.append([lo0, hi0, lo1, hi1]) + else: + kb.append(_lds4(rk + k_src + 64 * m)) + for m in cutlass.range_constexpr(8): + b = kb[m] + a_lo = (qa[m, 0], qb[m, 0], qa[m, 1], qb[m, 1]) + a_hi = (qa[m, 2], qb[m, 2], qa[m, 3], qb[m, 3]) + if const_expr(m % 2 == 0): + sa = self._mma(a_lo, (b[0], b[1]), sa) + sa = self._mma(a_hi, (b[2], b[3]), sa) + else: + sb = self._mma(a_lo, (b[0], b[1]), sb) + sb = self._mma(a_hi, (b[2], b[3]), sb) + + # mask + scale; tokens 8w + 2c, 8w + 2c + 1 + ea = _lds1(s_tok + (j * TILE + ta) * 4) + eb = _lds1(s_tok + (j * TILE + ta + 1) * 4) + s0 = cutlass.Float32(cutlass.select_(ea >= 0, (sa[0] + sb[0]) * score_scale, neg_inf)) + s1 = cutlass.Float32(cutlass.select_(eb >= 0, (sa[1] + sb[1]) * score_scale, neg_inf)) + s2 = cutlass.Float32(cutlass.select_(ea >= 0, (sa[2] + sb[2]) * score_scale, neg_inf)) + s3 = cutlass.Float32(cutlass.select_(eb >= 0, (sa[3] + sb[3]) * score_scale, neg_inf)) + mg = cute.arch.fmax(s0, s1) + mh = cute.arch.fmax(s2, s3) + mg = cute.arch.fmax(mg, cute.arch.shuffle_sync_bfly(mg, offset=1)) + mh = cute.arch.fmax(mh, cute.arch.shuffle_sync_bfly(mh, offset=1)) + mg = cute.arch.fmax(mg, cute.arch.shuffle_sync_bfly(mg, offset=2)) + mh = cute.arch.fmax(mh, cute.arch.shuffle_sync_bfly(mh, offset=2)) + if c == 0: + _sts1(red_g + warp * 4, _f32_bits(mg)) + _sts1(red_h + warp * 4, _f32_bits(mh)) + cute.arch.sync_threads() # [B] maxes; every warp is past its K reads + vw = [] + if const_expr(elem == 1): + # The raw e4m3 V rows into registers. Thread: token tidx // 4, dims 64q .. + # 64q + 63 (q = tidx % 4), its four 16-byte reads rotated by q // 2 and by the + # token's parity, so a quarter-warp's reads hit eight banks. + for i in cutlass.range_constexpr(4): + ii = (i + vq // 2 + 2 * (vt % 2)) % 4 + vw.append((ii, _lds4(rv + vt * V8_LD + 64 * vq + 16 * ii))) + rg0 = _lds4(red_g) + rg1 = _lds4(red_g + 16) + rh0 = _lds4(red_h) + rh1 = _lds4(red_h + 16) + tmg = st[0] + tmh = st[1] + for e in cutlass.range_constexpr(4): + tmg = cute.arch.fmax(tmg, cute.arch.fmax(_bits_f32(rg0[e]), _bits_f32(rg1[e]))) + tmh = cute.arch.fmax(tmh, cute.arch.fmax(_bits_f32(rh0[e]), _bits_f32(rh1[e]))) + safe_g = cutlass.Float32(cutlass.select_(tmg == neg_inf, cutlass.Float32(0.0), tmg)) + safe_h = cutlass.Float32(cutlass.select_(tmh == neg_inf, cutlass.Float32(0.0), tmh)) + corr_g = _ex2(st[0] - safe_g) + corr_h = _ex2(st[1] - safe_h) + p0 = _ex2(s0 - safe_g) + p1 = _ex2(s1 - safe_g) + p2 = _ex2(s2 - safe_h) + p3 = _ex2(s3 - safe_h) + if const_expr(elem == 1): + _sts1(p_st_g, _f2_pack(p0, p1)) + _sts1(p_st_h, _f2_pack(p2, p3)) + else: + _sts1(p_st_g, _bf2_pack(p0, p1)) + _sts1(p_st_h, _bf2_pack(p2, p3)) + st[0] = tmg + st[1] = tmh + st[2] = st[2] * corr_g + (p0 + p1) + st[3] = st[3] * corr_h + (p2 + p3) + cute.arch.sync_threads() # [C] P; every raw read of this slot is done + if const_expr(elem == 1): + # -- the e4m3 V tile widened (exact) into the f16 MMA tile over the slot + for ii, w in vw: + a0, a1 = _e4m3x4(w[0]) + a2, a3 = _e4m3x4(w[1]) + b0, b1 = _e4m3x4(w[2]) + b2, b3 = _e4m3x4(w[3]) + _sts4(rk + v_dst + 16 * ((2 * ii) ^ (2 * vq)), a0, a1, a2, a3) + _sts4(rk + v_dst + 16 * ((2 * ii + 1) ^ (2 * vq)), b0, b1, b2, b3) + cute.arch.sync_threads() # [D] the V tile + + # -- O += P V: this warp's dims 32w .. 32w + 31 (n-tiles of 8) + if cute.arch.vote_any_sync((corr_g != 1.0) | (corr_h != 1.0)): + for nt in cutlass.range_constexpr(4): + acc[nt, 0] = acc[nt, 0] * corr_g + acc[nt, 1] = acc[nt, 1] * corr_g + acc[nt, 2] = acc[nt, 2] * corr_h + acc[nt, 3] = acc[nt, 3] * corr_h + for kk in cutlass.range_constexpr(4): + a = _ldsm4(p_ld + 32 * kk) + b0 = _ldsm4t(sv + v_ld0 + 16 * kk * SV_LD) + b1 = _ldsm4t(sv + v_ld1 + 16 * kk * SV_LD) + r0 = self._mma(a, (b0[0], b0[1]), (acc[0, 0], acc[0, 1], acc[0, 2], acc[0, 3])) + r1 = self._mma(a, (b0[2], b0[3]), (acc[1, 0], acc[1, 1], acc[1, 2], acc[1, 3])) + r2 = self._mma(a, (b1[0], b1[1]), (acc[2, 0], acc[2, 1], acc[2, 2], acc[2, 3])) + r3 = self._mma(a, (b1[2], b1[3]), (acc[3, 0], acc[3, 1], acc[3, 2], acc[3, 3])) + for e in cutlass.range_constexpr(4): + acc[0, e] = r0[e] + acc[1, e] = r1[e] + acc[2, e] = r2[e] + acc[3, e] = r3[e] + + # ---- the running sums over the quad and the warps ----------------------------------- + lsg = st[2] + lsh = st[3] + lsg = lsg + cute.arch.shuffle_sync_bfly(lsg, offset=1) + lsh = lsh + cute.arch.shuffle_sync_bfly(lsh, offset=1) + lsg = lsg + cute.arch.shuffle_sync_bfly(lsg, offset=2) + lsh = lsh + cute.arch.shuffle_sync_bfly(lsh, offset=2) + cute.arch.sync_threads() + if c == 0: + _sts1(red_g + warp * 4, _f32_bits(lsg)) + _sts1(red_h + warp * 4, _f32_bits(lsh)) + cute.arch.sync_threads() + ug0 = _lds4(red_g) + ug1 = _lds4(red_g + 16) + uh0 = _lds4(red_h) + uh1 = _lds4(red_h + 16) + tsg = cutlass.Float32(0.0) + tsh = cutlass.Float32(0.0) + for e in cutlass.range_constexpr(4): + tsg = tsg + _bits_f32(ug0[e]) + tsh = tsh + _bits_f32(uh0[e]) + for e in cutlass.range_constexpr(4): + tsg = tsg + _bits_f32(ug1[e]) + tsh = tsh + _bits_f32(uh1[e]) + self._epilogue( + acc, + st, + tsg, + tsh, + row, + kvh, + split, + warp, + g, + c, + rows, + out_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + ) + + def _mma(self, a, b, c): + """The tile MMA in the pool's type: f16 over an e4m3 pool, bf16 over a bf16 one.""" + return _mma(a, b, c) if self.elem == 1 else _mma_bf16(a, b, c) + + @cute.jit + def _epilogue( + self, + acc, + st, + lg, + lh, + row, + kvh, + split, + warp, + g, + c, + rows, + out_base, + lse_base, + gate_base, + gate_stride, + gate_hs, + ): + """Heads g and g + 8's normalized outputs: the bf16 result (gated when the gate rides + this launch) with one split, else the fp32 partial and its log2-sum-exp.""" + group: cutlass.Constexpr = self.group + heads: cutlass.Constexpr = self.heads + splits: cutlass.Constexpr = self.splits + neg_inf = cutlass.Float32(-math.inf) + ig = cutlass.Float32(0.0) + ih = cutlass.Float32(0.0) + if lg > 0.0: + ig = 1.0 / lg + if lh > 0.0: + ih = 1.0 / lh + hg = kvh * group + g + hh = kvh * group + g + 8 + okg = g < group + okh = (g + 8) < group + for nt in cutlass.range_constexpr(4): + d = 32 * warp + 8 * nt + 2 * c + og0 = acc[nt, 0] * ig + og1 = acc[nt, 1] * ig + oh0 = acc[nt, 2] * ih + oh1 = acc[nt, 3] * ih + if const_expr(splits == 1): + ag = out_base + (cutlass.Int64(row) * heads + hg) * (HEAD_DIM * 2) + d * 2 + ah = out_base + (cutlass.Int64(row) * heads + hh) * (HEAD_DIM * 2) + d * 2 + if okg: + if const_expr(self.gated): + gw = _ldg1( + gate_base + (cutlass.Int64(row) * gate_stride + hg * gate_hs + d) * 2 + ) + og0 = _round_bf16(og0) * _sigmoid(_bf2_lo(gw)) + og1 = _round_bf16(og1) * _sigmoid(_bf2_hi(gw)) + _stg1(ag, _bf2_pack(og0, og1)) + if okh: + if const_expr(self.gated): + gw2 = _ldg1( + gate_base + (cutlass.Int64(row) * gate_stride + hh * gate_hs + d) * 2 + ) + oh0 = _round_bf16(oh0) * _sigmoid(_bf2_lo(gw2)) + oh1 = _round_bf16(oh1) * _sigmoid(_bf2_hi(gw2)) + _stg1(ah, _bf2_pack(oh0, oh1)) + else: + pg_ = ( + out_base + + ((cutlass.Int64(split) * rows + row) * heads + hg) * (HEAD_DIM * 4) + + d * 4 + ) + ph_ = ( + out_base + + ((cutlass.Int64(split) * rows + row) * heads + hh) * (HEAD_DIM * 4) + + d * 4 + ) + if okg: + _stg2f(pg_, og0, og1) + if okh: + _stg2f(ph_, oh0, oh1) + if const_expr(splits > 1): + if warp == 0: + if c == 0: + if okg: + lse_g = neg_inf + if lg > 0.0: + lse_g = st[0] + cute.math.log2(lg) + _stg1( + lse_base + ((cutlass.Int64(split) * rows + row) * heads + hg) * 4, + _f32_bits(lse_g), + ) + if okh: + lse_h = neg_inf + if lh > 0.0: + lse_h = st[1] + cute.math.log2(lh) + _stg1( + lse_base + ((cutlass.Int64(split) * rows + row) * heads + hh) * 4, + _f32_bits(lse_h), + ) + + +# --------------------------------------------------------------------------------------- +# merge kernel (splits > 1) +# --------------------------------------------------------------------------------------- + + +class _MergeKernel: + def __init__(self, *, heads, splits, gated, pdl): + assert HEAD_DIM % CHUNK == 0 and 1 < splits <= MAX_SPLITS + self.heads, self.splits, self.gated, self.pdl = heads, splits, gated, pdl + self.chunks = HEAD_DIM // CHUNK + # at least four warps: the final pass needs a thread per element of the chunk + self.warps = max(4, min(8, splits)) + self.spw = -(-splits // self.warps) + + @cute.jit + def __call__( + self, + gate_t: cute.Tensor, + part_ptr: cute.Pointer, + lse_ptr: cute.Pointer, + out_ptr: cute.Pointer, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + stream, + ): + lse = cute.make_tensor( + lse_ptr, + cute.make_layout( + (self.splits, rows, self.heads), stride=(rows * self.heads, self.heads, 1) + ), + ) + out = cute.make_tensor( + out_ptr, + cute.make_layout( + (rows, self.heads, HEAD_DIM), stride=(self.heads * HEAD_DIM, HEAD_DIM, 1) + ), + ) + self.kernel( + gate_t.iterator.toint(), part_ptr.toint(), lse, out, rows, gate_stride, gate_hs + ).launch( + grid=[rows * self.heads * self.chunks, 1, 1], + block=[self.warps * 32, 1, 1], + smem=self.warps * CHUNK * 4 + 64, + stream=stream, + use_pdl=self.pdl, + ) + + @cute.kernel + def kernel( + self, + gate_base: cutlass.Int64, + part_base: cutlass.Int64, + mLse, + mOut, + rows: cutlass.Int32, + gate_stride: cutlass.Int32, + gate_hs: cutlass.Int32, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + per_row: cutlass.Constexpr = self.heads * self.chunks + spw: cutlass.Constexpr = self.spw + row = bidx // per_row + head = (bidx % per_row) // self.chunks + c0 = (bidx % self.chunks) * CHUNK + red = cutlass.utils.SmemAllocator().allocate_tensor( + cutlass.Float32, + cute.make_layout((self.warps, CHUNK), stride=(CHUNK, 1)), + byte_alignment=16, + ) + gval = cutlass.Float32(0.0) + if const_expr(self.gated): + if tidx < CHUNK: + gw = _ldg1( + gate_base + + (cutlass.Int64(row) * gate_stride + head * gate_hs + c0 + (tidx & ~1)) * 2 + ) + gval = cutlass.Float32(cutlass.select_((tidx & 1) == 0, _bf2_lo(gw), _bf2_hi(gw))) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + neg_inf = cutlass.Float32(-math.inf) + l0 = neg_inf + l1 = neg_inf + if lane < self.splits: + l0 = mLse[lane, row, head] + if lane + 32 < self.splits: + l1 = mLse[lane + 32, row, head] + p = cute.make_rmem_tensor(cute.make_layout((spw, 4), stride=(4, 1)), cutlass.Float32) + for jj in cutlass.range_constexpr(spw): + sidx = cutlass.min(warp_i * spw + jj, self.splits - 1) + w4 = _ldg4( + part_base + + ((cutlass.Int64(sidx) * rows + row) * self.heads + head) * (HEAD_DIM * 4) + + (c0 + 4 * lane) * 4 + ) + for e in cutlass.range_constexpr(4): + p[jj, e] = _bits_f32(w4[e]) + mx = cute.arch.warp_reduction_max(cute.arch.fmax(l0, l1)) + w0 = cutlass.Float32(0.0) + w1 = cutlass.Float32(0.0) + if mx > neg_inf: + w0 = cute.math.exp2(l0 - mx) + w1 = cute.math.exp2(l1 - mx) + den = cute.arch.warp_reduction_sum(w0 + w1) + acc = cute.make_rmem_tensor(cute.make_layout((4,), stride=(1,)), cutlass.Float32) + acc.fill(0.0) + for jj in cutlass.range_constexpr(spw): + sidx = warp_i * spw + jj + wa = cute.arch.shuffle_sync(w0, sidx % 32) + wb = cute.arch.shuffle_sync(w1, sidx % 32) + w = cutlass.Float32(cutlass.select_(sidx < 32, wa, wb)) + w = cutlass.Float32(cutlass.select_(sidx < self.splits, w, cutlass.Float32(0.0))) + for e in cutlass.range_constexpr(4): + acc[e] = w * p[jj, e] + acc[e] + for e in cutlass.range_constexpr(4): + red[warp_i, 4 * lane + e] = acc[e] + cute.arch.sync_threads() + if tidx < CHUNK: + tot = cutlass.Float32(0.0) + for w_ in cutlass.range_constexpr(self.warps): + tot = tot + red[w_, tidx] + merged = cutlass.Float32(0.0) + if den > 0.0: + merged = tot / den + if const_expr(self.gated): + att = merged.to(cutlass.BFloat16).to(cutlass.Float32) + merged = att * (1.0 / (1.0 + cute.math.exp(-gval))) + mOut[row, head, c0 + tidx] = merged.to(cutlass.BFloat16) + + +# --------------------------------------------------------------------------------------- +# launch +# --------------------------------------------------------------------------------------- + +_COMPILED: dict = {} +#: (geometry, gated, gate tensor rank) forms whose every reachable split plan is compiled. +_PLANNED: set = set() + +#: Splits by (rows x KV heads), per pool element size, measured at the routed geometries +#: (`MAX_ROWS`): one tile a CTA while the grid is under about a wave, then fewer, longer splits; +#: one split past the last bound. +_SPLIT_TABLE = { + 2: ((8, 33), (16, 9)), + 1: ((4, 33), (8, 11), (32, 9), (64, 4), (256, 2)), +} +#: Past the table's last bound at one KV head over e4m3 (a tp2 / tep2 rank): two or three +#: shorter splits a row keep more latency-bound tile chains in flight than one split. +_WIDE_SPLITS_ONE_KV = {1: ((320, 2), (1024, 3))} +#: At one KV head over e4m3, from 65 pairs these replace the table's 2: split counts whose CTAs +#: (rows x splits, two an SM) fill whole waves. +_MID_SPLITS_ONE_KV = {1: ((96, 6), (160, 2), (256, 3))} + +_POOL_ELEM = {torch.bfloat16: 2, torch.float8_e4m3fn: 1} + + +def plan_splits(rows: int, kv_heads: int, width: int, elem: int) -> tuple[int, int]: + """(splits, tiles per split) for this many (row, KV head) pairs over a pool of `elem`-byte + elements.""" + n_tiles = -(-width // TILE) + pairs = rows * kv_heads + want = 1 + table = _SPLIT_TABLE[elem] + if kv_heads == 1: + mid = _MID_SPLITS_ONE_KV.get(elem, ()) + lead = tuple(e for e in table if not mid or e[0] <= 64) + rest = tuple(e for e in table if mid and e[0] > 256) + table = lead + mid + rest + _WIDE_SPLITS_ONE_KV.get(elem, ()) + for bound, sp in table: + if pairs <= bound: + want = sp + break + want = max(1, min(n_tiles, want, MAX_SPLITS)) + tps = -(-n_tiles // want) + return -(-n_tiles // tps), tps + + +def reachable_plans(max_rows: int, kv_heads: int, width: int, elem: int) -> list: + """Every `(splits, tiles per split)` `plan_splits` returns for 1..max_rows rows.""" + return sorted({plan_splits(r, kv_heads, width, elem) for r in range(1, max_rows + 1)}) + + +def _compile_split(geometry: tuple, splits: int, tps: int, gated: bool, kern_kw: dict, args): + """The split pass of `geometry` at plan `(splits, tps)` (`gated`: the one-split pass stores + the gated output), compiled once; `args()` builds the compile arguments.""" + key = geometry + (splits, tps, gated) + fn = _COMPILED.get(key) + if fn is None: + kern = _SplitKernel(splits=splits, tps=tps, gated=gated, **kern_kw) + fn = cute.compile(kern, *args(), options="--opt-level 2 --enable-tvm-ffi") + _COMPILED[key] = fn + return fn + + +def _compile_merge(heads: int, splits: int, gated: bool, pdl: bool, gdim: int, args): + """The merge of `splits` partials, compiled once per gate tensor rank `gdim` (the trunk's + `[rows, Hq * D]` gate, the draft layer's `[rows, Hq, D]` view, q when ungated); `args()` + builds the compile arguments.""" + key = ("merge", heads, splits, gated, gdim, pdl) + fn = _COMPILED.get(key) + if fn is None: + fn = cute.compile( + _MergeKernel(heads=heads, splits=splits, gated=gated, pdl=pdl), + *args(), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + +def _ptr(t: torch.Tensor, dtype, align=16): + return make_ptr(dtype, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + +def sparse_gqa_decode( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """`QsaSparseGqaSplitkCute.__call__`: split pass, then the merge when there are splits.""" + rows, num_q_heads, head_dim = q.shape + num_pages, num_kv_heads, tpb, dim = k_cache.shape + elem = _POOL_ELEM[k_cache.dtype] + assert head_dim == HEAD_DIM and dim == HEAD_DIM and tpb == tokens_per_block + assert q.dtype == torch.bfloat16 and q.stride(2) == 1 + assert v_cache.dtype == k_cache.dtype and v_cache.shape == k_cache.shape + assert k_cache.stride() == v_cache.stride() and k_cache.stride(3) == 1 + assert block_table.dtype == torch.int32 and block_table.stride(1) == 1 + assert block_table.stride(0) == block_table.shape[1], "row-contiguous block table" + assert selected_tokens.dtype == torch.int32 and selected_tokens.stride(1) == 1 + assert request_indices.is_contiguous() and request_indices.dtype in (torch.int32, torch.int64) + group = num_q_heads // num_kv_heads + width = selected_tokens.shape[1] + out = torch.empty((rows, num_q_heads, head_dim), dtype=torch.bfloat16, device=q.device) + if rows == 0: + return out + n_splits, tps = plan_splits(rows, num_kv_heads, width, elem) + use_pdl = bool(get_env_enable_pdl()) + gated = output_gate is not None + gate_hs = head_dim + if gated: + assert output_gate.dtype == torch.bfloat16 and output_gate.stride(-1) == 1 + if output_gate.dim() == 3: + assert output_gate.shape == q.shape + gate_hs = output_gate.stride(1) + else: + assert output_gate.shape == (rows, num_q_heads * head_dim) + req64 = request_indices.dtype == torch.int64 + if n_splits > 1: + part = torch.empty( + (n_splits, rows, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + lse = torch.empty((n_splits, rows, num_q_heads), dtype=torch.float32, device=q.device) + o_t, l_t = part, lse + else: + o_t, l_t = out, out + gate_t = output_gate if gated else q + gate_stride = output_gate.stride(0) if gated else 0 + # The compile key without the split plan (rows pick it) and the one-split gate. + geometry = ( + elem, + group, + num_kv_heads, + width, + tpb, + float(softmax_scale), + num_pages, + tuple(block_table.shape), + tuple(k_cache.stride()[:3]), + tuple(q.stride()[:2]), + req64, + use_pdl, + ) + kern_kw = dict( + elem=elem, + group=group, + kv_heads=num_kv_heads, + width=width, + tpb=tpb, + scale=float(softmax_scale), + num_pages=num_pages, + bt_width=block_table.shape[1], + num_requests=block_table.shape[0], + kv_strides=tuple(k_cache.stride()[:3]), + q_strides=tuple(q.stride()[:2]), + pdl=use_pdl, + ) + + def split_args(): + return ( + from_dlpack(selected_tokens, assumed_align=4).mark_layout_dynamic(leading_dim=1), + _ptr(q, cutlass.BFloat16), + _ptr(k_cache, cutlass.Uint8), + _ptr(v_cache, cutlass.Uint8), + _ptr(block_table, cutlass.Int32, 4), + _ptr(request_indices, cutlass.Int64 if req64 else cutlass.Int32, 8 if req64 else 4), + _ptr(o_t, cutlass.Float32), + _ptr(l_t, cutlass.Float32), + _ptr(gate_t, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(selected_tokens.stride(0)), + cutlass.Int32(gate_stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + + def merge_args(gate, stride): + # The partials' pointers only type the compile (a one-split call has none: `out`'s). + return lambda: ( + from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=gate.dim() - 1), + _ptr(o_t, cutlass.Float32), + _ptr(l_t, cutlass.Float32, 4), + _ptr(out, cutlass.BFloat16), + cutlass.Int32(rows), + cutlass.Int32(stride), + cutlass.Int32(gate_hs), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + ) + + # The plan depends on the call's row count: the first call of a geometry in a gate form + # compiles the split pass and merge of every plan its rows can reach (`MAX_ROWS`), so no + # serving step compiles one; a gated call also compiles the ungated flavours, which a mixed + # step's generation rows (and a draft-extend step's) call at the geometry a decode step + # calls gated. The engine's warmup runs each captured decode batch eagerly first. + form = geometry + (gated, gate_t.dim()) + if form not in _PLANNED: + _PLANNED.add(form) + cap = max(rows, MAX_ROWS.get((num_q_heads, num_kv_heads, k_cache.dtype), 0)) + for splits, per in reachable_plans(cap, num_kv_heads, width, elem): + _compile_split(geometry, splits, per, gated and splits == 1, kern_kw, split_args) + if splits == 1: + if gated: + _compile_split(geometry, splits, per, False, kern_kw, split_args) + continue + gm = merge_args(gate_t, gate_stride) + _compile_merge(num_q_heads, splits, gated, use_pdl, gate_t.dim(), gm) + if gated: + _compile_merge(num_q_heads, splits, False, use_pdl, q.dim(), merge_args(q, 0)) + fn = _compile_split(geometry, n_splits, tps, gated and n_splits == 1, kern_kw, split_args) + fn( + selected_tokens, + q.data_ptr(), + k_cache.data_ptr(), + v_cache.data_ptr(), + block_table.data_ptr(), + request_indices.data_ptr(), + o_t.data_ptr(), + l_t.data_ptr(), + gate_t.data_ptr(), + rows, + selected_tokens.stride(0), + gate_stride, + gate_hs, + ) + if n_splits == 1: + return out + mfn = _compile_merge( + num_q_heads, n_splits, gated, use_pdl, gate_t.dim(), merge_args(gate_t, gate_stride) + ) + mfn(gate_t, part.data_ptr(), lse.data_ptr(), out.data_ptr(), rows, gate_stride, gate_hs) + return out + + +def serves(q: torch.Tensor, k_cache: torch.Tensor, selected_tokens: torch.Tensor) -> bool: + """Whether the target sends this decode call (no prefill pair) here: a certified head + geometry and pool dtype, at most `MAX_ROWS` rows of it.""" + if q.dim() != 3 or k_cache.dim() != 4 or q.dtype != torch.bfloat16: + return False + rows, num_q_heads, head_dim = q.shape + cap = MAX_ROWS.get((num_q_heads, k_cache.shape[1], k_cache.dtype), 0) + return ( + 0 < rows <= cap + and head_dim == HEAD_DIM + and k_cache.shape[3] == HEAD_DIM + and selected_tokens.shape[1] > TILE + ) + + +#: Rows the target sends here per (q heads, KV heads, pool dtype), up to the row counts measured +#: faster than the composition: a tp4 / tep4 rank over bf16, a dep4 rank and a tp2 / tep2 rank +#: under kv_fp8. Anything else keeps `qsa_paged_sparse_gqa` (24 / 2 over bf16 measured slower: +#: a bf16 tile slot holds 70 KB, one CTA an SM). +MAX_ROWS = { + (6, 1, torch.bfloat16): 16, + (24, 2, torch.float8_e4m3fn): 128, + (12, 1, torch.float8_e4m3fn): 512, +} + +#: QSA geometry: 32-token pages, 2048 + 4 - 1 selected tokens a row, a `max_seq_len` 262144 table. +_QSA = dict(tokens_per_block=32, compress_ratio=4, token_topk=2048, head_dim=256, pages=8196) +#: The engine's trunk: a slot holds every attention layer's K and V role pages (two more under +#: attention DP with the draft layer); the calls read layer 11's. +_TEP4 = dict(_QSA, q_heads=6, kv_heads=1, slots=4, slot_pages=24, layer=11, kv=torch.bfloat16) +_DEP4 = dict( + _QSA, q_heads=24, kv_heads=2, slots=128, slot_pages=26, layer=11, kv=torch.float8_e4m3fn +) +_TEP2 = dict( + _QSA, q_heads=12, kv_heads=1, slots=128, slot_pages=24, layer=11, kv=torch.float8_e4m3fn +) +#: The element offset the cells' pools straddle in the layer's K and V planes. +_OFFSET_EDGE = 2**31 + + +def _verify(member: dict, requests: int, rows: int = 4, last=196607, step=4099, **kw) -> dict: + """A verify step of `requests` requests of `rows` rows each, deep in a long sequence: + request r's rows end at `last - step * r`.""" + segments = tuple((last + 1 - rows - step * r, rows) for r in range(requests)) + return dict(member, segments=segments, **kw) + + +#: The trunk's decode site: the gate is the fused input projection row's gate columns. +_TEP4_ROW = dict(gate="row", row_heads=17) +_DEP4_ROW = dict(gate="row", row_heads=55) +_TEP2_ROW = dict(gate="row", row_heads=29) +#: The draft layer's site: the gate half of its interleaved `[T, Hq, 2 x 256]` q projection. +_DRAFT = dict(gate="interleaved") + + +class QsaSparseGqaSplitkCute(OpWrapper): + """`qsa_paged_sparse_gqa` for decode rows (no prefill pair) over a bf16 or e4m3 paged + pool: sparse GQA over each row's explicitly selected tokens, `[rows, Hq, D]` bf16. With + `output_gate` (`[rows, Hq * D]`, or `[rows, Hq, D]` with any head stride; bf16, unit inner + stride) the result is `bf16(attn) * sigmoid(gate)` rounded once more to bf16, what + `fused_sigmoid_mul_inplace` makes of the ungated call's output. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ll_tep4 MTP3 verify: 1 request x 4 rows at position 196607, 6 q / 1 KV head " + "over the bf16 trunk, 33 splits + gated merge, captured", + spec=_verify(_TEP4, 1, splits=33, capture=True, **_TEP4_ROW), + ), + Cell( + why="tep4 largest captured batch: 4 requests x 4 rows = 16, 6 / 1 heads, bf16, " + "gated, captured", + spec=_verify(_TEP4, 4, capture=True, **_TEP4_ROW), + ), + Cell( + why="tep4 mixed step's generation rows: 3 rows off a wider selection (unaligned " + "slice), short sequences, holes, one foreign row, ungated", + spec=dict( + _TEP4, + segments=((77, 1), (2047, 1), (9000, 1)), + row_offset=1, + holes=True, + foreign=True, + ), + ), + Cell( + why="ht_dep4 b4 verify: 4 x 4 = 16 rows, e4m3, gated, captured", + spec=_verify(_DEP4, 4, capture=True, **_DEP4_ROW), + ), + Cell( + why="ht_dep4 draft layer: 8 x 4 = 32 rows, e4m3, interleaved gate", + spec=_verify(_DEP4, 8, **_DRAFT), + ), + Cell( + why="dep4 b1 verify: 1 x 4 rows, e4m3, gated", + spec=_verify(_DEP4, 1, **_DEP4_ROW), + ), + Cell( + why="dep4 b32 verify: 32 x 4 = 128 rows (the routed cap), e4m3, gated, captured", + spec=_verify(_DEP4, 32, last=60000, step=1700, capture=True, **_DEP4_ROW), + ), + Cell( + why="ht_tep2 verify: 4 requests x 4 rows = 16, 12 q / 1 KV head over the e4m3 " + "trunk, [128, 8196] table, gated, captured", + spec=_verify(_TEP2, 4, capture=True, **_TEP2_ROW), + ), + Cell( + why="tep2 b1 verify: 1 x 4 rows, e4m3, gated", + spec=_verify(_TEP2, 1, **_TEP2_ROW), + ), + Cell( + why="ht_tep2 MTP-2 verify: 160 x 3 = 480 rows (the 160-request graph, three splits " + "a row), e4m3, gated", + spec=_verify( + dict(_TEP2, slots=160), 160, rows=3, last=30000, step=150, splits=3, **_TEP2_ROW + ), + ), + Cell( + why="ht_tep2 draft-extend step's continuing rows: 128 x 1, the gate a contiguous " + "[rows, Hq * D] (rows gathered), e4m3", + spec=_verify(_TEP2, 128, rows=1, last=30000, step=150, gate="contiguous"), + ), + Cell( + why="dep4 mixed step's generation rows: 7 rows off a wider selection (unaligned " + "slice), short sequences, holes, one foreign row, e4m3, ungated", + spec=dict( + _DEP4, + segments=((77, 1), (2047, 1), (5000, 1), (9000, 1), (30000, 1), (1, 1), (4, 1)), + row_offset=1, + holes=True, + foreign=True, + ), + ), + ) + + note = """ + Source: CuTe DSL split pass and merge (TVM-FFI), JIT-compiled per key (pool dtype, heads, + selection width, split plan, pool page count, table shape, plane and q strides, gate fused + or not); the engine's warmup runs each captured batch eagerly, so no compile lands inside a + capture. Routed by `serves` / `MAX_ROWS` from the target's `_sparse_gqa`: the decode + step's trunk layers and the MTP draft layer (gated), and a mixed step's generation rows + (ungated). + + Preconditions: + - PDL: the split pass reads q, and the merge the gate, before their grid-dependency + wait, so both must be complete when the grid ahead releases this one (a launch + without PDL releases at its end; `qsa_topk_expand_decode` only after its own wait). + - a pool below 2^26 pages at 32 tokens a page: the token table's `page * tpb + slot` is + 32-bit (page indices are widened to 64 bits before the plane stride product). + + Numerics: bf16 pool as the Triton composition (bf16 MMAs, fp32 accumulation, bf16 P); e4m3 + pool with K/V widened exactly to f16, q rounded to f16 and f16 P. The merge runs in split + order, so the output differs from the composition's by an ulp on some elements. + Deterministic and graph-capturable (the row count is a runtime value). + + Not certified: other head geometries, head dims or page sizes (refused at compile: at most + 16 q heads a KV head, a power-of-two page); fp16 q or pools; NVFP4 pools + (`qsa_paged_sparse_gqa_nvfp4_decode`); selections of 64 tokens or fewer; int64 request + indices; the prefill pair; the one-split launch (no routed shape reaches it); at one KV + head over e4m3, the 4- and 6-split plans (33..96 rows). + """ + + serves = staticmethod(serves) + + def __call__( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return sparse_gqa_decode( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + tokens_per_block, + softmax_scale, + output_gate=output_gate, + ) + + def reference( + self, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + output_gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """fp32 sparse GQA over every resident selected token, rounded once (the + composition entry's reference); with `output_gate`, then gated in fp32 over that + rounding and rounded again.""" + out = _composition.reference( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + ) + if output_gate is None: + return out + gate = torch.sigmoid(output_gate.float().reshape(out.shape)) + return (out.float() * gate).to(out.dtype) + + +qsa_sparse_gqa_splitk_cute = QsaSparseGqaSplitkCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py new file mode 100644 index 000000000000..3207059e0382 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_spec_step_metadata__decode.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A captured verify step's input prologue in one launch: the overlap scheduler's accepted-count +correction of the mRoPE position ids and the KV lengths, and the QSA row mapping over the +corrected lengths. + +It replaces what `DecoderInputPreparer.preprocess_inputs` does on a generation-only verify step +(the position and KV-length corrections and `on_update_kv_lens()`'s row mapping); +`SpecStepMetadataPrologue` is the engine hook that calls it. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Largest batch the single program covers (requests, rows). +MAX_SEQS = 64 +MAX_ROWS = 512 +#: Requests a step of the row-mapping loop covers once the `[rows, requests]` comparison matrix +#: would pass CHUNK_MIN_ELEMS elements (a single larger int64 matrix spills). Same output bits. +SEQ_CHUNK = 16 +CHUNK_MIN_ELEMS = 8192 + + +@triton.jit +def _spec_step_metadata_kernel( + positions, + stride_coord, + pos_offsets, + num_tokens, + kv_lens, + kv_offsets, + seq_lens, + request_indices, + logical_positions, + sequence_lengths, + visible_blocks, + num_seqs, + pos_out, + COMPRESS_RATIO: tl.constexpr, + BLOCK_ROWS: tl.constexpr, + BLOCK_SEQS: tl.constexpr, + HAS_POS_OUT: tl.constexpr, + SEQ_CHUNK_: tl.constexpr, +): + rows = tl.arange(0, BLOCK_ROWS) + valid = rows < num_tokens + seqs = tl.arange(0, BLOCK_SEQS) + sv = seqs < num_seqs + # Every load before any store (the stores overwrite positions and lengths it reads). + off = tl.load(pos_offsets + rows, mask=valid, other=0) + kv_new = tl.load(kv_lens + seqs, mask=sv, other=0) + tl.load( + kv_offsets + seqs, mask=sv, other=0 + ) + # A row's request is the number of requests ending at or before it, its start the sum of + # their lengths (`_qsa_generation_token_mapping_kernel`'s arithmetic). + request = tl.zeros([BLOCK_ROWS], tl.int32) + request_start = tl.zeros([BLOCK_ROWS], tl.int64) + carry = tl.zeros([1], tl.int64) + rows64 = rows.to(tl.int64) + for s0 in tl.static_range(0, BLOCK_SEQS, SEQ_CHUNK_): + cseq = s0 + tl.arange(0, SEQ_CHUNK_) + csv = cseq < num_seqs + lengths = tl.load(seq_lens + cseq, mask=csv, other=0).to(tl.int64) + ends = tl.cumsum(lengths, axis=0) + carry + ended = (ends[None, :] <= rows64[:, None]) & csv[None, :] + request += tl.sum(ended.to(tl.int32), axis=1) + request_start += tl.sum(tl.where(ended, lengths[None, :], 0), axis=1) + carry += tl.sum(lengths, axis=0) + kv_len = tl.load(kv_lens + request, mask=valid, other=0) + tl.load( + kv_offsets + request, mask=valid, other=0 + ) + seq_len = tl.load(seq_lens + request, mask=valid, other=0) + p0 = tl.load(positions + rows, mask=valid, other=0) + p1 = tl.load(positions + stride_coord + rows, mask=valid, other=0) + p2 = tl.load(positions + 2 * stride_coord + rows, mask=valid, other=0) + tl.debug_barrier() + tl.store(positions + rows, p0 + off, mask=valid) + tl.store(positions + stride_coord + rows, p1 + off, mask=valid) + tl.store(positions + 2 * stride_coord + rows, p2 + off, mask=valid) + if HAS_POS_OUT: + tl.store(pos_out + rows, p0 + off, mask=valid) + tl.store(pos_out + num_tokens + rows, p1 + off, mask=valid) + tl.store(pos_out + 2 * num_tokens + rows, p2 + off, mask=valid) + tl.store(kv_lens + seqs, kv_new, mask=sv) + logical = kv_len.to(tl.int64) - seq_len.to(tl.int64) + rows.to(tl.int64) - request_start + visible_num = logical + 1 + visible = visible_num // COMPRESS_RATIO + visible = tl.where( + (visible_num % COMPRESS_RATIO != 0) & (visible_num < 0), visible - 1, visible + ) + tl.store(request_indices + rows, request, mask=valid) + tl.store(logical_positions + rows, logical, mask=valid) + tl.store(sequence_lengths + rows, kv_len, mask=valid) + tl.store(visible_blocks + rows, visible.to(tl.int32), mask=valid) + + +def _cells(): + return tuple( + Cell(why=f"{n} request(s) x {p} verify rows", spec=dict(n=n, p=p, capture=capture)) + for n, p, capture in ( + (1, 3, True), + (8, 4, False), + (40, 3, True), + (56, 3, True), + (64, 3, False), + ) + ) + + +class QsaSpecStepMetadata(OpWrapper): + """`qsa_spec_step_metadata(positions, pos_offsets, kv_lens, kv_offsets, seq_lens, mapping, + compress_ratio)`, in place: + + - `positions` `[C, 1, T]` int32 (the step's mRoPE positions, unit token stride, any + coordinate stride): `positions[:, :, :T] += pos_offsets[:T]`; + - `kv_lens` `[N]` int32: `kv_lens += kv_offsets[:N]`; + - `mapping = (request_indices, logical_positions, sequence_lengths, visible_blocks)` + (int32, int64, int32, int32, at least `T` long): every row's request, logical position, + KV length and visible-block count over the corrected lengths, with each request's + `seq_lens[i]` rows consecutive -- bitwise the composed corrections followed by + `triton_qsa_generation_token_mapping`. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: one Triton program (`_spec_step_metadata_kernel`), up to 64 requests and 512 rows; + integer arithmetic only. + """ + + def __call__( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ): + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + block_rows = max(16, triton.next_power_of_2(num_tokens)) + block_seqs = max(16, triton.next_power_of_2(num_seqs)) + chunked = block_rows * block_seqs > CHUNK_MIN_ELEMS + _spec_step_metadata_kernel[(1,)]( + positions, + positions.stride(0), + pos_offsets, + num_tokens, + kv_lens, + kv_offsets, + seq_lens, + *mapping, + num_seqs, + pos_out if pos_out is not None else positions, + COMPRESS_RATIO=compress_ratio, + BLOCK_ROWS=block_rows, + BLOCK_SEQS=block_seqs, + HAS_POS_OUT=pos_out is not None, + SEQ_CHUNK_=SEQ_CHUNK if chunked else block_seqs, + num_warps=8 if chunked else 4, + ) + + def reference( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ): + """The composition, in place: the two corrections, then the row mapping in torch.""" + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + positions += pos_offsets[:num_tokens] + if pos_out is not None: + pos_out.copy_(positions.reshape(positions.shape[0], num_tokens)) + kv_lens += kv_offsets[:num_seqs] + request_indices, logical_positions, sequence_lengths, visible_blocks = mapping + lens = seq_lens[:num_seqs].long() + req = torch.repeat_interleave(torch.arange(num_seqs, device=kv_lens.device), lens)[ + :num_tokens + ] + starts = torch.cumsum(lens, 0) - lens + rows = torch.arange(num_tokens, device=kv_lens.device) + kv = kv_lens[req].long() + logical = kv - lens[req] + rows - starts[req] + request_indices[:num_tokens] = req.to(request_indices.dtype) + logical_positions[:num_tokens] = logical.to(logical_positions.dtype) + sequence_lengths[:num_tokens] = kv.to(sequence_lengths.dtype) + visible_blocks[:num_tokens] = torch.div( + logical + 1, compress_ratio, rounding_mode="floor" + ).to(visible_blocks.dtype) + + def is_valid( + self, + positions, + pos_offsets, + kv_lens, + kv_offsets, + seq_lens, + mapping, + compress_ratio: int, + pos_out=None, + ) -> None: + num_tokens = positions.shape[-1] + num_seqs = kv_lens.shape[0] + assert ( + positions.dim() == 3 and positions.shape[:2] == (3, 1) and positions.stride(2) == 1 + ), ( + tuple(positions.shape), + positions.stride(), + ) + assert positions.dtype == pos_offsets.dtype == torch.int32 + assert kv_lens.dtype == kv_offsets.dtype == seq_lens.dtype == torch.int32 + assert kv_lens.dim() == 1 and kv_lens.is_contiguous() + assert 0 < num_seqs <= MAX_SEQS and 0 < num_tokens <= MAX_ROWS, (num_seqs, num_tokens) + assert pos_offsets.numel() >= num_tokens and kv_offsets.numel() >= num_seqs + assert seq_lens.numel() >= num_seqs + dtypes = (torch.int32, torch.int64, torch.int32, torch.int32) + for t, dt in zip(mapping, dtypes): + assert t.dtype == dt and t.numel() >= num_tokens and t.is_contiguous(), ( + t.dtype, + t.numel(), + ) + + +qsa_spec_step_metadata = QsaSpecStepMetadata() + + +class SpecStepMetadataPrologue: + """The engine's decode-prologue hook (`RuntimeHooks.decode_prologue`): on a generation-only + verify step with the overlap scheduler, runs `qsa_spec_step_metadata` plus + `on_update_kv_lens()`'s host statements and returns True; anything else returns False and + keeps the native code.""" + + def __call__( + self, preparer, inputs, *, enable_spec_decode: bool, runtime_draft_len: int + ) -> bool: + from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAAttentionMetadata + + md = inputs.get("attn_metadata") + if md is not None: + # A packet of an earlier step never reaches this step's forward. + md.spec_step_positions = None + if not enable_spec_decode or preparer._config.disable_overlap_scheduler: + return False + if not isinstance(md, QSAAttentionMetadata) or md.kv_cache_manager is None: + return False + if not md.qsa_has_local_layers or preparer.mapping.has_cp_helix(): + return False + num_seqs, num_tokens = md.num_seqs, md.num_tokens + if md.num_contexts != 0 or md.num_chunked_ctx_requests != 0 or md.num_ctx_tokens != 0: + return False + # the native mapping's generation path (not its one-row-a-request decode path) + if num_tokens == num_seqs or not (0 < num_seqs <= MAX_SEQS) or num_tokens > MAX_ROWS: + return False + pos = inputs["position_ids"] + if ( + pos.ndim != 3 + or tuple(pos.shape[:2]) != (3, 1) + or pos.stride(2) != 1 + or pos.dtype != torch.int32 + ): + return False + if inputs["input_ids"].shape[0] != num_tokens or pos.shape[2] != num_tokens: + return False + buffers = preparer._buffers + pos_off, kv_off = ( + buffers.previous_pos_id_offsets_cuda, + buffers.previous_kv_lens_offsets_cuda, + ) + kv_lens = md.kv_lens_cuda[:num_seqs] + # the mapping reads the runtime view: it must be these lengths + runtime = getattr(md, "kv_lens_cuda_runtime", None) + if runtime is None or runtime.data_ptr() != kv_lens.data_ptr(): + return False + if ( + pos_off.dtype != torch.int32 + or kv_off.dtype != torch.int32 + or kv_lens.dtype != torch.int32 + ): + return False + seq_lens = md.seq_lens_cuda[:num_seqs] + if seq_lens.dtype != torch.int32: + return False + mapping = ( + md.qsa_req_idx_per_token, + md.qsa_logical_positions, + md.qsa_sequence_lengths, + md.qsa_visible_blocks, + ) + if (mapping[0].dtype, mapping[1].dtype, mapping[2].dtype, mapping[3].dtype) != ( + torch.int32, + torch.int64, + torch.int32, + torch.int32, + ): + return False + # The corrected positions also as the contiguous `[C, T]` the decode forward reads + # (`take_step_positions`). + pos3 = torch.empty((pos.shape[0], num_tokens), dtype=torch.int32, device=pos.device) + qsa_spec_step_metadata( + pos, pos_off, kv_lens, kv_off, seq_lens, mapping, md.sparse_metadata_params.compress_ratio, + pos_out=pos3, + ) # fmt: skip + md.spec_step_positions = (pos, pos3) + # `on_update_kv_lens()`'s host statements (the QSA override adds only the mapping). + super(QSAAttentionMetadata, md).on_update_kv_lens() + return True + + +def take_step_positions(md, position_ids): + """The contiguous int32 `[C, T]` positions `SpecStepMetadataPrologue` wrote for exactly this + `position_ids` tensor in this step (consumed once), or None.""" + packet = getattr(md, "spec_step_positions", None) + if packet is None: + return None + md.spec_step_positions = None + return packet[1] if packet[0] is position_ids else None diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py new file mode 100644 index 000000000000..cb124b86c423 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_expand_cluster__decode.py @@ -0,0 +1,753 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA's decode block top-k and token expansion in one CuTe DSL launch, a cluster of CTAs a row. + +Each CTA holds a slice of the row's window in registers; radix passes (12, 11, 8 bits, top down) +narrow to the threshold class, every CTA adding its histogram into every CTA's copy through +distributed shared memory so the cluster narrows in step. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import dsl_user_op + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Consecutive scores a thread loads at once (one 16-byte load): a group. +KPT = 4 +#: The radix passes: (first bit, bits). 31 magnitude bits, top down. +_PASSES = ((19, 12), (8, 11), (0, 8)) +_HIST = 1 << 12 +#: Most rows a launch serves (148 CTAs in clusters of 4); more take the family's one-CTA kernel. +MAX_ROWS = 37 +#: Narrowest table this entry serves: narrower ones take the family's one-CTA kernel. +MIN_WIDTH = 4096 + KPT +#: Widest table (8192 + 128 pages): keeps a CTA's keys below 2^16, the width of its packed counts. +MAX_WIDTH = (8192 + 128) * 32 // 4 +#: CTA size bounds (the scans give each thread at most `_HIST / _MIN_THREADS` bins). +_MAX_THREADS = 1024 +_MIN_THREADS = 128 + + +@dsl_user_op +def _red_add_cluster(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + None, + [ + cutlass.Int32(addr).ir_value(loc=loc, ip=ip), + cutlass.Int32(val).ir_value(loc=loc, ip=ip), + ], + "red.relaxed.cluster.shared::cluster.add.u32 [$0], $1;", + "r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _st_cluster(addr, val, *, loc=None, ip=None): + llvm.inline_asm( + None, + [ + cutlass.Int32(addr).ir_value(loc=loc, ip=ip), + cutlass.Int32(val).ir_value(loc=loc, ip=ip), + ], + "st.shared::cluster.u32 [$0], $1;", + "r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _dsmem(ptr, rank): + """The cluster-shared address of `ptr`'s location in CTA `rank` of the cluster.""" + return cutlass.Int32(cute.arch.map_dsmem_ptr(ptr, rank).toint()) + + +def cluster_size(rows: int) -> int: + """CTAs a row: 8 up to 12 rows (16 clusters of 8 overflow GB300's GPCs in one wave), else 4.""" + return 8 if rows <= 12 else 4 + + +def geometry(width: int, cluster: int) -> tuple[int, int]: + """`(threads, groups)` of a CTA: its `ceil(width / 4 / cluster)` groups over the fewest groups + a thread 1024 threads allow (a group slot costs more than a wider CTA).""" + q4 = -(-(-(-width // KPT)) // cluster) + groups = max(1, -(-q4 // _MAX_THREADS)) + threads = max(_MIN_THREADS, -(-(-(-q4 // groups)) // 32) * 32) + assert threads <= _MAX_THREADS, (width, cluster, threads) + return threads, groups + + +class _TopkExpandClusterKernel: + def __init__(self, *, width, key_stride, block_topk, compress, token_topk, pdl, cluster): + self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( + width, + key_stride, + block_topk, + compress, + token_topk, + pdl, + ) + self.C = cluster + self.threads, self.groups = geometry(width, cluster) + self.warps = self.threads // 32 + self.final = token_topk + compress - 1 + + def smem_bytes(self) -> int: + words = _HIST + sum(1 << bits for _, bits in _PASSES) + 32 + 32 + 32 * self.groups + 16 + return words * 4 + 8 * 128 + + @cute.jit + def __call__( + self, + keys_t: cute.Tensor, + end_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + len_ptr: cute.Pointer, + topk_ptr: cute.Pointer, + sel_ptr: cute.Pointer, + rows: cutlass.Int32, + stream, + ): + # Keys arrive as a tensor (the TVM-FFI entry takes its stream from one); the layout is + # rebuilt with the static width and row stride the 16-byte loads need. + mKeys = cute.make_tensor( + keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) + ) + per_row = cute.make_layout((rows,), stride=(1,)) + mEnd = cute.make_tensor(end_ptr, per_row) + mPos = cute.make_tensor(pos_ptr, per_row) + mLen = cute.make_tensor(len_ptr, per_row) + mTopk = cute.make_tensor(topk_ptr, cute.make_layout((rows, self.k), stride=(self.k, 1))) + mSel = cute.make_tensor( + sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) + ) + ld_keys = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT + ) + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys).launch( + grid=[rows * self.C, 1, 1], + block=[self.threads, 1, 1], + cluster=[self.C, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + ) + + @cute.jit + def _exclusive_scan(self, own, warp, lane, wsum): + """`own`'s exclusive prefix over the CTA's threads in thread order (one barrier).""" + incl = own + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + wsum[warp] = incl + cute.arch.sync_threads() + above = cutlass.Int32(0) + if lane < warp: + above = wsum[lane] + return cute.arch.warp_reduction_sum(above) + incl - own + + @cute.jit + def _narrow( + self, hist, tidx, warp, lane, wsum, state, shift: cutlass.Constexpr, bits: cutlass.Constexpr + ): + """The cluster histogram of a pass read from the top bin down: the bin holding the + `need`-th key of the class becomes the class. Every CTA reads the same counts.""" + nbins: cutlass.Constexpr = 1 << bits + per: cutlass.Constexpr = -(-nbins // self.threads) + prefix = state[0] + mask = state[1] + need = state[2] + h = cute.make_rmem_tensor(cute.make_layout((per,), stride=(1,)), cutlass.Int32) + own = cutlass.Int32(0) + for j in cutlass.range_constexpr(per): + b = nbins - 1 - (tidx * per + j) + hv = cutlass.Int32(0) + if b >= 0: + hv = hist[b] + h[j] = hv + own = own + hv + before = self._exclusive_scan(own, warp, lane, wsum) + if before < need: + if need <= before + own: + run = before + for j in cutlass.range_constexpr(per): + if run < need: + if need <= run + h[j]: + b = nbins - 1 - (tidx * per + j) + rem = need - run + state[0] = prefix | (b << shift) + state[1] = mask | ((nbins - 1) << shift) + state[2] = rem + state[3] = h[j] + state[4] = cutlass.Int32( + cutlass.select_((h[j] == rem) | (shift == 0), 1, 0) + ) + run = run + h[j] + cute.arch.sync_threads() + + @cute.jit + def _pass( + self, + keys, + tidx, + warp, + lane, + local, + merged, + wsum, + state, + shift: cutlass.Constexpr, + bits: cutlass.Constexpr, + q4, + ): + """One radix pass unless narrowing is done: count this CTA's class keys, add the counts + into every CTA's `merged`, cluster barrier, scan. Slot j holds keys only while + `j * threads < q4`.""" + nbins: cutlass.Constexpr = 1 << bits + if state[4] == 0: + prefix = state[0] + mask = state[1] + for j in cutlass.range_constexpr(self.groups): + if j * self.threads < q4: + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + if (kv & mask) == prefix: + cute.arch.atomic_add( + local.iterator + ((kv >> shift) & (nbins - 1)), + cutlass.Int32(1), + scope="cta", + ) + cute.arch.sync_threads() + for b in cutlass.range_constexpr(-(-nbins // self.threads)): + i = tidx + b * self.threads + if i < nbins: + v = local[i] + if v != 0: + local[i] = cutlass.Int32(0) + for r in cutlass.range_constexpr(self.C): + _red_add_cluster(_dsmem(merged.iterator + i, r), v) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + self._narrow(merged, tidx, warp, lane, wsum, state, shift, bits) + + @cute.kernel + def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + rank = cute.arch.block_idx_in_cluster() + C: cutlass.Constexpr = self.C + nt: cutlass.Constexpr = self.threads + nw: cutlass.Constexpr = self.warps + G: cutlass.Constexpr = self.groups + k: cutlass.Constexpr = self.k + cr: cutlass.Constexpr = self.cr + row = bidx // C + smem = cutlass.utils.SmemAllocator() + local = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((_HIST,), stride=(1,)), byte_alignment=128 + ) + merged0 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[0][1],), stride=(1,)), byte_alignment=128 + ) + merged1 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[1][1],), stride=(1,)), byte_alignment=128 + ) + merged2 = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((1 << _PASSES[2][1],), stride=(1,)), byte_alignment=128 + ) + ex = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + wsum = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + wtot = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32 * G,), stride=(1,)), byte_alignment=128 + ) + state = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((16,), stride=(1,)), byte_alignment=128 + ) + + # 0. The histograms zeroed and the row's host-filled metadata read before the wait; + # the arrive releases the zeroed counts before any peer adds into them. + for b in cutlass.range_constexpr(-(-_HIST // nt)): + if tidx + b * nt < _HIST: + local[tidx + b * nt] = cutlass.Int32(0) + merged0[tidx + b * nt] = cutlass.Int32(0) + for b in cutlass.range_constexpr(-(-(1 << _PASSES[1][1]) // nt)): + if tidx + b * nt < (1 << _PASSES[1][1]): + merged1[tidx + b * nt] = cutlass.Int32(0) + for b in cutlass.range_constexpr(-(-(1 << _PASSES[2][1]) // nt)): + if tidx + b * nt < (1 << _PASSES[2][1]): + merged2[tidx + b * nt] = cutlass.Int32(0) + n = mEnd[row] + seqlen = mLen[row] + visible = cutlass.Int32(mPos[row]) + 1 + if tidx == 0: + state[0] = cutlass.Int32(0) # the class's fixed bits + state[1] = cutlass.Int32(0) # their mask + state[2] = cutlass.min(n, cutlass.Int32(k)) # still needed from the class + state[3] = n # keys in the class + state[4] = cutlass.Int32(cutlass.select_(n <= k, 1, 0)) # narrowing done + cute.arch.cluster_arrive() + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + # 1. This CTA's slice of the window: groups [g_lo, g_hi) of `KPT` consecutive keys, + # group `g_lo + j * nt + tidx` in slot j of thread tidx; -1 past the window. + n4 = (n + (KPT - 1)) // KPT + q4 = (n4 + (C - 1)) // C + g_lo = rank * q4 + g_hi = cutlass.min(g_lo + q4, n4) + gK = cute.logical_divide(mKeys, (None, KPT)) + keys = cute.make_rmem_tensor(cute.make_layout((KPT, G), stride=(1, KPT)), cutlass.Int32) + keys.fill(-1) + for j in cutlass.range_constexpr(G): + gl = g_lo + j * nt + tidx + if gl < g_hi: + cute.copy(ld_keys, gK[row, (None, gl)], keys[None, j]) + for c in cutlass.range_constexpr(KPT): + v = cutlass.max(keys[c, j], cutlass.Int32(0)) + if (gl >= g_hi) | (gl * KPT + c >= n): + v = cutlass.Int32(-1) + keys[c, j] = v + cute.arch.cluster_wait() + + self._pass( + keys, tidx, warp, lane, local, merged0, wsum, state, _PASSES[0][0], _PASSES[0][1], q4 + ) + self._pass( + keys, tidx, warp, lane, local, merged1, wsum, state, _PASSES[1][0], _PASSES[1][1], q4 + ) + self._pass( + keys, tidx, warp, lane, local, merged2, wsum, state, _PASSES[2][0], _PASSES[2][1], q4 + ) + prefix = state[0] + mask = state[1] + need = state[2] + + # 3. Block order within the CTA (slot j of every thread, then slot j + 1): each + # (thread, slot)'s counts above and in the class, packed (above + class << 16), and their + # exclusive prefix. Slots past `q4` hold no keys in any thread (uniform skip). + pre = cute.make_rmem_tensor(cute.make_layout((G,), stride=(1,)), cutlass.Int32) + pre.fill(0) + for j in cutlass.range_constexpr(G): + if j * nt < q4: + a = cutlass.Int32(0) + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + mk = kv & mask + if mk > prefix: + a = a + 1 + if mk == prefix: + a = a + (1 << 16) + incl = a + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + wtot[j * 32 + warp] = incl + pre[j] = incl - a + cute.arch.sync_threads() + run = cutlass.Int32(0) + for j in cutlass.range_constexpr(G): + if j * nt < q4: + v = cutlass.Int32(0) + if lane < nw: + v = wtot[j * 32 + lane] + above = cutlass.Int32(0) + if lane < warp: + above = v + pre[j] = pre[j] + run + cute.arch.warp_reduction_sum(above) + run = run + cute.arch.warp_reduction_sum(v) + + # 4. Every CTA's totals into every CTA (lane r of warp 0 writes CTA r's slot `rank`). + if tidx < C: + _st_cluster(_dsmem(ex.iterator + rank, tidx), run) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + # The class's `need` are its highest blocks, so later CTAs fill first. `base`: selected + # count of the CTAs before this one; `skip`: this CTA's lowest class keys left out. + base = cutlass.Int32(0) + skip = cutlass.Int32(0) + later = cutlass.Int32(0) + for rr in cutlass.range_constexpr(C): + r = C - 1 - rr + tot = ex[r] + e_r = tot >> 16 + take = cutlass.max(cutlass.min(need - later, e_r), cutlass.Int32(0)) + later = later + e_r + if r < rank: + base = base + (tot & 0xFFFF) + take + if r == rank: + skip = e_r - take + + # 5. Each selected block at its position, with its tokens below the sequence length. + for j in cutlass.range_constexpr(G): + gl = g_lo + j * nt + tidx + ab = pre[j] & 0xFFFF + cb = pre[j] >> 16 + for c in cutlass.range_constexpr(KPT): + kv = keys[c, j] + if kv >= 0: + mk = kv & mask + sel = cutlass.Int32(0) + pos = cutlass.Int32(0) + if mk > prefix: + sel = cutlass.Int32(1) + pos = base + ab + cutlass.max(cb - skip, cutlass.Int32(0)) + ab = ab + 1 + if mk == prefix: + if cb >= skip: + sel = cutlass.Int32(1) + pos = base + ab + cb - skip + cb = cb + 1 + if sel == 1: + col = gl * KPT + c + mTopk[row, pos] = col + for tt in cutlass.range_constexpr(cr): + tok = col * cr + tt + val = cutlass.Int32(-1) + if tok < seqlen: + val = tok + mSel[row, pos * cr + tt] = val + + # 6. Over the cluster's threads: the block list's -1 padding, then the causal tail and -1. + total = cutlass.min(n, cutlass.Int32(k)) + for b in cutlass.range_constexpr(-(-k // (C * nt))): + slot = (b * C + rank) * nt + tidx + if slot < k: + if slot >= total: + mTopk[row, slot] = cutlass.Int32(-1) + tail_start = (visible // cr) * cr + tail_count = visible - tail_start + body = cutlass.min(total * cr, cutlass.Int32(self.token_topk)) + for b in cutlass.range_constexpr(-(-self.final // (C * nt))): + col = (b * C + rank) * nt + tidx + if col < self.final: + if col >= body: + off = col - body + val = cutlass.Int32(-1) + if off < cr - 1: + if off < tail_count: + if tail_start + off < seqlen: + val = tail_start + off + mSel[row, col] = val + + +#: Compiled launches; the row count is a runtime argument. +_COMPILED: dict = {} +_INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} + + +def launch( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, +): + """Writes `topk_indices`; returns the int32 token selection (`out` when given).""" + rows, width = scores.shape + block_topk = topk_indices.shape[1] + sel = out + if sel is None: + sel = torch.empty( + (rows, token_topk + compress_ratio - 1), dtype=torch.int32, device=scores.device + ) + if rows == 0: + return sel + pdl = bool(get_env_enable_pdl()) + keys = scores.view(torch.int32) + cluster = cluster_size(rows) + key = ( + width, + scores.stride(0), + block_topk, + compress_ratio, + token_topk, + query_positions.dtype, + sequence_lengths.dtype, + pdl, + cluster, + ) + ptrs = ( + row_ends.data_ptr(), + query_positions.data_ptr(), + sequence_lengths.data_ptr(), + topk_indices.data_ptr(), + sel.data_ptr(), + ) + fn = _COMPILED.get(key) + if fn is None: + gm = cute.AddressSpace.gmem + kernel = _TopkExpandClusterKernel( + width=width, + key_stride=scores.stride(0), + block_topk=block_topk, + compress=compress_ratio, + token_topk=token_topk, + pdl=pdl, + cluster=cluster, + ) + fn = _COMPILED[key] = cute.compile( + kernel, + from_dlpack(keys, assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.Int32, ptrs[0], gm, assumed_align=4), + make_ptr( + _INDEX_TYPES[query_positions.dtype], + ptrs[1], + gm, + assumed_align=query_positions.element_size(), + ), + make_ptr( + _INDEX_TYPES[sequence_lengths.dtype], + ptrs[2], + gm, + assumed_align=sequence_lengths.element_size(), + ), + make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + cutlass.Int32(rows), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + fn(keys, *ptrs, rows) + return sel + + +def supports(scores) -> bool: + """Whether this entry serves a decode call over this score table (see `is_valid`).""" + rows, width = scores.shape + return 0 < rows <= MAX_ROWS and width % KPT == 0 and MIN_WIDTH <= width <= MAX_WIDTH + + +#: Score-table widths: 262144 tokens with / without MTP's extra pages, and 131072 tokens. +_TUNE_WIDTH = 65568 +_TUNE_WIDTH_NO_MTP = 65536 +_HALF_WIDTH = 32800 +#: Decode windows of 0, 1, 2 and 9 blocks, k - 1, k, k + 1, the whole table, and between. +_EDGES_16 = ( + 2, + 3, + 11, + 35, + 2047, + 2051, + 2055, + 16380, + 16388, + 99999, + 180224, + 262143, + 50000, + 150001, + 220000, + 7, +) + + +class QsaTopkExpandClusterDecode(OpWrapper): + """`qsa_topk_select` (zero row starts), `qsa_canonical_selection` and + `qsa_expand_block_indices` in one launch, for decode-sized calls over a wide score table. + + Per row: the `block_topk` largest of `scores[row, :row_ends[row]]` (ties to the higher + block) into `topk_indices[row]` in block order, -1 padded, in place. Returns the token + selection (int32 `[rows, token_topk + compress_ratio - 1]`, into `out` when given): selected + groups' tokens below `sequence_lengths[row]`, then the causal tail, then -1. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="ll_tep4 draft step: 1 row at ~96k tokens, 65568 wide, relu-sum, graph", + spec=dict(rows=1, pos=(96003,), relu_sum=True, width=_TUNE_WIDTH, graph=True), + ), + Cell( + why="37 rows (MAX_ROWS, 148 CTAs in clusters of 4): 9 requests x 4 verify rows " + "spread to ~200k tokens and a draft row", + spec=dict( + rows=37, + requests=tuple((1000 + 24000 * r, 4) for r in range(9)) + ((131072, 1),), + relu_sum=True, + width=_TUNE_WIDTH, + ), + ), + Cell( + why="4 rows of bf16-resolution log-normal scores: exact ties at the threshold, all " + "three passes, the tie rule across a cluster's CTAs", + spec=dict(rows=4, pos=(100000, 30000, 4095, 2047), lognormal=True, width=_TUNE_WIDTH), + ), + Cell( + why="2 relu-sparse rows (~1.4% positive, negative zeros among the zeros): the zero " + "class straddles the budget across CTAs", + spec=dict(rows=2, pos=(262143, 120000), relu_sparse=True, width=_TUNE_WIDTH_NO_MTP), + ), + Cell( + why="3 rows of equal scores (windows 25000, 513 and 512 blocks): the tie class is " + "the window, the highest blocks taken", + spec=dict(rows=3, pos=(100000, 2051, 2047), equal=True, width=_TUNE_WIDTH), + ), + Cell( + why="20 rows (clusters of 4) at the edges of k and the table, relu-sum, 65536 wide", + spec=dict( + rows=20, + pos=_EDGES_16 + (262140, 4, 2048, 131071), + relu_sum=True, + width=_TUNE_WIDTH_NO_MTP, + ), + ), + Cell( + why="a 131072-token engine's 32800-wide table: 4 verify rows up to its last position", + spec=dict( + rows=4, requests=((131068, 4),), relu_sum=True, width=_HALF_WIDTH, graph=True + ), + ), + ) + + note = """ + Source: this target's CuTe DSL kernel, `cute.compile`d (`--enable-tvm-ffi`) on first call + per table width and row stride, k, compress ratio, token budget, index dtypes, PDL and + cluster size. + + Preconditions: zero row starts; `scores` fp32 `[rows, W]`, unit column stride, 16-byte + aligned rows, `W % 4 == 0`; `topk_indices` int32 `[rows, K]` contiguous; `row_ends` int32 + (each at most `W`), positions and lengths int32 or int64, each a contiguous `[rows]`; the + call within `supports` (1..37 rows, 4100 <= W <= 66560). Ties go to the higher block; a + negative zero ranks as zero. Limits: rows x row stride below 2^31, positions read as Int32. + + Not certified: NaN scores inside the window (they rank above every finite score). + """ + + def __call__( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + return launch( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + def reference( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + """`(topk_indices_after, selection)` via a stable descending sort per row; ignores `out`.""" + rows, _ = scores.shape + k = topk_indices.shape[1] + device = scores.device + blocks = torch.full((rows, k), -1, dtype=torch.int64, device=device) + for r in range(rows): + n = int(row_ends[r]) + s = scores[r, :n].float().clamp(min=0.0) + # A stable sort of the reversed row puts the higher block first among equals. + order = (n - 1) - torch.sort(torch.flip(s, [0]), descending=True, stable=True).indices[ + :k + ] + chosen = torch.sort(order).values + blocks[r, : chosen.numel()] = chosen + final = token_topk + compress_ratio - 1 + lengths = sequence_lengths.long() + offsets = torch.arange(compress_ratio, device=device) + expanded = (blocks[:, :, None] * compress_ratio + offsets).reshape(rows, token_topk) + chosen = (blocks >= 0).repeat_interleave(compress_ratio, dim=1) + keep = chosen & (expanded < lengths[:, None]) + sel = torch.full((rows, final), -1, dtype=torch.int64, device=device) + sel[:, :token_topk] = torch.where(keep, expanded, -1) + first_free = (blocks >= 0).sum(dim=1) * compress_ratio + visible = query_positions.long() + 1 + tail_start = visible // compress_ratio * compress_ratio + row = torch.arange(rows, device=device) + for offset in range(compress_ratio - 1): + column = first_free + offset + token = tail_start + offset + live = (offset < visible - tail_start) & (token < lengths) & (column < final) + sel[row[live], column[live]] = token[live] + return blocks.to(torch.int32), sel.to(torch.int32) + + def is_valid( + self, + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, + ): + assert scores.dtype == torch.float32 and scores.stride(1) == 1, ( + "fp32 scores, unit column stride" + ) + assert scores.shape[1] % KPT == 0 and scores.shape[1] <= MAX_WIDTH, "W % 4 == 0, W <= 66560" + assert scores.stride(0) % KPT == 0 and scores.data_ptr() % 16 == 0, "16-byte aligned rows" + assert topk_indices.dtype == torch.int32 and topk_indices.is_contiguous(), ( + "int32 topk_indices" + ) + rows = scores.shape[0] + assert row_ends.dtype == torch.int32 and row_ends.shape == (rows,), "int32 row_ends [rows]" + for t in (query_positions, sequence_lengths): + assert t.dtype in _INDEX_TYPES and t.shape == (rows,), "int32/int64 [rows]" + assert all(t.is_contiguous() for t in (row_ends, query_positions, sequence_lengths)), ( + "contiguous row_ends, positions and lengths" + ) + assert token_topk == topk_indices.shape[1] * compress_ratio, ( + "token_topk = k * compress_ratio" + ) + assert out is None or ( + out.dtype == torch.int32 + and out.is_contiguous() + and out.shape == (scores.shape[0], token_topk + compress_ratio - 1) + ), "out: contiguous int32 [rows, token_topk + compress_ratio - 1]" + + +qsa_topk_expand_cluster = QsaTopkExpandClusterDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py new file mode 100644 index 000000000000..6f9262cf830b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_topk_rows_cute__prefill.py @@ -0,0 +1,695 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""QSA's block top-k and its expansion for a context chunk's rows over a wide score table. + +Same contract and bits as the family's `qsa_topk_expand_decode` (the `block_topk` largest of +each row's visible scores, ties to the higher block, in block order, -1 padded, then the token +expansion with the causal tail). One CTA a row reads the window twice from global, never staged: +a per-warp bound `lo` with at least `block_topk` keys at or above it, a candidate list of those +keys, radix passes that find the threshold's bin, and bitmaps whose scan orders the selected +blocks. Scores are non-negative fp32 read as int32 bit patterns; -0.0 ranks as zero. +""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute.runtime import from_dlpack + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( + qsa_topk_expand_decode as _family_topk, +) + +#: Consecutive scores a thread holds per slab (one 16-byte load). +KPT = 4 +#: Threads a row (a CTA). +THREADS = 512 +#: Slabs whose loads are issued together. +UNROLL = 4 +#: Candidate list entries: about 4 % of a window survives `lo`, so ~100k-block windows fit. +CAPC = 3072 +_HIST = 2048 +_I32_MAX = 0x7FFFFFFF +#: CTAs an SM the register allocation is held to. +MIN_BLOCKS = 3 +#: Score columns this kernel serves: wider than one family CTA's registers, up to a +#: 262144-token table with MTP's extra pages. +MIN_WIDTH = 4096 + KPT +MAX_WIDTH = 1 << 17 + + +class _TopkRowsKernel: + def __init__(self, *, width, key_stride, block_topk, compress, token_topk, pdl): + self.nt = THREADS + self.nw = THREADS // 32 + self.slab = KPT * THREADS + self.width, self.key_stride, self.k, self.cr, self.token_topk, self.pdl = ( + width, + key_stride, + block_topk, + compress, + token_topk, + pdl, + ) + self.final = token_topk + compress - 1 + self.words = -(-width // 32) + self.wpt = -(-self.words // THREADS) + self.per_warp = -(-block_topk // self.nw) + # Values a lane keeps for the bound: each warp needs `per_warp` of them at or above + # its value, from at most 32 * vpl. + self.vpl = max(2, -(-(3 * self.per_warp) // 64)) + self.union_words = max(2 * self.words, _HIST) + assert self.per_warp <= 32 * self.vpl, "too few lane values for the per-warp bound" + + def smem_bytes(self) -> int: + return (self.union_words + 2 * CAPC + 64 + 32 + 16) * 4 + 6 * 128 + + @cute.jit + def __call__( + self, + keys_t: cute.Tensor, + end_ptr: cute.Pointer, + pos_ptr: cute.Pointer, + len_ptr: cute.Pointer, + topk_ptr: cute.Pointer, + sel_ptr: cute.Pointer, + rows: cutlass.Int32, + stream, + ): + mKeys = cute.make_tensor( + keys_t.iterator, cute.make_layout((rows, self.width), stride=(self.key_stride, 1)) + ) + per_row = cute.make_layout((rows,), stride=(1,)) + mEnd = cute.make_tensor(end_ptr, per_row) + mPos = cute.make_tensor(pos_ptr, per_row) + mLen = cute.make_tensor(len_ptr, per_row) + mTopk = cute.make_tensor(topk_ptr, cute.make_layout((rows, self.k), stride=(self.k, 1))) + mSel = cute.make_tensor( + sel_ptr, cute.make_layout((rows, self.final), stride=(self.final, 1)) + ) + ld_keys = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), cutlass.Int32, num_bits_per_copy=32 * KPT + ) + self.kernel(mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys).launch( + grid=[rows, 1, 1], + block=[self.nt, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=MIN_BLOCKS, + ) + + @cute.jit + def _load(self, ld, gK, row, s, tidx, n, kr): + """Slab `s`'s four keys of thread `tidx` into `kr` (a negative as zero, -1 past the + window).""" + col4 = s * self.nt + tidx + e0 = col4 * KPT + kr.fill(-1) + if e0 < n: + cute.copy(ld, gK[row, (None, col4)], kr) + for c in cutlass.range_constexpr(KPT): + b = cutlass.max(kr[c], cutlass.Int32(0)) + if e0 + c >= n: + b = cutlass.Int32(-1) + kr[c] = b + + @cute.jit + def _insert(self, mv, b): + """`b` into the thread's descending `vpl` largest (`mv[0]` the largest).""" + for j in cutlass.range_constexpr(self.vpl - 1, 0, -1): + mv[j] = cutlass.max(mv[j], cutlass.min(mv[j - 1], b)) + mv[0] = cutlass.max(mv[0], b) + + @cute.jit + def _load_full(self, ld, gK, row, s, tidx, kr): + """`_load` of a slab wholly inside the window (no bounds).""" + cute.copy(ld, gK[row, (None, s * self.nt + tidx)], kr) + for c in cutlass.range_constexpr(KPT): + kr[c] = cutlass.max(kr[c], cutlass.Int32(0)) + + @cute.jit + def _compact(self, kr, s0, tidx, lane, lo, state, ckey, cidx): + """The UNROLL slabs' keys at or above `lo` appended to the list (a warp scan of the + counts, one shared atomic a warp).""" + cnt = cutlass.Int32(0) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + if kr[u][c] >= lo: + cnt = cnt + 1 + incl = cnt + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + base = cutlass.Int32(0) + if lane == 31: + if incl > 0: + base = cute.arch.atomic_add(state.iterator + 5, incl, scope="cta") + base = cute.arch.shuffle_sync(base, offset=31) + pos = base + incl - cnt + if cnt > 0: + for u in cutlass.range_constexpr(UNROLL): + e0 = ((s0 + u) * self.nt + tidx) * KPT + for c in cutlass.range_constexpr(KPT): + if kr[u][c] >= lo: + if pos < CAPC: + ckey[pos] = kr[u][c] + cidx[pos] = e0 + c + pos = pos + 1 + + @cute.jit + def _excl_scan(self, v, warp, lane, ws, half): + """Block-wide exclusive prefix of `v` and the total (one barrier; `ws[half..half+15]`).""" + incl = v + for off in (1, 2, 4, 8, 16): + up = cute.arch.shuffle_sync_up(incl, offset=off) + if lane >= off: + incl = incl + up + if lane == 31: + ws[half + warp] = incl + cute.arch.sync_threads() + above = cutlass.Int32(0) + if lane < warp: + above = ws[half + lane] + total = cutlass.Int32(0) + if lane < self.nw: + total = ws[half + lane] + pre = cute.arch.warp_reduction_sum(above) + incl - v + total = cute.arch.warp_reduction_sum(total) + return pre, total + + @cute.jit + def _narrow(self, tidx, warp, lane, hist, ws, state, prefix, mask, need, shift): + """One pass's histogram read (and cleared) from the top bin down: the bin holding the + `need`-th key becomes the threshold's (prefix, mask, need left, keys in the bin); the + passes stop once that bin holds exactly the keys still needed or no bits are left.""" + bpt: cutlass.Constexpr = _HIST // self.nt + h = cute.make_rmem_tensor(cute.make_layout((bpt,), stride=(1,)), cutlass.Int32) + h.fill(0) + own = cutlass.Int32(0) + for j in cutlass.range_constexpr(bpt): + h[j] = hist[_HIST - 1 - bpt * tidx - j] + hist[_HIST - 1 - bpt * tidx - j] = cutlass.Int32(0) + own = own + h[j] + before, _ = self._excl_scan(own, warp, lane, ws, 0) + if before < need: + if need <= before + own: + run = before + for j in cutlass.range_constexpr(bpt): + if run < need: + if need <= run + h[j]: + digit = cutlass.Int32(_HIST - 1 - bpt * tidx - j) + state[0] = prefix | (digit << shift) + state[1] = mask | (cutlass.Int32(_HIST - 1) << shift) + state[2] = need - run + state[3] = h[j] + state[4] = cutlass.Int32( + cutlass.select_((h[j] == need - run) | (shift == 0), 1, 0) + ) + state[6] = cutlass.max(shift - 11, cutlass.Int32(0)) + run = run + h[j] + cute.arch.sync_threads() + + @cute.kernel + def kernel(self, mKeys, mEnd, mPos, mLen, mTopk, mSel, ld_keys: cute.CopyAtom): + tidx, _, _ = cute.arch.thread_idx() + row, _, _ = cute.arch.block_idx() + warp = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + nt: cutlass.Constexpr = self.nt + k: cutlass.Constexpr = self.k + cr: cutlass.Constexpr = self.cr + words: cutlass.Constexpr = self.words + wpt: cutlass.Constexpr = self.wpt + smem = cutlass.utils.SmemAllocator() + # The radix histogram and the two bitmaps share one region: the passes clear every bin + # before the bitmaps are set. + region = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((self.union_words,), stride=(1,)), byte_alignment=128 + ) + hist = cute.make_tensor(region.iterator, cute.make_layout((_HIST,), stride=(1,))) + selw = cute.make_tensor(region.iterator, cute.make_layout((words,), stride=(1,))) + tiew = cute.make_tensor(region.iterator + words, cute.make_layout((words,), stride=(1,))) + ckey = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((CAPC,), stride=(1,)), byte_alignment=128 + ) + cidx = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((CAPC,), stride=(1,)), byte_alignment=128 + ) + ws = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((64,), stride=(1,)), byte_alignment=128 + ) + wv = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((32,), stride=(1,)), byte_alignment=128 + ) + state = smem.allocate_tensor( + cutlass.Int32, cute.make_layout((16,), stride=(1,)), byte_alignment=128 + ) + + # 0. Shared state and the row's host-filled metadata, before the wait. + for b in cutlass.range_constexpr(-(-self.union_words // nt)): + if tidx + b * nt < self.union_words: + region[tidx + b * nt] = cutlass.Int32(0) + if tidx == 0: + # A window of at most k blocks takes every block (mask 0, prefix 0, nothing skipped). + state[0] = cutlass.Int32(0) + state[1] = cutlass.Int32(0) + state[2] = cutlass.Int32(0) + state[3] = cutlass.Int32(0) + state[4] = cutlass.Int32(1) + state[5] = cutlass.Int32(0) # candidates compacted + state[6] = cutlass.Int32(0) + # Thread 0's reset of the candidate counter must be visible to every warp's `_compact` + # atomic below: the `n > k` path has a barrier of its own, a row of <= k blocks does not. + cute.arch.sync_threads() + n = mEnd[row] + seqlen = mLen[row] + visible = cutlass.Int32(mPos[row]) + 1 + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + cute.arch.griddepcontrol_launch_dependents() + + gK = cute.logical_divide(mKeys, (None, KPT)) + slab: cutlass.Constexpr = self.slab + nslabs = (n + (slab - 1)) // slab + # Slabs in whole UNROLL groups wholly inside the window (no per-key bounds). + nfull = (n // (slab * UNROLL)) * UNROLL + kr = [ + cute.make_rmem_tensor(cute.make_layout((KPT,), stride=(1,)), cutlass.Int32) + for _ in range(UNROLL) + ] + + # 1. Each thread's `vpl` largest keys; the per-warp bound; `lo`. + lo = cutlass.Int32(0) + hb = cutlass.Int32(-1) + if n > k: + mv = cute.make_rmem_tensor(cute.make_layout((self.vpl,), stride=(1,)), cutlass.Int32) + mv.fill(-1) + for s0 in cutlass.range(0, nfull, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load_full(ld_keys, gK, row, s0 + u, tidx, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + self._insert(mv, kr[u][c]) + for s0 in cutlass.range(nfull, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + self._insert(mv, kr[u][c]) + m1 = mv[0] + have_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(self.vpl): + have_t = have_t + cutlass.Int32(cutlass.select_(mv[j] >= 0, 1, 0)) + have = cute.arch.warp_redux_sync(have_t, "add") + v = cutlass.Int32(-1) + if have >= self.per_warp: + t = cutlass.Int32(0) + # The top 16 bits (exponent and 7 mantissa bits): a key at or below the + # exact per-warp value is still a valid bound, within 1/128 of it. + for bit in cutlass.range_constexpr(30, 14, -1): + c = t | cutlass.Int32(1 << bit) + cnt_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(self.vpl): + cnt_t = cnt_t + cutlass.Int32(cutlass.select_(mv[j] >= c, 1, 0)) + cnt = cute.arch.warp_redux_sync(cnt_t, "add") + if cnt >= self.per_warp: + t = c + v = t + wmax = cute.arch.warp_redux_sync(m1, "max") + if lane == 0: + wv[warp] = v + wv[16 + warp] = wmax + cute.arch.sync_threads() + bound = cutlass.Int32(_I32_MAX) + gmax = cutlass.Int32(-1) + if lane < self.nw: + bound = wv[lane] + gmax = wv[16 + lane] + bound = cute.arch.warp_reduction(bound, lambda x, y: cutlass.min(x, y)) + gmax = cute.arch.warp_reduction_max(gmax) + lo = cutlass.max(bound, cutlass.Int32(0)) + # Every candidate shares the bits above the highest bit where `gmax` and `lo` + # differ; the radix passes walk down from there, 11 bits at a time. + diff = gmax ^ lo + if diff != 0: + hb = 31 - cutlass.Int32(cute.arch.clz(diff)) + low_bits = (cutlass.Int64(1) << cutlass.Int64(hb + 1)) - 1 + mask0 = cutlass.Int32(cutlass.Int64(_I32_MAX) & (cutlass.Int64(-1) ^ low_bits)) + if tidx == 0: + state[0] = gmax & mask0 # threshold prefix + state[1] = mask0 # its mask + state[2] = cutlass.Int32(k) # still needed at the threshold + state[3] = cutlass.Int32(0) # keys in the threshold's bin + state[4] = cutlass.Int32(cutlass.select_(hb >= 0, 0, 1)) # narrowing done + state[6] = cutlass.max(hb - 10, cutlass.Int32(0)) # the next pass's shift + + # 2. The candidates (keys at or above `lo`), unordered, with their blocks. + for s0 in cutlass.range(0, nfull, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load_full(ld_keys, gK, row, s0 + u, tidx, kr[u]) + self._compact(kr, s0, tidx, lane, lo, state, ckey, cidx) + for s0 in cutlass.range(nfull, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + self._compact(kr, s0, tidx, lane, lo, state, ckey, cidx) + cute.arch.sync_threads() + m = state[5] + listed = m <= CAPC + + # 3. The threshold's bin, narrowed by radix passes over the list (or the window) + # until it holds exactly the keys still needed or every bit is fixed. + for _p in cutlass.range_constexpr(3): + if state[4] == 0: + prefix = state[0] + mask = state[1] + need = state[2] + shift = state[6] + if listed: + for i in cutlass.range(tidx, m, nt, unroll=1): + key = ckey[i] + if (key & mask) == prefix: + cute.arch.atomic_add( + hist.iterator + ((key >> shift) & (_HIST - 1)), + cutlass.Int32(1), + scope="cta", + ) + else: + for s0 in cutlass.range(0, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + for c in cutlass.range_constexpr(KPT): + key = kr[u][c] + if key >= 0: + if (key & mask) == prefix: + cute.arch.atomic_add( + hist.iterator + ((key >> shift) & (_HIST - 1)), + cutlass.Int32(1), + scope="cta", + ) + cute.arch.sync_threads() + self._narrow(tidx, warp, lane, hist, ws, state, prefix, mask, need, shift) + prefix = state[0] + mask = state[1] + need = state[2] + at_thr = state[3] + if n > k: + if hb < 0: + # Every candidate equals the largest: the bin is all of them. + at_thr = m + + # 4. Bitmaps of the blocks above the threshold's bin and in it. + if listed: + for i in cutlass.range(tidx, m, nt, unroll=1): + mk = ckey[i] & mask + e = cidx[i] + bit = cutlass.Int32(1) << (e & 31) + if mk > prefix: + cute.arch.atomic_or(selw.iterator + (e >> 5), bit, scope="cta") + elif mk == prefix: + cute.arch.atomic_or(tiew.iterator + (e >> 5), bit, scope="cta") + else: + for s0 in cutlass.range(0, nslabs, UNROLL, unroll=1): + for u in cutlass.range_constexpr(UNROLL): + self._load(ld_keys, gK, row, s0 + u, tidx, n, kr[u]) + for u in cutlass.range_constexpr(UNROLL): + e0 = ((s0 + u) * nt + tidx) * KPT + for c in cutlass.range_constexpr(KPT): + key = kr[u][c] + e = e0 + c + bit = cutlass.Int32(1) << (e & 31) + if key >= 0: + mk = key & mask + if mk > prefix: + cute.arch.atomic_or(selw.iterator + (e >> 5), bit, scope="cta") + elif mk == prefix: + cute.arch.atomic_or(tiew.iterator + (e >> 5), bit, scope="cta") + cute.arch.sync_threads() + + # 5. Block order: each thread folds its `wpt` words' kept tie blocks (the lowest `skip` + # dropped) into the selected words; a scan gives each word its first output position. + skip = cutlass.max(at_thr - need, cutlass.Int32(0)) + w0 = tidx * wpt + own_q = cutlass.Int32(0) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + own_q = own_q + cute.arch.popc(tiew[w0 + j]) + tied_before, _ = self._excl_scan(own_q, warp, lane, ws, 0) + fw = cute.make_rmem_tensor(cute.make_layout((wpt,), stride=(1,)), cutlass.Int32) + fw.fill(0) + own_t = cutlass.Int32(0) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + tw = tiew[w0 + j] + nq = cute.arch.popc(tw) + drop = cutlass.min(cutlass.max(skip - tied_before, cutlass.Int32(0)), nq) + # clear the lowest `drop` set bits of the word's tie class + for _d in cutlass.range(drop, unroll=1): + tw = tw & (tw - 1) + tied_before = tied_before + nq + f = selw[w0 + j] | tw + fw[j] = f + own_t = own_t + cute.arch.popc(f) + first, _ = self._excl_scan(own_t, warp, lane, ws, 32) + for j in cutlass.range_constexpr(wpt): + if w0 + j < words: + selw[w0 + j] = fw[j] + tiew[w0 + j] = first + first = first + cute.arch.popc(fw[j]) + cute.arch.sync_threads() + # Output position `o` (one a thread a round): its word by binary search over the words' + # first positions, its block by the in-word rank of the set bit. + total = n + if n > k: + total = cutlass.Int32(k) + for c in cutlass.range_constexpr(-(-k // nt)): + o = tidx + c * nt + if o < total: + lo_w = cutlass.Int32(0) + hi_w = cutlass.Int32(words - 1) + while lo_w < hi_w: + mid = (lo_w + hi_w + 1) >> 1 + if tiew[mid] <= o: + lo_w = mid + else: + hi_w = mid - 1 + x = selw[lo_w] + r = o - tiew[lo_w] + bit = cutlass.Int32(0) + for h in (16, 8, 4, 2, 1): + cnt_h = cute.arch.popc((x >> bit) & cutlass.Int32((1 << h) - 1)) + if r >= cnt_h: + r = r - cnt_h + bit = bit + h + e = lo_w * 32 + bit + mTopk[row, o] = e + for jj in cutlass.range_constexpr(cr): + tok = e * cr + jj + val = cutlass.Int32(-1) + if tok < seqlen: + val = tok + mSel[row, o * cr + jj] = val + + # 6. Padding of the block list; the causal tail after the selected tokens, then -1. + for c in cutlass.range_constexpr(-(-k // nt)): + slot = tidx + c * nt + if slot < k: + if slot >= total: + mTopk[row, slot] = cutlass.Int32(-1) + tail_start = (visible // cr) * cr + tail_count = visible - tail_start + body = total * cr + if body > self.token_topk: + body = cutlass.Int32(self.token_topk) + for c in cutlass.range_constexpr(-(-self.final // nt)): + col = tidx + c * nt + if col < self.final: + if col >= body: + off = col - body + val = cutlass.Int32(-1) + if off < cr - 1: + if off < tail_count: + if tail_start + off < seqlen: + val = tail_start + off + mSel[row, col] = val + + +_COMPILED: dict = {} +_INDEX_TYPES = {torch.int32: cutlass.Int32, torch.int64: cutlass.Int64} + + +def supports(scores, topk_indices) -> bool: + """Whether this kernel serves a call over this score table and `block_topk`.""" + width = scores.shape[1] + return ( + scores.dtype == torch.float32 + and scores.stride(1) == 1 + and width % KPT == 0 + and scores.stride(0) % KPT == 0 + and MIN_WIDTH <= width <= MAX_WIDTH + and -(-topk_indices.shape[1] // (THREADS // 32)) <= 96 + ) + + +def _qsa_topk_rows( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=None, +): + """`qsa_topk_expand_decode`'s contract, bit for bit, for wide tables (`supports`): writes + `topk_indices` and returns the token selection `[rows, token_topk + compress_ratio - 1]` + int32 (`out` when given).""" + rows, width = scores.shape + block_topk = topk_indices.shape[1] + sel = out + if sel is None: + sel = torch.empty( + (rows, token_topk + compress_ratio - 1), dtype=torch.int32, device=scores.device + ) + if rows == 0: + return sel + pdl = bool(get_env_enable_pdl()) + keys = scores.view(torch.int32) + key = ( + width, + scores.stride(0), + block_topk, + compress_ratio, + token_topk, + query_positions.dtype, + sequence_lengths.dtype, + pdl, + ) + ptrs = ( + row_ends.data_ptr(), + query_positions.data_ptr(), + sequence_lengths.data_ptr(), + topk_indices.data_ptr(), + sel.data_ptr(), + ) + fn = _COMPILED.get(key) + if fn is None: + gm = cute.AddressSpace.gmem + kernel = _TopkRowsKernel( + width=width, + key_stride=scores.stride(0), + block_topk=block_topk, + compress=compress_ratio, + token_topk=token_topk, + pdl=pdl, + ) + fn = _COMPILED[key] = cute.compile( + kernel, + from_dlpack(keys, assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.Int32, ptrs[0], gm, assumed_align=4), + make_ptr( + _INDEX_TYPES[query_positions.dtype], + ptrs[1], + gm, + assumed_align=query_positions.element_size(), + ), + make_ptr( + _INDEX_TYPES[sequence_lengths.dtype], + ptrs[2], + gm, + assumed_align=sequence_lengths.element_size(), + ), + make_ptr(cutlass.Int32, ptrs[3], gm, assumed_align=4), + make_ptr(cutlass.Int32, ptrs[4], gm, assumed_align=4), + cutlass.Int32(rows), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + fn(keys, *ptrs, rows) + return sel + + +class QsaTopkRows(OpWrapper): + """The family's `qsa_topk_expand_decode` on tables `supports` admits, the same bits: writes + `topk_indices` and returns the token selection `[rows, token_topk + compress_ratio - 1]` + int32 (`out` when given).""" + + def __call__( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + return _qsa_topk_rows( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out, + ) + + def reference( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """The family entry's reference: `(topk_indices_after, selection)`, fresh tensors.""" + return _family_topk.reference( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + def is_valid( + self, + scores: torch.Tensor, + row_ends: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + topk_indices: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, + ) -> None: + # The kernel is built for this width and checks none of it; past that, the family + # entry's contract and its silent failures are this kernel's. + assert supports(scores, topk_indices), "a call outside `supports`" + _family_topk.is_valid( + scores, + row_ends, + query_positions, + sequence_lengths, + topk_indices, + compress_ratio, + token_topk, + out=out, + ) + + +qsa_topk_rows = QsaTopkRows() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py new file mode 100644 index 000000000000..0265172be74a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qsa_triton_kernels__stable_jit.py @@ -0,0 +1,1356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This target's copies of the QSA Triton kernels for sparse GQA and the K/V append +(`tensorrt_llm._torch.attention.backends.sparse.qsa.kernels`), compiled once per head geometry. + +Extents that follow the engine or the step (pool pages, block-table shape and row stride, row +count) are `do_not_specialize` and the metadata pointers `do_not_specialize_on_alignment`; +geometry, strides and launch tiles stay constexpr. Outputs are bitwise equal to the shared +launchers': the bodies differ only in unsigned range compares under `tl.assume` (decode and +store kernels) and row-major split-K partials in the 64-split bucket. +""" + +from __future__ import annotations + +import math +import os + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_paged_kv_store, + triton_qsa_paged_sparse_gqa, +) +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.jit_variants import ( + compile_only, + first_time, + mark_warmed, +) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber +from tensorrt_llm._utils import get_sm_version + +_INTEGER_DTYPES = (torch.int32, torch.int64) +# Scores use base-2 exponentials for the online softmax (the shared kernels' constant). +_LOG2_E = tl.constexpr(1.4426950408889634) + + +def _require_same_cuda_device(name: str, *tensors: torch.Tensor) -> None: + if not tensors or not all(tensor.is_cuda for tensor in tensors): + raise ValueError(f"{name} requires CUDA tensors") + device = tensors[0].device + if any(tensor.device != device for tensor in tensors[1:]): + raise ValueError(f"{name} tensors must be on the same CUDA device") + + +def _pdl_enabled(rows: int) -> bool: + """The shared launcher's PDL rule: two- to four-row speculative graphs only.""" + return rows <= 4 and os.environ.get("TRTLLM_ENABLE_PDL", "1") == "1" and get_sm_version() >= 90 + + +#: Warps of a K/V append program (a store is a move: the warp count cannot change the result). +_KV_STORE_WARPS = 4 + + +def splitk_launch_config(rows: int, num_kv_heads: int, selected_width: int) -> tuple[int, int, int]: + """(BLOCK_N, splits, warps) by `rows * num_kv_heads`: the shared launcher's five buckets, + kept as they are so every launch reduces in the shared kernel's order.""" + base_programs = rows * num_kv_heads + if base_programs <= 4: + block_n, target_splits, num_warps = 16, 64, 4 + elif base_programs < 32: + block_n, target_splits, num_warps = 16, 32, 4 + elif base_programs <= 256: + block_n, target_splits, num_warps = 64, 8, 2 + elif base_programs <= 512: + block_n, target_splits, num_warps = 64, 4, 2 + else: + block_n, target_splits, num_warps = 64, 1, 2 + num_tiles = triton.cdiv(selected_width, block_n) + max_useful_splits = 1 << (num_tiles.bit_length() - 1) + return block_n, min(max_useful_splits, target_splits), num_warps + + +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=["request_indices", "logical_positions", "block_table"], +) +def _qsa_kv_store_stable_kernel( + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + num_rows, + num_pages, + num_requests, + page_table_width, + k_stride_row: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_row: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_dim: tl.constexpr, + k_cache_stride_page: tl.constexpr, + k_cache_stride_head: tl.constexpr, + k_cache_stride_token: tl.constexpr, + k_cache_stride_dim: tl.constexpr, + v_cache_stride_page: tl.constexpr, + v_cache_stride_head: tl.constexpr, + v_cache_stride_token: tl.constexpr, + v_cache_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + BLOCK_DIM: tl.constexpr, +): + # Extents are positive and below 2^31 (the launcher refuses empty ones), so each + # `0 <= x < extent` test is one unsigned compare. + tl.assume(num_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + row = tl.program_id(0) + head = tl.program_id(1) + dims = tl.arange(0, BLOCK_DIM) + valid = (row < num_rows) & (head < NUM_KV_HEADS) & (dims < HEAD_DIM) + + request = tl.load(request_indices + row).to(tl.int32) + logical = tl.load(logical_positions + row).to(tl.int32) + page_column = logical // TOKENS_PER_BLOCK + token_in_page = logical % TOKENS_PER_BLOCK + # `request` in [0, num_requests), `logical >= 0` and `page_column < page_table_width`. + valid_page_lookup = (request.to(tl.uint32) < num_requests.to(tl.uint32)) & ( + logical.to(tl.uint32) < (page_table_width * TOKENS_PER_BLOCK).to(tl.uint32) + ) + page32 = tl.load( + block_table + request * block_table_stride_request + page_column * block_table_stride_page, + mask=valid_page_lookup, + other=-1, + ) + valid &= page32.to(tl.uint32) < num_pages.to(tl.uint32) + page = page32.to(tl.int64) + + k_values = tl.load( + k + row * k_stride_row + head * k_stride_head + dims * k_stride_dim, + mask=valid, + other=0.0, + ) + v_values = tl.load( + v + row * v_stride_row + head * v_stride_head + dims * v_stride_dim, + mask=valid, + other=0.0, + ) + tl.store( + k_cache + + page * k_cache_stride_page + + head * k_cache_stride_head + + token_in_page * k_cache_stride_token + + dims * k_cache_stride_dim, + k_values, + mask=valid, + ) + tl.store( + v_cache + + page * v_cache_stride_page + + head * v_cache_stride_head + + token_in_page * v_cache_stride_token + + dims * v_cache_stride_dim, + v_values, + mask=valid, + ) + + +def _qsa_paged_kv_store( + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, +) -> None: + """`triton_qsa_paged_kv_store`'s contract and checks, on `_qsa_kv_store_stable_kernel`.""" + if k.shape != v.shape or k.ndim != 3: + raise ValueError("QSA K and V inputs must have matching [tokens, heads, dim] shapes") + if k_cache.shape != v_cache.shape or k_cache.ndim != 4: + raise ValueError("QSA K and V caches must have matching [pages, heads, tokens, dim] shapes") + rows, num_kv_heads, head_dim = k.shape + if num_kv_heads <= 0 or head_dim <= 0: + raise ValueError("QSA K/V store requires non-empty head geometry") + if k_cache.shape[1] != num_kv_heads or k_cache.shape[3] != head_dim: + raise ValueError("QSA K/V inputs and caches have incompatible head geometry") + if k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA K/V cache token dimension does not match tokens_per_block") + if tokens_per_block <= 0: + raise ValueError("QSA tokens_per_block must be positive") + if k.dtype != v.dtype or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA K/V pairs must use matching dtypes") + if k.dtype not in (torch.bfloat16, torch.float16): + raise ValueError("QSA K/V inputs must use BF16 or FP16") + if k_cache.dtype not in (torch.bfloat16, torch.float16, torch.float8_e4m3fn): + raise ValueError("QSA K/V cache must use BF16, FP16, or FP8 E4M3") + if request_indices.shape != (rows,) or logical_positions.shape != (rows,): + raise ValueError("QSA K/V store metadata must have one entry per input row") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA K/V store requires a two-dimensional int32 block table") + if ( + request_indices.dtype not in _INTEGER_DTYPES + or logical_positions.dtype not in _INTEGER_DTYPES + ): + raise ValueError("QSA K/V store request metadata must use integer storage") + _require_same_cuda_device( + "QSA paged K/V store", + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + ) + if rows == 0: + return + if k_cache.shape[0] == 0 or block_table.shape[0] == 0 or block_table.shape[1] == 0: + raise ValueError("QSA K/V store requires non-empty cache and block-table geometry") + + block_dim = triton.next_power_of_2(head_dim) + _qsa_kv_store_stable_kernel[(rows, num_kv_heads)]( + k, + v, + k_cache, + v_cache, + request_indices, + logical_positions, + block_table, + rows, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + TOKENS_PER_BLOCK=tokens_per_block, + BLOCK_DIM=block_dim, + num_warps=_KV_STORE_WARPS, + ) + + +class QsaPagedKvStoreStable(OpWrapper): + """`triton_qsa_paged_kv_store` on `_qsa_kv_store_stable_kernel`: every row's K and V into + its request's page of the HND pool, in place. The entry `qsa_paged_kv_store` (its own + module) calls this and carries the cells.""" + + def __call__( + self, + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> None: + _qsa_paged_kv_store( + k=k, + v=v, + k_cache=k_cache, + v_cache=v_cache, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + + def reference( + self, + *, + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(k_cache, v_cache)` after the shared launcher's store into copies (the same bits); + the inputs are not touched.""" + expected_k, expected_v = k_cache.clone(), v_cache.clone() + triton_qsa_paged_kv_store( + k=k, + v=v, + k_cache=expected_k, + v_cache=expected_v, + request_indices=request_indices, + logical_positions=logical_positions, + block_table=block_table, + tokens_per_block=tokens_per_block, + ) + return expected_k, expected_v + + +qsa_paged_kv_store = QsaPagedKvStoreStable() + + +@triton.jit( + do_not_specialize=[ + "num_cache_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=[ + "block_table", + "selected_tokens", + "request_indices", + "query_positions", + ], +) +def _qsa_prefill_pair_stable_kernel( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, + num_cache_pages, + num_requests, + page_table_width, + q_stride_row: tl.constexpr, + q_stride_head: tl.constexpr, + q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, + v_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, + selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + GROUP_SIZE: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + TOPK: tl.constexpr, + BLOCK_TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, +): + row = tl.program_id(0) + kv_head = tl.program_id(1) + tl.assume(num_cache_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + request = tl.load(request_indices + row).to(tl.int64) + # Signed range tests here, as in the shared kernel (the unsigned form is slower over e4m3). + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + query_heads = kv_head * GROUP_SIZE + head_offsets + query_values = tl.load( + q + + row * q_stride_row + + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=(head_offsets < GROUP_SIZE)[:, None], + other=0.0, + ) + + running_max = tl.full([BLOCK_M], -float("inf"), tl.float32) + running_sum = tl.zeros([BLOCK_M], tl.float32) + accumulator = tl.zeros([BLOCK_M, HEAD_DIM], tl.float32) + token_offsets = tl.arange(0, BLOCK_N) + visible_tokens = tl.load(query_positions + row) + 1 + selected_blocks = tl.minimum(visible_tokens // COMPRESS_RATIO, BLOCK_TOPK) + valid_tokens = selected_blocks * COMPRESS_RATIO + visible_tokens % COMPRESS_RATIO + valid_tiles = tl.cdiv(tl.minimum(valid_tokens, TOPK), BLOCK_N) + + for tile in tl.range(0, valid_tiles, num_stages=2): + selected_columns = tile * BLOCK_N + token_offsets + logical_tokens = tl.load( + selected_tokens + row * selected_stride_row + selected_columns * selected_stride_token, + mask=selected_columns < TOPK, + other=-1, + ) + safe_tokens = tl.maximum(logical_tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + valid = ( + valid_request + & (selected_columns < TOPK) + & (logical_tokens >= 0) + & (logical_pages < page_table_width) + ) + pages32 = tl.load( + block_table + + safe_request * block_table_stride_request + + tl.minimum(logical_pages, page_table_width - 1) * block_table_stride_page, + mask=valid, + other=-1, + ) + physical_pages = pages32.to(tl.int64) + valid &= (physical_pages >= 0) & (physical_pages < num_cache_pages) + safe_pages = tl.maximum(physical_pages, 0) + + keys = tl.load( + k_cache + + safe_pages[None, :] * k_stride_page + + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + + dim_offsets[:, None] * k_stride_dim, + mask=valid[None, :], + other=0.0, + ) + values = tl.load( + v_cache + + safe_pages[:, None] * v_stride_page + + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + + dim_offsets[None, :] * v_stride_dim, + mask=valid[:, None], + other=0.0, + ) + keys = keys.to(query_values.dtype) + values = values.to(query_values.dtype) + scores = tl.dot(query_values, keys) * softmax_scale * _LOG2_E + scores = tl.where(valid[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where( + valid[None, :], + tl.math.exp2(scores - next_max[:, None]), + 0.0, + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + accumulator * correction[:, None], + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + + normalized = tl.where( + running_sum[:, None] > 0, + accumulator / tl.maximum(running_sum[:, None], 1.0e-20), + 0.0, + ) + tl.store( + output + + row * output_stride_row + + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized, + mask=(head_offsets < GROUP_SIZE)[:, None], + ) + + +@triton.jit( + do_not_specialize=[ + "num_rows", + "num_cache_pages", + "num_requests", + "page_table_width", + "block_table_stride_request", + ], + do_not_specialize_on_alignment=["block_table", "selected_tokens", "request_indices"], +) +def _qsa_splitk_stable_kernel( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + num_rows, + num_cache_pages, + num_requests, + page_table_width, + q_stride_row: tl.constexpr, + q_stride_head: tl.constexpr, + q_stride_dim: tl.constexpr, + k_stride_page: tl.constexpr, + k_stride_head: tl.constexpr, + k_stride_token: tl.constexpr, + k_stride_dim: tl.constexpr, + v_stride_page: tl.constexpr, + v_stride_head: tl.constexpr, + v_stride_token: tl.constexpr, + v_stride_dim: tl.constexpr, + block_table_stride_request, + block_table_stride_page: tl.constexpr, + selected_stride_row: tl.constexpr, + selected_stride_token: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + GROUP_SIZE: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + TOKENS_PER_BLOCK: tl.constexpr, + TOPK: tl.constexpr, + NUM_SPLITS: tl.constexpr, + NUM_TILES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HEAD_DIM: tl.constexpr, + USE_PDL: tl.constexpr, + ROW_MAJOR: tl.constexpr, +): + row = tl.program_id(0) + kv_head = tl.program_id(1) + split = tl.program_id(2) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.assume(num_cache_pages > 0) + tl.assume(num_requests > 0) + tl.assume(page_table_width > 0) + request = tl.load(request_indices + row).to(tl.int64) + valid_request = request.to(tl.uint64) < num_requests.to(tl.uint64) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + # A selected token is resident-addressable when `0 <= t` and `t // TOKENS_PER_BLOCK < + # page_table_width`: one unsigned compare against the table's token span. + token_span = (page_table_width * TOKENS_PER_BLOCK).to(tl.uint32) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + token_offsets = tl.arange(0, BLOCK_N) + query_heads = kv_head * GROUP_SIZE + head_offsets + query_values = tl.load( + q + + row * q_stride_row + + query_heads[:, None] * q_stride_head + + dim_offsets[None, :] * q_stride_dim, + mask=(head_offsets < GROUP_SIZE)[:, None], + other=0.0, + ) + + running_max = tl.full([BLOCK_M], -float("inf"), tl.float32) + running_sum = tl.zeros([BLOCK_M], tl.float32) + accumulator = tl.zeros([BLOCK_M, HEAD_DIM], tl.float32) + split_tile_start = split * NUM_TILES // NUM_SPLITS + split_tile_end = (split + 1) * NUM_TILES // NUM_SPLITS + + for tile in range(split_tile_start, split_tile_end): + selected_columns = tile * BLOCK_N + token_offsets + logical_tokens = tl.load( + selected_tokens + row * selected_stride_row + selected_columns * selected_stride_token, + mask=selected_columns < TOPK, + other=-1, + ) + safe_tokens = tl.maximum(logical_tokens, 0) + logical_pages = safe_tokens // TOKENS_PER_BLOCK + token_in_page = safe_tokens % TOKENS_PER_BLOCK + valid = ( + valid_request & (selected_columns < TOPK) & (logical_tokens.to(tl.uint32) < token_span) + ) + pages32 = tl.load( + block_table + + safe_request * block_table_stride_request + + tl.minimum(logical_pages, page_table_width - 1) * block_table_stride_page, + mask=valid, + other=-1, + ) + valid &= pages32.to(tl.uint32) < num_cache_pages.to(tl.uint32) + safe_pages = tl.maximum(pages32.to(tl.int64), 0) + + keys = tl.load( + k_cache + + safe_pages[None, :] * k_stride_page + + kv_head * k_stride_head + + token_in_page[None, :] * k_stride_token + + dim_offsets[:, None] * k_stride_dim, + mask=valid[None, :], + other=0.0, + ).to(query_values.dtype) + values = tl.load( + v_cache + + safe_pages[:, None] * v_stride_page + + kv_head * v_stride_head + + token_in_page[:, None] * v_stride_token + + dim_offsets[None, :] * v_stride_dim, + mask=valid[:, None], + other=0.0, + ).to(query_values.dtype) + scores = tl.dot(query_values, keys) + scores *= softmax_scale * _LOG2_E + scores = tl.where(valid[None, :], scores, -float("inf")) + next_max = tl.maximum(running_max, tl.max(scores, axis=1)) + correction = tl.math.exp2(running_max - next_max) + probabilities = tl.where( + valid[None, :], + tl.math.exp2(scores - next_max[:, None]), + 0.0, + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + accumulator * correction[:, None], + ) + running_sum = running_sum * correction + tl.sum(probabilities, axis=1) + running_max = next_max + + has_values = running_sum > 0 + normalized = tl.where( + has_values[:, None], + accumulator / tl.maximum(running_sum[:, None], 1.0e-20), + 0.0, + ) + output_mask = (head_offsets < GROUP_SIZE)[:, None] + if NUM_SPLITS == 1: + tl.store( + output + + row * output_stride_row + + query_heads[:, None] * output_stride_head + + dim_offsets[None, :] * output_stride_dim, + normalized, + mask=output_mask, + ) + else: + partial_lse_values = tl.where( + has_values, + running_max + tl.math.log2(tl.maximum(running_sum, 1.0e-20)), + -float("inf"), + ) + # Partial slot of (split, row): row-major in the 64-split bucket, split-major (the shared + # layout) otherwise. Same values, merged in the same order. + if ROW_MAJOR: + slot = row * NUM_SPLITS + split + else: + slot = split * num_rows + row + tl.store( + partial_output + + (slot * NUM_QUERY_HEADS + query_heads[:, None]) * HEAD_DIM + + dim_offsets[None, :], + normalized, + mask=output_mask, + ) + tl.store( + partial_lse + slot * NUM_QUERY_HEADS + query_heads, + partial_lse_values, + mask=head_offsets < GROUP_SIZE, + ) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit(do_not_specialize=["num_rows"]) +def _qsa_merge_stable_kernel( + partial_output, + partial_lse, + output, + output_gate, + num_rows, + gate_stride_row: tl.constexpr, + gate_stride_head: tl.constexpr, + output_stride_row: tl.constexpr, + output_stride_head: tl.constexpr, + output_stride_dim: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + BLOCK_SPLITS: tl.constexpr, + HEAD_DIM: tl.constexpr, + USE_PDL: tl.constexpr, + FUSE_OUTPUT_GATE: tl.constexpr, + ROW_MAJOR: tl.constexpr, +): + row = tl.program_id(0) + head = tl.program_id(1) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + split_offsets = tl.arange(0, BLOCK_SPLITS) + dim_offsets = tl.arange(0, HEAD_DIM) + split_mask = split_offsets < NUM_SPLITS + if ROW_MAJOR: + slots = row * NUM_SPLITS + split_offsets + else: + slots = split_offsets * num_rows + row + lse = tl.load( + partial_lse + slots * NUM_QUERY_HEADS + head, + mask=split_mask, + other=-float("inf"), + ) + max_lse = tl.max(lse, axis=0) + has_values = max_lse > -float("inf") + weights = tl.math.exp2(tl.where(split_mask & has_values, lse - max_lse, -float("inf"))) + denominator = tl.sum(weights, axis=0) + partials = tl.load( + partial_output + + (slots[:, None] * NUM_QUERY_HEADS + head) * HEAD_DIM + + dim_offsets[None, :], + mask=split_mask[:, None], + other=0.0, + ) + merged = tl.sum(partials * weights[:, None], axis=0) + merged = tl.where(denominator > 0, merged / denominator, 0.0) + if FUSE_OUTPUT_GATE: + # The merge-then-gate contract: the attention result is rounded to its output dtype + # before the fp32 gate. + attention = merged.to(output.dtype.element_ty).to(tl.float32) + gate = tl.load( + output_gate + row * gate_stride_row + head * gate_stride_head + dim_offsets + ).to(tl.float32) + merged = attention * tl.sigmoid(gate) + tl.store( + output + + row * output_stride_row + + head * output_stride_head + + dim_offsets * output_stride_dim, + merged, + ) + + +def can_fuse_splitk_output_gate( + q: torch.Tensor, + k_cache: torch.Tensor, + selected_tokens: torch.Tensor, + output_gate: torch.Tensor, +) -> bool: + """Whether the split-K merge can consume an attention output gate (the shared rule).""" + if q.ndim != 3 or selected_tokens.ndim != 2 or selected_tokens.shape[1] <= 0: + return False + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or k_cache.ndim != 4: + return False + if not output_gate.is_cuda or output_gate.device != q.device or output_gate.dtype != q.dtype: + return False + if output_gate.stride(-1) != 1: + return False + if output_gate.ndim == 3: + if output_gate.shape != q.shape: + return False + elif output_gate.ndim == 2: + if output_gate.shape != (rows, num_q_heads * head_dim): + return False + else: + return False + _, num_splits, _ = splitk_launch_config(rows, k_cache.shape[1], selected_tokens.shape[1]) + return num_splits > 1 + + +def _qsa_paged_sparse_gqa( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, +) -> torch.Tensor: + """`triton_qsa_paged_sparse_gqa`'s contract, checks and branch choice, on these kernels.""" + if q.ndim != 3: + raise ValueError("QSA queries must have shape [rows, query_heads, head_dim]") + if k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA K/V caches must have matching HND paged shapes") + if block_table.ndim != 2 or block_table.dtype != torch.int32: + raise ValueError("QSA sparse GQA requires a two-dimensional int32 block table") + if selected_tokens.ndim != 2 or selected_tokens.dtype != torch.int32: + raise ValueError("QSA selected tokens must be a two-dimensional int32 tensor") + rows, num_q_heads, head_dim = q.shape + if selected_tokens.shape[0] != rows or selected_tokens.shape[1] <= 0: + raise ValueError("QSA selected-token rows must match non-empty query rows") + if request_indices.shape != (rows,) or request_indices.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA request indices must have one integer entry per row") + if not request_indices.is_contiguous(): + raise ValueError("QSA request indices must be contiguous") + if tokens_per_block <= 0 or k_cache.shape[2] != tokens_per_block: + raise ValueError("QSA cache token dimension must match positive tokens_per_block") + if k_cache.shape[3] != head_dim or k_cache.dtype != v_cache.dtype: + raise ValueError("QSA query and K/V cache head geometry is incompatible") + if q.dtype not in (torch.bfloat16, torch.float16): + raise ValueError("QSA queries must use BF16 or FP16") + if k_cache.dtype not in (torch.bfloat16, torch.float16, torch.float8_e4m3fn): + raise ValueError("QSA K/V cache must use BF16, FP16, or FP8 E4M3") + if head_dim <= 0 or head_dim & (head_dim - 1): + raise ValueError("QSA sparse GQA requires a power-of-two head dimension") + num_kv_heads = k_cache.shape[1] + if num_kv_heads <= 0 or num_q_heads <= 0 or num_q_heads % num_kv_heads: + raise ValueError("QSA query heads must be divisible by non-zero KV heads") + if k_cache.shape[0] <= 0 or block_table.shape[0] <= 0 or block_table.shape[1] <= 0: + raise ValueError("QSA sparse GQA requires non-empty cache and block-table geometry") + if not math.isfinite(softmax_scale) or softmax_scale <= 0: + raise ValueError("QSA sparse GQA softmax scale must be finite and positive") + _require_same_cuda_device( + "QSA paged sparse GQA", + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + ) + group_size = num_q_heads // num_kv_heads + block_m = max(16, triton.next_power_of_2(group_size)) + # Contiguous at every row count: the output strides are kernel constexprs, and `empty_like` + # would keep a one-row q's (dense) strides and compile a variant of its own. + output = torch.empty(q.shape, dtype=q.dtype, device=q.device) + if rows == 0: + return output + if output_gate is not None and not can_fuse_splitk_output_gate( + q, k_cache, selected_tokens, output_gate + ): + raise ValueError("QSA output-gate fusion requires a compatible split-K merge") + + if (query_positions is None) != (compress_ratio is None): + raise ValueError("QSA causal-prefix bounds require positions and compression ratio") + if query_positions is not None: + if query_positions.shape != (rows,): + raise ValueError("QSA query positions must match sparse-attention rows") + if query_positions.dtype not in _INTEGER_DTYPES: + raise ValueError("QSA query positions must use integer storage") + _require_same_cuda_device("QSA causal-prefix bounds", q, query_positions) + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + + base_programs = rows * num_kv_heads + if query_positions is not None and base_programs > 512: + final_topk = selected_tokens.shape[1] + token_topk = final_topk - compress_ratio + 1 + if token_topk <= 0 or token_topk % compress_ratio: + raise ValueError("QSA selected-token width is incompatible with its compression ratio") + _qsa_prefill_pair_stable_kernel[(rows, num_kv_heads)]( + *_pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions.contiguous(), + output, + softmax_scale, + ), + **_pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio), + ) + return output + + block_n, num_splits, num_warps = splitk_launch_config( + rows, num_kv_heads, selected_tokens.shape[1] + ) + use_pdl = num_splits > 1 and _pdl_enabled(rows) + # Row-major partials in the 64-split bucket (`splitk_launch_config`: rows x KV heads <= 4). + row_major = rows * num_kv_heads <= 4 + if num_splits == 1: + partial_output = output + partial_lse = output + else: + lead = (rows, num_splits) if row_major else (num_splits, rows) + partial_output = torch.empty( + (*lead, num_q_heads, head_dim), dtype=torch.float32, device=q.device + ) + partial_lse = torch.empty((*lead, num_q_heads), dtype=torch.float32, device=q.device) + gate = output if output_gate is None else output_gate + _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + output, + gate, + output_gate, + softmax_scale, + group_size, + block_m, + tokens_per_block, + None if query_positions is None else (compress_ratio, query_positions.dtype), + ) + _qsa_splitk_stable_kernel[(rows, num_kv_heads, num_splits)]( + *_splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, + ), + **_splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, + ), + ) + if num_splits == 1: + return output + _qsa_merge_stable_kernel[(rows, num_q_heads)]( + *_merge_args(partial_output, partial_lse, output, gate, output_gate, rows, head_dim), + **_merge_constexprs(num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate), + ) + return output + + +class QsaPagedSparseGqaStable(OpWrapper): + """`triton_qsa_paged_sparse_gqa` on this module's prefill-pair, split-K and merge kernels: + `[rows, Hq, D]` sparse GQA over each row's selected tokens. The entry `qsa_paged_sparse_gqa` + (its own module) calls this and carries the cells.""" + + def __call__( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, + ) -> torch.Tensor: + return _qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + def reference( + self, + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + query_positions: torch.Tensor | None = None, + compress_ratio: int | None = None, + output_gate: torch.Tensor | None = None, + ) -> torch.Tensor: + """The shared launcher's output on the same arguments (the same bits).""" + return triton_qsa_paged_sparse_gqa( + q=q, + k_cache=k_cache, + v_cache=v_cache, + block_table=block_table, + selected_tokens=selected_tokens, + request_indices=request_indices, + tokens_per_block=tokens_per_block, + softmax_scale=softmax_scale, + query_positions=query_positions, + compress_ratio=compress_ratio, + output_gate=output_gate, + ) + + +qsa_paged_sparse_gqa = QsaPagedSparseGqaStable() + + +def _pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, +) -> tuple: + return ( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + query_positions, + output, + softmax_scale, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + selected_tokens.stride(0), + selected_tokens.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio) -> dict: + final_topk = selected_tokens.shape[1] + token_topk = final_topk - compress_ratio + 1 + return dict( + GROUP_SIZE=group_size, + TOKENS_PER_BLOCK=k_cache.shape[2], + TOPK=final_topk, + BLOCK_TOPK=token_topk // compress_ratio, + COMPRESS_RATIO=compress_ratio, + BLOCK_M=block_m, + BLOCK_N=64, + HEAD_DIM=q.shape[2], + num_warps=2, + num_stages=2, + ) + + +def _splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, +) -> tuple: + return ( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + partial_output, + partial_lse, + output, + softmax_scale, + rows, + k_cache.shape[0], + block_table.shape[0], + block_table.shape[1], + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + block_table.stride(0), + block_table.stride(1), + selected_tokens.stride(0), + selected_tokens.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, +) -> dict: + return dict( + GROUP_SIZE=group_size, + NUM_QUERY_HEADS=q.shape[1], + TOKENS_PER_BLOCK=tokens_per_block, + TOPK=selected_tokens.shape[1], + NUM_SPLITS=num_splits, + NUM_TILES=triton.cdiv(selected_tokens.shape[1], block_n), + BLOCK_M=block_m, + BLOCK_N=block_n, + HEAD_DIM=q.shape[2], + USE_PDL=use_pdl, + ROW_MAJOR=row_major, + num_warps=num_warps, + num_stages=2, + launch_pdl=use_pdl, + ) + + +def _merge_args(partial_output, partial_lse, output, gate, output_gate, rows, head_dim) -> tuple: + if output_gate is not None and output_gate.ndim == 3: + gate_stride_row = output_gate.stride(0) + gate_stride_head = output_gate.stride(1) + else: + gate_stride_row = gate.stride(0) + gate_stride_head = head_dim + return ( + partial_output, + partial_lse, + output, + gate, + rows, + gate_stride_row, + gate_stride_head, + output.stride(0), + output.stride(1), + output.stride(2), + ) + + +def _merge_constexprs(num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate) -> dict: + return dict( + NUM_QUERY_HEADS=num_q_heads, + NUM_SPLITS=num_splits, + BLOCK_SPLITS=triton.next_power_of_2(num_splits), + HEAD_DIM=head_dim, + USE_PDL=use_pdl, + FUSE_OUTPUT_GATE=output_gate is not None, + ROW_MAJOR=row_major, + num_warps=2, + num_stages=1, + launch_pdl=use_pdl, + ) + + +#: Row counts reaching every split-K launch variant at one and two KV heads. +_BUCKET_ROWS = (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 128, 129, 256, 257, 512, 513) + + +def decode_bucket_variants(num_kv_heads: int, selected_width: int) -> list[tuple]: + """`(block_n, num_splits, num_warps, use_pdl, row_major)` of every split-K launch, any rows.""" + seen = [] + for rows in _BUCKET_ROWS: + block_n, num_splits, num_warps = splitk_launch_config(rows, num_kv_heads, selected_width) + variant = ( + block_n, + num_splits, + num_warps, + num_splits > 1 and _pdl_enabled(rows), + rows * num_kv_heads <= 4, + ) + if variant not in seen: + seen.append(variant) + return seen + + +def _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + output, + gate, + output_gate, + softmax_scale, + group_size, + block_m, + tokens_per_block, + pair=None, +) -> None: + """Compile (not run) the split-K and merge kernels for every row bucket of this call's + geometry the first time it is seen, so no serving step compiles one.""" + key = ( + q.dtype, + q.shape[1:], + q.stride(), + q.data_ptr() % 16 == 0, + k_cache.dtype, + k_cache.shape[1:], + k_cache.stride(), + v_cache.stride(), + k_cache.data_ptr() % 16 == 0 and v_cache.data_ptr() % 16 == 0, + block_table.stride(1), + selected_tokens.shape[1], + selected_tokens.stride(), + request_indices.dtype, + None if output_gate is None else (output_gate.shape[1:], output_gate.stride()), + tokens_per_block, + pair, + ) + if not first_time("qsa_stable_decode_buckets", key): + return + if output is None: + output = torch.empty(q.shape, dtype=q.dtype, device=q.device) + gate = output + rows = q.shape[0] + num_q_heads, head_dim = q.shape[1], q.shape[2] + scratch = torch.empty(16, dtype=torch.float32, device=q.device) + variants = decode_bucket_variants(k_cache.shape[1], selected_tokens.shape[1]) + splitk, merge = [], [] + for block_n, num_splits, num_warps, use_pdl, row_major in variants: + partials = (output, output) if num_splits == 1 else (scratch, scratch) + splitk.append( + ( + _splitk_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + *partials, + output, + softmax_scale, + rows, + ), + _splitk_constexprs( + q, + selected_tokens, + group_size, + block_m, + tokens_per_block, + block_n, + num_splits, + num_warps, + use_pdl, + row_major, + ), + ) + ) + if num_splits > 1: + merge.append( + ( + _merge_args(scratch, scratch, output, gate, output_gate, rows, head_dim), + _merge_constexprs( + num_q_heads, head_dim, num_splits, use_pdl, row_major, output_gate + ), + ) + ) + for args, constexprs in splitk: + compile_only(_qsa_splitk_stable_kernel, args, **constexprs) + for args, constexprs in merge: + compile_only(_qsa_merge_stable_kernel, args, **constexprs) + if pair is not None: + # A context launch past 512 programs over the same pool (the prefill pair kernel). + compress_ratio, positions_dtype = pair + positions = torch.empty(1, dtype=positions_dtype, device=q.device) + compile_only( + _qsa_prefill_pair_stable_kernel, + _pair_args( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + positions, + output, + softmax_scale, + ), + **_pair_constexprs(q, k_cache, selected_tokens, group_size, block_m, compress_ratio), + ) + mark_warmed("qsa_stable_decode_buckets", key) + + +def warm_decode_buckets( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + selected_tokens: torch.Tensor, + request_indices: torch.Tensor, + tokens_per_block: int, + softmax_scale: float, + compress_ratio: int, +) -> None: + """Compile (not run) the split-K composition for every row bucket of a decode call's + geometry, and the prefill pair kernel (int64 positions), for a caller that serves part of + the row range with another kernel.""" + num_q_heads = q.shape[1] + group_size = num_q_heads // k_cache.shape[1] + block_m = max(16, triton.next_power_of_2(group_size)) + _warm_decode_buckets( + q, + k_cache, + v_cache, + block_table, + selected_tokens, + request_indices, + None, + None, + None, + softmax_scale, + group_size, + block_m, + tokens_per_block, + (compress_ratio, torch.int64), + ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py new file mode 100644 index 000000000000..95fa342e654c --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/qwen4_exp_attn_decode_prologue__decode.py @@ -0,0 +1,700 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A sparse decode attention layer's work between its input projection and its index +scores, one PDL launch, at one rank's heads over a bf16 or e4m3 K/V pool. + +This target's copy of the NVFP4 default target's entry of the same name, plus an optional row +map for the index query. It fuses both planes' `fused_qk_norm_rope`, `qsa_paged_index_store`, +`qsa_prefill_compress`, the packed-QKV copy and `qsa_paged_kv_store`. Per row: `HPB`-head main +programs (q, k, K pool), one index-query program, one index-key / coordinate / compression +program, one program per v head. + +With PDL, every operand not written since the sublayer's hyper-connection up-gate-mean (page and +slot, rotation factors, norm weights, a group's earlier keys from the index plane) is read +before `griddepcontrol.wait`; after it a program reads only projection rows. The row mapping +and positions are written ahead of that up-gate-mean, which never releases its dependents early. +""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + + +@triton.jit +def _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, D: tl.constexpr, ROT: tl.constexpr, + S1: tl.constexpr, S2: tl.constexpr): # fmt: skip + """A D-wide head's rotation factors at the row's positions (interleaved mRoPE: pair j + reads coordinate 1 when j % 3 == 1 and j < 3 * S1, coordinate 2 when j % 3 == 2 and + j < 3 * S2, else 0): cos, and sin signed for the NeoX halves; identity past ROT.""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + j = d % HALF + p0 = tl.load(positions + token) + p1 = tl.load(positions + num_tokens + token) + p2 = tl.load(positions + 2 * num_tokens + token) + pos = tl.where((j % 3 == 1) & (j < 3 * S1), p1, tl.where((j % 3 == 2) & (j < 3 * S2), p2, p0)) + row = cos_sin + pos.to(tl.int64) * stride_cs + c = tl.load(row + j, mask=rot, other=1.0) + s = tl.load(row + HALF + j, mask=rot, other=0.0) + s = tl.where(d < HALF, -s, s) + return c, s + + +@triton.jit +def _norm_weights( + heads, weight, weight_k, D: tl.constexpr, ROT: tl.constexpr, WEIGHT_SPLIT: tl.constexpr +): + """`1 + w` for a tile's elements and for their rotation partners; heads at or past + WEIGHT_SPLIT (when > 0) take the second weight (the k heads).""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + partner = tl.where(d < HALF, d + HALF, tl.where(rot, d - HALF, d)) + w = tl.load(weight + d).to(tl.float32) + wp = tl.load(weight + partner).to(tl.float32) + if WEIGHT_SPLIT > 0: + is_k = (heads >= WEIGHT_SPLIT)[:, None] + wk = tl.load(weight_k + d).to(tl.float32) + wkp = tl.load(weight_k + partner).to(tl.float32) + w2 = tl.where(is_k, wk[None, :], w[None, :]) + 1.0 + wp2 = tl.where(is_k, wkp[None, :], wp[None, :]) + 1.0 + else: + w2 = w[None, :] + 1.0 + wp2 = wp[None, :] + 1.0 + return w2, wp2 + + +@triton.jit +def _norm_rope_apply(src, dst, heads, head_mask, w2, wp2, c, s, eps, D: tl.constexpr, ROT: tl.constexpr, + dst_head_stride): # fmt: skip + """rms over each head, `* (1 + w)`, rotate; fp32 throughout, one rounding to `dst`'s + dtype, stored at `dst`. Returns the stored (rounded) tile.""" + HALF: tl.constexpr = ROT // 2 + d = tl.arange(0, D) + rot = d < ROT + partner = tl.where(d < HALF, d + HALF, tl.where(rot, d - HALF, d)) + hm = head_mask[:, None] + x = tl.load(src + heads[:, None] * D + d[None, :], mask=hm, other=0.0).to(tl.float32) + xp = tl.load(src + heads[:, None] * D + partner[None, :], mask=hm & rot[None, :], other=0.0).to( + tl.float32 + ) + rstd = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps) + y = x * rstd[:, None] * w2 + yp = xp * rstd[:, None] * wp2 + out = (y * c[None, :] + yp * s[None, :]).to(dst.dtype.element_ty) + tl.store(dst + heads[:, None] * dst_head_stride + d[None, :], out, mask=hm) + return out + + +@triton.jit +def _compress_norm_rope(x, weight, cosine, sine, eps, HEAD_DIM: tl.constexpr, ROTARY_PAIRS: tl.constexpr): # fmt: skip + """QSA's `_qsa_gemma_norm_rope` for one row with its static factors given (`weight` is + `1 + w`; the fp32 table makes its intermediate rounding to the table's dtype a no-op).""" + x_f32 = x.to(tl.float32) + reciprocal_rms = tl.rsqrt(tl.sum(x_f32 * x_f32, axis=1) / HEAD_DIM + eps) + normalized = x_f32 * reciprocal_rms[:, None] * weight[None, :] + rotated, passthrough = tl.split( + tl.permute(tl.reshape(normalized, (1, 2, HEAD_DIM // 2)), (0, 2, 1)) + ) + first, second = tl.split(tl.permute(tl.reshape(rotated, (1, 2, ROTARY_PAIRS)), (0, 2, 1))) + output_first = first * cosine[None, :] - second * sine[None, :] + output_second = second * cosine[None, :] + first * sine[None, :] + rotated = tl.reshape( + tl.permute(tl.join(output_first, output_second), (0, 2, 1)), (1, HEAD_DIM // 2) + ) + return tl.reshape( + tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), + (1, HEAD_DIM), + ).to(tl.bfloat16) + + +@triton.jit(do_not_specialize=["num_tokens", "num_requests", "table_width", "num_pages"]) +def _attn_decode_prologue_kernel( + proj, + stride_proj, + out, + stride_out, + index_q, + stride_iq_token, + stride_iq_head, + positions, + num_tokens, + cos_sin, + stride_cs, + q_weight, + k_weight, + index_q_weight, + index_k_weight, + eps, + k_cache, + v_cache, + s_kc_page, + s_kc_head, + s_kc_tok, + s_vc_page, + s_vc_head, + s_vc_tok, + request_indices, + logical_positions, + block_table, + s_bt_req, + s_bt_col, + num_requests, + table_width, + num_pages, + index_cache, + s_ic_page, + s_ic_slot, + position_cache, + s_pc_page, + s_pc_slot, + s_pc_axis, + coords, + s_c_row, + s_c_axis, + index_q_rows, + HQ: tl.constexpr, + HK: tl.constexpr, + D: tl.constexpr, + IH: tl.constexpr, + ID: tl.constexpr, + IQ_COL: tl.constexpr, + ROT: tl.constexpr, + S1: tl.constexpr, + S2: tl.constexpr, + HPB: tl.constexpr, + MAIN_PROGRAMS: tl.constexpr, + TPB: tl.constexpr, + CR: tl.constexpr, + ROTARY_PAIRS: tl.constexpr, + IN_LAUNCH_GROUPS: tl.constexpr, + IQ_ROW_MAP: tl.constexpr, + USE_PDL: tl.constexpr, +): + token = tl.program_id(0).to(tl.int64) + program = tl.program_id(1) + if USE_PDL: + # The consumer (the index scores) reads nothing of this launch before its own wait. + tl.extra.cuda.gdc_launch_dependents() + row = proj + token * stride_proj + out_row = out + token * stride_out + + # The row's page and slot (shared by the K/V pool, index plane and coordinates); an unmapped + # row writes no cache. Written before the up-gate-mean, so read before the wait. + request = tl.load(request_indices + token).to(tl.int64) + logical = tl.load(logical_positions + token).to(tl.int64) + column = logical // TPB + slot = logical % TPB + live = (request >= 0) & (request < num_requests) & (logical >= 0) & (column < table_width) + page = tl.load(block_table + request * s_bt_req + column * s_bt_col, mask=live, other=-1).to( + tl.int64 + ) + live = live & (page >= 0) & (page < num_pages) + + if program < MAIN_PROGRAMS: + main_heads = program * HPB + tl.arange(0, HPB) + c, s = _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, D, ROT, S1, S2) + w2, wp2 = _norm_weights(main_heads, q_weight, k_weight, D, ROT, HQ) + if USE_PDL: + gdc_wait_with_memory_clobber() + rotated = _norm_rope_apply( + row, out_row, main_heads, main_heads < HQ + HK, w2, wp2, c, s, eps, D, ROT, D + ) + if program * HPB + HPB > HQ: + # The rounded rows the composition's store read back from the packed buffer. + is_k = (main_heads >= HQ) & (main_heads < HQ + HK) & live + k_heads = (main_heads - HQ).to(tl.int64)[:, None] + d = tl.arange(0, D) + tl.store( + k_cache + page * s_kc_page + k_heads * s_kc_head + slot * s_kc_tok + d[None, :], + rotated.to(k_cache.dtype.element_ty), + mask=is_k[:, None], + ) + elif program == MAIN_PROGRAMS: + index_heads = tl.arange(0, IH) + c, s = _rope_factors(positions, num_tokens, token, cos_sin, stride_cs, ID, ROT, 0, 0) + w2, wp2 = _norm_weights(index_heads, index_q_weight, index_q_weight, ID, ROT, 0) + if USE_PDL: + gdc_wait_with_memory_clobber() + iq_token = token + if IQ_ROW_MAP: + # The row's slot in the split index selection's permuted order. + iq_token = tl.load(index_q_rows + token).to(tl.int64) + _norm_rope_apply( + row + IQ_COL, index_q + iq_token * stride_iq_token, index_heads, index_heads < IH, + w2, wp2, c, s, eps, ID, ROT, stride_iq_head, + ) # fmt: skip + elif program >= MAIN_PROGRAMS + 2: + # One v head a program: the V pool, as the K/V append stores it, and the packed buffer. + h = program - MAIN_PROGRAMS - 2 + d = tl.arange(0, D) + if USE_PDL: + gdc_wait_with_memory_clobber() + v = tl.load(row + (HQ + HK + h) * D + d) + tl.store(out_row + (HQ + HK + h) * D + d, v) + if live: + tl.store( + v_cache + page * s_vc_page + h * s_vc_head + slot * s_vc_tok + d, + v.to(v_cache.dtype.element_ty), + ) + else: + # Before the wait: the coordinates and, at a group's last token, the group's earlier raw + # keys, the first token's coordinate 0, its rotation and the key norm weight. + axes = tl.arange(0, 4) + coordinate = tl.load(coords + token * s_c_row + axes * s_c_axis, mask=axes < 3, other=0) + dims = tl.arange(0, ID) + group_end = live & ((logical + 1) % CR == 0) + offsets = tl.arange(0, CR) + members = logical - (CR - 1) + offsets + member_pages = tl.load( + block_table + request * s_bt_req + (tl.maximum(members, 0) // TPB) * s_bt_col, + mask=group_end, + other=0, + ).to(tl.int64) + earlier = group_end & (offsets < CR - 1) + if IN_LAUNCH_GROUPS: + # A request's rows are consecutive at consecutive positions: an earlier member at + # row `token - (CR - 1) + o` of the same request is written by this launch, so its + # raw key (and, as the first member, its coordinates) come from its own row. + member_rows = token - (CR - 1) + offsets + maybe = earlier & (member_rows >= 0) + member_requests = tl.load(request_indices + member_rows, mask=maybe, other=-1) + member_logical = tl.load(logical_positions + member_rows, mask=maybe, other=-1) + in_launch = ( + maybe + & (member_requests.to(tl.int64) == request) + & (member_logical.to(tl.int64) == members) + ) + else: + in_launch = offsets < 0 + group = tl.load( + index_cache + + member_pages[:, None] * s_ic_page + + (tl.maximum(members, 0) % TPB)[:, None] * s_ic_slot + + dims[None, :], + mask=(earlier & ~in_launch)[:, None], + other=0.0, + ) + first = tl.maximum(logical - (CR - 1), 0) + first_page = tl.load( + block_table + request * s_bt_req + (first // TPB) * s_bt_col, mask=group_end, other=0 + ).to(tl.int64) + first_base = position_cache + first_page * s_pc_page + (first % TPB) * s_pc_slot + if IN_LAUNCH_GROUPS: + # A first member written by this launch has its coordinates in its row. + first_in_launch = tl.sum(tl.where(offsets == 0, in_launch, False).to(tl.int32)) > 0 + cached = group_end & ~first_in_launch + pos_t = tl.load(coords + (token - (CR - 1)) * s_c_row, mask=first_in_launch, other=0) + pos_t = tl.where(cached, tl.load(first_base, mask=cached, other=0), pos_t) + else: + pos_t = tl.load(first_base, mask=group_end, other=0) + # The compressed key rotates on the first token's coordinate 0 (QSA's plain rope). + pairs = tl.arange(0, ROTARY_PAIRS) + cs_row = cos_sin + pos_t.to(tl.int64) * stride_cs + cosine = tl.load(cs_row + pairs, mask=group_end, other=0.0) + sine = tl.load(cs_row + ROTARY_PAIRS + pairs, mask=group_end, other=0.0) + key_weight = tl.load(index_k_weight + dims).to(tl.float32) + 1.0 + if USE_PDL: + gdc_wait_with_memory_clobber() + if live: + # The coordinates, as the index store writes them. + tl.store( + position_cache + page * s_pc_page + slot * s_pc_slot + axes * s_pc_axis, + coordinate.to(position_cache.dtype.element_ty), + mask=axes < 3, + ) + # The raw index key -- or, at a group's last token, the group compressed in + # its place (the compression kernel overwrites the same slot). + current_k = tl.load(row + IQ_COL + IH * ID + dims) + stored_k = current_k.to(index_cache.dtype.element_ty) + if (logical + 1) % CR == 0: + if IN_LAUNCH_GROUPS: + launch_k = tl.load( + proj + + member_rows[:, None] * stride_proj + + IQ_COL + + IH * ID + + dims[None, :], + mask=in_launch[:, None], + other=0.0, + ) + group = tl.where( + in_launch[:, None], launch_k.to(index_cache.dtype.element_ty), group + ) + group = tl.where(offsets[:, None] == CR - 1, stored_k[None, :], group) + pooled = (tl.sum(group.to(tl.float32), axis=0) / CR).to(tl.bfloat16) + stored_k = tl.reshape( + _compress_norm_rope( + pooled[None, :], key_weight, cosine, sine, eps, ID, ROTARY_PAIRS + ), + (ID,), + ) + tl.store(index_cache + page * s_ic_page + slot * s_ic_slot + dims, stored_k) + + +def _launch(num_tokens: int) -> Tuple[int, int]: + """Main-plane heads per program and warps per program, by row count.""" + if num_tokens <= 16: + return 1, 2 + if num_tokens <= 64: + return 2, 2 + return 8, 8 + + +#: Rank geometries: tp4 / tep4 6 q heads and 1 KV head; tp2 / tep2 12 and 1; dep4 / dep2 / tp1 +#: 24 and 2 (heads of 256, a 4 x 128 index plane). +_TP4 = dict(geom="tp4") +_TP2 = dict(geom="tp2") +_DEP4 = dict(geom="dep4") + + +class Qwen4ExpAttnDecodePrologue(OpWrapper): + """The main and index planes' Gemma norm + rope, the packed-QKV copy, the paged K/V append, + the index key and coordinate store and the group compression of one sparse decode + attention layer, in one launch. Returns `(out, index_q)`. + + `proj` `[T, W]` (read only): q heads `[0, Hq)`, k heads `[Hq, Hq + Hk)` and v heads + `[Hq + Hk, Hq + 2 Hk)` of `head_dim`, the index query heads at `index_q_col` and the raw + index key right after them. Writes `out` `[T, (Hq + 2 Hk) * head_dim]` (q and k normalized + and rotated, v copied); `index_q` `[T, index_heads, index_head_dim]`; the rotated k and raw + v into `k_cache` / `v_cache` `[pages, Hk, tpb, head_dim]` (bf16 or e4m3); the raw index key + (or, at a group's last token, the compressed group) and the coordinates into the index + plane and the position cache. + + The rotation is NeoX over the first `rotary_dim` dims, factors from `cos_sin` (fp32 + `[P, rotary_dim]`, cos block then sin block). The main plane uses interleaved mRoPE over + `position_ids` `[3, T]` (pair `j` reads coordinate 1 when `j % 3 == 1 and j < 3 * + mrope_section1`, 2 when `j % 3 == 2 and j < 3 * mrope_section2`, else 0); the index plane + and the compressed key (`qsa_prefill_compress`'s, at the group's first token) use + coordinate 0. `rows_per_request` > 1 declares a verify step: each request's rows + consecutive, at consecutive positions. With `index_q_rows` (`[T]` int), row t's index query + goes to `index_q[index_q_rows[t]]`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="tep4 rank, verify 2 requests x 4 rows: a group end on a page's last slot " + "with one member in the plane, one with all three in the launch; zero, -0.0 and " + "outlier blocks in k and v", + spec=dict(_TP4, pos=(4093, 8696), rows_per_request=4, edge=True), + ), + Cell( + why="tep4 rank, MTP-7 verify 2 requests x 8 rows, one up to position 262143 " + "(the table's last row)", + spec=dict(_TP4, pos=(1001, 262136), rows_per_request=8), + ), + Cell( + why="tp4 rank, decode 3 rows: a group end, a mid-group row, an unmapped row", + spec=dict(_TP4, pos=(8699, 1201, 31), unmapped=2), + ), + Cell( + why="tp4 rank, decode 72 rows (eight-head programs: the one main program masks " + "its eighth head)", + spec=dict(_TP4, pos=tuple(1000 + 113 * r for r in range(72))), + ), + Cell( + why="tep4 rank, engine-laid pool: K/V the role planes of layer slot 11 of a " + "24-role page, the index plane strided by 12 layers' pages, rows on pages past " + "2^31 elements of the pool", + spec=dict(_TP4, pos=(900, 1203), rows_per_request=4, engine=True, large=True), + ), + Cell( + why="tep2 rank, 32 requests x 4 MTP-3 verify rows (a generation server's 128-row " + "graph), e4m3 pool", + spec=dict( + _TP2, pos=tuple(3000 + 6007 * r for r in range(32)), rows_per_request=4, fp8=True + ), + ), + Cell( + why="dep4 rank, 128 requests x 4 MTP-3 verify rows (ht_dep4's largest graph " + "batch, eight-head programs), e4m3 pool", + spec=dict( + _DEP4, + pos=tuple(3000 + 1531 * r for r in range(128)), + rows_per_request=4, + fp8=True, + ), # fmt: skip + ), + Cell( + why="dep2 / tp1 rank, decode 32 requests over the core's bf16 pool", + spec=dict(_DEP4, pos=tuple(2000 + 151 * r for r in range(32))), + ), + ) + + note = """ + Source: the Triton kernel above, grid `(T, cdiv(Hq + Hk, HPB) + 2 + Hk)`; PDL when the + engine enables it. Pages, rows and request slots index in int64 (pools past 2^31 elements). + + Preconditions: the index plane one KV head of `index_head_dim` (bf16); the position cache + int32 `[pages, tpb, >=3]`; `compress_ratio` divides `tokens_per_block`; one row per request, + or with `rows_per_request` > 1 each request's rows consecutive at consecutive positions. A + row whose page lookup fails writes no cache; its packed row and index query are still written. + + Numerics: fp32, one rounding at each store; v, raw keys and coordinates bitwise. The rotation + reads the fp32 cos/sin table, not `fused_qk_norm_rope`'s `__sincosf`. + + Not certified: NVFP4 pools (the forward keeps the composition there); non-zero index-plane + mRoPE sections; mRoPE on the compressed key; `T == 0` (returns at once); head geometries other + than 6/1, 12/1 and 24/2 x 256 with a 4 x 128 index plane. + """ + + def __call__( + self, + proj: torch.Tensor, + out: torch.Tensor, + index_q: torch.Tensor, + position_ids: torch.Tensor, + coords: torch.Tensor, + cos_sin: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + index_q_weight: torch.Tensor, + index_k_weight: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + index_heads: int, + index_head_dim: int, + index_q_col: int, + rotary_dim: int, + eps: float, + mrope_section1: int, + mrope_section2: int, + rows_per_request: int = 1, + index_q_rows: torch.Tensor | None = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + num_tokens = proj.shape[0] + if num_tokens == 0: + return out, index_q + hpb, warps = _launch(num_tokens) + main_programs = triton.cdiv(num_heads_q + num_heads_k, hpb) + pdl = get_env_enable_pdl() + _attn_decode_prologue_kernel[(num_tokens, main_programs + 2 + num_heads_k)]( + proj, + proj.stride(0), + out, + out.stride(0), + index_q, + index_q.stride(0), + index_q.stride(1), + position_ids, + num_tokens, + cos_sin, + cos_sin.stride(0), + q_weight, + k_weight, + index_q_weight, + index_k_weight, + eps, + k_cache, + v_cache, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + request_indices, + logical_positions, + block_table, + block_table.stride(0), + block_table.stride(1), + block_table.shape[0], + block_table.shape[1], + k_cache.shape[0], + index_cache, + index_cache.stride(0), + index_cache.stride(1), + position_cache, + position_cache.stride(0), + position_cache.stride(1), + position_cache.stride(2), + coords, + coords.stride(0), + coords.stride(1), + proj if index_q_rows is None else index_q_rows, + HQ=num_heads_q, + HK=num_heads_k, + D=head_dim, + IH=index_heads, + ID=index_head_dim, + IQ_COL=index_q_col, + ROT=rotary_dim, + S1=mrope_section1, + S2=mrope_section2, + HPB=hpb, + MAIN_PROGRAMS=main_programs, + TPB=tokens_per_block, + CR=compress_ratio, + ROTARY_PAIRS=cos_sin.shape[1] // 2, + IN_LAUNCH_GROUPS=rows_per_request > 1, + IQ_ROW_MAP=index_q_rows is not None, + USE_PDL=pdl, + num_warps=warps, + launch_pdl=pdl, + ) + return out, index_q + + def reference( + self, + proj: torch.Tensor, + out: torch.Tensor, + index_q: torch.Tensor, + position_ids: torch.Tensor, + coords: torch.Tensor, + cos_sin: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + index_q_weight: torch.Tensor, + index_k_weight: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + index_cache: torch.Tensor, + position_cache: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + compress_ratio: int, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + index_heads: int, + index_head_dim: int, + index_q_col: int, + rotary_dim: int, + eps: float, + mrope_section1: int, + mrope_section2: int, + rows_per_request: int = 1, + index_q_rows: torch.Tensor | None = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Plain torch, fp32, writing the given buffers: both planes' norm + rope, then every row's + stores, then the compression of every group a row ends (reading the plane those left).""" + rows, dev = proj.shape[0], proj.device + half = rotary_dim // 2 + pos = position_ids.long() + table = cos_sin.float() + j = torch.arange(half, device=dev) + + def factors(s1: int, s2: int): + coord = torch.zeros(half, dtype=torch.long, device=dev) + coord[(j % 3 == 1) & (j < 3 * s1)] = 1 + coord[(j % 3 == 2) & (j < 3 * s2)] = 2 + p = pos[coord, :].t() # [T, half] + return table[p, j][:, None, :], table[p, half + j][:, None, :] + + def plane(x: torch.Tensor, w: torch.Tensor, cos, sin) -> torch.Tensor: + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + w) + x1, x2 = x[..., :half], x[..., half:rotary_dim] + return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin, x[..., rotary_dim:]], -1) + + main_heads = num_heads_q + num_heads_k + width = main_heads * head_dim + v_width = num_heads_k * head_dim + weights = torch.cat( + [ + q_weight.float().expand(num_heads_q, head_dim), + k_weight.float().expand(num_heads_k, head_dim), + ] + ) + main = plane( + proj[:, :width].float().reshape(rows, main_heads, head_dim), + weights, + *factors(mrope_section1, mrope_section2), + ).to(out.dtype) + out[:, :width] = main.reshape(rows, width) + v = proj[:, width : width + v_width] + out[:, width : width + v_width] = v + iq = proj[:, index_q_col : index_q_col + index_heads * index_head_dim].float() + iq = plane( + iq.reshape(rows, index_heads, index_head_dim), index_q_weight.float(), *factors(0, 0) + ) + iq = iq.to(index_q.dtype) + if index_q_rows is None: + index_q.copy_(iq) + else: + index_q[index_q_rows.long()] = iq + key_col = index_q_col + index_heads * index_head_dim + raw = proj[:, key_col : key_col + index_head_dim] + + requests = request_indices.long().tolist() + logical = logical_positions.long().tolist() + num_requests, table_width = block_table.shape + + def page_of(request: int, position: int): + column = position // tokens_per_block + if not (0 <= request < num_requests and position >= 0 and column < table_width): + return None + page = int(block_table[request, column]) + return page if 0 <= page < k_cache.shape[0] else None + + k_rows = main[:, num_heads_q:] + v_rows = v.reshape(rows, num_heads_k, head_dim) + slots = [page_of(r, p) for r, p in zip(requests, logical)] + for t, page in enumerate(slots): + if page is None: + continue + slot = logical[t] % tokens_per_block + k_cache[page, :, slot] = k_rows[t].to(k_cache.dtype) + v_cache[page, :, slot] = v_rows[t].to(v_cache.dtype) + index_cache[page, slot, 0] = raw[t].to(index_cache.dtype) + position_cache[page, slot, :3] = coords[t].to(position_cache.dtype) + + key_weight = 1.0 + index_k_weight.float() + rot = 2 * (cos_sin.shape[1] // 2) + for t, page in enumerate(slots): + if page is None or (logical[t] + 1) % compress_ratio: + continue + members = [logical[t] - compress_ratio + 1 + o for o in range(compress_ratio)] + keys = torch.stack( + [ + index_cache[page_of(requests[t], m), m % tokens_per_block, 0].float() + for m in members + ] + ) + pooled = (keys.sum(0) / compress_ratio).to(torch.bfloat16).float() + first_page = page_of(requests[t], members[0]) + first = int(position_cache[first_page, members[0] % tokens_per_block, 0]) + x = pooled * torch.rsqrt(pooled.pow(2).mean() + eps) * key_weight + cos, sin = table[first, : rot // 2], table[first, rot // 2 : rot] + x1, x2 = x[: rot // 2], x[rot // 2 : rot] + key = torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin, x[rot:]]) + index_cache[page, logical[t] % tokens_per_block, 0] = key.to(index_cache.dtype) + return out, index_q + + def is_valid(self, proj, out, index_q, position_ids, coords, cos_sin, *args, **kwargs) -> None: + assert proj.stride(1) == 1 and proj.dtype == torch.bfloat16, "bf16 proj, unit column stride" + assert out.stride(1) == 1 and out.dtype == torch.bfloat16, "bf16 out, unit column stride" + assert index_q.stride(2) == 1, "index_q must have a unit dim stride" + assert position_ids.is_contiguous() and position_ids.shape == (3, proj.shape[0]), ( + "contiguous [3, T] positions (the kernel reads coordinate a of row t at a * T + t)" + ) + assert position_ids.dtype == torch.int32, "int32 positions" + assert coords.shape == (proj.shape[0], 3), "coords [T, 3]" + assert cos_sin.dtype == torch.float32 and cos_sin.stride(1) == 1, ( + "an fp32 cos|sin table with a unit column stride" + ) + + +qwen4_exp_attn_decode_prologue = Qwen4ExpAttnDecodePrologue() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py new file mode 100644 index 000000000000..ec4d712847df --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/spec_rollback_begin__verify.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A captured verify step's QSA rollback staging in one launch: every generation row's QSA slot +(`block_table[req, pos // tpb]`, `pos % tpb`), the pre-step contents of that slot in every +attention layer's index-K plane, and in the shared position plane. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: One index-K row: 128 bf16 a token (one 128-wide head). +ROW = 128 +#: A position-plane row: the token's three mRoPE coordinates, int32. +POS_W = 3 + + +# Program (l, r) resolves row r's slot and copies its row of plane l; the program past the last +# index plane copies the position triple and writes the slot. +@triton.jit( + do_not_specialize=["rows"], + do_not_specialize_on_alignment=["req_idx", "logical"], +) +def _begin_kernel( + req_idx, + logical, + block_table, + bt_stride, + ptrs, + page_strides, + position, + pos_s0, + pos_s1, + pages_out, + within_out, + index_out, + position_out, + rows, + TPB: tl.constexpr, + ROW_: tl.constexpr, + PLANES: tl.constexpr, + POS_W_: tl.constexpr, +): + plane = tl.program_id(0) + r = tl.program_id(1) + pos = tl.load(logical + r).to(tl.int64) + req = tl.load(req_idx + r).to(tl.int64) + page = tl.load(block_table + req * bt_stride + pos // TPB).to(tl.int64) + tok = pos % TPB + if plane < PLANES: + base = tl.load(ptrs + plane).to(tl.pointer_type(tl.bfloat16)) + stride = tl.load(page_strides + plane) + offs = tl.arange(0, ROW_) + vals = tl.load(base + page * stride + tok * ROW_ + offs) + tl.store(index_out + (plane.to(tl.int64) * rows + r) * ROW_ + offs, vals) + else: + w = tl.arange(0, 4) + wm = w < POS_W_ + v = tl.load(position + page * pos_s0 + tok * pos_s1 + w, mask=wm) + tl.store(position_out + r * POS_W_ + w, v, mask=wm) + tl.store(pages_out + r, page) + tl.store(within_out + r, tok) + + +def _cells(): + return tuple( + Cell( + why=f"tep4 MTP verify, {n} request(s) x {p} rows over 12 planes", + spec=dict(requests=n, p=p, planes=12, capture=capture), + ) + for n, p, capture in ( + (1, 3, False), + (4, 3, True), + (4, 4, False), + ) + ) + + +class SpecRollbackBegin(OpWrapper): + """`(pages, within, index_all, position_old) = spec_rollback_begin(req_idx, logical, + block_table, tokens_per_block, ptrs, page_strides, position)` for a verify step's rows. + + - `req_idx` `[R]` int32 / int64 and `logical` `[R]` int64: the generation rows' request + index (into `block_table`) and logical position (`qsa_req_idx_per_token`, + `qsa_logical_positions`); + - `block_table` `[S, max_pages]` int32 (unit column stride), `tokens_per_block` its page size; + - `ptrs` / `page_strides`: `plane_table(planes)` of the attention layers' index-K planes; + - `position` `[pages, tpb, 3]` int32 (unit innermost stride): the shared position plane. + + Returns `pages` / `within` `[R]` int64 (`block_table[req, pos // tpb]`, `pos % tpb`), + `index_all` `[L, R, 128]` bf16 (every row's slot of every plane, plane order -- what + `qsa_rollback_gather` returns) and `position_old` `[R, 3]` int32 (`position[pages, within]`). + Nothing but the outputs is written. Run it before any layer of the step writes a plane. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: one Triton kernel (`_begin_kernel`), JIT-compiled per (tokens_per_block, plane + count); grid (planes + 1, R). Copies and integer arithmetic only: bitwise the composed torch + path. + + Preconditions: the dtypes, ranks and strides `is_valid` checks; every slot inside its pool + (positions non-negative and inside the table) is the caller's. + """ + + def __call__( + self, req_idx, logical, block_table, tokens_per_block: int, ptrs, page_strides, position + ): + rows = logical.shape[0] + planes = ptrs.shape[0] + dev = logical.device + pages = torch.empty([rows], dtype=torch.int64, device=dev) + within = torch.empty([rows], dtype=torch.int64, device=dev) + index_all = torch.empty([planes, rows, ROW], dtype=torch.bfloat16, device=dev) + position_old = torch.empty([rows, POS_W], dtype=torch.int32, device=dev) + if rows: + _begin_kernel[(planes + 1, rows)]( + req_idx, + logical, + block_table, + block_table.stride(0), + ptrs, + page_strides, + position, + position.stride(0), + position.stride(1), + pages, + within, + index_all, + position_old, + rows, + TPB=tokens_per_block, + ROW_=ROW, + PLANES=planes, + POS_W_=POS_W, + num_warps=1, + ) + return pages, within, index_all, position_old + + def reference(self, req_idx, logical, block_table, tokens_per_block: int, planes, position): + """The composed torch path; takes the planes themselves where `__call__` takes their + `plane_table` (torch cannot dereference raw addresses).""" + positions = logical.to(torch.long) + pages = block_table[req_idx.to(torch.long), positions // tokens_per_block].to(torch.long) + within = positions % tokens_per_block + index_all = torch.stack([t[pages, within].reshape(-1, ROW) for t in planes]) + return pages, within, index_all, position[pages, within] + + def is_valid( + self, req_idx, logical, block_table, tokens_per_block: int, ptrs, page_strides, position + ) -> None: + rows = logical.shape[0] + assert logical.dtype == torch.int64 and logical.dim() == 1 and rows >= 1, ( + logical.dtype, + logical.shape, + ) + assert req_idx.dtype in (torch.int32, torch.int64) and tuple(req_idx.shape) == (rows,), ( + req_idx.dtype, + req_idx.shape, + ) + assert ( + block_table.dtype == torch.int32 + and block_table.dim() == 2 + and block_table.stride(1) == 1 + ), ( + block_table.dtype, + block_table.stride(), + ) + assert tokens_per_block > 0 + assert ( + ptrs.dtype == page_strides.dtype == torch.int64 and ptrs.shape == page_strides.shape + ), "ptrs / page_strides must be plane_table's int64 tables" + assert ( + position.dtype == torch.int32 + and position.dim() == 3 + and position.shape[2] == POS_W + and position.stride(2) == 1 + ), (position.dtype, position.shape, position.stride()) + + +spec_rollback_begin = SpecRollbackBegin() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py new file mode 100644 index 000000000000..512ea7574cbd --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/attention/vision_rope_2d_packed_qk.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, Tri Dao. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Adapted from flash-attn 2 (https://github.com/Dao-AILab/flash-attention), +# `flash_attn/ops/triton/rotary.py`: `rotary_kernel` and `apply_rotary`. +"""Vision-tower 2-D rotary, in place on the q|k prefix of the packed qkv projection. + +The caller builds the per-token `cos` / `sin` table once per tower forward from `grid_thw`. +The kernel is flash-attn 2's `rotary_kernel` (header above) cut down to one sequence, NeoX +pairing, int64 token offsets and in-place writes; the registered rope ops take 1-D positions only. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: This checkpoint's bf16 tower: 1152 / 16 heads = a 72-wide head, a qkv row of 3 * 16 * 72. +NUM_HEADS = 16 +HEAD_DIM = 72 + +#: Heads x tokens per program. +_BLOCK_H = 4 +_BLOCK_M = 8 + + +@triton.jit(do_not_specialize=["seqlen", "seqlen_ro"]) +def _rotary_neox_inplace_kernel( + X, + COS, + SIN, + seqlen, + nheads, + seqlen_ro, + stride_x_seqlen, + stride_x_nheads, + stride_x_headdim, + # Constexpr so the compiler sees the mask is constant every 8 elements (128-bit loads). + ROTARY_DIM: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_M: tl.constexpr, +): + ROTARY_DIM_HALF: tl.constexpr = ROTARY_DIM // 2 + pid_head = tl.program_id(axis=0) + pid_m = tl.program_id(axis=1) + if pid_m * BLOCK_M >= seqlen: + return + + rh = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + # Token offsets in int64: rows x row stride passes 2^31 at 621379 rows of a 3456-wide row. + rm64 = rm.to(tl.int64) + rk_half = tl.arange(0, BLOCK_K // 2) + + COS = COS + (rm64[:, None] * ROTARY_DIM_HALF + rk_half[None, :]) + SIN = SIN + (rm64[:, None] * ROTARY_DIM_HALF + rk_half[None, :]) + mask_cs = (rm[:, None] < seqlen_ro) & (rk_half[None, :] < ROTARY_DIM_HALF) + cos = tl.load(COS, mask=mask_cs, other=1.0).to(tl.float32) + sin = tl.load(SIN, mask=mask_cs, other=0.0).to(tl.float32) + + # Pairs (i, i + ROTARY_DIM_HALF), stored back where they were read. + X = X + ( + rh[:, None, None] * stride_x_nheads + + rm64[None, :, None] * stride_x_seqlen + + rk_half[None, None, :] * stride_x_headdim + ) + mask = ( + (rh[:, None, None] < nheads) + & (rm[None, :, None] < seqlen) + & (rk_half[None, None, :] < ROTARY_DIM_HALF) + ) + x0 = tl.load(X, mask=mask, other=0.0).to(tl.float32) + x1 = tl.load(X + ROTARY_DIM_HALF * stride_x_headdim, mask=mask, other=0.0).to(tl.float32) + o0 = x0 * cos - x1 * sin + o1 = x0 * sin + x1 * cos + tl.store(X, o0, mask=mask) + tl.store(X + ROTARY_DIM_HALF * stride_x_headdim, o1, mask=mask) + + +class VisionRope2dPackedQk(OpWrapper): + """Rotate every head of `qk` by its token's angles, in place; return `qk`. + + half = head_dim // 2 + qk[t, h, i] <- x[i] cos[t, i] - x[i + half] sin[t, i] + qk[t, h, i + half] <- x[i] sin[t, i] + x[i + half] cos[t, i] i < half + + In fp32, rounded once on the store. `qk` is the `[T, 2 * heads, head_dim]` q|k prefix of + the packed `[T, 3 * heads, head_dim]` qkv, addressed through its strides (the v heads are + never touched). `cos` / `sin` are fp32 `[T, half]`: the height angles, then the width angles. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: grid -- `grid_thw` rows the test builds the angle table from; grid_dtype -- its + # host dtype; dtype -- the qkv dtype. Heads 16 x 72, merge 2, theta 1e4 throughout. + CELLS: tuple[Cell, ...] = ( + Cell( + why="62x90 with the processor's int64 grid_thw: 5580 rows", + spec=dict(grid=((1, 62, 90),), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + Cell( + why="video (3,8,8) + image (1,16,24), two grid rows: 576 rows", + spec=dict(grid=((3, 8, 8), (1, 16, 24)), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + Cell( + why="video (64,16,32): 32768 rows, the tower's per-call ceiling", + spec=dict(grid=((64, 16, 32),), grid_dtype=torch.int64, dtype=torch.bfloat16), + ), + ) + + note = """ + Source: the Triton kernel above. Called eagerly by every vision block; the tower is + replicated, so every member runs these shapes. Bitwise equal to `reference` at every cell. + + Preconditions: `qk` 3-D, any strides; `cos` and `sin` fp32, contiguous, `[T, head_dim // 2]`. + `is_valid` refuses the silent failures: a non-fp32 table (rotates by rounded angles), a + short table (tail tokens unrotated), a narrow or wide table (partial rotation, or the last + k head writes into the first v head), a strided table (wrong angles). + + Not certified: float16 / float32 / float8 q|k; head geometries other than 16 x 72; partial + rotary; calls past 32768 rows (the tower's per-call ceiling); T = 0; CUDA-graph capture; + out-of-place use. + """ + + def __call__(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + seqlen, nheads, _ = qk.shape + rotary_dim = 2 * cos.shape[-1] + grid = (triton.cdiv(nheads, _BLOCK_H), triton.cdiv(seqlen, _BLOCK_M)) + _rotary_neox_inplace_kernel[grid]( + qk, + cos, + sin, + seqlen, + nheads, + cos.shape[0], + qk.stride(0), + qk.stride(1), + qk.stride(2), + ROTARY_DIM=rotary_dim, + BLOCK_K=triton.next_power_of_2(rotary_dim), + BLOCK_H=_BLOCK_H, + BLOCK_M=_BLOCK_M, + ) + return qk + + def reference(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + """Each head's halves as one complex number times `cos + i sin`, in fp32; `qk` is not + mutated.""" + half = qk.shape[-1] // 2 + x = qk.float() + rotor = torch.complex(cos.float(), sin.float()).unsqueeze(-2) + rotated = torch.complex(x[..., :half], x[..., half:]) * rotor + return torch.cat((rotated.real, rotated.imag), dim=-1).to(qk.dtype) + + def is_valid(self, qk: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> None: + assert cos.dtype == torch.float32 and sin.dtype == torch.float32, ( + f"cos and sin must be fp32; got {cos.dtype} / {sin.dtype} -- the kernel widens a " + "narrower table and rotates by its rounded angles" + ) + half = qk.shape[-1] // 2 + assert cos.shape == (qk.shape[0], half) and sin.shape == cos.shape, ( + f"cos and sin must be [T, head_dim // 2] = [{qk.shape[0]}, {half}]; got " + f"{tuple(cos.shape)} / {tuple(sin.shape)} -- a short table leaves the tail tokens " + "unrotated, a narrow one rotates only a prefix of each head" + ) + assert cos.is_contiguous() and sin.is_contiguous(), ( + "cos and sin must be contiguous; the kernel indexes them as t * half + i and a " + "strided table reads the wrong angles" + ) + + +vision_rope_2d_packed_qk = VisionRope2dPackedQk() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py new file mode 100644 index 000000000000..1b545b7bf590 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/ce_exchange__sp_pair.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Two-rank row exchange on the copy engines: each rank pushes a tensor into the peer's receive +buffer over NVLink with `cudaMemcpyAsync` and the pair signals with stream memory operations, so a +GEMM it overlaps keeps every SM. + +Protocol (`exchange`), on the caller's stream, every flag in the memory of the rank that waits on +it, every value a constant (so a CUDA graph replays it): + + 1. write peer.ready = 1 my receive buffer is free (everything before on this stream is done) + 2. wait my.ready == 1; write my.ready = 0 the peer's buffer is free + 3. copy my rows -> the peer's receive buffer (copy engine) + 4. write peer.data = 1 (the write's default memory barrier orders it after the copy) + 5. wait my.data == 1; write my.data = 0 the peer's rows are in my buffer + +A rank's step-1 signal for exchange k is issued after its step 5 of exchange k - 1, which follows +the peer's reset of `ready` for k - 1 (step 2 precedes the peer's step 4); likewise for `data`, so no +signal is lost or seen twice. The result is a view of the receive buffer, valid until the next +exchange's step 1. Both ranks must call `exchange` identically (same calls, order and sizes). +""" + +from __future__ import annotations + +import ctypes +import sys +from typing import Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm.logger import logger + +#: Bytes kept for the two flags after the data region (one 128-byte line each). +_FLAG_BYTES = 256 +#: The buffer's size is a multiple of this (2 MiB), as for the stock IPC buffers. +_ALIGN = 1 << 21 +#: Bytes of a `cudaIpcMemHandle_t` (CUDA_IPC_HANDLE_SIZE). +_HANDLE_BYTES = 64 +HIDDEN = 2560 +#: Words of one QSA selection row: the target's `token_topk + compress_ratio - 1`. +_SELECTION_WIDTH = 2051 + + +def _checked(runtime, result): + """The value of a `cuda.bindings.runtime` call's `(err, value)` result; raises RuntimeError + (and clears the error) when `err` is not success.""" + err = result[0] + if err != runtime.cudaError_t.cudaSuccess: + runtime.cudaGetLastError() + raise RuntimeError(f"CUDA runtime error: {err!r}") + return result[1] if len(result) > 1 else None + + +def _release(runtime, local: int, peer: int) -> None: + """Close the peer's mapping and free this rank's buffer (either may be 0).""" + if peer: + runtime.cudaIpcCloseMemHandle(peer) + if local: + runtime.cudaFree(local) + + +def _open_pair(runtime, rank: int, nbytes: int, allgather) -> tuple: + """`(local, peer)` pointers of the pair's buffers, or None on both ranks. Each rank allocates + and exports its zeroed buffer, `allgather((ok, handle))` gives every rank both outcomes, a rank + opens the peer's handle only when both exports succeeded, and `allgather(opened)` makes a + failed open on either rank fail both. `allgather` is the pair's object allgather (rank order); + every rank calls it exactly twice. Whatever a rank allocated or opened is released when it + returns None.""" + size = -(-nbytes // _ALIGN) * _ALIGN + local = peer = 0 + handle = b"" + try: + local = _checked(runtime, runtime.cudaMalloc(size)) + _checked(runtime, runtime.cudaMemset(local, 0, size)) + # The peer writes the flags after the agreement below; the zeroing must land first. + _checked(runtime, runtime.cudaDeviceSynchronize()) + exported = _checked(runtime, runtime.cudaIpcGetMemHandle(local)) + handle = ctypes.string_at(exported.getPtr(), _HANDLE_BYTES) + except RuntimeError as error: + logger.warning(f"copy-engine exchange: no IPC buffer ({error})") + peer_ok, peer_handle = allgather((bool(handle), handle))[1 - rank] + if handle and peer_ok: + try: + imported = runtime.cudaIpcMemHandle_t() + ctypes.memmove(imported.getPtr(), peer_handle, _HANDLE_BYTES) + peer = _checked( + runtime, + runtime.cudaIpcOpenMemHandle(imported, runtime.cudaIpcMemLazyEnablePeerAccess), + ) + except RuntimeError as error: + logger.warning(f"copy-engine exchange: the peer's IPC buffer did not open ({error})") + if all(allgather(bool(peer))): + return local, peer + _release(runtime, local, peer) + return None + + +class _CudaArray: + """`__cuda_array_interface__` over a raw device allocation (uint8), for `torch.as_tensor`.""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "shape": (nbytes,), + "typestr": "|u1", + "data": (ptr, False), + "version": 3, + "strides": None, + } + + +class CopyEngineExchange: + """The pair's buffers and flags: `exchange(x)` -> the peer's `x` (`ce_exchange`, a view of + this rank's receive buffer, same shape and dtype). `capacity` bytes a call at most. Built on + the attention-TP group of a two-rank `mapping` (IPC memory inside a node); `create` returns + None where that does not hold. Owns this rank's buffer and its mapping of the peer's, and + releases both when collected.""" + + def __init__(self, local: int, peer: int, capacity: int): + from cuda.bindings import driver, runtime + + self._driver, self._runtime = driver, runtime + self.capacity = capacity + self._local, self._peer = local, peer + self._my_ready, self._my_data = local + capacity, local + capacity + 128 + self._peer_ready, self._peer_data = peer + capacity, peer + capacity + 128 + self._recv = torch.as_tensor(_CudaArray(local, capacity), device=torch.device("cuda")) + + def __del__(self): + if not sys.is_finalizing() and self._local: + _release(self._runtime, self._local, self._peer) + self._local = self._peer = 0 + + @classmethod + def create(cls, mapping, capacity: int): + """The exchange over `mapping`'s attention-TP pair, or None (not two ranks, ranks on + different nodes, or IPC memory unavailable on either rank: `_open_pair`). A collective + of the pair: both ranks build it or neither, and both make the same collective calls.""" + if mapping.tp_size != 2 or mapping.tp_size > mapping.gpus_per_node: + return None + from cuda.bindings import runtime + + from tensorrt_llm._torch.distributed.communicator import Distributed + + ptrs = _open_pair( + runtime, + mapping.tp_rank, + capacity + _FLAG_BYTES, + Distributed.get(mapping).tp_allgather, + ) + if ptrs is None: + logger.warning("copy-engine exchange unavailable on the pair; the pair runs NCCL") + return None + return cls(*ptrs, capacity) + + def exchange(self, x: torch.Tensor) -> torch.Tensor: + """`ce_exchange(self, x)`.""" + return ce_exchange(self, x) + + +def _write(driver, stream, addr: int, value: int) -> None: + (err,) = driver.cuStreamWriteValue32( + stream, + driver.CUdeviceptr(addr), + value, + driver.CUstreamWriteValue_flags.CU_STREAM_WRITE_VALUE_DEFAULT, + ) + assert err == driver.CUresult.CUDA_SUCCESS, err + + +def _wait(driver, stream, addr: int, value: int) -> None: + (err,) = driver.cuStreamWaitValue32( + stream, + driver.CUdeviceptr(addr), + value, + driver.CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ, + ) + assert err == driver.CUresult.CUDA_SUCCESS, err + + +class CeExchange(OpWrapper): + """The peer's `x` over `pair` (a `CopyEngineExchange`), by the module docstring's protocol on + the caller's stream: a view of this rank's receive buffer, same shape and dtype, valid until + the next exchange. Both ranks call it identically.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell(why=f"sp2: {why}", spec=dict(world=2, rows=rows, width=width, dtype=dtype)) + for dtype, width, what in ( + (torch.bfloat16, HIDDEN, "bf16 hidden rows"), + (torch.int32, _SELECTION_WIDTH, "int32 QSA selection rows"), + ) + for rows, why in ( + (8192, f"half of a 16384-token step's {what}, the buffer's largest exchange"), + (4096, f"half of an 8192-token step's {what}, also replayed from a CUDA graph chain"), + (1024, f"half of a 2048-token chunk's {what}"), + (77, f"an odd row count of {what}"), + ) + ) + + note = """ + Certified byte-exact against the peer's input, eagerly and replayed from a CUDA graph chain + with new inputs. Not certified: pairs spanning nodes, groups other than two ranks. + """ + + def __call__(self, pair: CopyEngineExchange, x: torch.Tensor) -> torch.Tensor: + assert x.is_contiguous() and x.is_cuda + nbytes = x.numel() * x.element_size() + assert nbytes <= pair.capacity, (nbytes, pair.capacity) + d, r = pair._driver, pair._runtime + stream = d.CUstream(torch.cuda.current_stream().cuda_stream) + _write(d, stream, pair._peer_ready, 1) + _wait(d, stream, pair._my_ready, 1) + _write(d, stream, pair._my_ready, 0) + (err,) = r.cudaMemcpyAsync( + pair._peer, + x.data_ptr(), + nbytes, + r.cudaMemcpyKind.cudaMemcpyDefault, + r.cudaStream_t(torch.cuda.current_stream().cuda_stream), + ) + assert err == r.cudaError_t.cudaSuccess, err + _write(d, stream, pair._peer_data, 1) + _wait(d, stream, pair._my_data, 1) + _write(d, stream, pair._my_data, 0) + return pair._recv[:nbytes].view(x.dtype).view(x.shape) + + def reference(self, xs_in_rank_order: Sequence[torch.Tensor], rank: int) -> torch.Tensor: + """The other rank's `x`. The documented collective exception to mirroring `__call__`.""" + return xs_in_rank_order[1 - rank] + + +ce_exchange = CeExchange() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py new file mode 100644 index 000000000000..17a1abc423f0 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/hc_allreduce_inject_norm__tp_sum.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode block's TP sum and the hyper-connection site's injection + grouped norm in one launch. + +`native_ops.hc_allreduce_inject_norm` (`csrc/hcAllreduceInjectNorm.cu`) does the IPC one-shot +Lamport all-reduce of `catalog/comm/allreduce` over the same workspace and protocol (so it +interleaves with the plain all-reduce's calls), then `_hc_inject_norm_launch`'s injection and +norm from the summed row, its sum of squares in the Triton program's reduction tree at +`program_warps` warps. Both outputs are bitwise the all-reduce followed by the Triton kernel. + +The sum accumulates in fp32 only, the all-reduce's default. Under +`ALL_REDUCE_FUSION_KERNEL_ACC_FP32=0` the all-reduce sums in bf16 and this kernel does not +match it, so the target does not call it then (`modeling._hc_ar_inject`). + +Only the norm weights are read before the grid-dependency wait; the residual, injection weights +and shared-expert operands are read after it. +""" + +from __future__ import annotations + +from typing import Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, +) + +from ... import native_ops + +HIDDEN = 2560 +HC_COUNT = 4 + +#: Decode-site row counts certified at 2 and 4 ranks, each with and without the MoE combine, +#: and why each is there. +_ROWS = ( + (33, "the smallest fused site (`_HC_AR_INJECT_MIN_ROWS`)"), + (96, "a decode batch on 4-warp programs"), + (168, "a decode batch on 4-warp programs"), + (192, "a decode batch on 4-warp programs"), + (216, "a decode batch on 4-warp programs"), + (288, "a decode batch on 4-warp programs"), + (384, "a decode batch on 4-warp programs"), + (480, "a decode batch on 4-warp programs"), + (511, "the last row count on 4-warp programs"), + (512, "the first row count on 2-warp programs"), + (576, "a decode batch on 2-warp programs"), + (640, "a decode batch on 2-warp programs"), + (1024, "a decode batch on 2-warp programs"), +) + + +def _aligned(t: torch.Tensor) -> bool: + return t.data_ptr() % 16 == 0 and t.stride(-1) == 1 and t.stride(0) % 8 == 0 + + +def fits(hc, partial, inject, shared=None, gate=None) -> bool: + """Whether the native kernel takes these operands (else the caller sums and injects apart).""" + rows = hc.shape[0] + if partial.dim() != 2 or partial.shape != (rows, HIDDEN) or not partial.is_contiguous(): + return False + if hc.shape != (rows, HC_COUNT * HIDDEN) or not _aligned(hc): + return False + if inject is None or inject.shape[0] != rows or inject.shape[1] < HC_COUNT: + return False + if inject.stride(-1) != 1: + return False + if (shared is None) != (gate is None): + return False + if shared is not None and (shared.shape != (rows, HIDDEN) or not _aligned(shared)): + return False + dtypes = {t.dtype for t in (hc, partial, inject, shared, gate) if t is not None} + return dtypes == {torch.bfloat16} + + +class HcAllreduceInjectNorm(OpWrapper): + """`(hc_out, normed)` of the site after a block whose per-rank output is `partial`: bitwise + `_hc_inject_norm_launch(hc, tp_sum(partial), inject, norm, eps, 4, shared, gate, + program_warps)`. `workspace`, `rank` and `nranks` are the stock IPC all-reduce's + (`catalog/comm/allreduce`); `fits` says which operands the kernel takes.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell( + why=f"tp{world}: {rows} rows, {why}", + spec=dict( + world=world, rows=rows, hidden=HIDDEN, hc_count=HC_COUNT, dtype=torch.bfloat16 + ), + ) + for world in (2, 4) + for rows, why in _ROWS + ) + + note = """ + Certified bitwise the IPC all-reduce followed by the Triton injection, with and without the + MoE combine, interleaved with plain all-reduces and in a replayed CUDA graph chain. + Not certified: hc_count other than 4, hidden other than 2560, groups spanning nodes. + """ + + def __call__( + self, + partial: torch.Tensor, + workspace: torch.Tensor, + rank: int, + nranks: int, + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + program_warps: int = 4, + ) -> tuple[torch.Tensor, torch.Tensor]: + hc_out, normed = native_ops.hc_allreduce_inject_norm( + partial, + workspace, + int(rank), + int(nranks), + hc, + inject, + shared, + gate, + norm, + float(eps), + int(program_warps), + # trigger_completion_at_end=False: release the next launch right after this kernel's + # wait. + False, + ) + return hc_out, normed + + def reference( + self, + partials_in_rank_order: Sequence[torch.Tensor], + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """The fp32 sum of every rank's `partial`, in rank order, rounded once, then + `qwen4_exp_hc_inject_norm`'s reference of the site. The documented collective exception + to mirroring `__call__`; `program_warps` only orders the kernel's sum of squares.""" + total = torch.zeros_like(partials_in_rank_order[0], dtype=torch.float32) + for partial in partials_in_rank_order: + total += partial.float() + block = total.to(partials_in_rank_order[0].dtype) + return qwen4_exp_hc_inject_norm.reference( + hc, block, inject[:, :HC_COUNT], norm, eps, HC_COUNT, shared, gate + ) + + def is_valid( + self, + partial: torch.Tensor, + workspace: torch.Tensor, + rank: int, + nranks: int, + hc: torch.Tensor, + inject: torch.Tensor, + norm: torch.Tensor, + eps: float, + shared: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + program_warps: int = 4, + ) -> None: + assert fits(hc, partial, inject, shared, gate), "operands the kernel does not take" + assert nranks in (2, 4) and 0 <= rank < nranks, (rank, nranks) + assert program_warps in (2, 4), program_warps + + +hc_allreduce_inject_norm = HcAllreduceInjectNorm() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py new file mode 100644 index 000000000000..c8246c7a77cd --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/lamport_allgather__index_split.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The gather of a decode step's QSA selection split by request over the TP group, one launch. + +Each rank selects some requests' rows (`_decode_index_selection`); one Triton launch gathers the +int32 selection rows in rank order as a one-shot Lamport exchange over CUDA IPC buffers: each +rank stores its rows into every peer's buffer (NVLink) and its own output, then copies the +peers' rows out of its own buffer as they land. Bitwise the NCCL gather. + +Buffers: `[SLOTS, max_words]` int32 a rank, every word `SENTINEL` (int32 min; tokens are >= -1) +until a peer's store lands. Slot protocol: every program reads the call's slot (`state[0]`) and +the previous call's share (`state[1]`) after its grid-dependency wait and takes a ticket +(`state[2]`, acq_rel) after its stores; the last ticket sets the next slot and share and resets +the ticket. Each call clears the previous call's slot back to `SENTINEL`. A peer stores into +call k's slot only after reading this rank's call k - 1 stores, made after call k - 2 completed +(stream order), so call k's slot (call k - 3's) was cleared by call k - 2. Every rank of the +group must make the same sequence of calls (the split depends only on the row count). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Sequence + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + +SLOTS = 3 +SENTINEL = -(2**31) +_BLOCK = 1024 +_PROGRAMS = 256 +#: Words of one selection row: the target's `token_topk + compress_ratio - 1`. +_SELECTION_WIDTH = 2051 + + +if TYPE_CHECKING: + from tensorrt_llm._ipc_utils import IpcMemory + + +@dataclass(frozen=True) +class SelectionLamport: + """One rank's view of the group's buffers: `buffers` the `IpcMemory` (kept alive here; it + frees this rank's buffer and closes the peers' with the object), `state` the `[4]` int32 + slot state (`state[0]` the next call's slot, `state[1]` the previous call's share in words, + `state[2]` the ticket counter), `args` the `[8]` int64 the kernel reads: the peers' buffer + addresses at [0, world), this rank's own at `rank`.""" + + rank: int + world: int + max_words: int + buffers: IpcMemory + state: torch.Tensor + args: torch.Tensor + + def fits(self, words: int) -> bool: + """Whether a call gathering `words` int32 in all (every rank's share) fits a slot.""" + return 0 < words <= self.max_words + + +def build(mapping, max_words: int) -> SelectionLamport: + """Allocate this rank's buffers and open the peers' over `mapping`'s TP group (CUDA IPC, + one node, `IpcMemory`). A TP collective: every rank calls `build` at the same point, before + any CUDA graph capture.""" + from tensorrt_llm._ipc_utils import IpcMemory, cuda + from tensorrt_llm._torch.distributed.communicator import Distributed + + rank, world = mapping.tp_rank, mapping.tp_size + assert 2 <= world <= 4, "the kernel addresses 2 to 4 ranks" + ipc = IpcMemory(mapping, SLOTS * max_words * 4, open_ipc=True) + assert ipc.open_ipc, "the TP group spans nodes: no CUDA IPC" + (err,) = cuda.cuMemsetD32(ipc.local_ptr, SENTINEL & 0xFFFFFFFF, SLOTS * max_words) + if err != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"cuMemsetD32 of the selection buffers: {err!r}") + dev = torch.device("cuda", torch.cuda.current_device()) + state = torch.zeros(4, dtype=torch.int32, device=dev) + peers = [int(ptr) for ptr in ipc.peer_ptrs] + args = torch.tensor(peers + [0] * (8 - world), dtype=torch.int64, device=dev) + torch.cuda.synchronize() + # every rank's buffer is filled and opened before any rank's first push + Distributed.get(mapping).tp_allgather(b"") + return SelectionLamport( + rank=rank, + world=world, + max_words=max_words, + buffers=ipc, + state=state, + args=args, + ) + + +@triton.jit +def _dst_words(dst_rows, g, m, width, ROW_MAP: tl.constexpr): + """Output word of gathered word `g`: itself, or with `ROW_MAP` the same column of the row + `dst_rows` names for its row.""" + if ROW_MAP: + # int32 arithmetic (a slot holds < 2^31 words): cheaper than a 64-bit division per word. + g32 = g.to(tl.int32) + row = g32 // width + to = tl.load(dst_rows + row, mask=m, other=0).to(tl.int64) + return to * width + (g32 - row * width) + return g + + +@triton.jit(do_not_specialize=["share", "rank", "slot_words", "width"]) +def _lamport_allgather_kernel( + src, + dst, + args, + state, + share, + rank, + slot_words, + dst_rows, + width, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, + ROW_MAP: tl.constexpr, + USE_PDL: tl.constexpr, +): + """Program p moves words p * BLOCK, (p + P) * BLOCK, ... of each share (`share` words a + rank): pushes this rank's from `src` to every peer's slot and to `dst`, clears the previous + call's peer shares of the previous slot, then copies each peer's share out of this rank's + slot as it lands. With `ROW_MAP`, gathered row `j` (of `width` words) lands in row + `dst_rows[j]` of `dst` instead of row `j`.""" + pid = tl.program_id(0) + nprog = tl.num_programs(0) + if USE_PDL: + # With a memory clobber: no load of the predecessor's output (`src`) or of the state is + # hoisted above the wait. + gdc_wait_with_memory_clobber() + slot = tl.load(state) + prev_share = tl.load(state + 1) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + offs = tl.arange(0, BLOCK) + local = tl.load(args + rank).to(tl.pointer_type(tl.int32)) + slot_off = slot.to(tl.int64) * slot_words + own0 = rank.to(tl.int64) * share + # 1. this rank's share: to every peer's slot (same offset) and to the output + for start in range(pid * BLOCK, share, nprog * BLOCK): + w = start + offs + m = w < share + out = dst + _dst_words(dst_rows, own0 + w, m, width, ROW_MAP) + v = tl.load(src + w, mask=m, other=0) + tl.store(out, v, mask=m) + for q in tl.static_range(WORLD): + if q != rank: + peer = tl.load(args + q).to(tl.pointer_type(tl.int32)) + tl.store(peer + slot_off + own0 + w, v, mask=m) + # 2. the previous call's peer shares of the previous slot back to the marker + prev_off = ((slot + 2) % 3).to(tl.int64) * slot_words + for q in tl.static_range(WORLD): + if q != rank: + q0 = q * prev_share.to(tl.int64) + for start in range(pid * BLOCK, prev_share, nprog * BLOCK): + w = start + offs + tl.store( + local + prev_off + q0 + w, + tl.full([BLOCK], -2147483648, tl.int32), + mask=w < prev_share, + ) + # Every program has read the state (at its start): the last ticket moves it on. + ticket = tl.atomic_add(state + 2, 1, sem="acq_rel", scope="gpu") + if ticket == nprog - 1: + tl.store(state, (slot + 1) % 3) + tl.store(state + 1, share) + tl.store(state + 2, 0) + # 3. each peer's share, once it has landed in this rank's slot, to the output + for q in tl.static_range(WORLD): + if q != rank: + q0 = q * share.to(tl.int64) + for start in range(pid * BLOCK, share, nprog * BLOCK): + w = start + offs + m = w < share + out = dst + _dst_words(dst_rows, q0 + w, m, width, ROW_MAP) + v = tl.load(local + slot_off + q0 + w, mask=m, other=0, volatile=True) + pending = tl.sum(((v == -2147483648) & m).to(tl.int32), axis=0) + while pending > 0: + v = tl.load(local + slot_off + q0 + w, mask=m, other=0, volatile=True) + pending = tl.sum(((v == -2147483648) & m).to(tl.int32), axis=0) + tl.store(out, v, mask=m) + + +class LamportAllgather(OpWrapper): + """`src`'s rows from every rank of `lam`'s group concatenated in rank order (the NCCL + `allgather` with equal shares): `src` is this rank's contiguous int32 `[rows, width]` + share. Returns a fresh `[world * rows, width]` int32 tensor. With `dst_rows` (int32 + `[world * rows]`, a permutation, the same on every rank) the gathered row `j` is stored at + row `dst_rows[j]` instead (`out[dst_rows] = gathered`), in the same launch. `lam` comes from + `build`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = tuple( + Cell( + why=f"tep{world}: a served step of {requests} requests, 12 trunk calls at " + f"{3 * requests} rows and 2 draft calls at {requests}", + spec=dict(world=world, requests=requests, width=_SELECTION_WIDTH, dtype=torch.int32), + ) + for world in (2, 4) + for requests in (16, 32, 64, 128, 192) + ) + + note = """ + Certified bitwise NCCL's gather over a long mixed sequence of each cell's calls (the three + slots rotating), with and without the row map, eagerly and replayed from a CUDA graph. + Not certified: groups spanning nodes, world sizes other than 2 and 4, widths other than + the target's selection rows. + """ + + def __call__( + self, src: torch.Tensor, lam: SelectionLamport, dst_rows: torch.Tensor | None = None + ) -> torch.Tensor: + assert src.dtype == torch.int32 and src.is_contiguous() and src.ndim == 2 + rows, width = src.shape + share = rows * width + assert lam.fits(share * lam.world), (share, lam.world, lam.max_words) + dst = torch.empty([lam.world * rows, width], dtype=torch.int32, device=src.device) + if dst_rows is not None: + assert dst_rows.dtype == torch.int32 and dst_rows.is_contiguous() + assert dst_rows.numel() == lam.world * rows + pdl = get_env_enable_pdl() + _lamport_allgather_kernel[(_PROGRAMS,)]( + src, + dst, + lam.args, + lam.state, + share, + lam.rank, + lam.max_words, + dst if dst_rows is None else dst_rows, + width, + WORLD=lam.world, + BLOCK=_BLOCK, + ROW_MAP=dst_rows is not None, + USE_PDL=pdl, + num_warps=4, + launch_pdl=pdl, + ) + return dst + + def reference( + self, srcs_in_rank_order: Sequence[torch.Tensor], dst_rows: torch.Tensor | None = None + ) -> torch.Tensor: + """Every rank's `src` concatenated in rank order, its row `j` at row `dst_rows[j]` with + `dst_rows`. The documented collective exception to mirroring `__call__`.""" + gathered = torch.cat(list(srcs_in_rank_order)) + if dst_rows is None: + return gathered + out = torch.empty_like(gathered) + out[dst_rows.long()] = gathered + return out + + +lamport_allgather = LamportAllgather() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py new file mode 100644 index 000000000000..8e241359265a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/mnnvl_allreduce__tp_sum.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Sum across the TP group through a multicast NVLink (MNNVL) workspace, nothing fused. + +This target's variant of `catalog/comm/mnnvl_allreduce` (which certifies only the residual + +RMSNorm fusion) for the decode-sized calls of `tp_sum`: the op's plain sum (fusion `NONE`).""" + +from __future__ import annotations + +from typing import Sequence + +import torch + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.catalog.comm.mnnvl_allreduce import ( + mnnvl_workspace_args, + workspace_bytes, +) +from tensorrt_llm.functional import AllReduceFusionOp + +NONE = int(AllReduceFusionOp.NONE) + +HIDDEN = 2560 + +__all__ = ["mnnvl_tp_sum", "mnnvl_workspace_args"] + +_TP_SUM = dict(hidden=HIDDEN, dtype=torch.bfloat16) + + +class MnnvlTpSum(OpWrapper): + """Sum `input` (`[rows, HIDDEN]` bf16) over the TP group; returns a fresh tensor. The + workspace is built once by `mnnvl_workspace_args` (a TP collective) for a row budget the + kernel does not check; `is_valid` does.""" + + ARCHS = frozenset({Arch.SM_100, Arch.SM_103}) + + CELLS: tuple[Cell, ...] = tuple( + Cell(why=f"tp_sum, {why}, tp{world}", spec=dict(world=world, rows=rows, **_TP_SUM)) + for world, rows, why in ( + (4, 1, "one draft row"), + (4, 32, "an MTP-3 verify of 8 requests"), + (2, 4, "an MTP-3 verify of 1 request"), + ) + ) + + def __call__( + self, input: torch.Tensor, comm_buffer: torch.Tensor, buffer_flags: torch.Tensor, world: int + ) -> torch.Tensor: + """`world` is not passed to the op, which reads it from the workspace; it is here so + `is_valid` can size the call against the buffer.""" + return torch.ops.trtllm.mnnvl_fusion_allreduce( + input, None, None, 1e-6, comm_buffer, buffer_flags, False, None, NONE + )[0] + + def reference(self, inputs_in_rank_order: Sequence[torch.Tensor]) -> torch.Tensor: + """The fp32 sum of every rank's `input`, in rank order, rounded once. The documented + collective exception to mirroring `__call__`.""" + total = torch.zeros_like(inputs_in_rank_order[0], dtype=torch.float32) + for x in inputs_in_rank_order: + total += x.float() + return total.to(inputs_in_rank_order[0].dtype) + + def is_valid( + self, input: torch.Tensor, comm_buffer: torch.Tensor, buffer_flags: torch.Tensor, world: int + ) -> None: + hidden = input.shape[-1] + need = workspace_bytes(input.numel() // hidden, hidden, world, input.dtype) + have = comm_buffer.shape[-1] * comm_buffer.element_size() + assert need <= have, (need, have) + + +mnnvl_tp_sum = MnnvlTpSum() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py new file mode 100644 index 000000000000..f5a0a05b2da7 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/comm/moe_a2a__dp_dispatch_combine.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The attention-DP MoE round trip as an all-to-all (`NVLinkOneSided`), run when +`TRTLLM_FORCE_COMM_METHOD=NVLINK_ONE_SIDED` (`modeling.MOE_COMM_METHOD_ENV`) in place of the +default all-gather / reduce-scatter. `moe_a2a_dispatch` sends every rank the rows with a top-k +expert in its whole-expert window, with their top-k ids and weights; `moe_a2a_combine` returns +each partial to the row's rank and sums them there. The strategy (`build_strategy`) holds the +MNNVL workspace and the state between a dispatch and its combine: build it once, pass it to every +call.""" + +from __future__ import annotations + +from typing import List, NamedTuple, Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +#: The id the strategy writes over a received slot that holds no token: outside every window. +NO_EXPERT = -1 + + +def build_strategy(mapping, max_num_tokens_per_rank: int): + """The stock `NVLinkOneSided` strategy over the attention-DP group (`mapping`'s TP group, + every rank one EP rank), or None where the platform has no MNNVL. Collective: every rank of + the group builds it at the same point.""" + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided + + if not NVLinkOneSided.is_platform_supported(): + return None + return NVLinkOneSided( + mapping, + num_slots=NUM_EXPERTS, + top_k=TOPK, + max_num_tokens_per_rank=max_num_tokens_per_rank, + hidden_size=HIDDEN, + dtype=torch.bfloat16, + ) + + +def _cell(world: int, counts: tuple, why: str) -> Cell: + """One round trip over an attention-DP group of `world` ranks holding `counts` rows.""" + return Cell( + why=why, + spec=dict(world=world, counts=counts, hidden=HIDDEN, topk=TOPK, dtype=torch.bfloat16), + ) + + +_CELLS = ( + _cell(2, (1, 1), "dep2: a decode step at batch 1"), + _cell(2, (5, 300), "dep2: a ragged eager step, one rank holding a context chunk"), + _cell(4, (0, 17, 3, 0), "dep4: a ragged step where two ranks hold no rows"), + _cell(4, (2048,) * 4, "dep4: a prefill chunk of 2048 rows a rank"), +) + + +class Dispatched(NamedTuple): + """What `moe_a2a_dispatch` hands this rank, `[world * max(counts), ...]` each: the received + `rows`, their `topk_ids` (global expert ids; `NO_EXPERT` on a slot that holds no token) and + `topk_weights`. Block `s` holds source rank `s`'s tokens in an order the kernel picks.""" + + rows: torch.Tensor + topk_ids: torch.Tensor + topk_weights: torch.Tensor + + +class MoeA2aDispatch(OpWrapper): + """Send every rank the rows routed into its window; see the module docstring.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _CELLS + + note = """ + Source: `NVLinkOneSided.dispatch` (`torch.ops.trtllm.moe_a2a_dispatch`): each token written + once to every rank one of its top-k experts lives on, into that rank's block of the sender + (slot order inside a block is not fixed), the slots past each block's count stamped + `NO_EXPERT`. Measured on GB300; received rows compared with `reference` as a set. + Not certified: world sizes other than 2 and 4, groups spanning nodes, sm_100, post-quant + dispatch, low-precision combine, EPLB statistics. + """ + + def __call__( + self, + strategy, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + all_rank_num_tokens: List[int], + ) -> Dispatched: + # The weights travel as fp32 (bf16 -> fp32 -> bf16 is exact). + rows, _, ids, weights = strategy.dispatch( + x, None, topk_ids, topk_weights.float(), list(all_rank_num_tokens) + ) + return Dispatched(rows, ids, weights.to(topk_weights.dtype)) + + def reference( + self, + per_rank_inputs: Sequence[Sequence[torch.Tensor]], + all_rank_num_tokens: List[int], + experts_per_rank: int, + rank: int = 0, + ) -> List[tuple]: + """What `rank` receives from every rank's `(x, topk_ids, topk_weights)` (ascending rank + order): per source, its tokens with a top-k id in `rank`'s window, in token order. + Compare as a set: the kernel's order inside a block is its own.""" + lo, hi = rank * experts_per_rank, (rank + 1) * experts_per_rank + out = [] + for x, ids, w in per_rank_inputs: + hit = ((ids >= lo) & (ids < hi)).any(1) + out.append((x[hit], ids[hit], w[hit])) + return out + + def is_valid( + self, + strategy, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + all_rank_num_tokens: List[int], + ) -> None: + assert x.shape[0] == topk_ids.shape[0] == topk_weights.shape[0], ( + x.shape, + topk_ids.shape, + topk_weights.shape, + ) + assert x.shape[0] <= strategy.max_num_tokens_per_rank, ( + f"{x.shape[0]} rows past the workspace's {strategy.max_num_tokens_per_rank}" + ) + assert topk_ids.dtype == torch.int32, topk_ids.dtype + + +class MoeA2aCombine(OpWrapper): + """Return each partial to the rank its row came from and sum a row's partials there. + `partials` is `[world * max(counts), hidden]` in the order `moe_a2a_dispatch` handed the rows + (a slot that holds no token may hold anything); returns this rank's `[rows, hidden]`.""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _CELLS + + note = """ + Source: `NVLinkOneSided.combine` (`torch.ops.trtllm.moe_a2a_combine`). Must follow the + strategy's `dispatch` (the strategy keeps the slot map between them). Not certified: as + `MoeA2aDispatch`. + """ + + def __call__(self, strategy, partials: torch.Tensor) -> torch.Tensor: + return strategy.combine(partials) + + def reference(self, per_rank_token_partials: Sequence[torch.Tensor]) -> torch.Tensor: + """This rank's rows: the partial every rank computed for each of them (zero where the + row was not sent there), `[rows, hidden]` each in ascending rank order, summed in fp32 + and rounded once.""" + out = torch.zeros_like(per_rank_token_partials[0], dtype=torch.float32) + for partial in per_rank_token_partials: + out += partial.float() + return out.to(per_rank_token_partials[0].dtype) + + def is_valid(self, strategy, partials: torch.Tensor) -> None: + assert partials.dim() == 2 and partials.shape[1] == HIDDEN, partials.shape + + +moe_a2a_dispatch = MoeA2aDispatch() +moe_a2a_combine = MoeA2aCombine() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py new file mode 100644 index 000000000000..a84b7d349b2a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/embedding/embed_rows__decode.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's token embedding in one launch: the rows of the table at the step's ids, +optionally with negative ids read as row 0, written `copies` times side by side (once per +hyper-connection stream). +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +BLOCK = 512 + + +# `ids` is a view of the step's token buffer at any row (a gather index, read by scalar loads). +@triton.jit(do_not_specialize_on_alignment=["ids"]) +def _embed_rows_kernel( + ids, + table, + out, + stride_table, + stride_out, + H: tl.constexpr, + COPIES: tl.constexpr, + CLAMP: tl.constexpr, + BLOCK_: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.program_id(1) * BLOCK_ + tl.arange(0, BLOCK_) + mask = cols < H + token = tl.load(ids + row).to(tl.int64) + if CLAMP: + token = tl.maximum(token, 0) + vals = tl.load(table + token * stride_table + cols, mask=mask) + for c in tl.static_range(COPIES): + tl.store(out + row.to(tl.int64) * stride_out + c * H + cols, vals, mask=mask) + + +class EmbedRows(OpWrapper): + """`out = embed_rows(ids, table, copies, clamp)`: `[T, copies * H]`, block `c` of row `t` + `table[ids[t]]` (`table[max(ids[t], 0)]` with `clamp`); `ids` `[T]` int32 / int64 + contiguous, `table` `[V, H]` with unit column stride. Bitwise the composed `F.embedding` + path. Ids past the table (or negative without `clamp`) read out of bounds: the caller's. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="batch-4 verify: 12 rows over 4 streams", spec=dict(rows=12, copies=4, clamp=False)), + Cell(why="draft decode step: 1 row, clamped", spec=dict(rows=1, copies=1, clamp=True)), + ) # fmt: skip + + note = """ + Source: one Triton kernel (`_embed_rows_kernel`), grid `(T, ceil(H / 512))`, JIT-compiled per + (H, copies, clamp, id dtype); the engine's warmup forward compiles it before capture. Copies + only: bitwise the composed path for every in-range id. + """ + + def __call__( + self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool + ) -> torch.Tensor: + rows = ids.shape[0] + hidden = table.shape[1] + out = torch.empty([rows, copies * hidden], dtype=table.dtype, device=table.device) + if rows: + _embed_rows_kernel[(rows, triton.cdiv(hidden, BLOCK))]( + ids, + table, + out, + table.stride(0), + out.stride(0), + H=hidden, + COPIES=copies, + CLAMP=clamp, + BLOCK_=BLOCK, + num_warps=4, + ) + return out + + def reference( + self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool + ) -> torch.Tensor: + h = torch.nn.functional.embedding(ids.clamp_min(0) if clamp else ids, table) + return torch.cat([h] * copies, -1) + + def is_valid(self, ids: torch.Tensor, table: torch.Tensor, copies: int, clamp: bool) -> None: + assert ids.dim() == 1 and ids.is_contiguous() and ids.dtype in (torch.int32, torch.int64), ( + ids.shape, + ids.dtype, + ) + assert table.dim() == 2 and table.stride(1) == 1, (tuple(table.shape), table.stride()) + assert copies >= 1 + + +embed_rows = EmbedRows() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py new file mode 100644 index 000000000000..0f5a1e65b502 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/gemm/low_m_bf16_mm__decode.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Decode-shaped bf16 projection: one kernel per call, its source picked per (M, N, K). + +At decode M a dense bf16 projection streams its weight once, and FlashInfer's low-M CuTe-DSL +kernels (`dense_bf16_gemm_direct.run_direct_dense`, the register-prefetch kernel, and +`dense_bf16_gemm_warp_splitk.run_warp_splitk_dense`, cp.async + mma.sync warp split-K) beat +cuBLAS there. A per-(N, K) table by M bucket picks one and its tactic; every other shape and +every M above the table stays on the trtllm cuBLASLt op. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple + +import torch +from flashinfer.gemm.kernels import dense_bf16_gemm_direct as _fi_direct +from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fi_warp + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + + +class _Plan(NamedTuple): + """`kind` is "direct", "warp" or "cublas"; `tactic` is the kernel's own tactic tuple.""" + + kind: str + tactic: tuple = () + + +def _d(block_size: int, outputs_per_block: int) -> tuple: + """A direct-kernel choice; rows_per_block is the kernel's default for the runtime M.""" + return ("direct", block_size, outputs_per_block) + + +def _w(output_tile: int, token_tile: int, k_tile: int, stages: int, b_loaders: int) -> tuple: + """A warp split-K choice.""" + return ("warp", output_tile, token_tile, k_tile, stages, b_loaders) + + +#: (N, K) -> ((M upper bound, choice), ...) in ascending M; other shapes and larger M run on +#: cuBLAS. The decode projections of a tp4/tep4 rank (1..16 rows) and a tp2/tep2 and tp1 rank's +#: attention in / out (3..16 rows), each raced on GB300 over the kernel's `autotune_tactics`. +_TABLE: dict[tuple[int, int], tuple[tuple[int, tuple], ...]] = { + # Attention input projection [q|gate 6 x 256 | k | v | index q|k 640 | pad 128] = 4352. + (4352, 2560): ((1, _d(64, 2)), (8, _w(16, 8, 256, 9, 2)), (16, _w(16, 16, 256, 6, 2))), + # GDN input projection 4120, padded to 4128 (`weights.IN_PROJ_ROW_ALIGN`) so the warp split-K's + # 16-wide output tile divides it. + (4128, 2560): ( + (2, _d(64, 2)), + (4, _w(16, 8, 256, 8, 2)), + (8, _w(16, 8, 128, 11, 2)), + (16, _w(16, 16, 256, 6, 2)), + ), + # Attention o and GDN out (1536 in), the draft layer's o too. + (2560, 1536): ((1, _d(192, 2)), (2, _d(64, 2)), (16, _w(16, 8, 128, 11, 1))), + # MoE router (512 experts), trunk and draft layer. + (512, 2560): ((4, _d(64, 1)), (16, _w(16, 8, 256, 10, 1))), + # Shared expert gate|up|gate logit|pad, 1288: the warp kernel cannot tile it. + (1288, 2560): ((4, _d(64, 2)),), + # Shared expert down projection. + (2560, 640): ((4, _w(16, 8, 128, 5, 2)),), + # Draft layer `fc_embedding` (M rows) and `fc_hidden` (4M rows, one per stream). + (2560, 2560): ((1, _d(64, 2)), (4, _w(16, 8, 256, 9, 1)), (16, _w(32, 16, 256, 7, 1))), + # Draft layer q|gate. + (3072, 2560): ((1, _d(64, 2)), (4, _w(16, 8, 256, 9, 1))), + # Draft layer k and v. + (256, 2560): ((4, _d(64, 1)),), + # Draft layer index q|k. + (640, 2560): ((4, _d(64, 1)),), + # tp2/tep2 attention input projection (7424). Its GDN input projection (8240) is not a whole + # number of 32-wide tiles and stays on cuBLAS. + (7424, 2560): ((8, _w(32, 8, 256, 5, 2)), (16, _w(32, 16, 256, 4, 2))), + # tp2/tep2 attention o and GDN out (3072 in). + (2560, 3072): ((8, _w(32, 8, 256, 11, 2)), (16, _w(32, 16, 256, 9, 2))), + # tp1 attention input projection (14080); cuBLAS catches up from 12 rows, and GDN in (16480) + # stays on cuBLAS. + (14080, 2560): ((8, _w(32, 8, 256, 5, 2)),), + # tp1 attention o and GDN out (6144 in). + (2560, 6144): ((8, _w(32, 8, 256, 11, 2)), (16, _w(32, 16, 256, 9, 2))), +} + + +@functools.lru_cache(maxsize=None) +def plan(m: int, n: int, k: int) -> _Plan: + """The kernel and tactic `LowMBf16Mm` launches for an [M, K] x [K, N] product.""" + for bound, choice in _TABLE.get((n, k), ()): + if m <= bound: + if choice[0] == "direct": + t = _fi_direct.DirectTactic( + choice[1], choice[2], _fi_direct.default_tactic(m, n, k).rows_per_block + ) + _fi_direct.validate_tactic(t, m, n, k) + return _Plan("direct", tuple(choice[1:]) + (t.rows_per_block,)) + _fi_warp.validate_tactic(_fi_warp.WarpSplitKTactic(*choice[1:]), m, n, k) + return _Plan("warp", tuple(choice[1:])) + return _Plan("cublas") + + +def _cell(why: str, m: int, n: int, k: int, kind: str, pdl: bool = True) -> Cell: + """`kind` is the branch `plan` must pick for this shape; the test holds it to that.""" + return Cell(why=why, spec=dict(m=m, n=n, k=k, kind=kind, pdl=pdl)) + + +class LowMBf16Mm(OpWrapper): + """`mat_a @ mat_b` in bf16 with fp32 accumulation: one kernel launch per call. + + A drop-in for `catalog/gemm/cublas_mm` without bias or `out_dtype`: `mat_a` dense row-major + `[M, K]`, `mat_b` dense column-major `[K, N]` (a weight's `.t()`), the result a fresh + `[M, N]` bf16 tensor. `plan(M, N, K)` picks the FlashInfer direct kernel (M <= 2, and up to 4 + on narrow outputs), its warp split-K kernel (M 2..16 where 16 divides N) or cuBLAS. + + `pdl` launches the FlashInfer kernels with programmatic dependent launch: each streams its + weight before `griddepcontrol.wait` and reads `mat_a` and writes its output only after it, + so it is safe after any kernel; it changes nothing on the cuBLAS branch. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # One cell per `_TABLE` tactic (PDL on), N=1288, both cuBLAS fallbacks and PDL off for each + # FlashInfer kernel; `kind` pins the branch (the test's plan pins hold the other M buckets). + CELLS: tuple[Cell, ...] = ( + _cell("tep4 attn in M=4 (MTP-3 verify, 1 request): warp split-K", 4, 4352, 2560, "warp"), + _cell("tep4 attn in M=16 (MTP-3, 4 requests): warp 16-token tile", 16, 4352, 2560, "warp"), + _cell("tep4 attn in M=1 (decode, no MTP): direct", 1, 4352, 2560, "direct"), + _cell("tep4 attn in M=24: cuBLAS above the table", 24, 4352, 2560, "cublas"), + _cell("tep4 GDN in (padded) M=8 (MTP-3, 2 requests): warp 128-K", 8, 4128, 2560, "warp"), + _cell("tep4 GDN in (padded) M=1, PDL off: direct", 1, 4128, 2560, "direct", pdl=False), + _cell("tep4 GDN in (padded) M=3, PDL off: warp", 3, 4128, 2560, "warp", pdl=False), + _cell("tep4 o / GDN out M=3 (MTP-2 verify): warp, ragged tile", 3, 2560, 1536, "warp"), + _cell("tep4 o / GDN out M=1: direct 192-thread", 1, 2560, 1536, "direct"), + _cell("dep4 GDN in M=32: cuBLAS (not in the table)", 32, 16480, 2560, "cublas"), + _cell("tep4 router M=4 (MTP-3 verify / draft extend): direct", 4, 512, 2560, "direct"), + _cell("tep4 router M=16 (MTP-3, 4 requests): warp", 16, 512, 2560, "warp"), + _cell("tep4 shared gate|up M=4: direct", 4, 1288, 2560, "direct"), + _cell("tep4 shared down M=4: warp", 4, 2560, 640, "warp"), + _cell("tep4 draft fc_embedding M=4 (draft extend): warp", 4, 2560, 2560, "warp"), + _cell("tep4 draft fc_hidden M=16 (4 rows x 4 streams): warp 32x16", 16, 2560, 2560, "warp"), + _cell("tep2 attn in M=3 (MTP-2 verify): warp 32-wide", 3, 7424, 2560, "warp"), + _cell("tep2 attn in M=16 (4 requests x 4 rows): warp 16-token", 16, 7424, 2560, "warp"), + _cell("tep2 o / GDN out M=12: warp 16-token", 12, 2560, 3072, "warp"), + _cell("tp1 o / GDN out M=1: warp", 1, 2560, 6144, "warp"), + ) + + note = """ + Preconditions: bf16, `mat_a` dense row-major, `mat_b` dense column-major, both 32-byte + aligned (the FlashInfer kernels refuse padded operands; cuBLAS reads them as dense, see + `is_valid`). The FlashInfer kernels are JIT-compiled per (N, K, tactic, pdl; direct also per + rows_per_block) on the first call, which must come before CUDA-graph capture, and cost tens + of microseconds of host time a call: route only captured decode calls here. + + Certified on GB300 within the test's 4 / 2 bf16-ulp band at the cells: M 1, 3, 4, 8, 12, 16 + on the FlashInfer kernels, 24 and 32 on cuBLAS. Not certified: M above 16 on FlashInfer. + """ + + def __call__(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> torch.Tensor: + m, k = mat_a.shape + n = mat_b.shape[1] + p = plan(m, n, k) + if p.kind == "cublas": + return torch.ops.trtllm.cublas_mm(mat_a, mat_b, None, None, 0, None) + out = torch.empty((m, n), dtype=mat_a.dtype, device=mat_a.device) + if p.kind == "direct": + return _fi_direct.run_direct_dense( + mat_a, mat_b, out, pdl, _fi_direct.DirectTactic(*p.tactic) + ) + return _fi_warp.run_warp_splitk_dense( + mat_a, mat_b, out, pdl, _fi_warp.WarpSplitKTactic(*p.tactic) + ) + + def reference(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> torch.Tensor: + """fp32-accumulated, TF32 off; `pdl` is a launch attribute and is ignored.""" + with true_fp32_matmul(): + return (mat_a.float() @ mat_b.float()).to(mat_a.dtype) + + def is_valid(self, mat_a: torch.Tensor, mat_b: torch.Tensor, pdl: bool = True) -> None: + # The cuBLAS branch reads a row-padded mat_a or column-padded mat_b as dense: silently a + # different product. The FlashInfer branches raise on those. + assert mat_a.dtype == torch.bfloat16 and mat_b.dtype == torch.bfloat16, "bf16 operands only" + assert mat_a.is_contiguous(), "mat_a must be dense row-major [M, K]" + assert mat_b.stride(0) == 1 and mat_b.stride(1) == mat_b.shape[0], ( + "mat_b must be dense column-major [K, N] (e.g. weight.t())" + ) + + +low_m_bf16_mm = LowMBf16Mm() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py new file mode 100644 index 000000000000..22b9e89025f7 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_rows_tc__decode.py @@ -0,0 +1,1060 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Decode hyper-connection site for 1..16 rows on warp tensor cores: two PDL launches, the +first emitting the bottleneck `low`, the second the up projection, gate and stream mean. +The NVFP4 tp1 target's `qwen4_exp_hc_decode_rows_tc` kernels, bf16 variants only (the routed +experts' output arrives finalized and summed; the mix feeds bf16 blocks). + +first (`_First2`): a (2 K halves x 4 streams) cluster per 24-column chunk forms the injection, +the per-stream sums of squares and `y = bf16(x * (1 + w))` and multiplies `y` by the weight +rows on mma.sync; cluster rank o reduces rows (2o, 2o+1) into `rstd_s = (ss_s / H + eps)^-1/2`, +the logits `z = sum_s rstd_s (P_s,0 + P_s,1) / C`, `low = bf16(silu(z))` (columns < 320) and +the injection weights `2 sigmoid(z)` (columns 320..323). mix (`_Mix2`): the up projection of +`low`, gate x normed, the stream mean. + +`hc_out` is bitwise `qwen4_exp_hc_inject_norm`'s; the rest is bf16 operands on tensor cores +with fp32 accumulation (1-2 bf16 ulp of the float64 composition). +""" + +from __future__ import annotations + +import functools + +import cuda.bindings.driver as cuda_drv +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass.cute import experimental as cute_ext +from cutlass.cute.nvgpu import cpasync, warp +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.experimental import primitives as prims + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( + _asm, + _mapa, + _sigmoid, +) + +#: This checkpoint's hyper-connection bottleneck width. +RANK = 320 +#: Rows a launch serves (one m16 MMA tile). +MAX_ROWS = 16 +M = 16 +VEC = 8 +PAD = 8 +KH = 2 # K halves +KHW = 1280 +NC = 24 # columns a CTA +NCH = 14 # 14 x 24 = 336 >= 324 +WARPS = 8 +THREADS = WARPS * 32 +KW = KHW // WARPS # K a warp +CL = KH * 4 # cluster size +RPO = M // CL +#: Item iterations a thread whose activation loads issue together (all of them at <= 16 rows). +GROUP_IT = 10 +#: Rows of First2's A stage allocated at least; the footprint leaves an SM room for the mix CTAs. +A_MIN_ROWS = 8 +BLOCK_H = 16 + + +@dsl_user_op +def _st_async_f32(addr, val, mbar, *, loc=None, ip=None): + """4 bytes into a cluster CTA's shared memory, completing on that CTA's mbarrier.""" + _asm( + T.i32(), + [cutlass.Int32(addr), cutlass.Float32(val), cutlass.Int32(mbar)], + "{ st.async.shared::cluster.mbarrier::complete_tx::bytes.f32 [$1], $2, [$3]; " + "mov.u32 $0, 0; }", + "=r,r,f,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _mbar_init_tx(mbar, nbytes, *, loc=None, ip=None): + """Init an mbarrier (one arrival), make it visible to the cluster, arrive expecting + `nbytes` of transactions.""" + _asm( + T.i32(), + [cutlass.Int32(mbar), cutlass.Int32(nbytes)], + "{ mbarrier.init.shared::cta.b64 [$1], 1; fence.mbarrier_init.release.cluster; " + "mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _mbar_wait0(mbar, *, loc=None, ip=None): + """Wait (CTA scope) for phase 0 of an mbarrier that this CTA's own bulk copies complete.""" + _asm( + T.i32(), + [cutlass.Int32(mbar)], + "{ .reg .pred p; WAIT: mbarrier.try_wait.parity.shared::cta.b64 p, [$1], 0; " + "@!p bra WAIT; mov.u32 $0, 0; }", + "=r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _bulk_g2s(dst, src, nbytes: int, mbar, *, loc=None, ip=None): + """One non-tensor bulk copy global -> this CTA's shared memory, completing on `mbar`.""" + _asm( + T.i32(), + [cutlass.Int32(dst), cutlass.Int64(src), cutlass.Int32(mbar)], + "{ cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], " + + str(nbytes) + + ", [$3]; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side_effects=True, + ) + + +class _First2: + def __init__(self, *, hidden, rank, n_cols, rows, inject, combine, pdl): + assert hidden == KH * KHW and n_cols <= NC * NCH and rows <= M + assert inject or not combine, "the combine forms the injected block output" + self.hidden, self.rank, self.n_cols, self.rows = hidden, rank, n_cols, rows + self.inject, self.combine, self.pdl = inject, combine, pdl + self.ld = KHW + PAD + self.rld = NC + 2 # pushed slot: NC partials + the sum of squares (+pad) + + def smem_bytes(self): + def r(n): + return -(-n // 128) * 128 + + assert WARPS * M * NC * 4 + M * NC * 4 <= NC * self.ld * 2 + b = r(max(self.rows, A_MIN_ROWS) * self.ld * 2) + r(NC * self.ld * 2) + b += r(self.rows * (KHW // VEC) * 4) + b += r(CL * RPO * self.rld * 4) + r(M * 4) + b += r(KHW * 2) # the norm-weight slice + return b + 256 + 128 + + @cute.jit + def __call__( + self, + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps: cutlass.Float32, + stream: cuda_drv.CUstream, + ): + mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(cutlass.BFloat16, cutlass.Float32, (16, 8, 16)), + cute.make_layout((1, 1, 1)), + ) + g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.BFloat16, num_bits_per_copy=128) + self.kernel( + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps, + mma, + g2s, + ).launch( + grid=[KH, 4, NCH], + block=[THREADS, 1, 1], + cluster=[KH, 4, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + mHc, + mBlock, + mInj, + mShared, + mGate, + mNorm, + mW, + mHcOut, + mLow, + mInjOut, + mRstd, + eps: cutlass.Float32, + mma: cute.TiledMma, + g2s: cute.CopyAtom, + ): + tidx, _, _ = cute.arch.thread_idx() + kh, s, nc = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + rows: cutlass.Constexpr = self.rows + hidden: cutlass.Constexpr = self.hidden + rank: cutlass.Constexpr = self.rank + n_cols: cutlass.Constexpr = self.n_cols + ld: cutlass.Constexpr = self.ld + rld: cutlass.Constexpr = self.rld + nvec: cutlass.Constexpr = KHW // VEC + k0 = kh * KHW + n0 = nc * NC + me = kh + KH * s # cluster rank + + smem = cutlass.utils.SmemAllocator() + # sA holds the R live rows only; the MMA's 16-row A view runs on into sW (rows past R + # are garbage, their outputs discarded: the MMA's rows are independent). + sA = smem.allocate_tensor( + cutlass.BFloat16, + cute.make_layout((max(rows, A_MIN_ROWS), KHW), stride=(ld, 1)), + byte_alignment=128, + ) + sW = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((NC, KHW), stride=(ld, 1)), byte_alignment=128 + ) + sSs = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((rows, nvec), stride=(nvec, 1)), byte_alignment=128 + ) + # The warps' partials and their sum reuse sW once every warp holds its B fragment. + wp_ptr = cute.recast_ptr(sW.iterator, dtype=cutlass.Float32) + sWp = cute.make_tensor(wp_ptr, cute.make_layout((WARPS, M, NC), stride=(M * NC, NC, 1))) + sAcc = cute.make_tensor(wp_ptr + WARPS * M * NC, cute.make_layout((M, NC), stride=(NC, 1))) + sRed = smem.allocate_tensor( + cutlass.Float32, + cute.make_layout((CL, RPO, rld), stride=(RPO * rld, rld, 1)), + byte_alignment=128, + ) + sSsr = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M,), stride=(1,)), byte_alignment=128 + ) + sNw = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((1, KHW), stride=(KHW, 1)), byte_alignment=128 + ) + sBar = smem.allocate_tensor(cutlass.Int64, cute.make_layout((2,)), byte_alignment=8) + rbar = cutlass.Int32(sBar.iterator.toint()) + wbar = rbar + 8 # the weight rows' bulk copies + rbar_ptr = sBar.iterator # pointer form for the cluster-scope wait below + # This CTA receives rows [me * RPO, me * RPO + RPO) of the cluster's partials from all CL + # CTAs, by st.async on `rbar` (no cluster barrier after the pushes). + rows_me = cutlass.max(cutlass.min(cutlass.Int32(RPO), rows - me * RPO), cutlass.Int32(0)) + if tidx == 0: + _mbar_init_tx(rbar, rows_me * (CL * (NC + 1) * 4)) + _mbar_init_tx(wbar, NC * KHW * 2) + cute.arch.cluster_arrive() + + # 1. Before the wait: the norm weight's slice (its own cp.async group, waited on right + # after the PDL wait), then the CTA's 24 weight rows over its K half. + nw_vec = cute.logical_divide(mNorm, (None, VEC)) + snw_vec = cute.logical_divide(sNw, (None, VEC)) + if tidx < KHW // VEC: + cute.copy( + g2s, + nw_vec[0, (None, (s * hidden + k0) // VEC + tidx)], + snw_vec[0, (None, tidx)], + ) + cute.arch.cp_async_commit_group() + # One bulk copy a weight row (2560 B) into its padded shared row, completing on `wbar`. + col0 = (s * hidden + k0) // VEC + if tidx == 0: + w_base = mW.iterator.toint() + cutlass.Int64(col0 * VEC * 2) + sw_base = cutlass.Int32(sW.iterator.toint()) + for n in cutlass.range_constexpr(NC): + _bulk_g2s( + sw_base + n * (ld * 2), + w_base + cutlass.Int64(n0 + n) * (4 * hidden * 2), + KHW * 2, + wbar, + ) + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.griddepcontrol_wait() + + # 2. The K half's injection, sums of squares, y (bf16) into sA. A thread issues all + # loads of a group of GROUP_IT items before using the first: one L2 round trip a group. + hc_vec = cute.logical_divide(mHc, (None, VEC)) + b_vec = cute.logical_divide(mBlock, (None, VEC)) + sh_vec = cute.logical_divide(mShared, (None, VEC)) + o_vec = cute.logical_divide(mHcOut, (None, VEC)) + sa_vec = cute.logical_divide(sA, (None, VEC)) + items: cutlass.Constexpr = rows * nvec + n_it: cutlass.Constexpr = -(-items // THREADS) + cute.arch.cp_async_wait_group(0) # the norm slice (the weights land on `wbar`) + cute.arch.sync_threads() + nwr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + outr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + yr = cute.make_rmem_tensor(cute.make_layout((VEC,), stride=(1,)), cutlass.BFloat16) + for g0 in cutlass.range_constexpr(0, n_it, GROUP_IT): + g_n: cutlass.Constexpr = min(GROUP_IT, n_it - g0) + xg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + bg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + shg = cute.make_rmem_tensor( + cute.make_layout((g_n, VEC), stride=(VEC, 1)), cutlass.BFloat16 + ) + ig = cute.make_rmem_tensor(cute.make_layout((g_n,), stride=(1,)), cutlass.Float32) + zg = cute.make_rmem_tensor(cute.make_layout((g_n,), stride=(1,)), cutlass.Float32) + for k in cutlass.range_constexpr(g_n): + # Past the last item a thread reads the last item's operands (unused). + c = cutlass.min(tidx + (g0 + k) * THREADS, cutlass.Int32(items - 1)) + r = c // nvec + p = c % nvec + cute.autovec_copy(hc_vec[r, (None, col0 + p)], xg[k, None]) + if const_expr(self.inject): + cute.autovec_copy(b_vec[r, (None, k0 // VEC + p)], bg[k, None]) + ig[k] = mInj[r, s].to(cutlass.Float32) + if const_expr(self.combine): + cute.autovec_copy(sh_vec[r, (None, k0 // VEC + p)], shg[k, None]) + zg[k] = mGate[r, 0].to(cutlass.Float32) + for k in cutlass.range_constexpr(g_n): + c = tidx + (g0 + k) * THREADS + if c < items: + r = c // nvec + p = c % nvec + cute.autovec_copy(snw_vec[0, (None, p)], nwr) + ssum = cutlass.Float32(0.0) + if const_expr(self.inject): + g = ig[k] + if const_expr(self.combine): + # `tl.sigmoid`'s instructions, so the combine (and so `hc_out`) + # is bitwise `qwen4_exp_hc_inject_norm`'s. + sg = _sigmoid(zg[k]) + for v in cutlass.range_constexpr(VEC): + bv = bg[k, v].to(cutlass.Float32) + if const_expr(self.combine): + # The MoE block output, rounded once. + bv = ( + cute.math.fma(sg, shg[k, v].to(cutlass.Float32), bv) + .to(cutlass.BFloat16) + .to(cutlass.Float32) + ) + outr[v] = cute.math.fma(bv, g, xg[k, v].to(cutlass.Float32)).to( + cutlass.BFloat16 + ) + if nc == 0: + cute.autovec_copy(outr, o_vec[r, (None, col0 + p)]) + else: + for v in cutlass.range_constexpr(VEC): + outr[v] = xg[k, v] + for v in cutlass.range_constexpr(VEC): + xv = outr[v].to(cutlass.Float32) + ssum = xv * xv + ssum + yr[v] = (xv * (1.0 + nwr[v].to(cutlass.Float32))).to(cutlass.BFloat16) + cute.autovec_copy(yr, sa_vec[r, (None, p)]) + sSs[r, p] = ssum + _mbar_wait0(wbar) + cute.arch.sync_threads() + + # 3. Warp w: rows x its 160-wide K slice x the 24 columns; the warps' partials summed in + # warp order; the K half's sums of squares per row. + # The warp's K slice: 320-byte offsets, so 16-byte aligned (ldmatrix rows). + a_w = cute.make_tensor( + cute.make_ptr( + cutlass.BFloat16, + sA.iterator.toint() + warp_i * (KW * 2), + cute.AddressSpace.smem, + assumed_align=16, + ), + cute.make_layout((M, KW), stride=(ld, 1)), + ) + b_w = cute.make_tensor( + cute.make_ptr( + cutlass.BFloat16, + sW.iterator.toint() + warp_i * (KW * 2), + cute.AddressSpace.smem, + assumed_align=16, + ), + cute.make_layout((NC, KW), stride=(ld, 1)), + ) + thr = mma.get_slice(lane) + tCsA = thr.partition_A(a_w) + tCrA = thr.make_fragment_A(tCsA) + tCsB = thr.partition_B(b_w) + tCrB = thr.make_fragment_B(tCsB) + # Fragments by ldmatrix (4 / 2 8x8 matrices an instruction). + ldsm_a = cute.make_tiled_copy_A( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 4), cutlass.BFloat16), mma + ) + ldsm_b = cute.make_tiled_copy_B( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 2), cutlass.BFloat16), mma + ) + cpa = ldsm_a.get_slice(lane) + cpb = ldsm_b.get_slice(lane) + cute.copy(ldsm_a, cpa.partition_S(a_w), cpa.retile(tCrA)) + cute.copy(ldsm_b, cpb.partition_S(b_w), cpb.retile(tCrB)) + acc = cute.make_rmem_tensor(thr.partition_shape_C((M, NC)), cutlass.Float32) + acc.fill(0.0) + cute.gemm(mma, acc, tCrA, tCrB, acc) + cute.arch.sync_threads() # every warp's fragments are loaded: sW may be reused + co = thr.partition_C(cute.make_identity_tensor((M, NC))) + for j in cutlass.range_constexpr(cute.size(acc)): + rr, nn = co[j] + sWp[warp_i, rr, nn] = acc[j] + # Sums of squares: 16 threads a row (10 vectors each), then a 16-lane butterfly. + segs: cutlass.Constexpr = THREADS // M + per_seg: cutlass.Constexpr = nvec // segs + rq = tidx // segs + sq = tidx % segs + t = cutlass.Float32(0.0) + if rq < rows: + for p in cutlass.range_constexpr(per_seg): + t = t + sSs[rq, sq * per_seg + p] + t = cute.arch.warp_reduction_sum(t, threads_in_group=segs) + if sq == 0: + if rq < rows: + sSsr[rq] = t + cute.arch.sync_threads() + for i in cutlass.range_constexpr(-(-(M * NC) // THREADS)): + e = tidx + i * THREADS + if e < M * NC: + rr = e // NC + nn = e % NC + t = cutlass.Float32(0.0) + for w in cutlass.range_constexpr(WARPS): + t = t + sWp[w, rr, nn] + sAcc[rr, nn] = t + cute.arch.sync_threads() + + # 4. Push rows (2o, 2o+1) (and their sums of squares) to cluster rank o, slot `me`. + cute.arch.cluster_wait() + red_addr = cutlass.Int32(sRed.iterator.toint()) + per: cutlass.Constexpr = RPO * (NC + 1) + for it in cutlass.range_constexpr(-(-(CL * per) // THREADS)): + c = tidx + it * THREADS + if c < CL * per: + o = c // per + rem = c % per + rl = rem // (NC + 1) + nn = rem % (NC + 1) + rr = o * RPO + rl + if rr < rows: + v = cutlass.Float32(0.0) + if nn < NC: + v = sAcc[rr, nn] + else: + v = sSsr[rr] + dst = red_addr + ((me * RPO + rl) * rld + nn) * 4 + _st_async_f32(_mapa(dst, o), v, _mapa(rbar, o)) + # The cluster's partials for this rank's rows have landed: the peers pushed them by + # st.async, whose complete-tx releases at cluster scope, so a relaxed cluster-scope wait + # followed by an acquire fence restricted to shared::cluster is the ISA's acquire pattern + # for it (and, unlike .acquire.cluster on the wait, costs no L1 invalidate). + while not prims.mbarrier_try_wait_parity( + rbar_ptr, + 0, + scope=prims.MBarrierScope.CLUSTER, + order=prims.MemOrder.RELAXED, + ): + pass + prims.fence_sync_restrict(prims.MemOrder.ACQUIRE) + + # 5. Rank `me` finishes rows (2 me, 2 me + 1): rstd per stream, the logits, low / injection. + for it in cutlass.range_constexpr(-(-(RPO * NC) // THREADS)): + c = tidx + it * THREADS + if c < RPO * NC: + rl = c // NC + nn = c % NC + rr = me * RPO + rl + col = n0 + nn + if rr < rows: + if col < n_cols: + z = cutlass.Float32(0.0) + for st in cutlass.range_constexpr(4): + ssv = sRed[KH * st, rl, NC] + sRed[KH * st + 1, rl, NC] + rs = cute.math.rsqrt(ssv / hidden + eps) + z = (sRed[KH * st, rl, nn] + sRed[KH * st + 1, rl, nn]) * rs + z + z = z * 0.25 + sig = 1.0 / (1.0 + cute.math.exp(-z)) + if col < rank: + mLow[rr, col] = (z * sig).to(cutlass.BFloat16) + else: + mInjOut[rr, col - rank] = (2.0 * sig).to(cutlass.BFloat16) + if nc == 0: + if tidx < RPO * 4: + rl = tidx // 4 + st = tidx % 4 + rr = me * RPO + rl + if rr < rows: + ssv = sRed[KH * st, rl, NC] + sRed[KH * st + 1, rl, NC] + mRstd[rr, st] = cute.math.rsqrt(ssv / hidden + eps) + + +def _dl(t: torch.Tensor, align: int = 16): + return from_dlpack(t, assumed_align=align) + + +@functools.lru_cache(maxsize=None) +def _compiled_first( + hidden, + hc_count, + rank, + n_cols, + n_wrows, + rows, + inject, + combine, + pdl, + hc_stride, + block_stride, + gate_stride, +): + kern = _First2( + hidden=hidden, + rank=rank, + n_cols=n_cols, + rows=rows, + inject=inject, + combine=combine, + pdl=pdl, + ) + dev, bf = "cuda", torch.bfloat16 + width = hidden * hc_count + hc = torch.empty(rows, hc_stride, dtype=bf, device=dev)[:, :width] + block = torch.empty(rows, block_stride, dtype=bf, device=dev)[:, :hidden] + inj = torch.empty(rows, hc_count, dtype=bf, device=dev) + shared = torch.empty(rows, hidden, dtype=bf, device=dev) + gate = torch.empty(rows, gate_stride, dtype=bf, device=dev)[:, :1] + norm = torch.empty(1, width, dtype=bf, device=dev) + w = torch.empty(n_wrows, width, dtype=bf, device=dev) + hc_out = torch.empty(rows, width, dtype=bf, device=dev) + low = torch.empty(rows, rank, dtype=bf, device=dev) + inj_out = torch.empty(rows, hc_count, dtype=bf, device=dev) + rstd = torch.empty(rows, hc_count, dtype=torch.float32, device=dev) + stream = cuda_drv.CUstream(torch.cuda.current_stream().cuda_stream) + return cute_ext.compile( + kern, + _dl(hc), + _dl(block), + _dl(inj, 2), + _dl(shared), + _dl(gate, 2), + _dl(norm), + _dl(w), + _dl(hc_out), + _dl(low), + _dl(inj_out, 2), + _dl(rstd, 4), + cutlass.Float32(1e-6), + stream, + ) + + +def first_rows( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + rank=RANK, +): + """Returns `(hc_out, low [R, rank] bf16, inject_out [R, C] bf16 or None, rstd [R, C] fp32)`; + `n_cols` = rank + C with the injection, rank without (the stack-final mixer).""" + rows, width = hyper_input.shape + hidden = width // hc_count + dev = hyper_input.device + has_inject = block_out is not None + has_shared = shared is not None + hc_out = ( + torch.empty((rows, width), dtype=hyper_input.dtype, device=dev) + if has_inject + else hyper_input + ) + low = torch.empty((rows, rank), dtype=hyper_input.dtype, device=dev) + inj_out = torch.empty((rows, hc_count), dtype=hyper_input.dtype, device=dev) + rstd = torch.empty((rows, hc_count), dtype=torch.float32, device=dev) + weight = first.t() + assert weight.shape[0] >= NC * NCH + blk = block_out if has_inject else hyper_input.new_empty(rows, hidden) + inj = inject if has_inject else hyper_input.new_empty(rows, hc_count) + sh = shared if has_shared else hyper_input.new_empty(rows, hidden) + # The gate logit is a [R, 1] column (the shared expert's gate|up output carries it). + gate = gate_logits.reshape(rows, 1) if has_shared else hyper_input.new_empty(rows, 1) + fn = _compiled_first( + hidden, + hc_count, + rank, + n_cols, + weight.shape[0], + rows, + has_inject, + has_shared, + bool(get_env_enable_pdl()), + hyper_input.stride(0), + blk.stride(0), + gate.stride(0), + ) + stream = cuda_drv.CUstream(torch.cuda.current_stream(dev).cuda_stream) + fn( + _dl(hyper_input), + _dl(blk), + _dl(inj, 2), + _dl(sh), + _dl(gate, 2), + _dl(norm_weight.reshape(1, -1)), + _dl(weight), + _dl(hc_out), + _dl(low), + _dl(inj_out, 2), + _dl(rstd, 4), + cutlass.Float32(eps), + stream, + ) + return hc_out, low, (inj_out if n_cols > rank else None), rstd + + +class _Mix2: + def __init__(self, *, hidden, streams, rank, rows, pdl): + assert streams * BLOCK_H == WARPS * 8 and rows <= M and rank % 16 == 0 + self.hidden, self.streams, self.rank, self.rows, self.pdl = hidden, streams, rank, rows, pdl + self.ld = rank + PAD + + def smem_bytes(self): + def r(n): + return -(-n // 128) * 128 + + nrow = self.streams * BLOCK_H + b = r(nrow * self.ld * 2) + r(M * self.ld * 2) + b += r(M * nrow * 4) + r(M * nrow * 2) + r(M * self.streams * 4) + r(nrow * 4) + return b + 256 + + @cute.jit + def __call__(self, mLow, mRstd, mUp, mHc, mNorm, mMixed, stream: cuda_drv.CUstream): + mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(cutlass.BFloat16, cutlass.Float32, (16, 8, 16)), + cute.make_layout((1, WARPS, 1)), + ) + g2s = cute.make_copy_atom(cpasync.CopyG2SOp(), cutlass.BFloat16, num_bits_per_copy=128) + self.kernel(mLow, mRstd, mUp, mHc, mNorm, mMixed, mma, g2s).launch( + grid=[self.hidden // BLOCK_H, 1, 1], + block=[THREADS, 1, 1], + smem=self.smem_bytes(), + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=2, + ) + + @cute.kernel + def kernel( + self, + mLow, + mRstd, + mUp, + mHc, + mNorm, + mMixed, + mma: cute.TiledMma, + g2s: cute.CopyAtom, + ): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp_i = cute.arch.warp_idx() + lane = cute.arch.lane_idx() + hidden: cutlass.Constexpr = self.hidden + streams: cutlass.Constexpr = self.streams + rank: cutlass.Constexpr = self.rank + rows: cutlass.Constexpr = self.rows + ld: cutlass.Constexpr = self.ld + nrow: cutlass.Constexpr = streams * BLOCK_H + i0 = bidx * BLOCK_H + + smem = cutlass.utils.SmemAllocator() + sU = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((nrow, rank), stride=(ld, 1)), byte_alignment=128 + ) + sL = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((M, rank), stride=(ld, 1)), byte_alignment=128 + ) + sV = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M, nrow), stride=(nrow, 1)), byte_alignment=128 + ) + sX = smem.allocate_tensor( + cutlass.BFloat16, cute.make_layout((M, nrow), stride=(nrow, 1)), byte_alignment=128 + ) + sR = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((M, streams), stride=(streams, 1)), byte_alignment=128 + ) + sG = smem.allocate_tensor( + cutlass.Float32, cute.make_layout((nrow,), stride=(1,)), byte_alignment=128 + ) + + u_vec = cute.logical_divide(mUp, (None, VEC)) + su_vec = cute.logical_divide(sU, (None, VEC)) + pieces: cutlass.Constexpr = nrow * (rank // VEC) + for i in cutlass.range_constexpr(-(-pieces // THREADS)): + c = tidx + i * THREADS + if c < pieces: + n = c // (rank // VEC) + p = c % (rank // VEC) + jrow = (n // BLOCK_H) * hidden + i0 + n % BLOCK_H + cute.copy(g2s, u_vec[jrow, (None, p)], su_vec[n, (None, p)]) + cute.arch.cp_async_commit_group() + if tidx < nrow: + sG[tidx] = 1.0 + mNorm[0, (tidx // BLOCK_H) * hidden + i0 + tidx % BLOCK_H].to( + cutlass.Float32 + ) + # Rows of sL past R are never written (MMA rows are independent; discarded). Before the + # wait the up rows land and every thread loads its B fragments into registers. + if const_expr(self.pdl): + cute.arch.griddepcontrol_launch_dependents() + cute.arch.cp_async_wait_group(0) + cute.arch.sync_threads() + thr = mma.get_slice(tidx) + tCsB = thr.partition_B(sU) + tCrB = thr.make_fragment_B(tCsB) + cute.autovec_copy(tCsB, tCrB) + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + # low rows (16-byte pieces), rstd, this CTA's residual elements. + l_vec = cute.logical_divide(mLow, (None, VEC)) + sl_vec = cute.logical_divide(sL, (None, VEC)) + lp: cutlass.Constexpr = rows * (rank // VEC) + for i in cutlass.range_constexpr(-(-lp // THREADS)): + c = tidx + i * THREADS + if c < lp: + rr = c // (rank // VEC) + p = c % (rank // VEC) + cute.copy(g2s, l_vec[rr, (None, p)], sl_vec[rr, (None, p)]) + cute.arch.cp_async_commit_group() + if tidx < rows * streams: + sR[tidx // streams, tidx % streams] = mRstd[tidx // streams, tidx % streams] + h_vec = cute.logical_divide(mHc, (None, VEC)) + sx_vec = cute.logical_divide(sX, (None, VEC)) + xp: cutlass.Constexpr = rows * (nrow // VEC) + if tidx < xp: + rr = tidx // (nrow // VEC) + q = tidx % (nrow // VEC) + cute.copy( + g2s, + h_vec[ + rr, + ( + None, + ((q * VEC) // BLOCK_H) * (hidden // VEC) + + (i0 + (q * VEC) % BLOCK_H) // VEC, + ), + ], + sx_vec[rr, (None, q)], + ) + cute.arch.cp_async_commit_group() + cute.arch.cp_async_wait_group(0) + cute.arch.sync_threads() + + tCsA = thr.partition_A(sL) + tCrA = thr.make_fragment_A(tCsA) + ldsm_a = cute.make_tiled_copy_A( + cute.make_copy_atom(warp.LdMatrix8x8x16bOp(False, 4), cutlass.BFloat16), mma + ) + cpa = ldsm_a.get_slice(tidx) + cute.copy(ldsm_a, cpa.partition_S(sL), cpa.retile(tCrA)) + acc = cute.make_rmem_tensor(thr.partition_shape_C((M, nrow)), cutlass.Float32) + acc.fill(0.0) + cute.gemm(mma, acc, tCrA, tCrB, acc) + co = thr.partition_C(cute.make_identity_tensor((M, nrow))) + for j in cutlass.range_constexpr(cute.size(acc)): + rr, n = co[j] + if rr < rows: + gate = 1.0 / (1.0 + cute.math.exp(-acc[j])) + x = sX[rr, n].to(cutlass.Float32) + sV[rr, n] = gate * (x * sR[rr, n // BLOCK_H] * sG[n]) + cute.arch.sync_threads() + + for rb in cutlass.range_constexpr(0, rows, WARPS): + r = rb + warp_i + if r < rows: + if lane < BLOCK_H: + tot = cutlass.Float32(0.0) + for t in cutlass.range_constexpr(streams): + tot = tot + sV[r, t * BLOCK_H + lane] + mMixed[r, i0 + lane] = (tot * (1.0 / streams)).to(cutlass.BFloat16) + + +@functools.lru_cache(maxsize=None) +def _compiled_mix(hidden, streams, rank, rows, hc_stride, pdl): + kern = _Mix2(hidden=hidden, streams=streams, rank=rank, rows=rows, pdl=pdl) + dev, bf = "cuda", torch.bfloat16 + width = hidden * streams + low = torch.empty(rows, rank, dtype=bf, device=dev) + rstd = torch.empty(rows, streams, dtype=torch.float32, device=dev) + up = torch.empty(width, rank, dtype=bf, device=dev) + hc = torch.empty(rows, hc_stride, dtype=bf, device=dev)[:, :width] + norm = torch.empty(1, width, dtype=bf, device=dev) + mixed = torch.empty(rows, hidden, dtype=bf, device=dev) + stream = cuda_drv.CUstream(torch.cuda.current_stream().cuda_stream) + return cute_ext.compile( + kern, + _dl(low), + _dl(rstd, 4), + _dl(up), + _dl(hc, 16), + _dl(norm, 2), + _dl(mixed, 2), + stream, + ) + + +def mix_rows(low, rstd, up, hc_out, norm_weight, hc_count): + rows, width = hc_out.shape + hidden = width // hc_count + rank = up.shape[0] + dev, dt = hc_out.device, hc_out.dtype + mixed = torch.empty((rows, hidden), dtype=dt, device=dev) + fn = _compiled_mix(hidden, hc_count, rank, rows, hc_out.stride(0), bool(get_env_enable_pdl())) + stream = cuda_drv.CUstream(torch.cuda.current_stream(dev).cuda_stream) + fn( + _dl(low), + _dl(rstd, 4), + _dl(up.t()), + _dl(hc_out, 16), + _dl(norm_weight.reshape(1, -1), 2), + _dl(mixed, 2), + stream, + ) + return mixed + + +def _reference_first( + hyper_input, block_out, inject, norm_weight, first, hc_count, n_cols, eps, shared, gate_logits +): + """float64: the combine and the injection each rounded once as `qwen4_exp_hc_inject_norm` + does, the grouped RMS, the projection operand `y = x * (1 + w)` rounded once to bf16 (the + MMA's A operand), the logits `z = sum_s rstd_s (y_s @ W_s) / C`, `low = silu(z)` and + `2 sigmoid(z)` each rounded once, rstd in fp32.""" + dt = hyper_input.dtype + rows, width = hyper_input.shape + hidden = width // hc_count + if block_out is None: + hc_out = hyper_input + else: + blk = block_out.double() + if shared is not None: + sg = torch.sigmoid(gate_logits.double().reshape(rows, 1)) + blk = (blk + sg * shared.double()).to(dt).double() + streams = hyper_input.double().unflatten(-1, (hc_count, hidden)) + hc_out = (streams + blk.unsqueeze(-2) * inject.double().unsqueeze(-1)).flatten(-2).to(dt) + x = hc_out.double().unflatten(-1, (hc_count, hidden)) + rstd = torch.rsqrt(x.pow(2).mean(-1) + eps) + y = (x * (1.0 + norm_weight.double().unflatten(-1, (hc_count, hidden)))).to(dt).double() + z = (y * rstd.unsqueeze(-1)).flatten(-2) @ first[:, :n_cols].double() / hc_count + low = torch.nn.functional.silu(z[:, :RANK]).to(dt) + inj = (2.0 * torch.sigmoid(z[:, RANK:])).to(dt) if n_cols > RANK else None + return hc_out, low, inj, rstd.float() + + +class Qwen4ExpHcDecodeRowsFirst(OpWrapper): + """First launch of a 1..16-row decode hyper-connection site: the previous block's injection + (and, with `shared` / `gate_logits`, the MoE block's combine `routed + sigmoid(gate) * shared` + rounded once), the grouped RMS, the first projection over `[down; block_inject]`, the SiLU + bottleneck and the injection weights. Returns `(hc_out, low [R, 320] bf16, inject_out [R, C] + or None, rstd [R, C] fp32)`; `hc_out` *is* `hyper_input` without `block_out`. + + `hc_out` is bitwise `qwen4_exp_hc_inject_norm`'s; `low` and the injection weights are within + one bf16 step of the float64 composition over `y = bf16(x * (1 + w))` (`reference`). + + Fusion boundary: `qwen4_exp_hc_inject_norm` + `qwen4_exp_hc_decode_tc_first`, with the rstd + handed on instead of the `[R, C * H]` normed stream. + """ + + ARCHS = frozenset({Arch.SM_103}) + # spec: rows, site (combine: after a MoE block; inject: after an attention / GDN block or + # the draft front's embedding add; norm: no pending block), cols (324, or 320 for the + # stack-final / draft-head mixer); the checkpoint's geometry, bf16. + CELLS: tuple[Cell, ...] = ( + Cell( + why="1 row after a MoE block: a one-request decode step / draft pass", + spec=dict(rows=1, site="combine", cols=324), + ), + Cell( + why="1 row after an attention / GDN block: a draft pass's MoE site", + spec=dict(rows=1, site="inject", cols=324), + ), + Cell( + why="4 rows, the stack-final mixer after the last MoE block (320 cols)", + spec=dict(rows=4, site="combine", cols=320), + ), + Cell( + why="4 rows, norm only: the stack's first site (324 cols)", + spec=dict(rows=4, site="norm", cols=324), + ), + Cell( + why="16 rows after a MoE block: a 4-request MTP-3 verify step (MAX_ROWS)", + spec=dict(rows=16, site="combine", cols=324), + ), + Cell( + why="16 rows after an attention / GDN block: a 16-request rank graph's draft MoE site", + spec=dict(rows=16, site="inject", cols=324), + ), + Cell( + why="16 rows, norm only, 320 cols (MAX_ROWS)", + spec=dict(rows=16, site="norm", cols=320), + ), + ) + note = """ + Source: `_First2` above (CuTe DSL). Grid (2 K halves, 4 streams, 14 N chunks), cluster + (2, 4, 1); the norm slice and the CTA's 24 weight rows staged before the PDL wait, + dependents released there. The gate logit is read through its row stride (a column of the + shared expert's gate|up output). One compile per row count, variant and operand row + strides (lru_cache, first use: compile before capture). + Not certified: hc_count other than 4, hidden other than 2560, rank other than 320. + """ + + def __call__( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ): + return first_rows( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared, + gate_logits, + ) + + def reference( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ): + """float64 (`_reference_first`).""" + return _reference_first( + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared, + gate_logits, + ) + + def is_valid( + self, + hyper_input, + block_out, + inject, + norm_weight, + first, + hc_count, + n_cols, + eps, + shared=None, + gate_logits=None, + ) -> None: + rows, width = hyper_input.shape + assert 1 <= rows <= MAX_ROWS, f"1..{MAX_ROWS} rows" + assert hyper_input.dtype == torch.bfloat16 and hyper_input.stride(1) == 1 + assert hc_count == 4 and width // hc_count == KH * KHW, "hc_count 4, hidden 2560" + assert n_cols in (RANK, RANK + hc_count), "the bottleneck (+ the injection) columns" + assert first.stride(0) == 1 and first.shape[1] >= NC * NCH, ( + "the padded [width, 384] .t() view" + ) + assert (block_out is None) == (inject is None) + assert (shared is None) == (gate_logits is None) + assert shared is None or block_out is not None, "the combine needs the routed rows" + if block_out is not None: + assert block_out.shape == (rows, width // hc_count) and block_out.stride(1) == 1 + assert inject.is_contiguous() and inject.shape == (rows, hc_count) + if shared is not None: + assert shared.is_contiguous() and shared.shape == block_out.shape + assert gate_logits.numel() == rows and gate_logits.stride(-1) == 1 + for t in (hyper_input, block_out, shared, first): + assert t is None or t.data_ptr() % 16 == 0, "16-byte aligned rows" + assert hyper_input.stride(0) % VEC == 0, "16-byte aligned rows" + assert block_out is None or block_out.stride(0) % VEC == 0, "16-byte aligned rows" + + +class Qwen4ExpHcDecodeRowsMix(OpWrapper): + """Second launch of a 1..16-row decode hyper-connection site: the up projection of `low`, + the sigmoid gate on the normed stream (`hc_out * rstd * (1 + w)`, fp32) and the stream + mean, rounded once. Returns `mixed` `[R, hidden]`. Fusion boundary: + `qwen4_exp_hc_up_gate_mean`, from `hc_out` and its rstd instead of the normed stream. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="1 row: a one-request decode step / draft pass", spec=dict(rows=1)), + Cell(why="16 rows: a 4-request MTP-3 verify step (MAX_ROWS)", spec=dict(rows=16)), + ) + note = """ + Source: `_Mix2` above (CuTe DSL), after `qwen4_exp_hc_decode_rows_first`. The up weight's + 64 rows staged and their B fragments loaded before the PDL wait; dependents released at + the start. + """ + + def __call__(self, low, rstd, up, hc_out, norm_weight, hc_count): + return mix_rows(low, rstd, up, hc_out, norm_weight, hc_count) + + def reference(self, low, rstd, up, hc_out, norm_weight, hc_count): + """float64 mixed, rounded once.""" + hidden = hc_out.shape[1] // hc_count + gate = torch.sigmoid(low.double() @ up.double()) + x = hc_out.double().unflatten(-1, (hc_count, hidden)) * rstd.double().unsqueeze(-1) + normed = x.flatten(-2) * (1.0 + norm_weight.double()) + return (gate * normed).unflatten(-1, (hc_count, hidden)).mean(-2).to(hc_out.dtype) + + def is_valid(self, low, rstd, up, hc_out, norm_weight, hc_count) -> None: + rows = hc_out.shape[0] + assert 1 <= rows <= MAX_ROWS and low.shape == (rows, RANK) and low.is_contiguous() + assert rstd.shape == (rows, hc_count) and rstd.is_contiguous() + assert hc_count == 4 and hc_out.shape[1] == hc_count * KH * KHW, "hc_count 4, hidden 2560" + assert up.shape == (RANK, hc_out.shape[1]) and up.t().is_contiguous(), ( + "up the [R, width] view of a row-major [width, R] weight" + ) + assert hc_out.stride(1) == 1 and hc_out.stride(0) % VEC == 0 + + +qwen4_exp_hc_decode_rows_first = Qwen4ExpHcDecodeRowsFirst() +qwen4_exp_hc_decode_rows_mix = Qwen4ExpHcDecodeRowsMix() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py new file mode 100644 index 000000000000..9bf11039fcd2 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/hc/qwen4_exp_hc_decode_tc_first__decode.py @@ -0,0 +1,794 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hyper-connection mixer first half at decode row counts (1..640) as one tcgen05 split-K +GEMM: `normed @ [down; block_inject]^T` with the SiLU and injection activations in its +epilogue. The NVFP4 default target's kernel; HC is replicated, so only the rows vary. + +A CTA computes a `[128, BN]` partial over `K / KS` (grid z is the M tile): a TMA warp streams +the normed rows (past `T` zero-filled) and the `n_cols` real weight rows (the padding is never +read), one MMA warp accumulates in tensor memory, four epilogue warps push the fp32 partial +into the shared memory of the cluster CTA that reduces those rows; after one cluster barrier +each CTA sums its rows' `KS` partials in K-slice order (deterministic). PDL: the weight +streams before the grid-dependency wait, and the consumer is released at once. +""" + +from __future__ import annotations + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.pipeline as pipeline +import cutlass.utils as utils +import torch +import torch.nn.functional as F +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.kernels.blackwell.custom_pipeline import PipelineTmaUmma, PipelineUmmaAsync +from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Largest row count this entry is certified for: five 128-row MMA tiles. +MAX_ROWS = 640 + +#: Programmatic dependent launch (sm_90+), off with TRTLLM_ENABLE_PDL=0. +_PDL = get_env_enable_pdl() + + +def _asm(ret, args, text, constraints, loc, ip, side_effects=False): + return llvm.inline_asm( + ret, + [a.ir_value(loc=loc, ip=ip) for a in args], + text, + constraints, + has_side_effects=side_effects, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _mapa(addr, rank, *, loc=None, ip=None): + """This CTA's shared-memory address `addr` in cluster CTA `rank`'s window.""" + return cutlass.Int32( + _asm( + T.i32(), + [cutlass.Int32(addr), cutlass.Int32(rank)], + "mapa.shared::cluster.u32 $0, $1, $2;", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sts_v4_f32(addr, a, b, c, d, *, loc=None, ip=None): + _asm( + T.i32(), + [cutlass.Int32(addr)] + [cutlass.Float32(v) for v in (a, b, c, d)], + "{ st.shared.v4.f32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,f,f,f,f", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _bulk_to_peer(dst, src, nbytes, mbar, peer, *, loc=None, ip=None): + """`nbytes` of this CTA's shared memory at `src` into cluster CTA `peer`'s shared + memory at `dst`, completing on `peer`'s mbarrier at `mbar` (`dst` / `mbar` are offsets + in this CTA's window: every CTA has the same layout).""" + _asm( + T.i32(), + [cutlass.Int32(x) for x in (dst, src, nbytes, mbar, peer)], + "{ .reg .b32 rd, rb; mapa.shared::cluster.u32 rd, $1, $5; " + "mapa.shared::cluster.u32 rb, $4, $5; " + "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes " + "[rd], [$2], $3, [rb]; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side_effects=True, + ) + + +@dsl_user_op +def _lds_v4_f32(addr, *, loc=None, ip=None): + """16 bytes of this CTA's shared memory; side-effecting, so it stays behind the + cluster barrier that makes them valid.""" + ty = T.f32() + res = _asm( + llvm.StructType.get_literal([ty] * 4), + [cutlass.Int32(addr)], + "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,r", + loc, + ip, + side_effects=True, + ) + return [cutlass.Float32(llvm.extractvalue(ty, res, [i], loc=loc, ip=ip)) for i in range(4)] + + +@dsl_user_op +def _bf16x2(lo, hi, *, loc=None, ip=None): + """Two fp32 values rounded (RN, as `.to(BFloat16)`) into one bf16x2 word, `lo` low.""" + return cutlass.Int32( + _asm( + T.i32(), + [cutlass.Float32(hi), cutlass.Float32(lo)], + "cvt.rn.bf16x2.f32 $0, $1, $2;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg_v2(addr, a, b, *, loc=None, ip=None): + _asm( + T.i32(), + [cutlass.Int64(addr), cutlass.Int32(a), cutlass.Int32(b)], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side_effects=True, + ) + + +class HcDecodeTcFirstKernel: + """`Qwen4ExpHcDecodeTcFirst`'s kernel. Compile-time: `bn` (MMA N, a multiple of 16), + `ks` (K split = cluster size), `hc`, `rank`, `n_cols`, `k`, `with_inject`; runtime: the + row count (one grid-z M tile per 128 rows) and the pointers.""" + + def __init__(self, bn, ks, hc, rank, n_cols, k, with_inject, epi_cols=16): + assert bn % epi_cols == 0 and bn % 16 == 0 and 16 <= bn <= 256 + assert k % (64 * ks) == 0, "K must split into 64-wide tiles per CTA" + assert rank % 4 == 0 and hc % 4 == 0, "the reduction writes 4-column groups" + self.bn, self.ks, self.hc, self.rank, self.n_cols, self.k = bn, ks, hc, rank, n_cols, k + self.with_inject = with_inject + self.n_tiles = (n_cols + bn - 1) // bn + self.acc_dtype = cutlass.Float32 + self.io_dtype = cutlass.BFloat16 + self.mma_tiler = (128, bn, 1) + self.epi_cols = epi_cols + self.cta_group = tcgen05.CtaGroup.ONE + # The MMA itself is single-CTA: no multicast; the launch cluster is along K only. + self.cluster_shape_mn = (1, 1) + self.epilogue_warp_id = (0, 1, 2, 3) + self.mma_warp_id = 4 + self.tma_warp_id = 5 + self.threads_per_cta = 32 * 6 + self.epilog_sync_bar_id = 1 + self.tmem_alloc_sync_bar_id = 2 + self.num_acc_stage = 1 + # Accumulator rows of a column tile each cluster CTA reduces (the last may own fewer). + self.rows_per = (128 + ks - 1) // ks + # 4-column groups a CTA reduces, at most, per thread (all six warps). + groups = self.rows_per * (bn // 4) + self.red_iters = (groups + self.threads_per_cta - 1) // self.threads_per_cta + + def _setup(self, a_major, b_major): + tiled_mma = utils.sm100.make_trivial_tiled_mma( + self.io_dtype, a_major, b_major, self.acc_dtype, self.cta_group, self.mma_tiler[:2] + ) + inst_k = cute.size(tiled_mma.shape_mnk, mode=[2]) + self.mma_tiler = (self.mma_tiler[0], self.mma_tiler[1], inst_k * 4) + self.cta_tile_shape_mnk = self.mma_tiler + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout((*self.cluster_shape_mn, 1)), (tiled_mma.thr_id.shape,) + ) + self.epi_tile = (self.mma_tiler[0], self.epi_cols) + a_one = utils.sm100.make_smem_layout_a(tiled_mma, self.mma_tiler, self.io_dtype, 1) + b_one = utils.sm100.make_smem_layout_b(tiled_mma, self.mma_tiler, self.io_dtype, 1) + ab_bytes = cute.size_in_bytes(self.io_dtype, a_one) + cute.size_in_bytes( + self.io_dtype, b_one + ) + smem_capacity = utils.get_smem_capacity_in_bytes() + self.k_tiles = self.k // self.mma_tiler[2] // self.ks + # The receive buffer: for each K slice j (row block j) the `rows_per` accumulator rows + # this CTA reduces, as its peer j pushed them, padded by 16 bytes a row. + self.p_stride = self.bn + 4 + p_bytes = self.ks * self.rows_per * self.p_stride * 4 + self.num_ab_stage = max(2, min(self.k_tiles, (smem_capacity - 2048 - p_bytes) // ab_bytes)) + self.n_pre = min(self.num_ab_stage, self.k_tiles) + self.a_smem_layout_staged = utils.sm100.make_smem_layout_a( + tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage + ) + self.b_smem_layout_staged = utils.sm100.make_smem_layout_b( + tiled_mma, self.mma_tiler, self.io_dtype, self.num_ab_stage + ) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + acc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + self.num_tmem_alloc_cols = utils.get_num_tmem_alloc_cols(acc_fake, arch="sm_100") + return tiled_mma + + @cute.jit + def __call__( + self, + m: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + low_t: cute.Tensor, + inj_ptr: cute.Pointer, + stream: cuda.CUstream, + ): + k, ks = self.k, self.ks + a = cute.make_tensor(a_ptr, cute.make_layout((m, k, 1), stride=(k, 1, 0))) + b = cute.make_tensor(b_ptr, cute.make_layout((self.n_cols, k, 1), stride=(k, 1, 0))) + # `low` arrives as a tensor (the TVM-FFI entry takes its stream from a GPU tensor). + low = cute.make_tensor( + low_t.iterator, cute.make_layout((m, self.rank), stride=(self.rank, 1)) + ) + inj = cute.make_tensor(inj_ptr, cute.make_layout((m, self.hc), stride=(self.hc, 1))) + k_major = utils.LayoutEnum.ROW_MAJOR.mma_major_mode() + tiled_mma = self._setup(k_major, k_major) + + a_op = utils.sm100.cluster_shape_to_tma_atom_A(self.cluster_shape_mn, tiled_mma.thr_id) + a_smem = cute.slice_(self.a_smem_layout_staged, (None, None, None, 0)) + tma_atom_a, tma_a = cute.nvgpu.make_tiled_tma_atom_A( + a_op, a, a_smem, self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape + ) + b_op = utils.sm100.cluster_shape_to_tma_atom_B(self.cluster_shape_mn, tiled_mma.thr_id) + b_smem = cute.slice_(self.b_smem_layout_staged, (None, None, None, 0)) + tma_atom_b, tma_b = cute.nvgpu.make_tiled_tma_atom_B( + b_op, b, b_smem, self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape + ) + self.num_tma_load_bytes = cute.size_in_bytes(self.io_dtype, a_smem) + cute.size_in_bytes( + self.io_dtype, b_smem + ) + self.kernel( + tiled_mma, + tma_atom_a, + tma_a, + tma_atom_b, + tma_b, + low, + inj, + m, + self.cluster_layout_vmnk, + self.a_smem_layout_staged, + self.b_smem_layout_staged, + self.epi_tile, + ).launch( + grid=[ks, self.n_tiles, (m + 127) // 128], + block=[self.threads_per_cta, 1, 1], + cluster=[ks, 1, 1], + stream=stream, + use_pdl=_PDL, + ) + + @cute.kernel + def kernel( + self, + tiled_mma: cute.TiledMma, + tma_atom_a: cute.CopyAtom, + mA_mkl: cute.Tensor, + tma_atom_b: cute.CopyAtom, + mB_nkl: cute.Tensor, + mLow: cute.Tensor, + mInj: cute.Tensor, + m: cutlass.Int32, + cluster_layout_vmnk: cute.Layout, + a_smem_layout_staged: cute.ComposedLayout, + b_smem_layout_staged: cute.ComposedLayout, + epi_tile: cute.Tile, + ): + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + # The dependent may start now: it streams its own weight, then waits for this grid. + cute.arch.griddepcontrol_launch_dependents() + if warp_idx == self.tma_warp_id: + cpasync.prefetch_descriptor(tma_atom_a) + cpasync.prefetch_descriptor(tma_atom_b) + tidx, _, _ = cute.arch.thread_idx() + bidx, bidy, bidz = cute.arch.block_idx() + kr = bidx # K slice == rank in the (KS, 1, 1) cluster + nt = bidy # column tile + mt = bidz # 128-row M tile + + @cute.struct + class SharedStorage: + ab_full_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_ab_stage * 2] + acc_full_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2] + tmem_dealloc_mbar: cutlass.Int64 + recv_full: cutlass.Int64 + tmem_holding_buf: cutlass.Int32 + + smem = utils.SmemAllocator() + storage = smem.allocate(SharedStorage) + # Completes once every K slice's partial of this CTA's rows has landed. + if tidx == 0: + cute.arch.mbarrier_init(storage.recv_full, 1) + cute.arch.mbarrier_init_fence() + ab_producer, ab_consumer = PipelineTmaUmma.create( + barrier_storage=storage.ab_full_mbar_ptr.data_ptr(), + num_stages=self.num_ab_stage, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread), + consumer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, 1), + tx_count=self.num_tma_load_bytes, + cta_layout_vmnk=cluster_layout_vmnk, + ).make_participants() + acc_pipeline = PipelineUmmaAsync.create( + barrier_storage=storage.acc_full_mbar_ptr.data_ptr(), + num_stages=self.num_acc_stage, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread), + consumer_group=pipeline.CooperativeGroup( + pipeline.Agent.Thread, len(self.epilogue_warp_id) + ), + cta_layout_vmnk=cluster_layout_vmnk, + ) + tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=self.tmem_alloc_sync_bar_id, + num_threads=32 * len((self.mma_warp_id, *self.epilogue_warp_id)), + ) + tmem = utils.TmemAllocator( + storage.tmem_holding_buf.ptr, + barrier_for_retrieve=tmem_alloc_barrier, + allocator_warp_id=self.epilogue_warp_id[0], + is_two_cta=False, + two_cta_tmem_dealloc_mbar_ptr=storage.tmem_dealloc_mbar.ptr, + ) + pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True) + + sA = smem.allocate_tensor( + element_type=self.io_dtype, + layout=a_smem_layout_staged.outer, + byte_alignment=128, + swizzle=a_smem_layout_staged.inner, + ) + sB = smem.allocate_tensor( + element_type=self.io_dtype, + layout=b_smem_layout_staged.outer, + byte_alignment=128, + swizzle=b_smem_layout_staged.inner, + ) + sP = smem.allocate_tensor( + element_type=self.acc_dtype, + layout=cute.make_layout(self.ks * self.rows_per * self.p_stride), + byte_alignment=16, + ) + p_base = cutlass.Int32(sP.iterator.toint()) + row0 = mt * 128 + nreal = cutlass.min(m - row0, 128) + + gA_mkl = cute.local_tile( + mA_mkl, cute.slice_(self.mma_tiler, (None, 0, None)), (None, None, None) + ) + gB_nkl = cute.local_tile( + mB_nkl, cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) + ) + k0 = kr * self.k_tiles + + thr_mma = tiled_mma.get_slice(0) + tCgA = thr_mma.partition_A(gA_mkl) + tCgB = thr_mma.partition_B(gB_nkl) + tAsA, tAgA = cpasync.tma_partition( + tma_atom_a, + 0, + cute.make_layout(1), + cute.group_modes(sA, 0, 3), + cute.group_modes(tCgA, 0, 3), + ) + tBsB, tBgB = cpasync.tma_partition( + tma_atom_b, + 0, + cute.make_layout(1), + cute.group_modes(sB, 0, 3), + cute.group_modes(tCgB, 0, 3), + ) + tCrA = tiled_mma.make_fragment_A(sA) + tCrB = tiled_mma.make_fragment_B(sB) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + tCtAcc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + + pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn) + # Start-up cluster barrier (split): a peer pushes partials into this CTA only after + # its wait, i.e. after this CTA initialized `recv_full`. + cute.arch.cluster_arrive_relaxed() + if tidx == 0: + cute.arch.mbarrier_arrive_and_expect_tx( + storage.recv_full, self.ks * self._rows_of(kr, nreal) * self.p_stride * 4 + ) + + if warp_idx == self.tma_warp_id: + tAgA_slice = tAgA[(None, mt, None, 0)] + tBgB_slice = tBgB[(None, nt, None, 0)] + ab_producer.reset() + # The first stages' weight tiles stream before the grid-dependency wait; the + # normed rows, the previous kernel's output, after it. + pre = [] + for kk in cutlass.range_constexpr(self.n_pre): + handle = ab_producer.acquire_and_advance() + cute.copy( + tma_atom_b, + tBgB_slice[(None, k0 + kk)], + tBsB[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + pre.append(handle) + cute.arch.griddepcontrol_wait() + for kk in cutlass.range_constexpr(self.n_pre): + handle = pre[kk] + cute.copy( + tma_atom_a, + tAgA_slice[(None, k0 + kk)], + tAsA[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + for k_tile in cutlass.range(self.n_pre, self.k_tiles, 1, unroll=1): + handle = ab_producer.acquire_and_advance() + cute.copy( + tma_atom_a, + tAgA_slice[(None, k0 + k_tile)], + tAsA[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + cute.copy( + tma_atom_b, + tBgB_slice[(None, k0 + k_tile)], + tBsB[(None, handle.index)], + tma_bar_ptr=handle.barrier, + ) + ab_producer.tail() + + if warp_idx == self.mma_warp_id: + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + acc_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_acc_stage + ) + tCtAcc = tCtAcc_base[(None, None, None, acc_producer_state.index)] + ab_consumer.reset() + peek = ab_consumer.try_wait() + acc_pipeline.producer_acquire(acc_producer_state) + tiled_mma.set(tcgen05.Field.ACCUMULATE, False) + for k_tile in cutlass.range(0, self.k_tiles, 1, unroll=1): + handle = ab_consumer.wait_and_advance(peek) + num_kblocks = cute.size(tCrA, mode=[2]) + for kb in cutlass.range(num_kblocks, unroll_full=True): + kcrd = (None, None, kb, handle.index) + cute.gemm(tiled_mma, tCtAcc, tCrA[kcrd], tCrB[kcrd], tCtAcc) + tiled_mma.set(tcgen05.Field.ACCUMULATE, True) + handle.release() + peek = cutlass.Boolean(1) + if handle.count + 1 < self.k_tiles: + peek = ab_consumer.try_wait() + acc_pipeline.producer_commit(acc_producer_state) + acc_producer_state.advance() + acc_pipeline.producer_tail(acc_producer_state) + + if warp_idx < self.mma_warp_id: + tmem.allocate(self.num_tmem_alloc_cols) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + acc_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_acc_stage + ) + copy_atom_t2r = utils.sm100.get_tmem_load_op( + self.cta_tile_shape_mnk, + utils.LayoutEnum.ROW_MAJOR, + self.acc_dtype, + self.acc_dtype, + epi_tile, + False, + ) + tAcc_epi = cute.flat_divide(tCtAcc_base[((None, None), 0, 0, None)], epi_tile) + tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + tTR_tAcc_base = thr_copy_t2r.partition_S(tAcc_epi) + tTR_cP = thr_copy_t2r.partition_D( + cute.flat_divide(cute.make_identity_tensor((128, self.bn)), epi_tile) + ) + tTR_rAcc = cute.make_rmem_tensor(tTR_cP[(None, None, None, 0, 0)].shape, self.acc_dtype) + tTR_tAcc = tTR_tAcc_base[(None, None, None, None, None, acc_consumer_state.index)] + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + # 32dp32b: warp w reads TMEM lanes [32w, 32w + 32), so this thread holds accumulator + # row `tidx`. Rows are staged in row order in the operand ring (dead once the + # accumulator is complete), so each owner's rows are one contiguous block. + send_base = cutlass.Int32(sA.iterator.toint()) + acc_pipeline.consumer_wait(acc_consumer_state) + for sub in cutlass.range_constexpr(self.bn // self.epi_cols): + cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, sub)], tTR_rAcc) + if tidx < nreal: + for q in cutlass.range_constexpr(self.epi_cols // 4): + _sts_v4_f32( + send_base + (tidx * self.p_stride + sub * self.epi_cols + 4 * q) * 4, + tTR_rAcc[4 * q], + tTR_rAcc[4 * q + 1], + tTR_rAcc[4 * q + 2], + tTR_rAcc[4 * q + 3], + ) + cute.arch.fence_proxy("async.shared", space="cta") + with cute.arch.elect_one(): + acc_pipeline.consumer_release(acc_consumer_state) + acc_consumer_state.advance() + + cute.arch.barrier( + barrier_id=self.epilog_sync_bar_id, + number_of_threads=32 * len(self.epilogue_warp_id), + ) + tmem.relinquish_alloc_permit() + tmem.free(tmem_ptr) + # Peers' `recv_full` are initialized (the start-up barrier; warp-aligned wait). + cute.arch.cluster_wait() + if tidx == 0: + # One bulk copy per owner (this CTA included): its rows, into its block `kr`. + for o in cutlass.range_constexpr(self.ks): + rows_o = self._rows_of(o, nreal) + if rows_o > 0: + _bulk_to_peer( + p_base + kr * self.rows_per * self.p_stride * 4, + send_base + o * self.rows_per * self.p_stride * 4, + rows_o * self.p_stride * 4, + storage.recv_full.toint(), + o, + ) + else: + cute.arch.cluster_wait() + + # Every K slice's partial of this CTA's rows is local once `recv_full` completes; + # no DSMEM access follows (a CTA may exit once its own copies have read their source). + cute.arch.mbarrier_wait(storage.recv_full, 0) + + # This CTA's rows [kr * rows_per, ...) of the tile's real rows, in 4-column groups, + # all 6 warps; an element's `KS` partials summed in K-slice order (deterministic). + r_lo = kr * self.rows_per + mine = self._rows_of(kr, nreal) + g4 = self.bn // 4 + groups = mine * g4 + inv_hc = cutlass.Float32(1.0 / self.hc) + low_base = mLow.iterator.toint() + inj_base = mInj.iterator.toint() + for it in cutlass.range_constexpr(self.red_iters): + g = tidx + it * self.threads_per_cta + if g < groups: + rl = g // g4 + c = (g - rl * g4) * 4 + src = p_base + (rl * self.p_stride + c) * 4 + parts = [ + _lds_v4_f32(src + j * self.rows_per * self.p_stride * 4) for j in range(self.ks) + ] + n = nt * self.bn + c + row = cutlass.Int64(row0 + r_lo + rl) + lows, injs = [], [] + for e in cutlass.range_constexpr(4): + acc = parts[0][e] + for j in cutlass.range_constexpr(1, self.ks): + acc = acc + parts[j][e] + z = acc * inv_hc + sg = cutlass.Float32(1.0) / (cutlass.Float32(1.0) + cute.math.exp(-z)) + lows.append(z * sg) + injs.append(cutlass.Float32(2.0) * sg) + if n < self.rank: + _stg_v2( + low_base + (row * self.rank + n) * 2, + _bf16x2(lows[0], lows[1]), + _bf16x2(lows[2], lows[3]), + ) + if cutlass.const_expr(self.with_inject): + if n >= self.rank and n < self.rank + self.hc: + _stg_v2( + inj_base + (row * self.hc + n - self.rank) * 2, + _bf16x2(injs[0], injs[1]), + _bf16x2(injs[2], injs[3]), + ) + # No CTA may exit (releasing its shared memory) while a peer's push out of it is still + # reading it: every CTA arrives only after all pushes into it landed, so the cluster + # barrier completes once every push has read its source. + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + + def _rows_of(self, owner, nreal): + """Real rows of the tile that cluster CTA `owner` reduces (0 past the real rows).""" + lo = owner * self.rows_per + hi = cutlass.min(lo + self.rows_per, 128) + return cutlass.max(cutlass.min(nreal, hi) - lo, 0) + + +_COMPILED: dict = {} + + +def _launcher(bn, ks, hc, rank, n_cols, k, with_inject, ptrs): + """Compile once per static configuration (the row count is a runtime argument); the + entry launches on the caller's current stream. `ptrs`: `(normed, weight, low, inject)`.""" + key = (bn, ks, hc, rank, n_cols, k, with_inject) + fn = _COMPILED.get(key) + if fn is None: + kern = HcDecodeTcFirstKernel(bn, ks, hc, rank, n_cols, k, with_inject) + gmem = cute.AddressSpace.gmem + cptrs = [ + make_ptr(cutlass.BFloat16, ptrs[0], gmem, assumed_align=16), + make_ptr(cutlass.BFloat16, ptrs[1], gmem, assumed_align=16), + from_dlpack(ptrs[2], assumed_align=16).mark_layout_dynamic(leading_dim=1), + make_ptr(cutlass.BFloat16, ptrs[3], gmem, assumed_align=2), + ] + fn = cute.compile( + kern, + MAX_ROWS, + *cptrs, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + +def hc_decode_tc_first(normed, first, rank, hc_count, with_inject, bn, ks): + """Launch with an explicit tile (`bn` columns, `ks`-way K split). `first` is + the `[K, >= n_cols]` transposed view of the row-major weight, as + `cublas_mm(normed, first)` takes it. Returns `(low, inject)`.""" + rows, k = normed.shape + n_cols = rank + (hc_count if with_inject else 0) + dev, dt = normed.device, normed.dtype + low = torch.empty((rows, rank), dtype=dt, device=dev) + inject = torch.empty((rows, hc_count), dtype=dt, device=dev) if with_inject else None + if rows > 0: + ptrs = ( + normed.data_ptr(), + first.data_ptr(), + low, + (inject if with_inject else low).data_ptr(), + ) + _launcher(bn, ks, hc_count, rank, n_cols, k, with_inject, ptrs)(rows, *ptrs) + return low, inject + + +#: Tile `(BN, KS)` by row count: 48 weight rows per CTA (7 tiles cover the 324 real rows) +#: and a `KS`-way K split, one cluster per column tile, so the grid stays about one wave of +#: one CTA an SM (70 to 140 CTAs). +_TILES = ( + (4, (48, 10)), + (128, (48, 16)), + (256, (48, 8)), + (384, (48, 5)), + (512, (48, 4)), + (MAX_ROWS, (48, 4)), +) +#: Every `_TILES` K split divides K into 64-wide tiles per CTA when K divides by this. +_K_ALIGN = 64 * 80 + + +def _tile(rows: int) -> tuple[int, int]: + return next(t for hi, t in _TILES if rows <= hi) + + +def precompile(k: int, rank: int, hc_count: int, device) -> None: + """Compile every `(tile, with_inject)` variant for a `[*, k]` stream without launching, + so no serving step stalls on a compile. The placeholder operands' contents are unread.""" + normed = torch.empty((MAX_ROWS, k), dtype=torch.bfloat16, device=device) + first = torch.empty((rank + hc_count, k), dtype=torch.bfloat16, device=device) + low = torch.empty((MAX_ROWS, rank), dtype=torch.bfloat16, device=device) + ptrs = (normed.data_ptr(), first.data_ptr(), low, low.data_ptr()) + for _, (bn, ks) in _TILES: + for with_inject in (True, False): + n_cols = rank + (hc_count if with_inject else 0) + _launcher(bn, ks, hc_count, rank, n_cols, k, with_inject, ptrs) + + +class Qwen4ExpHcDecodeTcFirst(OpWrapper): + """With `R = rank`, `C = hc_count`, `normed` the `[T, K]` grouped-normed + stream and `first` the `[K, >= R (+ C)]` transposed view of the load-time + `[down; block_inject; 0]` weight, `L = normed @ first` in fp32:: + + low = silu(L[:, :R] / C) # [T, R], normed's dtype + inject = 2 * sigmoid(L[:, R : R + C] / C) # [T, C], only with_inject + + fp32 accumulation (a fixed-order sum of the K-split partials), each activation rounded + once. Returns fresh `(low, inject)`; `inject` is None exactly when `with_inject` is + False. Fusion boundary: `cublas_mm(normed, first, None, fp32)` + `qwen4_exp_hc_lowrank_act`. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: rows, inject (True); the checkpoint's geometry (hc 4, rank 320, the padded + # [384, K] weight, bf16). With inject one cell per `_TILES` bucket at partial and full + # M tiles; without, three. + CELLS: tuple[Cell, ...] = ( + Cell( + why="1 row: a one-request decode step without MTP (tp1..dep4), a draft pass", + spec=dict(rows=1), + ), + Cell( + why="16 rows: ll_tep4's 4-request MTP-3 verify step; a dep4 rank's draft pass", + spec=dict(rows=16), + ), + Cell( + why="128 rows: ht_dep4's 32-request rank graph cap (MTP-3 verify), one M tile", + spec=dict(rows=128), + ), + Cell(why="129 rows: a 1-row second M tile", spec=dict(rows=129)), + Cell( + why="288 rows: tep2 gen 96-request MTP-2 verify step, three M tiles", + spec=dict(rows=288), + ), + Cell( + why="480 rows: tep2 gen 160-request MTP-2 verify step, four M tiles", + spec=dict(rows=480), + ), + Cell(why="640 rows: MAX_ROWS, five full M tiles", spec=dict(rows=640)), + Cell( + why="stack-final mixer: 320 cols, no inject, 200 rows (two M tiles)", + spec=dict(rows=200, inject=False), + ), + Cell( + why="stack-final mixer: 320 cols, no inject, 384 rows (tep2 gen b128 verify)", + spec=dict(rows=384, inject=False), + ), + Cell( + why="MTP draft head mixer: 320 cols, no inject, 1 row (ll draft step)", + spec=dict(rows=1, inject=False), + ), + ) + note = """ + Source: `HcDecodeTcFirstKernel` above (CuTe DSL). Preconditions (bf16): `normed` `[T, K]` + contiguous, T <= 640; `first` the `.t()` view of a row-major `[N, K]` weight with + N >= R (+ C); 16-byte aligned; K a multiple of 64 * 80. Only the R (+ C) real weight rows + are read. One CTA an SM (~227 KB shared memory); the partials meet in distributed shared + memory, no global workspace. PDL off with TRTLLM_ENABLE_PDL=0. Graph-capturable (current + stream, no host sync, no persistent state); compiles once per (BN, KS, C, R, N, K, + inject), so call `precompile` before capture. + Certified on GB300 at the cells' rows, graph replay bitwise the eager launch. + Not certified: hc_count other than 4, an unpadded weight (N = R + C), zero rows. + """ + + def __call__( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ): + return hc_decode_tc_first( + normed, first, rank, hc_count, with_inject, *_tile(normed.shape[0]) + ) + + def reference( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ): + """float64 projection and activations, one rounding per output.""" + n_cols = rank + (hc_count if with_inject else 0) + lg = normed.double() @ first[:, :n_cols].double() + low = F.silu(lg[:, :rank] / hc_count).to(normed.dtype) + inject = None + if with_inject: + inject = (2.0 * torch.sigmoid(lg[:, rank:n_cols] / hc_count)).to(normed.dtype) + return low, inject + + def is_valid( + self, + normed: torch.Tensor, + first: torch.Tensor, + rank: int, + hc_count: int, + with_inject: bool, + ) -> None: + rows, k = normed.shape + n_cols = rank + (hc_count if with_inject else 0) + assert normed.dtype == first.dtype == torch.bfloat16, "bf16 operands" + assert rows <= MAX_ROWS, f"at most {MAX_ROWS} rows (five MMA tiles)" + assert normed.stride() == (k, 1), "normed contiguous [T, K]" + assert first.shape[0] == k and first.shape[1] >= n_cols, "first [K, >= n_cols]" + assert first.stride() == (1, k), "first must be the .t() view of a row-major [N, K] weight" + assert k % _K_ALIGN == 0, f"K must be a multiple of {_K_ALIGN}" + for t in (normed, first): + assert t.data_ptr() % 16 == 0, "16-byte aligned operands" + + +qwen4_exp_hc_decode_tc_first = Qwen4ExpHcDecodeTcFirst() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py new file mode 100644 index 000000000000..3b33c577d312 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/blockwise_gemm_grouped.py @@ -0,0 +1,2076 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is copied and modified from cutlass example https://github.com/NVIDIA/cutlass/blob/main/examples/python/CuTeDSL/blackwell/blockwise_gemm/blockwise_gemm.py + +"""The target's copy of the stock CuTe DSL `Sm100BlockwiseGemmKernel` +(`tensorrt_llm/_torch/kernels/blackwell/blockwise_gemm/blockwise_gemm.py`), reduced to the +contiguous grouped mode `moe_prefill_grouped__prefill` runs (1-CTA instructions, fp32 +accumulator): + +- A / SFA / C are M tiles of rows back to back, each tile one group's rows; B / SFB are batched by + group and `tile_group[l]` names tile l's group; tiles from the device scalar `num_tiles[0]` on + are skipped, so a launch over an upper bound of tiles needs no host synchronization; +- with `epi_quant=True` (`wrapper_grouped_quant`), C holds each row's 128-column block as e4m3 + codes and `c_scale` its fp32 scale instead of the accumulator. + +C = (SFA * A) * (SFB * B), fp8 e4m3 A / B, fp32 1x128 SFA and 128x128 SFB. Warp roles: a scheduler +warp, a TMA warp (A / B), a scale warp (SFA / SFB through cp.async), an MMA warp (tcgen05.mma into +TMEM), four acc-update warps (scale each k tile's partial and sum it in registers, then store the +sum back to TMEM) and four epilogue warps (TMEM -> registers -> SMEM -> TMA store). + +Constraints: MMA tiler M 64 / 128 / 256, N 128 (the SFB granularity); cluster M / N powers of 2, +at most 16 CTAs; the contiguous dimension of A / B / C 16-byte aligned. +""" + +import math +from typing import Optional, Tuple, Type, Union + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.pipeline as pipeline +import cutlass.utils as utils +import cutlass.utils.blackwell_helpers as sm100_utils +import torch +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import from_dlpack, make_ptr +from cutlass.cutlass_dsl import T, dsl_user_op +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait + +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper, true_fp32_matmul + +from .fp8_block_scale_moe_runner import dequantize_block_weight, requantize_1x128 + +#: Scale block edge: 1x128 SFA, 128x128 SFB. +BLOCK = 128 + + +@dsl_user_op +def _div_rn(a, b, *, loc=None, ip=None): + """IEEE fp32 division (div.rn.f32), as Triton's `tl.math.div_rn`.""" + return cutlass.Float32( + llvm.inline_asm( + T.f32(), + [ + cutlass.Float32(a).ir_value(loc=loc, ip=ip), + cutlass.Float32(b).ir_value(loc=loc, ip=ip), + ], + "div.rn.f32 $0, $1, $2;", + "=f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +class Sm100BlockwiseGemmKernel: + """The grouped blockwise FP8 GEMM; see the module docstring for the operand layouts.""" + + def __init__( + self, + mma_tiler_mn: Tuple[int, int], + cluster_shape_mn: Tuple[int, int], + epi_quant: bool = False, + ): + """`epi_quant` (128-column tiles only): store each tile row's 1x128 e4m3 round trip + instead of the accumulator: C holds the codes `acc * (448 / amax)` and + `c_scale[n_tile, row]` the fp32 scale `amax / 448` (amax floored at 1e-10).""" + self.epi_quant = epi_quant + assert not epi_quant or mma_tiler_mn[1] == 128 + self.acc_dtype: Type[cutlass.Numeric] = cutlass.Float32 + self.cluster_shape_mn = cluster_shape_mn + # K is set in _setup_attributes. + self.mma_tiler = (*mma_tiler_mn, 1) + self.cta_group = tcgen05.CtaGroup.ONE + + self.occupancy = 1 + self.acc_update_warp_id = (0, 1, 2, 3) + self.epilog_warp_id = (4, 5, 6, 7) + self.mma_warp_id = 8 + self.tma_warp_id = 9 + self.scale_warp_id = 10 + self.sched_warp_id = 11 + self.threads_per_warp = 32 + self.threads_per_cta = self.threads_per_warp * len( + ( + *self.acc_update_warp_id, + *self.epilog_warp_id, + self.mma_warp_id, + self.tma_warp_id, + self.scale_warp_id, + self.sched_warp_id, + ) + ) + self.threads_wo_sched = self.threads_per_warp * len( + ( + *self.acc_update_warp_id, + *self.epilog_warp_id, + self.mma_warp_id, + self.tma_warp_id, + self.scale_warp_id, + ) + ) + self.num_regs_uniform_warps = 64 + self.num_regs_sched_warps = 64 + self.num_regs_epilogue_warps = 216 + self.num_regs_acc_update_warps = 216 + + self.epilog_sync_barrier = pipeline.NamedBarrier( + barrier_id=1, + num_threads=32 * len(self.epilog_warp_id), + ) + self.tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=2, + num_threads=32 + * len((self.mma_warp_id, *self.epilog_warp_id, *self.acc_update_warp_id)), + ) + self.sched_sync_barrier = pipeline.NamedBarrier( + barrier_id=3, + num_threads=self.threads_per_warp, + ) + self.num_smem_capacity = utils.get_smem_capacity_in_bytes("sm_100") + # TMEM column of the acc-update warps' summed accumulator (the epilogue's source). + self.tmem_final_offset = 384 + + def _setup_attributes(self): + """Tile shapes, stage counts and SMEM layouts that depend on the operands' dtypes and + majorness (set by `__call__`).""" + tiled_mma = sm100_utils.make_trivial_tiled_mma( + self.a_dtype, + self.a_major_mode, + self.b_major_mode, + self.acc_dtype, + self.cta_group, + self.mma_tiler[:2], + ) + + mma_inst_shape_k = cute.size(tiled_mma.shape_mnk, mode=[2]) + mma_inst_tile_k = 4 + self.mma_tiler = ( + self.mma_tiler[0], + self.mma_tiler[1], + mma_inst_shape_k * mma_inst_tile_k, + ) + self.cta_tile_shape_mnk = ( + self.mma_tiler[0] // cute.size(tiled_mma.thr_id.shape), + self.mma_tiler[1], + self.mma_tiler[2], + ) + + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout((*self.cluster_shape_mn, 1)), + (tiled_mma.thr_id.shape,), + ) + + self.scale_granularity_m = 1 + self.scale_granularity_n = 128 + self.scale_granularity_k = 128 + self.scale_m_per_tile = self.cta_tile_shape_mnk[0] // self.scale_granularity_m + self.scale_n_per_tile = self.cta_tile_shape_mnk[1] // self.scale_granularity_n + self.scale_k_per_tile = self.cta_tile_shape_mnk[2] // self.scale_granularity_k + + if self.scale_k_per_tile != 1: + raise ValueError("scale_k_per_tile must be 1") + if self.scale_m_per_tile != self.cta_tile_shape_mnk[0]: + raise ValueError("scale_m_per_tile must be cta_tile_m") + if self.scale_n_per_tile != 1: + raise ValueError("scale_n_per_tile must be 1") + + self.num_mcast_ctas_a = cute.size(self.cluster_layout_vmnk.shape[2]) + self.num_mcast_ctas_b = cute.size(self.cluster_layout_vmnk.shape[1]) + self.is_a_mcast = self.num_mcast_ctas_a > 1 + self.is_b_mcast = self.num_mcast_ctas_b > 1 + + self.epi_tile = sm100_utils.compute_epilogue_tile_shape( + self.cta_tile_shape_mnk, + False, + self.c_layout, + self.c_dtype, + ) + + ( + self.num_acc_stage, + self.num_ab_stage, + self.num_c_stage, + self.num_scale_stage, + self.num_tile_stage, + ) = self._compute_stages( + tiled_mma, + self.mma_tiler, + self.a_dtype, + self.b_dtype, + self.epi_tile, + self.c_dtype, + self.c_layout, + self.sfa_dtype, + self.sfb_dtype, + self.scale_m_per_tile * self.scale_k_per_tile, + self.scale_n_per_tile * self.scale_k_per_tile, + self.num_smem_capacity, + self.occupancy, + ) + + self.a_smem_layout_staged = sm100_utils.make_smem_layout_a( + tiled_mma, + self.mma_tiler, + self.a_dtype, + self.num_ab_stage, + ) + self.b_smem_layout_staged = sm100_utils.make_smem_layout_b( + tiled_mma, + self.mma_tiler, + self.b_dtype, + self.num_ab_stage, + ) + self.c_smem_layout_staged = sm100_utils.make_smem_layout_epi( + self.c_dtype, + self.c_layout, + self.epi_tile, + self.num_c_stage, + ) + self.sfa_smem_layout_staged = cute.make_layout( + ( + (self.scale_granularity_m, self.scale_m_per_tile), + (self.scale_granularity_k, self.scale_k_per_tile), + self.num_scale_stage, + ), + stride=( + (0, self.scale_k_per_tile), + (0, 1), + self.scale_k_per_tile * self.scale_m_per_tile, + ), + ) + self.sfb_smem_layout_staged = cute.make_layout( + ( + (self.scale_granularity_n, self.scale_n_per_tile), + (self.scale_granularity_k, self.scale_k_per_tile), + self.num_scale_stage, + ), + stride=( + (0, self.scale_k_per_tile), + (0, 1), + self.scale_k_per_tile * self.scale_n_per_tile, + ), + ) + self.num_tmem_alloc_cols = 512 + + @cute.jit + def __call__( + self, + a: cute.Tensor, + b: cute.Tensor, + c: cute.Tensor, + sfa: cute.Tensor, + sfb: cute.Tensor, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + tile_group: Optional[cute.Tensor] = None, + num_tiles: Optional[cute.Tensor] = None, + c_scale: Optional[cute.Tensor] = None, + ): + """Build the TMA atoms, grid and shared storage for these operands and launch `kernel` + on `stream`. Called through `wrapper_grouped` / `wrapper_grouped_quant`.""" + self.a_dtype: Type[cutlass.Numeric] = a.element_type + self.b_dtype: Type[cutlass.Numeric] = b.element_type + self.c_dtype: Type[cutlass.Numeric] = c.element_type + self.sfa_dtype: Type[cutlass.Numeric] = sfa.element_type + self.sfb_dtype: Type[cutlass.Numeric] = sfb.element_type + self.a_major_mode = utils.LayoutEnum.from_tensor(a).mma_major_mode() + self.b_major_mode = utils.LayoutEnum.from_tensor(b).mma_major_mode() + self.c_layout = utils.LayoutEnum.from_tensor(c) + + if cutlass.const_expr(self.a_dtype != self.b_dtype): + raise TypeError(f"Type must match: {self.a_dtype} != {self.b_dtype}") + + self._setup_attributes() + + tiled_mma = sm100_utils.make_trivial_tiled_mma( + self.a_dtype, + self.a_major_mode, + self.b_major_mode, + self.acc_dtype, + self.cta_group, + self.mma_tiler[:2], + ) + atom_thr_size = cute.size(tiled_mma.thr_id.shape) + + a_op = self._get_tma_atom_kind(atom_thr_size, self.is_a_mcast) + a_smem_layout = cute.slice_(self.a_smem_layout_staged, (None, None, None, 0)) + tma_atom_a, tma_tensor_a = cute.nvgpu.make_tiled_tma_atom_A( + a_op, + a, + a_smem_layout, + self.mma_tiler, + tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=(cutlass.TFloat32 if a.element_type is cutlass.Float32 else None), + ) + + b_op = self._get_tma_atom_kind(atom_thr_size, self.is_b_mcast) + b_smem_layout = cute.slice_(self.b_smem_layout_staged, (None, None, None, 0)) + tma_atom_b, tma_tensor_b = cute.nvgpu.make_tiled_tma_atom_B( + b_op, + b, + b_smem_layout, + self.mma_tiler, + tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=(cutlass.TFloat32 if b.element_type is cutlass.Float32 else None), + ) + + a_copy_size = cute.size_in_bytes(self.a_dtype, a_smem_layout) + b_copy_size = cute.size_in_bytes(self.b_dtype, b_smem_layout) + self.num_tma_load_bytes = (a_copy_size + b_copy_size) * atom_thr_size + + c_cta_v_layout = cute.composition(cute.make_identity_layout(c.shape), self.epi_tile) + epi_smem_layout = cute.slice_(self.c_smem_layout_staged, (None, None, 0)) + tma_atom_c, tma_tensor_c = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileS2GOp(), + c, + epi_smem_layout, + c_cta_v_layout, + ) + + tensor_sfa = cute.make_tensor( + sfa.iterator, + cute.make_layout( + ( + (self.scale_granularity_m, sfa.shape[0]), + (self.scale_granularity_k, sfa.shape[1]), + sfa.shape[2], + ), + stride=( + (0, sfa.layout.stride[0]), + (0, sfa.layout.stride[1]), + sfa.layout.stride[2], + ), + ), + ) + tensor_sfb = cute.make_tensor( + sfb.iterator, + cute.make_layout( + ( + (self.scale_granularity_n, sfb.shape[0]), + (self.scale_granularity_k, sfb.shape[1]), + sfb.shape[2], + ), + stride=( + (0, sfb.layout.stride[0]), + (0, sfb.layout.stride[1]), + sfb.layout.stride[2], + ), + ), + ) + + self.tile_sched_params, grid = self._compute_grid( + c, self.cta_tile_shape_mnk, self.cluster_shape_mn, max_active_clusters + ) + + self.buffer_align_bytes = 1024 + + c_smem_size = cute.cosize(self.c_smem_layout_staged.outer) + + @cute.struct + class SharedStorage: + sInfo: cute.struct.Align[ + cute.struct.MemRange[cutlass.Int32, 4 * self.num_tile_stage], + 1, + ] + ab_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_ab_stage * 2] + scale_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_scale_stage * 2] + acc_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_acc_stage * 2] + tile_info_mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.num_tile_stage * 2] + epi_mbar_ptr: cute.struct.MemRange[cutlass.Int64, 1 * 2] + tmem_dealloc_mbar_ptr: cutlass.Int64 + tmem_holding_buf: cutlass.Int32 + sC: cute.struct.Align[ + cute.struct.MemRange[ + self.c_dtype, + c_smem_size, + ], + self.buffer_align_bytes, + ] + sA: cute.struct.Align[ + cute.struct.MemRange[self.a_dtype, cute.cosize(self.a_smem_layout_staged.outer)], + self.buffer_align_bytes, + ] + sB: cute.struct.Align[ + cute.struct.MemRange[self.b_dtype, cute.cosize(self.b_smem_layout_staged.outer)], + self.buffer_align_bytes, + ] + sSFA: cute.struct.Align[ + cute.struct.MemRange[self.sfa_dtype, cute.cosize(self.sfa_smem_layout_staged)], + self.buffer_align_bytes, + ] + sSFB: cute.struct.Align[ + cute.struct.MemRange[self.sfb_dtype, cute.cosize(self.sfb_smem_layout_staged)], + self.buffer_align_bytes, + ] + + self.shared_storage = SharedStorage + + self.kernel( + tiled_mma, + tma_atom_a, + tma_tensor_a, + tma_atom_b, + tma_tensor_b, + tma_atom_c, + tma_tensor_c, + tensor_sfa, + tensor_sfb, + self.cluster_layout_vmnk, + self.a_smem_layout_staged, + self.b_smem_layout_staged, + self.c_smem_layout_staged, + self.sfa_smem_layout_staged, + self.sfb_smem_layout_staged, + self.epi_tile, + self.tile_sched_params, + tile_group, + num_tiles, + c_scale, + ).launch( + grid=grid, + block=[self.threads_per_cta, 1, 1], + cluster=(*self.cluster_shape_mn, 1), + smem=self.shared_storage.size_in_bytes(), + stream=stream, + min_blocks_per_mp=1, + ) + return + + @cute.kernel + def kernel( + self, + tiled_mma: cute.TiledMma, + tma_atom_a: cute.CopyAtom, + mA_mkl: cute.Tensor, + tma_atom_b: cute.CopyAtom, + mB_nkl: cute.Tensor, + tma_atom_c: cute.CopyAtom, + mC_mnl: cute.Tensor, + mSFA_mkl: cute.Tensor, + mSFB_nkl: cute.Tensor, + cluster_layout_vmnk: cute.Layout, + a_smem_layout_staged: cute.ComposedLayout, + b_smem_layout_staged: cute.ComposedLayout, + c_smem_layout_staged: Union[cute.Layout, cute.ComposedLayout, None], + sfa_smem_layout_staged: cute.Layout, + sfb_smem_layout_staged: cute.Layout, + epi_tile: cute.Tile, + tile_sched_params: utils.PersistentTileSchedulerParams, + tile_group: Optional[cute.Tensor], + num_tiles: Optional[cute.Tensor], + mCS: Optional[cute.Tensor], + ): + """The persistent warp-specialized kernel; `mCS` is `c_scale` (`epi_quant` only).""" + warp_idx = cute.arch.warp_idx() + warp_idx = cute.arch.make_warp_uniform(warp_idx) + lane_idx = cute.arch.lane_idx() + + if warp_idx == self.tma_warp_id: + cpasync.prefetch_descriptor(tma_atom_a) + cpasync.prefetch_descriptor(tma_atom_b) + cpasync.prefetch_descriptor(tma_atom_c) + + bidx, bidy, bidz = cute.arch.block_idx() + mma_tile_coord_v = bidx % cute.size(tiled_mma.thr_id.shape) + is_leader_cta = mma_tile_coord_v == 0 + cta_rank_in_cluster = cute.arch.make_warp_uniform(cute.arch.block_idx_in_cluster()) + block_in_cluster_coord_vmnk = cluster_layout_vmnk.get_flat_coord(cta_rank_in_cluster) + tidx, _, _ = cute.arch.thread_idx() + + smem = utils.SmemAllocator() + storage = smem.allocate(self.shared_storage) + + tmem_dealloc_mbar_ptr = storage.tmem_dealloc_mbar_ptr + tmem_holding_buf = storage.tmem_holding_buf + + ab_pipeline_producer_group = pipeline.CooperativeGroup(pipeline.Agent.Thread) + num_tma_producer = self.num_mcast_ctas_a + self.num_mcast_ctas_b - 1 + ab_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, num_tma_producer + ) + ab_pipeline = pipeline.PipelineTmaUmma.create( + barrier_storage=storage.ab_mbar_ptr.data_ptr(), + num_stages=self.num_ab_stage, + producer_group=ab_pipeline_producer_group, + consumer_group=ab_pipeline_consumer_group, + tx_count=self.num_tma_load_bytes, + cta_layout_vmnk=cluster_layout_vmnk, + defer_sync=True, + ) + + scale_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * 1, + ) + scale_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.epilog_warp_id), + ) + scale_pipeline = pipeline.PipelineCpAsync.create( + barrier_storage=storage.scale_mbar_ptr.data_ptr(), + num_stages=self.num_scale_stage, + producer_group=scale_pipeline_producer_group, + consumer_group=scale_pipeline_consumer_group, + defer_sync=True, + ) + + acc_pipeline_producer_group = pipeline.CooperativeGroup(pipeline.Agent.Thread) + num_acc_consumer_threads = len(self.epilog_warp_id) + acc_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, num_acc_consumer_threads + ) + acc_pipeline = pipeline.PipelineUmmaAsync.create( + barrier_storage=storage.acc_mbar_ptr.data_ptr(), + num_stages=self.num_acc_stage, + producer_group=acc_pipeline_producer_group, + consumer_group=acc_pipeline_consumer_group, + cta_layout_vmnk=cluster_layout_vmnk, + defer_sync=True, + ) + + epi_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.acc_update_warp_id), + ) + epi_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * len(self.epilog_warp_id), + ) + epi_pipeline = pipeline.PipelineAsync.create( + barrier_storage=storage.epi_mbar_ptr.data_ptr(), + num_stages=1, + producer_group=epi_pipeline_producer_group, + consumer_group=epi_pipeline_consumer_group, + defer_sync=True, + ) + + tile_info_pipeline_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_per_warp * 1, + ) + tile_info_pipeline_consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + self.threads_wo_sched, + ) + tile_info_pipeline = pipeline.PipelineAsync.create( + barrier_storage=storage.tile_info_mbar_ptr.data_ptr(), + num_stages=self.num_tile_stage, + producer_group=tile_info_pipeline_producer_group, + consumer_group=tile_info_pipeline_consumer_group, + defer_sync=True, + ) + + tmem = utils.TmemAllocator( + tmem_holding_buf, + barrier_for_retrieve=self.tmem_alloc_barrier, + allocator_warp_id=self.epilog_warp_id[0], + is_two_cta=False, + two_cta_tmem_dealloc_mbar_ptr=tmem_dealloc_mbar_ptr, + ) + + pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True) + + sC = storage.sC.get_tensor(c_smem_layout_staged.outer, swizzle=c_smem_layout_staged.inner) + sA = storage.sA.get_tensor(a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner) + sB = storage.sB.get_tensor(b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner) + sSFA = storage.sSFA.get_tensor(sfa_smem_layout_staged) + sSFB = storage.sSFB.get_tensor(sfb_smem_layout_staged) + info_layout = cute.make_layout((4, self.num_tile_stage), stride=(1, 4)) + sInfo = storage.sInfo.get_tensor(info_layout) + + a_full_mcast_mask = None + b_full_mcast_mask = None + if cutlass.const_expr(self.is_a_mcast or self.is_b_mcast): + a_full_mcast_mask = cpasync.create_tma_multicast_mask( + cluster_layout_vmnk, block_in_cluster_coord_vmnk, mcast_mode=2 + ) + b_full_mcast_mask = cpasync.create_tma_multicast_mask( + cluster_layout_vmnk, block_in_cluster_coord_vmnk, mcast_mode=1 + ) + + gA_mkl = cute.local_tile( + mA_mkl, cute.slice_(self.mma_tiler, (None, 0, None)), (None, None, None) + ) + gB_nkl = cute.local_tile( + mB_nkl, cute.slice_(self.mma_tiler, (0, None, None)), (None, None, None) + ) + gC_mnl = cute.local_tile( + mC_mnl, cute.slice_(self.mma_tiler, (None, None, 0)), (None, None, None) + ) + gSFA_mkl = cute.local_tile( + mSFA_mkl, + cute.slice_(self.cta_tile_shape_mnk, (None, 0, None)), + (None, None, None), + ) + gSFB_nkl = cute.local_tile( + mSFB_nkl, + cute.slice_(self.cta_tile_shape_mnk, (0, None, None)), + (None, None, None), + ) + cSFA_mkl = cute.make_identity_tensor(cute.shape(mSFA_mkl)) + cSFB_nkl = cute.make_identity_tensor(cute.shape(mSFB_nkl)) + cSFA = cute.local_tile( + cSFA_mkl, + cute.slice_(self.cta_tile_shape_mnk, (None, 0, None)), + (None, None, None), + ) + cSFB = cute.local_tile( + cSFB_nkl, + cute.slice_(self.cta_tile_shape_mnk, (0, None, None)), + (None, None, None), + ) + k_tile_cnt = cute.size(gA_mkl, mode=[3]) + + thr_mma = tiled_mma.get_slice(mma_tile_coord_v) + tCgA = thr_mma.partition_A(gA_mkl) + tCgB = thr_mma.partition_B(gB_nkl) + tCgC = thr_mma.partition_C(gC_mnl) + + sSFA_view_as_C_layout = cute.make_layout( + ( + (self.scale_granularity_m, self.scale_m_per_tile), + self.cta_tile_shape_mnk[1], + self.num_scale_stage, + ), + stride=((0, 1), 0, self.scale_m_per_tile), + ) + sSFB_view_as_C_layout = cute.make_layout( + ( + self.cta_tile_shape_mnk[0], + (self.scale_granularity_n, self.scale_n_per_tile), + self.num_scale_stage, + ), + stride=(0, (0, 1), self.scale_n_per_tile), + ) + sSFA_view_as_C = cute.make_tensor(sSFA.iterator, sSFA_view_as_C_layout) + sSFB_view_as_C = cute.make_tensor(sSFB.iterator, sSFB_view_as_C_layout) + + a_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, 0, None, 0)).shape) + tAsA, tAgA = cpasync.tma_partition( + tma_atom_a, + block_in_cluster_coord_vmnk[2], + a_cta_layout, + cute.group_modes(sA, 0, 3), + cute.group_modes(tCgA, 0, 3), + ) + b_cta_layout = cute.make_layout(cute.slice_(cluster_layout_vmnk, (0, None, 0, 0)).shape) + tBsB, tBgB = cpasync.tma_partition( + tma_atom_b, + block_in_cluster_coord_vmnk[1], + b_cta_layout, + cute.group_modes(sB, 0, 3), + cute.group_modes(tCgB, 0, 3), + ) + + atom_copy = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp(), + mSFA_mkl.element_type, + num_bits_per_copy=mSFA_mkl.element_type.width, + ) + tiled_copy_sfa = cute.make_tiled_copy_tv( + atom_copy, cute.make_layout((32,)), cute.make_layout((1,)) + ) + tiled_copy_sfb = cute.make_tiled_copy_tv( + atom_copy, cute.make_layout((32,)), cute.make_layout((1,)) + ) + thr_copy_sfa = tiled_copy_sfa.get_slice(lane_idx) + thr_copy_sfb = tiled_copy_sfb.get_slice(lane_idx) + tAgSFA_mkl = thr_copy_sfa.partition_S(gSFA_mkl) + tAsSFA = thr_copy_sfa.partition_D(sSFA) + tAcSFA = thr_copy_sfa.partition_S(cSFA) + tBgSFB_nkl = thr_copy_sfb.partition_S(gSFB_nkl) + tBsSFB = thr_copy_sfb.partition_D(sSFB) + tBcSFB = thr_copy_sfb.partition_S(cSFB) + + tCrA = tiled_mma.make_fragment_A(sA) + tCrB = tiled_mma.make_fragment_B(sB) + acc_shape = tiled_mma.partition_shape_C(self.mma_tiler[:2]) + tCtAcc_fake = tiled_mma.make_fragment_C(cute.append(acc_shape, self.num_acc_stage)) + + # Cluster wait before tensor memory alloc + pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn) + + # Specialized Schedule warp + if warp_idx == self.sched_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_sched_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + tile_info_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_tile_stage + ) + + sched_valid = cutlass.Int32(work_tile.is_valid_tile) + if work_tile.tile_idx[2] >= num_tiles[0]: + sched_valid = cutlass.Int32(0) + + while sched_valid == 1: + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + + tile_info_pipeline.producer_acquire(tile_info_producer_state) + + cur_tile_coord = work_tile.tile_idx + sched_valid = cutlass.Int32(work_tile.is_valid_tile) + # Tiles are visited in increasing batch order, so the first tile past the + # live ones ends this CTA's schedule. + if cur_tile_coord[2] >= num_tiles[0]: + sched_valid = cutlass.Int32(0) + with cute.arch.elect_one(): + sInfo[(0, tile_info_producer_state.index)] = cur_tile_coord[0] + sInfo[(1, tile_info_producer_state.index)] = cur_tile_coord[1] + sInfo[(2, tile_info_producer_state.index)] = cur_tile_coord[2] + sInfo[(3, tile_info_producer_state.index)] = sched_valid + + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + self.sched_sync_barrier.arrive_and_wait() + tile_info_pipeline.producer_commit(tile_info_producer_state) + tile_info_producer_state.advance() + + tile_info_pipeline.producer_tail(tile_info_producer_state) + + # Specialized TMA load warp + if warp_idx == self.tma_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + ab_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_ab_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + mma_tile_coord_mnl = ( + tile_info[0] // cute.size(tiled_mma.thr_id.shape), + tile_info[1], + tile_info[2], + ) + a_batch = mma_tile_coord_mnl[2] + tAgA_slice = tAgA[(None, mma_tile_coord_mnl[0], None, a_batch)] + b_batch = tile_group[mma_tile_coord_mnl[2]] + tBgB_slice = tBgB[(None, mma_tile_coord_mnl[1], None, b_batch)] + + # Peek (try_wait) AB buffer empty for k_tile = prefetch_k_tile_cnt + ab_producer_state.reset_count() + peek_ab_empty_status = cutlass.Boolean(1) + if ab_producer_state.count < k_tile_cnt: + peek_ab_empty_status = ab_pipeline.producer_try_acquire(ab_producer_state) + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tAgA_k = tAgA_slice[(None, ab_producer_state.count)] + tBgB_k = tBgB_slice[(None, ab_producer_state.count)] + tAsA_pipe = tAsA[(None, ab_producer_state.index)] + tBsB_pipe = tBsB[(None, ab_producer_state.index)] + + tma_bar = ab_pipeline.producer_get_barrier(ab_producer_state) + + ab_pipeline.producer_acquire(ab_producer_state, peek_ab_empty_status) + + # Peek (try_wait) AB buffer empty for k_tile = prefetch_k_tile_cnt + k_tile + 1 + ab_producer_state.advance() + peek_ab_empty_status = cutlass.Boolean(1) + if ab_producer_state.count < k_tile_cnt: + peek_ab_empty_status = ab_pipeline.producer_try_acquire(ab_producer_state) + + cute.copy( + tma_atom_a, + tAgA_k, + tAsA_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=a_full_mcast_mask, + ) + cute.copy( + tma_atom_b, + tBgB_k, + tBsB_pipe, + tma_bar_ptr=tma_bar, + mcast_mask=b_full_mcast_mask, + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + ab_pipeline.producer_tail(ab_producer_state) + + # Specialized Scale load warp + if warp_idx == self.scale_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + scale_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_scale_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + sfa_batch = tile_info[2] + sfb_batch = tile_group[tile_info[2]] + tApSFA = cute.make_rmem_tensor( + cute.make_layout( + cute.filter_zeros(cute.slice_(tAsSFA, (None, None, None, 0))).shape + ), + cutlass.Boolean, + ) + tBpSFB = cute.make_rmem_tensor( + cute.make_layout( + cute.filter_zeros(cute.slice_(tBsSFB, (None, None, None, 0))).shape + ), + cutlass.Boolean, + ) + + # Peek (try_wait) SCALE buffer empty + scale_producer_state.reset_count() + peek_scale_empty_status = cutlass.Boolean(1) + if scale_producer_state.count < k_tile_cnt: + peek_scale_empty_status = scale_pipeline.producer_try_acquire( + scale_producer_state + ) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tAsSFA_pipe = cute.filter_zeros( + tAsSFA[(None, None, None, scale_producer_state.index)] + ) + tBsSFB_pipe = cute.filter_zeros( + tBsSFB[(None, None, None, scale_producer_state.index)] + ) + tAgSFA_k = cute.filter_zeros( + tAgSFA_mkl[ + ( + None, + None, + None, + tile_info[0], + scale_producer_state.count, + sfa_batch, + ) + ] + ) + tBgSFB_k = cute.filter_zeros( + tBgSFB_nkl[ + ( + None, + None, + None, + tile_info[1], + scale_producer_state.count, + sfb_batch, + ) + ] + ) + + tAcSFA_compact = cute.filter_zeros( + cute.slice_( + tAcSFA, + ( + None, + None, + None, + tile_info[0], + scale_producer_state.count, + sfa_batch, + ), + ) + ) + tBcSFB_compact = cute.filter_zeros( + cute.slice_( + tBcSFB, + ( + None, + None, + None, + tile_info[1], + scale_producer_state.count, + sfb_batch, + ), + ) + ) + for i in cutlass.range_constexpr(cute.size(tApSFA, mode=[1])): + tApSFA[((0, 0), i, (0, 0))] = cute.elem_less( + tAcSFA_compact[(i)][0], mSFA_mkl.shape[0] + ) + for i in cutlass.range_constexpr(cute.size(tBpSFB, mode=[1])): + tBpSFB[((0, 0), i, (0, 0))] = cute.elem_less( + tBcSFB_compact[(i)][0], mSFB_nkl.shape[0] + ) + + scale_pipeline.producer_acquire(scale_producer_state, peek_scale_empty_status) + + cute.copy(tiled_copy_sfa, tAgSFA_k, tAsSFA_pipe, pred=tApSFA) + cute.copy(tiled_copy_sfb, tBgSFB_k, tBsSFB_pipe, pred=tBpSFB) + + scale_pipeline.producer_commit(scale_producer_state) + + # Peek (try_wait) Scale buffer empty + scale_producer_state.advance() + peek_scale_empty_status = cutlass.Boolean(1) + if scale_producer_state.count < k_tile_cnt: + peek_scale_empty_status = scale_pipeline.producer_try_acquire( + scale_producer_state + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + scale_pipeline.producer_tail(scale_producer_state) + + # Specialized MMA warp + if warp_idx == self.mma_warp_id: + cute.arch.warpgroup_reg_dealloc(self.num_regs_uniform_warps) + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + ab_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_ab_stage + ) + acc_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.num_acc_stage + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + # Peek (try_wait) AB buffer full for k_tile = 0 + ab_consumer_state.reset_count() + peek_ab_full_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt and is_leader_cta: + peek_ab_full_status = ab_pipeline.consumer_try_wait(ab_consumer_state) + + # Peek (try_wait) Acc buffer empty for k_tile = 0 + acc_producer_state.reset_count() + peek_acc_empty_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt and is_leader_cta: + peek_acc_empty_status = acc_pipeline.producer_try_acquire(acc_producer_state) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tCtAcc = tCtAcc_base[(None, None, None, acc_producer_state.index)] + + if is_leader_cta: + acc_pipeline.producer_acquire(acc_producer_state, peek_acc_empty_status) + + tiled_mma.set(tcgen05.Field.ACCUMULATE, False) + + if is_leader_cta: + ab_pipeline.consumer_wait(ab_consumer_state, peek_ab_full_status) + + # tCtAcc += tCrA * tCrB + num_kblocks = cute.size(tCrA, mode=[2]) + for kblock_idx in cutlass.range(num_kblocks, unroll_full=True): + kblock_coord = ( + None, + None, + kblock_idx, + ab_consumer_state.index, + ) + + cute.gemm( + tiled_mma, + tCtAcc, + tCrA[kblock_coord], + tCrB[kblock_coord], + tCtAcc, + ) + # Enable accumulate on tCtAcc after first kblock + tiled_mma.set(tcgen05.Field.ACCUMULATE, True) + + ab_pipeline.consumer_release(ab_consumer_state) + + # Peek (try_wait) AB buffer full for k_tile = k_tile + 1 + ab_consumer_state.advance() + peek_ab_full_status = cutlass.Boolean(1) + if ab_consumer_state.count < k_tile_cnt: + if is_leader_cta: + peek_ab_full_status = ab_pipeline.consumer_try_wait(ab_consumer_state) + + if is_leader_cta: + acc_pipeline.producer_commit(acc_producer_state) + + # Peek (try_wait) Acc buffer empty for k_tile = k_tile + 1 + acc_producer_state.advance() + if acc_producer_state.count < k_tile_cnt: + if is_leader_cta: + peek_acc_empty_status = acc_pipeline.producer_try_acquire( + acc_producer_state + ) + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + acc_pipeline.producer_tail(acc_producer_state) + + # Specialized acc update warps + if warp_idx <= self.acc_update_warp_id[-1]: + cute.arch.warpgroup_reg_alloc(self.num_regs_acc_update_warps) + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + tCtAcc_final = cute.make_tensor( + tCtAcc_base.iterator + self.tmem_final_offset, tCtAcc_base.layout + ) + + epi_tidx = tidx % 128 + ( + tiled_copy_t2r, + tiled_copy_r2t, + tTR_tAcc_base, + tTR_rAcc, + tTR_rAcc_final, + tTR_sSFA, + tTR_sSFB, + tRT_rAcc, + tRT_tAcc_base, + ) = self.acc_update_tmem_copy_and_partition( + epi_tidx, + tCtAcc_base, + tCtAcc_final, + tCgC, + sSFA_view_as_C, + sSFB_view_as_C, + epi_tile, + ) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + acc_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_acc_stage + ) + + scale_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_scale_stage + ) + + epi_producer_state = pipeline.make_pipeline_state(pipeline.PipelineUserType.Producer, 1) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + while is_valid_tile: + tTR_rAcc_final.fill(0.0) + + tTR_rSFA = cute.make_rmem_tensor( + cute.slice_(tTR_sSFA, (None, None, None, 0, None, 0)).shape, + self.acc_dtype, + ) + tTR_rSFB = cute.make_rmem_tensor( + cute.slice_(tTR_sSFB, (None, None, None, 0, None, 0)).shape, + self.acc_dtype, + ) + + scale_consumer_state.reset_count() + peek_scale_full_status = cutlass.Boolean(1) + if scale_consumer_state.count < k_tile_cnt: + peek_scale_full_status = scale_pipeline.consumer_try_wait(scale_consumer_state) + + acc_consumer_state.reset_count() + peek_acc_full_status = cutlass.Boolean(1) + if acc_consumer_state.count < k_tile_cnt: + peek_acc_full_status = acc_pipeline.consumer_try_wait(acc_consumer_state) + + for k_tile in cutlass.range(0, k_tile_cnt, 1, unroll=1): + tTR_tAcc = tTR_tAcc_base[ + (None, None, None, None, None, acc_consumer_state.index) + ] + + scale_pipeline.consumer_wait(scale_consumer_state, peek_scale_full_status) + + tTR_sSFA_slice = cute.slice_( + tTR_sSFA, + (None, None, None, 0, None, scale_consumer_state.index), + ) + tTR_sSFB_slice = cute.slice_( + tTR_sSFB, + (None, None, None, 0, None, scale_consumer_state.index), + ) + + scale_atom_copy = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + self.acc_dtype, + num_bits_per_copy=self.acc_dtype.width, + ) + + cute.copy(scale_atom_copy, tTR_sSFA_slice, tTR_rSFA) + cute.copy(scale_atom_copy, tTR_sSFB_slice, tTR_rSFB) + + acc_pipeline.consumer_wait(acc_consumer_state, peek_acc_full_status) + + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + + subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) + for subtile_idx in cutlass.range(subtile_cnt): + tTR_tAcc_mn = tTR_tAcc[(None, None, None, subtile_idx)] + cute.copy(tiled_copy_t2r, tTR_tAcc_mn, tTR_rAcc) + + tTR_rAcc_subtile = tTR_rAcc_final[(None, None, None, subtile_idx)] + tTR_rSFA_subtile = tTR_rSFA[(None, None, None, subtile_idx)] + tTR_rSFB_subtile = tTR_rSFB[(None, None, None, subtile_idx)] + + acc_vec = tTR_rAcc.load() + final_vec = tTR_rAcc_subtile.load() + scale_a = tTR_rSFA_subtile.load() + scale_b = tTR_rSFB_subtile.load() + scale = scale_a * scale_b + final_vec = acc_vec * scale + final_vec + tTR_rAcc_subtile.store(final_vec.to(self.acc_dtype)) + + scale_pipeline.consumer_release(scale_consumer_state) + scale_consumer_state.advance() + + peek_scale_full_status = cutlass.Boolean(1) + if scale_consumer_state.count < k_tile_cnt: + peek_scale_full_status = scale_pipeline.consumer_try_wait( + scale_consumer_state + ) + with cute.arch.elect_one(): + acc_pipeline.consumer_release(acc_consumer_state) + acc_consumer_state.advance() + + peek_acc_full_status = cutlass.Boolean(1) + if acc_consumer_state.count < k_tile_cnt: + peek_acc_full_status = acc_pipeline.consumer_try_wait(acc_consumer_state) + + tRT_tAcc = tRT_tAcc_base[(None, None, None, None, None, 0)] + tRT_tAcc = cute.group_modes(tRT_tAcc, 3, cute.rank(tRT_tAcc)) + + epi_pipeline.producer_acquire(epi_producer_state) + + cute.copy(tiled_copy_r2t, tTR_rAcc_final, tRT_tAcc) + cute.arch.fence_view_async_tmem_store() + + epi_pipeline.producer_commit(epi_producer_state) + epi_producer_state.advance() + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + # Specialized epilogue warps + if warp_idx <= self.epilog_warp_id[-1] and warp_idx >= self.epilog_warp_id[0]: + cute.arch.warpgroup_reg_alloc(self.num_regs_epilogue_warps) + tmem.allocate(self.num_tmem_alloc_cols) + + tmem.wait_for_alloc() + + tmem_ptr = tmem.retrieve_ptr(self.acc_dtype) + tCtAcc_base_ = cute.make_tensor(tmem_ptr, tCtAcc_fake.layout) + tCtAcc_final = cute.make_tensor( + tCtAcc_base_.iterator + self.tmem_final_offset, tCtAcc_base_.layout + ) + + epi_tidx = tidx % 128 + ( + tiled_copy_t2r, + tTR_tAcc_base, + tTR_rAcc, + ) = self.epilog_tmem_copy_and_partition(epi_tidx, tCtAcc_final, tCgC, epi_tile) + + tTR_rC = None + tiled_copy_r2s = None + tRS_rC = None + tRS_sC = None + bSG_sC = None + bSG_gC_partitioned = None + tTR_rC = cute.make_rmem_tensor(tTR_rAcc.shape, self.c_dtype) + tiled_copy_r2s, tRS_rC, tRS_sC = self.epilog_smem_copy_and_partition( + tiled_copy_t2r, tTR_rC, epi_tidx, sC + ) + ( + tma_atom_c, + bSG_sC, + bSG_gC_partitioned, + ) = self.epilog_gmem_copy_and_partition(epi_tidx, tma_atom_c, tCgC, epi_tile, sC) + + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + + epi_consumer_state = pipeline.make_pipeline_state(pipeline.PipelineUserType.Consumer, 1) + + c_pipeline = None + c_producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, + 32 * len(self.epilog_warp_id), + ) + c_pipeline = pipeline.PipelineTmaStore.create( + num_stages=self.num_c_stage, + producer_group=c_producer_group, + ) + + tile_info_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.num_tile_stage + ) + + tile_info = cute.make_rmem_tensor((4,), cutlass.Int32) + cur_tile_coord = work_tile.tile_idx + tile_info[0] = cur_tile_coord[0] + tile_info[1] = cur_tile_coord[1] + tile_info[2] = cur_tile_coord[2] + tile_info[3] = work_tile.is_valid_tile + if tile_info[2] >= num_tiles[0]: + tile_info[3] = cutlass.Int32(0) + + is_valid_tile = cutlass.Boolean(1) + is_valid_tile = tile_info[3] == 1 + + num_prev_subtiles = cutlass.Int32(0) + + while is_valid_tile: + mma_tile_coord_mnl = ( + tile_info[0] // cute.size(tiled_mma.thr_id.shape), + tile_info[1], + tile_info[2], + ) + bSG_gC = None + bSG_gC = bSG_gC_partitioned[ + ( + None, + None, + None, + mma_tile_coord_mnl[0], + mma_tile_coord_mnl[1], + mma_tile_coord_mnl[2], + ) + ] + + tTR_tAcc = tTR_tAcc_base[(None, None, None, None, None, epi_consumer_state.index)] + + epi_pipeline.consumer_wait(epi_consumer_state) + + tTR_tAcc = cute.group_modes(tTR_tAcc, 3, cute.rank(tTR_tAcc)) + bSG_gC = cute.group_modes(bSG_gC, 1, cute.rank(bSG_gC)) + + subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) + q_inv = cutlass.Float32(1.0) + if cutlass.const_expr(self.epi_quant): + # A thread holds one row of the 128-column tile (lane = row of the 32dp TMEM + # loads), so the row's 1x128 amax is thread-local: one pass over the subtiles + # for it, the store pass below scales by 448 / amax. + q_amax = cutlass.Float32(0.0) + for subtile_idx in cutlass.range_constexpr(subtile_cnt): + cute.copy( + tiled_copy_t2r, tTR_tAcc[(None, None, None, subtile_idx)], tTR_rAcc + ) + for i in cutlass.range_constexpr(cute.size(tTR_rAcc)): + q_v = tTR_rAcc[i] + q_amax = cutlass.max( + q_amax, cutlass.max(q_v, cutlass.Float32(0.0) - q_v) + ) + q_amax = cutlass.max(q_amax, cutlass.Float32(1.0e-10)) + q_inv = _div_rn(cutlass.Float32(448.0), q_amax) + mCS[ + ( + mma_tile_coord_mnl[1], + mma_tile_coord_mnl[2] * self.cta_tile_shape_mnk[0] + epi_tidx, + ) + ] = _div_rn(q_amax, cutlass.Float32(448.0)) + for subtile_idx in cutlass.range(subtile_cnt): + tTR_tAcc_mn = tTR_tAcc[(None, None, None, subtile_idx)] + cute.copy(tiled_copy_t2r, tTR_tAcc_mn, tTR_rAcc) + + acc_vec = tiled_copy_r2s.retile(tTR_rAcc).load() + if cutlass.const_expr(self.epi_quant): + acc_vec = (acc_vec * q_inv).to(self.c_dtype) + else: + acc_vec = acc_vec.to(self.c_dtype) + tRS_rC.store(acc_vec) + + num_prev_subtiles = num_prev_subtiles + 1 + c_buffer = num_prev_subtiles % self.num_c_stage + cute.copy( + tiled_copy_r2s, + tRS_rC, + tRS_sC[(None, None, None, c_buffer)], + ) + # Fence and barrier to make sure shared memory store is visible to TMA store + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + self.epilog_sync_barrier.arrive_and_wait() + + if warp_idx == self.epilog_warp_id[0]: + cute.copy( + tma_atom_c, + bSG_sC[(None, c_buffer)], + bSG_gC[(None, subtile_idx)], + ) + # Fence and barrier to make sure shared memory store is visible to TMA store + c_pipeline.producer_commit() + c_pipeline.producer_acquire() + self.epilog_sync_barrier.arrive_and_wait() + + epi_pipeline.consumer_release(epi_consumer_state) + epi_consumer_state.advance() + + tile_info_pipeline.consumer_wait(tile_info_consumer_state) + for idx in cutlass.range(4, unroll_full=True): + tile_info[idx] = sInfo[(idx, tile_info_consumer_state.index)] + is_valid_tile = tile_info[3] == 1 + cute.arch.fence_proxy( + "async.shared", + space="cta", + ) + tile_info_pipeline.consumer_release(tile_info_consumer_state) + tile_info_consumer_state.advance() + + tmem.relinquish_alloc_permit() + self.epilog_sync_barrier.arrive_and_wait() + tmem.free(tmem_ptr) + c_pipeline.producer_tail() + + def acc_update_tmem_copy_and_partition( + self, + tidx: cutlass.Int32, + tAcc: cute.Tensor, + tAcc_final: cute.Tensor, + gC_mnl: cute.Tensor, + sSFA: cute.Tensor, + sSFB: cute.Tensor, + epi_tile: cute.Tile, + ) -> Tuple[ + cute.TiledCopy, + cute.TiledCopy, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + cute.Tensor, + ]: + """TMEM <-> register copies of the acc-update warps: `(tiled_copy_t2r, tiled_copy_r2t, + tTR_tAcc, tTR_rAcc, tTR_rAcc_final, tTR_sSFA, tTR_sSFB, tRT_rAcc_final, tRT_tAcc_final)` -- + each k tile's partial loaded from `tAcc`, the running sum held in registers and stored to + `tAcc_final`, and SFA / SFB partitioned like the partial.""" + tmem_load_atom = None + tmem_store_atom = None + if cutlass.const_expr(self.mma_tiler[0] == 64): + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld16x256bOp(tcgen05.copy.Repetition(8)), + self.acc_dtype, + ) + else: + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), + self.acc_dtype, + ) + if cutlass.const_expr(self.mma_tiler[0] == 64): + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St16x256bOp(tcgen05.copy.Repetition(8)), + self.acc_dtype, + ) + else: + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), + self.acc_dtype, + ) + + tAcc_epi = cute.flat_divide(tAcc[((None, None), 0, 0, None)], epi_tile) + tAcc_final_epi = cute.flat_divide(tAcc_final[((None, None), 0, 0, None)], epi_tile) + + tiled_copy_t2r = tcgen05.make_tmem_copy(tmem_load_atom, tAcc_epi[(None, None, 0, 0, 0)]) + tiled_copy_r2t = tcgen05.make_tmem_copy( + tmem_store_atom, tAcc_final_epi[(None, None, 0, 0, 0)] + ) + + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + thr_copy_r2t = tiled_copy_r2t.get_slice(tidx) + + tTR_tAcc = thr_copy_t2r.partition_S(tAcc_epi) + gC_mnl_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + sSFA_epi = cute.flat_divide(sSFA, epi_tile) + sSFB_epi = cute.flat_divide(sSFB, epi_tile) + tTR_gC = thr_copy_t2r.partition_D(gC_mnl_epi) + tTR_sSFA = thr_copy_t2r.partition_D(sSFA_epi) + tTR_sSFB = thr_copy_t2r.partition_D(sSFB_epi) + tTR_rAcc = cute.make_rmem_tensor( + tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype + ) + tTR_rAcc_final_ = cute.make_rmem_tensor( + tTR_gC[(None, None, None, None, None, 0, 0, 0)].shape, self.acc_dtype + ) + tTR_rAcc_final = cute.group_modes(tTR_rAcc_final_, 3, cute.rank(tTR_rAcc_final_)) + + tRT_gC = thr_copy_r2t.partition_S(gC_mnl_epi) + tRT_tAcc_final = thr_copy_r2t.partition_D(tAcc_final_epi) + tRT_rAcc_final_ = cute.make_rmem_tensor( + tRT_gC[(None, None, None, None, None, 0, 0, 0)].shape, self.acc_dtype + ) + tRT_rAcc_final = cute.group_modes(tRT_rAcc_final_, 3, cute.rank(tRT_rAcc_final_)) + + return ( + tiled_copy_t2r, + tiled_copy_r2t, + tTR_tAcc, + tTR_rAcc, + tTR_rAcc_final, + tTR_sSFA, + tTR_sSFB, + tRT_rAcc_final, + tRT_tAcc_final, + ) + + def epilog_tmem_copy_and_partition( + self, + tidx: cutlass.Int32, + tAcc: cute.Tensor, + gC_mnl: cute.Tensor, + epi_tile: cute.Tile, + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: + """`(tiled_copy_t2r, tTR_tAcc, tTR_rAcc)`: the epilogue warps' TMEM -> register copy over + `epi_tile` subtiles.""" + copy_atom_t2r = sm100_utils.get_tmem_load_op( + self.cta_tile_shape_mnk, + self.c_layout, + self.c_dtype, + self.acc_dtype, + epi_tile, + False, + ) + tAcc_epi = cute.flat_divide( + tAcc[((None, None), 0, 0, None)], + epi_tile, + ) + tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tAcc_epi[(None, None, 0, 0, 0)]) + + thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) + tTR_tAcc = thr_copy_t2r.partition_S(tAcc_epi) + + gC_mnl_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + tTR_gC = thr_copy_t2r.partition_D(gC_mnl_epi) + + tTR_rAcc = cute.make_rmem_tensor( + tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype + ) + + return tiled_copy_t2r, tTR_tAcc, tTR_rAcc + + def epilog_smem_copy_and_partition( + self, + tiled_copy_t2r: cute.TiledCopy, + tTR_rC: cute.Tensor, + tidx: cutlass.Int32, + sC: cute.Tensor, + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: + """`(tiled_copy_r2s, tRS_rC, tRS_sC)`: the register -> SMEM copy of a C subtile.""" + copy_atom_r2s = sm100_utils.get_smem_store_op( + self.c_layout, self.c_dtype, self.acc_dtype, tiled_copy_t2r + ) + tiled_copy_r2s = cute.make_tiled_copy_D(copy_atom_r2s, tiled_copy_t2r) + thr_copy_r2s = tiled_copy_r2s.get_slice(tidx) + tRS_sC = thr_copy_r2s.partition_D(sC) + tRS_rC = tiled_copy_r2s.retile(tTR_rC) + return tiled_copy_r2s, tRS_rC, tRS_sC + + def epilog_gmem_copy_and_partition( + self, + tidx: cutlass.Int32, + atom: Union[cute.CopyAtom, cute.TiledCopy], + gC_mnl: cute.Tensor, + epi_tile: cute.Tile, + sC: cute.Tensor, + ) -> Tuple[cute.CopyAtom, cute.Tensor, cute.Tensor]: + """`(tma_atom_c, bSG_sC, bSG_gC)`: the TMA store partition of a C subtile, SMEM to GMEM.""" + gC_epi = cute.flat_divide(gC_mnl[((None, None), 0, 0, None, None, None)], epi_tile) + tma_atom_c = atom + sC_for_tma_partition = cute.group_modes(sC, 0, 2) + gC_for_tma_partition = cute.group_modes(gC_epi, 0, 2) + bSG_sC, bSG_gC = cpasync.tma_partition( + tma_atom_c, + 0, + cute.make_layout(1), + sC_for_tma_partition, + gC_for_tma_partition, + ) + return tma_atom_c, bSG_sC, bSG_gC + + @staticmethod + def _compute_stages( + tiled_mma: cute.TiledMma, + mma_tiler_mnk: Tuple[int, int, int], + a_dtype: Type[cutlass.Numeric], + b_dtype: Type[cutlass.Numeric], + epi_tile: cute.Tile, + c_dtype: Type[cutlass.Numeric], + c_layout: utils.LayoutEnum, + sfa_dtype: Type[cutlass.Numeric], + sfb_dtype: Type[cutlass.Numeric], + sfa_count: int, + sfb_count: int, + num_smem_capacity: int, + occupancy: int, + ) -> Tuple[int, int, int]: + """`(ACC, A/B, C, scale, tile info)` stage counts for the CTA's SMEM at `occupancy` CTAs an + SM.""" + num_acc_stage = 3 if mma_tiler_mnk[0] / tiled_mma.thr_id.shape == 128 else 6 + + num_c_stage = 2 + + num_scale_stage = 10 + + num_tile_stage = 2 + + a_smem_layout_stage_one = sm100_utils.make_smem_layout_a( + tiled_mma, + mma_tiler_mnk, + a_dtype, + 1, # a tmp 1 stage is provided + ) + b_smem_layout_staged_one = sm100_utils.make_smem_layout_b( + tiled_mma, + mma_tiler_mnk, + b_dtype, + 1, # a tmp 1 stage is provided + ) + c_smem_layout_staged_one = sm100_utils.make_smem_layout_epi( + c_dtype, + c_layout, + epi_tile, + 1, + ) + + ab_bytes_per_stage = cute.size_in_bytes( + a_dtype, a_smem_layout_stage_one + ) + cute.size_in_bytes(b_dtype, b_smem_layout_staged_one) + # 1024B alignment + mbar_helpers_bytes = 1024 + c_bytes_per_stage = cute.size_in_bytes(c_dtype, c_smem_layout_staged_one) + c_bytes = c_bytes_per_stage * num_c_stage + sfa_bytes = sfa_count * (sfa_dtype.width // 8) * num_scale_stage + sfb_bytes = sfb_count * (sfb_dtype.width // 8) * num_scale_stage + scale_bytes = math.ceil((sfa_bytes + sfb_bytes) / 1024) * 1024 + + # A/B stages fill what is left of the CTA's SMEM after the barriers, scales and C stages; + # the C stages then take what the A/B stages leave. + num_ab_stage = ( + num_smem_capacity // occupancy - (mbar_helpers_bytes + c_bytes + scale_bytes) + ) // ab_bytes_per_stage + + num_c_stage += ( + num_smem_capacity + - occupancy * ab_bytes_per_stage * num_ab_stage + - occupancy * (mbar_helpers_bytes + c_bytes + scale_bytes) + ) // (occupancy * c_bytes_per_stage) + return num_acc_stage, num_ab_stage, num_c_stage, num_scale_stage, num_tile_stage + + @staticmethod + def _compute_grid( + c: cute.Tensor, + cta_tile_shape_mnk: Tuple[int, int, int], + cluster_shape_mn: Tuple[int, int], + max_active_clusters: cutlass.Constexpr, + ) -> Tuple[utils.PersistentTileSchedulerParams, Tuple[int, int, int]]: + """`(tile_sched_params, grid)` of the persistent tile scheduler over C's tiles.""" + c_shape = cute.slice_(cta_tile_shape_mnk, (None, None, 0)) + gc = cute.zipped_divide(c, tiler=c_shape) + num_ctas_mnl = gc[(0, (None, None, None))].shape + cluster_shape_mnl = (*cluster_shape_mn, 1) + + tile_sched_params = utils.PersistentTileSchedulerParams(num_ctas_mnl, cluster_shape_mnl) + grid = utils.StaticPersistentTileScheduler.get_grid_shape( + tile_sched_params, max_active_clusters + ) + + return tile_sched_params, grid + + @staticmethod + def _get_tma_atom_kind( + atom_sm_cnt: cutlass.Int32, mcast: cutlass.Boolean + ) -> Union[cpasync.CopyBulkTensorTileG2SMulticastOp, cpasync.CopyBulkTensorTileG2SOp]: + """The TMA G2S copy op for `atom_sm_cnt` SMs, multicast or not; ValueError otherwise.""" + if atom_sm_cnt == 2 and mcast: + return cpasync.CopyBulkTensorTileG2SMulticastOp(tcgen05.CtaGroup.TWO) + elif atom_sm_cnt == 2 and not mcast: + return cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.TWO) + elif atom_sm_cnt == 1 and mcast: + return cpasync.CopyBulkTensorTileG2SMulticastOp(tcgen05.CtaGroup.ONE) + elif atom_sm_cnt == 1 and not mcast: + return cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE) + + raise ValueError(f"Invalid atom_sm_cnt: {atom_sm_cnt} and {mcast}") + + @cute.jit + def wrapper_grouped( + self, + tiles: cutlass.Int32, + n: cutlass.Int32, + k: cutlass.Int32, + sf_m: cutlass.Int32, + sf_n: cutlass.Int32, + sf_k: cutlass.Int32, + num_groups: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + c_tensor: cute.Tensor, + tile_group_ptr: cute.Pointer, + num_tiles_ptr: cute.Pointer, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + ): + """The contiguous grouped GEMM over raw pointers: A holds `tiles` M tiles of + rows back to back (`[tiles * M_tile, k]`, k-major), each tile of one group; SFA is M-major + over all of those rows (`[sf_k, sf_m]`, `sf_m >= tiles * M_tile`); B `[num_groups, n, k]` + and SFB `[num_groups, sf_n, sf_k]` are batched by group; C is `(M_tile, n, tiles)` over + `[tiles * M_tile, n]`. `tile_group[l]` (int32 `[tiles]`) names tile l's group; tiles from + `num_tiles[0]` (int32 `[1]`) on are skipped (so `tiles` can be a capture-time upper bound). + Visiting order: every N tile of an M tile, then the next M tile -- a group's tiles back to + back, its B from L2. + """ + mt = self.mma_tiler[0] + a_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((mt, k, tiles), order=(1, 0, 2)), + ) + b_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, num_groups), order=(1, 0, 2)), + ) + sfa_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_layout((mt, sf_k, tiles), stride=(1, sf_m, mt)), + ) + sfb_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout((sf_n, sf_k, num_groups), order=(1, 0, 2)), + ) + tile_group = cute.make_tensor(tile_group_ptr, layout=cute.make_layout((tiles,))) + num_tiles = cute.make_tensor(num_tiles_ptr, layout=cute.make_layout((1,))) + self( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + max_active_clusters, + stream, + tile_group=tile_group, + num_tiles=num_tiles, + ) + + @cute.jit + def wrapper_grouped_quant( + self, + tiles: cutlass.Int32, + n: cutlass.Int32, + k: cutlass.Int32, + sf_m: cutlass.Int32, + sf_n: cutlass.Int32, + sf_k: cutlass.Int32, + num_groups: cutlass.Int32, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + c_ptr: cute.Pointer, + c_scale: cute.Tensor, + tile_group_ptr: cute.Pointer, + num_tiles_ptr: cute.Pointer, + max_active_clusters: cutlass.Constexpr, + stream: cuda.CUstream, + ): + """`wrapper_grouped` for `epi_quant=True`: C `[tiles * M_tile, n]` e4m3 codes (`c_ptr`), + `c_scale` `[n / 128, >= tiles * M_tile]` fp32 (row-major) the scale of each row's 128-column + block.""" + mt = self.mma_tiler[0] + a_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((mt, k, tiles), order=(1, 0, 2)), + ) + b_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, num_groups), order=(1, 0, 2)), + ) + sfa_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_layout((mt, sf_k, tiles), stride=(1, sf_m, mt)), + ) + sfb_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout((sf_n, sf_k, num_groups), order=(1, 0, 2)), + ) + c_tensor = cute.make_tensor( + c_ptr, + layout=cute.make_ordered_layout((mt, n, tiles), order=(1, 0, 2)), + ) + tile_group = cute.make_tensor(tile_group_ptr, layout=cute.make_layout((tiles,))) + num_tiles = cute.make_tensor(num_tiles_ptr, layout=cute.make_layout((1,))) + self( + a_tensor, + b_tensor, + c_tensor, + sfa_tensor, + sfb_tensor, + max_active_clusters, + stream, + tile_group=tile_group, + num_tiles=num_tiles, + c_scale=c_scale, + ) + + +_GEMMS: dict = {} + + +def _gmem_ptr(t: torch.Tensor, dtype, align: int = 16): + """A global-memory CuTe pointer at `t`'s data.""" + return make_ptr(dtype, t.data_ptr(), cute.AddressSpace.gmem, assumed_align=align) + + +def _grouped_reference(a, w, sfa, sfw, rows, group, ntiles) -> torch.Tensor: + """fp32 `[rows, n]`: tile `t < ntiles[0]` of `rows / len(group)` rows times its group's + dequantized weight; the rows of skipped tiles NaN (unspecified).""" + _, n, k = w.shape + m_tile = rows // group.numel() + act = ( + a[:rows].float().reshape(rows, k // BLOCK, BLOCK) + * sfa[:, :rows].t().float().reshape(rows, k // BLOCK, 1) + ).reshape(rows, k) + out = torch.full((rows, n), float("nan"), dtype=torch.float32, device=a.device) + with true_fp32_matmul(): + for t, e in enumerate(group[: int(ntiles[0])].tolist()): + r = slice(t * m_tile, (t + 1) * m_tile) + out[r] = act[r] @ dequantize_block_weight(w[e], sfw[e]).t() + return out + + +class BlockwiseGemmGrouped(OpWrapper): + """`c[r, n] = sum_k (sfa * a)[r, k] (sfw * w)[group[r // M_tile], n, k]` over `tiles` M tiles + (`M_tile` the tactic's MMA M) of which the first `ntiles[0]` (device scalar) are live: `a` + `[tiles * M_tile, k]` e4m3, `sfa` `[k / 128, sf_m]` fp32 (M-major), `w` `[groups, n, k]` e4m3, + `sfw` `[groups, n / 128, k / 128]` fp32, `c` `[tiles * M_tile, n]` written in place; `tactic` + is `(mma_tiler_mn, cluster_shape_mn)`. Compiled once per `c` dtype and tactic.""" + + def __call__( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + c: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> None: + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + tiler, cluster = tactic + c3 = c.view(tiles, tiler[0], n).permute(1, 2, 0) + key = (c.dtype, tactic) + fn = _GEMMS.get(key) + if fn is None: + gemm = Sm100BlockwiseGemmKernel(mma_tiler_mn=tiler, cluster_shape_mn=cluster) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + fn = cute.compile( + gemm.wrapper_grouped, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + _gmem_ptr(a, cutlass.Float8E4M3FN), + _gmem_ptr(w, cutlass.Float8E4M3FN), + _gmem_ptr(sfa, cutlass.Float32), + _gmem_ptr(sfw, cutlass.Float32), + from_dlpack(c3).mark_layout_dynamic(leading_dim=1), + _gmem_ptr(group, cutlass.Int32, 4), + _gmem_ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + c3, + group.data_ptr(), + ntiles.data_ptr(), + ) + + def reference( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + c: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> torch.Tensor: + """`c` after the call, as a fresh tensor of its dtype (fp32 GEMMs over the dequantized + operands, rounded once); the rows of skipped tiles NaN (unspecified). `sf_m` and + `tactic` pick a layout and a launch, not a result.""" + return _grouped_reference(a, w, sfa, sfw, c.shape[0], group[:tiles], ntiles).to(c.dtype) + + +blockwise_gemm_grouped = BlockwiseGemmGrouped() + + +class BlockwiseGemmGroupedQuant(OpWrapper): + """`blockwise_gemm_grouped` with the 1x128 e4m3 round trip in the epilogue (`epi_quant`): + `cq[r, n]` e4m3 codes and `cs[n // 128, r]` fp32 scales (row-major `[n / 128, sf_m]`) + instead of `c`. Compiled once per tactic.""" + + def __call__( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + cq: torch.Tensor, + cs: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> None: + groups, n, k = w.shape + sf_n, sf_k = -(-n // BLOCK), -(-k // BLOCK) + key = ("q", tactic) + fn = _GEMMS.get(key) + if fn is None: + tiler, cluster = tactic + gemm = Sm100BlockwiseGemmKernel( + mma_tiler_mn=tiler, cluster_shape_mn=cluster, epi_quant=True + ) + mac = cutlass.utils.HardwareInfo().get_max_active_clusters(cluster[0] * cluster[1]) + fn = cute.compile( + gemm.wrapper_grouped_quant, + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + _gmem_ptr(a, cutlass.Float8E4M3FN), + _gmem_ptr(w, cutlass.Float8E4M3FN), + _gmem_ptr(sfa, cutlass.Float32), + _gmem_ptr(sfw, cutlass.Float32), + _gmem_ptr(cq, cutlass.Float8E4M3FN), + from_dlpack(cs).mark_layout_dynamic(leading_dim=1), + _gmem_ptr(group, cutlass.Int32, 4), + _gmem_ptr(ntiles, cutlass.Int32, 4), + max_active_clusters=mac, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _GEMMS[key] = fn + fn( + tiles, + n, + k, + sf_m, + sf_n, + sf_k, + groups, + a.data_ptr(), + w.data_ptr(), + sfa.data_ptr(), + sfw.data_ptr(), + cq.data_ptr(), + cs, + group.data_ptr(), + ntiles.data_ptr(), + ) + + def reference( + self, + a: torch.Tensor, + w: torch.Tensor, + sfa: torch.Tensor, + sfw: torch.Tensor, + cq: torch.Tensor, + cs: torch.Tensor, + group: torch.Tensor, + ntiles: torch.Tensor, + tiles: int, + sf_m: int, + tactic: tuple, + ) -> torch.Tensor: + """The values the call's codes and scales encode, `cq.float() * cs[n // 128, r]`, as + fp32 `[rows, n]`: `blockwise_gemm_grouped.reference`'s fp32 product through the MoE + runner's 1x128 e4m3 round trip; the rows of skipped tiles NaN (unspecified). `sf_m` and + `tactic` pick a layout and a launch, not a result.""" + return requantize_1x128( + _grouped_reference(a, w, sfa, sfw, cq.shape[0], group[:tiles], ntiles) + ) + + +blockwise_gemm_grouped_quant = BlockwiseGemmGroupedQuant() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py new file mode 100644 index 000000000000..e7b4b40f2bd2 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/fp8_block_scale_moe_runner.py @@ -0,0 +1,611 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""DeepSeek-style FP8 block-scale mixture-of-experts layer: top-k routed grouped FC1 GEMM + +SwiGLU + FP8 requantization + grouped FC2 GEMM + routing-weighted combine, in one trtllm-gen call. + +This target's copy of the family entry `models/qwen4_exp/ops/moe/fp8_block_scale_moe_runner.py`, +with the logits-routed decode front end (`fp8_block_scale_moe_front`) and pinned decode tactics.""" + +from __future__ import annotations + +from typing import Callable, Optional + +import torch + +import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* +from tensorrt_llm._torch.autotuner import AutoTuner +from tensorrt_llm._torch.custom_ops.trtllm_gen_custom_ops import FP8BlockScaleMoERunner +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ... import native_ops + +#: Block edge on every scale axis: 1x128 activations, 128x128 weights. +BLOCK = 128 + +#: Largest finite e4m3 magnitude. +E4M3_MAX = 448.0 + +#: The activation kernel's floor on a block amax (`kDsActAmaxEpsilon` in +#: trtllmGenKernels/blockScaleMoe/DevKernel.cu). Only an all-zero block reaches it. +ACT_AMAX_FLOOR = 1.0e-10 + + +def dequantize_block_weight(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """One expert's `[M, K]` fp32 weight from e4m3 codes and `[M/128, K/128]` scales.""" + m, k = codes.shape + blocks = codes.float().reshape(m // BLOCK, BLOCK, k // BLOCK, BLOCK) + return (blocks * scales.float().reshape(m // BLOCK, 1, k // BLOCK, 1)).reshape(m, k) + + +def requantize_1x128(x: torch.Tensor) -> torch.Tensor: + """The e4m3 round trip the kernel applies to both intermediates: per row and 128 columns, + `scale = max(amax, 1e-10) / 448` (not a power of two), round-to-nearest-even saturating at 448. + """ + shape = x.shape + blocks = x.reshape(*shape[:-1], shape[-1] // BLOCK, BLOCK) + scale = blocks.abs().amax(dim=-1, keepdim=True).clamp(min=ACT_AMAX_FLOOR) / E4M3_MAX + codes = (blocks / scale).clamp(-E4M3_MAX, E4M3_MAX).to(torch.float8_e4m3fn) + return (codes.float() * scale).reshape(shape) + + +def _geom(hidden, inter, num_experts, top_k, offset=0, local=None, capture=False): + """A cell's MoE geometry (every expert local by default; `capture`: a captured graph).""" + return dict( + hidden=hidden, + inter=inter, + num_experts=num_experts, + top_k=top_k, + local_expert_offset=offset, + local_num_experts=num_experts if local is None else local, + capture=capture, + ) + + +def _whole(capture: bool = False) -> dict: + """The checkpoint's routed experts (H 2560, I 640, 512, top-10), all local (tp1).""" + return _geom(2560, 640, 512, 10, capture=capture) + + +_CKPT = _whole() + + +def _window(offset: int, local: int = 128, capture: bool = False) -> dict: + """One rank's window of the 512 (128 or 256 wide); `num_experts` stays 512.""" + return _geom(2560, 640, 512, 10, offset=offset, local=local, capture=capture) + + +_CUSTOM_OP = "trtllm::fp8_block_scale_moe_runner" + +#: The geometry `DECODE_TACTICS` was measured on: (num_experts, top_k, intermediate, routing). +DECODE_TACTICS_GEOMETRY = (512, 10, 640, 1) + +#: trtllm-gen tactic `(tileN, config index)` per (window width, rows) for logits-routed decode +#: calls (`routing_logits=`, Renormalize). The engine's tuning pass profiles pre-routed shapes +#: only, so the AutoTuner holds nothing for these and would hand them the slower cold fallback +#: `[32, 0]`. Every tactic gives the fallback's bits: the choice moves time only. A call not +#: listed takes the AutoTuner's tactic for the same call pre-routed (`_dispatch`), else the fallback. +DECODE_TACTICS: dict[tuple[int, int], tuple[int, int]] = { + (128, 1): (8, 13), + (128, 2): (8, 13), + (128, 3): (8, 13), + (128, 4): (8, 13), + (128, 6): (8, 12), + (128, 8): (8, 12), +} + + +#: The tactic logits-routed decode calls over a 256-expert window (tp2 / tep2) run from +#: `WIDE_DECODE_MIN_ROWS` rows on. The AutoTuner profiles near-uniform routing, where tileN 8 wins; +#: this checkpoint's skewed routing makes hot experts restream weight tiles at tileN 8, so tileN 32 +#: is faster from 168 rows (GB300, recorded routing). Every tactic writes the same bits. Config 8 +#: pairs the persistent-scheduler FC1 (`u2`) and FC2 cubins. +WIDE_DECODE_TACTIC: tuple[int, int] = (32, 8) +WIDE_DECODE_MIN_ROWS = 168 +WIDE_DECODE_WIDTH = 256 + +#: The same calls below `WIDE_DECODE_MIN_ROWS`, by the row count each tactic starts at: what a +#: full-size tuning call picks. Pinned because a small first tuning call profiles every bucket on +#: a few experts (the tuner repeats its rows' `topk_ids`) and picks tiles slower under real routing. +WIDE_DECODE_SMALL_TACTICS: tuple[tuple[int, tuple[int, int]], ...] = ( + (1, (8, 13)), + (4, (8, 12)), + (8, (8, 8)), +) + + +#: `(tactic, min_rows)` of the 256-wide decode calls; None (tests set it) leaves them to the AutoTuner. +_WIDE_DECODE: Optional[tuple[tuple[int, int], int]] = (WIDE_DECODE_TACTIC, WIDE_DECODE_MIN_ROWS) + + +def _pinned_tactic(geometry: tuple, inputs: list, runner) -> Optional[list]: + """`DECODE_TACTICS`'s tactic for a logits-routed call at its geometry, if listed, or for a + 256-wide window `WIDE_DECODE_TACTIC` from `WIDE_DECODE_MIN_ROWS` rows on and + `WIDE_DECODE_SMALL_TACTICS` below -- either only if valid at the call's rows; else None.""" + experts, top_k, inter, _, local, routing = geometry[:6] + if inputs[0] is None or inputs[9] is not None: + return None + if (experts, top_k, inter, routing) != DECODE_TACTICS_GEOMETRY: + return None + rows = inputs[2].shape[0] + tactic = DECODE_TACTICS.get((local, rows)) + if tactic is None and _WIDE_DECODE is not None and local == WIDE_DECODE_WIDTH: + if rows >= _WIDE_DECODE[1]: + tactic = _WIDE_DECODE[0] + else: + tactic = [t for start, t in WIDE_DECODE_SMALL_TACTICS if rows >= start][-1] + if tactic is None: + return None + hidden = inputs[2].shape[1] + valid = runner.get_runner().get_valid_configs(top_k, None, hidden, inter, local, rows) + if list(tactic) not in [[int(v) for v in t] for t in valid]: + return None + return list(tactic) + + +# The custom op's AutoTuner choice, keyed on everything its lookup reads (geometry, input shapes, +# the profiling cache's size: a lookup made before the tuning pass is not reused after it). +_DISPATCH: dict = {} + + +def _dispatch(geometry: tuple, inputs: list) -> Optional[tuple]: + """`(runner, tactic)` as `torch.ops.trtllm.fp8_block_scale_moe_runner` would choose them, + remembered; None while the AutoTuner tunes or captures (the custom op then runs as is).""" + tuner = AutoTuner.get() + if tuner.is_tuning_mode or tuner._active_capture is not None: + return None + shapes = tuple(None if t is None else tuple(t.shape) for t in inputs) + key = (geometry, shapes, len(tuner.profiling_cache)) + hit = _DISPATCH.get(key) + if hit is not None: + return hit + experts, top_k, inter, offset, local, routing, act, clamp, tune_max, use_dp = geometry + runner = FP8BlockScaleMoERunner( + experts, + top_k, + None, + None, + None, + inter, + offset, + local, + None, + routing, + act, + tune_max_num_tokens=tune_max, + use_dp=use_dp, + gemm1_clamp_limit_value=clamp, + ) + pinned = _pinned_tactic(geometry, inputs, runner) + if pinned is not None: + _DISPATCH[key] = (runner, pinned) + return runner, pinned + lookup = inputs + if inputs[0] is not None and inputs[9] is None: + # Routed from logits, which the tuning pass never profiles: use the tactic held for the + # same call pre-routed (the GEMMs' shapes are the same). + rows, device = inputs[2].shape[0], inputs[2].device + lookup = list(inputs) + lookup[0] = None + lookup[8] = torch.empty((rows, top_k), dtype=torch.bfloat16, device=device) + lookup[9] = torch.empty((rows, top_k), dtype=torch.int32, device=device) + runner, tactic = tuner.choose_one(_CUSTOM_OP, [runner], runner.tuning_config, lookup) + if tactic == -1: + tactic = runner.get_fallback_tactic(inputs[2].shape[1], inputs[2].shape[0]) + _DISPATCH[key] = (runner, tactic) + return runner, tactic + + +def fp8_block_scale_moe_front( + routing_logits: torch.Tensor, + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routing_method_type: int, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, +) -> Optional[Callable[[Optional[torch.Tensor]], torch.Tensor]]: + """The logits-routed runner call through this target's one-launch decode routing + (`native_ops.fp8_block_scale_moe_front`, Renormalize bitwise the runner's) at the tactic the + custom op would choose. Returns a callable taking the bf16 hidden states (then also quantized + into `hidden_states` / `hidden_states_scale`, bitwise `fp8_quantize_1x128`) or None; returns + None while the AutoTuner tunes or captures (the caller makes the plain call).""" + inputs = [ + routing_logits, + None, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + None, + None, + ] + geometry = ( + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + 0, + None, + tune_max_num_tokens, + use_dp, + ) + chosen = _dispatch(geometry, inputs) + if chosen is None: + return None + runner, tactic = chosen + tactic = [int(v) for v in tactic] + if len(tactic) != 2 or tactic[0] <= 0 or tactic[1] < 0: + return None + + def call(hidden_states_bf16: Optional[torch.Tensor]) -> torch.Tensor: + return native_ops.fp8_block_scale_moe_front( + routing_logits, + hidden_states_bf16, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + tactic[0], + tactic[1], + ) + + return call + + +class Fp8BlockScaleMoeRunner(OpWrapper): + """One FP8 block-scale MoE layer over e4m3 activations, entered pre-routed. + + Per token `t` and slot `j`, with `A[t]` the fp32 activations + (`hidden_states[t, k] * hidden_states_scale[k // 128, t]`) and `W_*` the + dequantized 128x128-block weights of local expert `e = topk_ids[t, j] - + local_expert_offset` (slots outside `[0, local_num_experts)` contribute + nothing): + + up, gate = requant(A[t] @ [W_up; W_gate].T).split(I) + gate, up = min(gate, lim), clamp(up, -lim, lim) # gemm1_clamp_limit + out[t] += topk_weights[t, j] * (requant(silu(gate) * up) @ W_down.T) + + accumulated in fp32 and stored bf16, `requant` being `requantize_1x128`. + FC1 rows are `[up ; gate]` (the loader's `[w3; w1]`). The router and top-k, the input + quantization (`fp8_quantize_1x128(x, use_ue8m0=False)`), the shared expert and any + collective are the caller's. + + `hidden_states_scale` is `[H/128, T]`, as `fp8_quantize_1x128` returns it: at `T == H/128` + (20 tokens) its transpose has the same shape, passes every check and is wrong. + + Returns a fresh `[T, H]` bf16 tensor, or an empty `[0]` tensor when `output` is given (the + result is then in `output`). + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + # Each window width (512 / 256 / 128) at its smallest and largest rows, first and last + # offsets, captured where the engine captures, and one non-power-of-two decode batch. + Cell( + why="MTPLayer._moe decode: one token, 10 of 512 experts", spec=dict(tokens=1, **_CKPT) + ), + Cell( + why="20 tokens == H/128: the activation scale buffer is square", + spec=dict(tokens=20, **_CKPT), + ), + Cell( + why="16384 rows: max_num_tokens of the tuned config; past the last tuning bucket", + spec=dict(tokens=16384, **_CKPT), + ), + Cell( + why="tp1 captured decode, 128 rows: the whole 512-expert stack", + spec=dict(tokens=128, **_whole(capture=True)), + ), + Cell( + why="tep2 captured decode, 1 row: 256-expert window 0", + spec=dict(tokens=1, **_window(0, 256, capture=True)), + ), + Cell( + why="dep2 captured decode, 2 x 128 gathered rows: 256-expert window 0", + spec=dict(tokens=256, **_window(0, 256, capture=True)), + ), + Cell( + why="tep2 prefill, a 16384-row chunk (max_num_tokens 16384): 256-expert window 256", + spec=dict(tokens=16384, **_window(256, 256)), + ), + Cell( + why="tep4 captured decode, 1 row: 128-expert window 256", + spec=dict(tokens=1, **_window(256, capture=True)), + ), + Cell( + why="tp4 captured decode, 128 rows: 128-expert window 0", + spec=dict(tokens=128, **_window(0, capture=True)), + ), + Cell( + why="dep4 captured MTP3 verify, 4 x 128 x 4 gathered rows: 128-expert window 384", + spec=dict(tokens=2048, **_window(384, capture=True)), + ), + Cell( + why="tep2/tp2 captured decode, batch 40: 256-expert window 0", + spec=dict(tokens=40, **_window(0, 256, capture=True)), + ), + Cell( + why="tep4/tp4 captured decode, batch 56: 128-expert window 128", + spec=dict(tokens=56, **_window(128, capture=True)), + ), + ) + + note = """ + Source: the custom op `trtllm::fp8_block_scale_moe_runner` + (`_torch/custom_ops/trtllm_gen_custom_ops.py`) over `torch.classes.trtllm.FP8BlockScaleMoERunner` + (`cpp/tensorrt_llm/thop/moe/trtllmGen/fp8BlockScaleMoe.cpp`): trtllm-gen's precompiled + batched-GEMM cubins, nothing JIT-compiled. A warm tuned tactic is bitwise the cold fallback; + the tactic is chosen on the host at capture and fixed in the graph, and the runner's + workspaces come from the graph pool, so a replay is bitwise the eager call. + + Preconditions: `hidden_states` `[T, H]` e4m3, `hidden_states_scale` `[H/128, T]` fp32 (see + the class docstring); plain row-major weights, no shuffle or swizzle: `gemm1_weights` + `[E, 2I, H]` e4m3 rows `[up ; gate]`, `gemm2_weights` `[E, H, I]`, scales `[E, 2I/128, H/128]` + and `[E, H/128, I/128]` fp32; `topk_ids` `[T, top_k]` int32 and `topk_weights` `[T, top_k]` + bf16, used as given. H and I multiples of 128, `top_k <= 32` (the op raises otherwise). Ids + must be distinct per row: a repeated id counts once or twice by the routing kernel the row + count picks. + + Windows: local slot i is global id `local_expert_offset + i`; `num_experts` stays the routing + space (512) and the weights hold the window's experts only; a token with no slot in the + window gets zero. The window's own count passed as `num_experts` silently zeroes or reads out + of bounds; `is_valid` refuses a window starting outside `[0, num_experts)`. + `gemm1_clamp_limit` is live; `routing_method_type`, `n_group` / `topk_group`, + `routed_scaling_factor`, `tune_max_num_tokens` and `use_dp` are inert on the pre-routed path. + The registered fake returns `[T, 2H]` (upstream defect; eager is unaffected). + + Certified on GB300 with near-uniform random routing at the checkpoint's E 512, H 2560, I 640, + top-10, within the test's 16 / 0.75 bf16-ulp gate: the whole stack, 256-wide windows at + offsets 0 / 256 and 128-wide at 0 / 128 / 256 / 384, at the cells' rows (1 to 16384, captured + where the cells say). Not certified: other windows or geometries, rows past 16384 (the target + chunks at 8192 under attention DP and 16384 under TP), captured rows other than the cells', + repeated or out-of-range ids, `gemm1_clamp_limit`, `output=`, skewed routing beyond the + 256-wide logits-routed calls the test holds, multi-rank execution, sm_100. + """ + + def __call__( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> torch.Tensor: + # Outside a tuning pass, a pre-routed (or plain logits-routed) unfused call enters the + # runner directly at the custom op's own choice (`_dispatch`), skipping its dispatch and + # AutoTuner lookup (~0.1 ms of host time a call). + if ( + (routing_logits is None) != (topk_ids is None) + and routing_bias is None + and num_fused_shared_experts is None + and n_group is None + and topk_group is None + and routed_scaling_factor is None + ): + inputs = [ + routing_logits, + None, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + topk_weights, + topk_ids, + ] + geometry = ( + num_experts, + top_k, + intermediate_size, + local_expert_offset, + local_num_experts, + routing_method_type, + act_type, + gemm1_clamp_limit, + tune_max_num_tokens, + use_dp, + ) + chosen = _dispatch(geometry, inputs) + if chosen is not None: + runner, tactic = chosen + result = runner.forward(inputs, tactic=tactic, output=output) + if output is not None: + return torch.empty(0, device=result.device, dtype=result.dtype) + return result + return torch.ops.trtllm.fp8_block_scale_moe_runner( + routing_logits, + routing_bias, + hidden_states, + hidden_states_scale, + gemm1_weights, + gemm1_weights_scale, + gemm2_weights, + gemm2_weights_scale, + num_experts, + top_k, + num_fused_shared_experts, + n_group, + topk_group, + intermediate_size, + local_expert_offset, + local_num_experts, + routed_scaling_factor, + routing_method_type, + topk_weights=topk_weights, + topk_ids=topk_ids, + act_type=act_type, + gemm1_clamp_limit=gemm1_clamp_limit, + output=output, + tune_max_num_tokens=tune_max_num_tokens, + use_dp=use_dp, + ) + + def reference( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> torch.Tensor: + """The layer over bit-identical operands, fp32 GEMMs, bf16 store. Pre-routed only + (`routing_logits` None); the arguments inert on that path are ignored, and the `[T, H]` + result is returned even when `output` is given.""" + assert routing_logits is None and topk_ids is not None and topk_weights is not None, ( + "the reference models the pre-routed path only" + ) + num_tokens, hidden = hidden_states.shape + nb = hidden // BLOCK + act_in = ( + hidden_states.float().reshape(num_tokens, nb, BLOCK) + * hidden_states_scale.t().reshape(num_tokens, nb, 1).float() + ).reshape(num_tokens, hidden) + out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=hidden_states.device) + local = topk_ids.long() - local_expert_offset + live = local[(local >= 0) & (local < local_num_experts)] + with true_fp32_matmul(): + for e in torch.unique(live).tolist(): + tok, slot = (local == e).nonzero(as_tuple=True) + w1 = dequantize_block_weight(gemm1_weights[e], gemm1_weights_scale[e]) + y1 = requantize_1x128(act_in[tok] @ w1.t()) + up, gate = y1[:, :intermediate_size], y1[:, intermediate_size:] + if gemm1_clamp_limit is not None: + gate = gate.clamp(max=gemm1_clamp_limit) + up = up.clamp(-gemm1_clamp_limit, gemm1_clamp_limit) + act = requantize_1x128(torch.nn.functional.silu(gate) * up) + w2 = dequantize_block_weight(gemm2_weights[e], gemm2_weights_scale[e]) + y = act @ w2.t() + out.index_add_(0, tok, y * topk_weights[tok, slot].float().unsqueeze(1)) + return out.to(torch.bfloat16) + + def is_valid( + self, + routing_logits: Optional[torch.Tensor], + routing_bias: Optional[torch.Tensor], + hidden_states: torch.Tensor, + hidden_states_scale: torch.Tensor, + gemm1_weights: torch.Tensor, + gemm1_weights_scale: torch.Tensor, + gemm2_weights: torch.Tensor, + gemm2_weights_scale: torch.Tensor, + num_experts: int, + top_k: int, + num_fused_shared_experts: Optional[int], + n_group: Optional[int], + topk_group: Optional[int], + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + topk_weights: Optional[torch.Tensor] = None, + topk_ids: Optional[torch.Tensor] = None, + act_type: int = 0, + gemm1_clamp_limit: Optional[float] = None, + output: Optional[torch.Tensor] = None, + tune_max_num_tokens: int = 8192, + use_dp: bool = False, + ) -> None: + """Inputs the op accepts and answers wrongly.""" + # The kernel takes raw data pointers: a strided view is read as dense. + for name, tensor in ( + ("routing_logits", routing_logits), + ("routing_bias", routing_bias), + ("hidden_states", hidden_states), + ("hidden_states_scale", hidden_states_scale), + ("gemm1_weights", gemm1_weights), + ("gemm1_weights_scale", gemm1_weights_scale), + ("gemm2_weights", gemm2_weights), + ("gemm2_weights_scale", gemm2_weights_scale), + ("topk_weights", topk_weights), + ("topk_ids", topk_ids), + ("output", output), + ): + if tensor is not None: + assert tensor.is_contiguous(), ( + f"{name} must be contiguous; a strided view is read as if dense " + "and silently produces wrong results" + ) + # The Python runner overwrites the activation with SwiGlu. + assert act_type == 0, ( + f"act_type={act_type} is silently ignored -- this op always computes " + "SwiGLU. Use a different runner for a non-gated activation" + ) + # Once topk_ids carry the routing, nothing reads the bias. + if routing_bias is not None: + assert topk_ids is None, ( + "routing_bias is ignored when topk_ids/topk_weights are given " + "(routing has already happened); fold it into the router logits" + ) + # `num_experts` is the routing space, not the window: a window starting at or past it + # holds no routable id (silent zeros, or an out-of-bounds read at 16 rows). + assert 0 <= local_expert_offset < num_experts, ( + f"local_expert_offset={local_expert_offset} is outside the routing space " + f"[0, num_experts={num_experts}); num_experts is every routed expert " + "(512 for a 128-expert window of 512), not the window's count" + ) + + +fp8_block_scale_moe_runner = Fp8BlockScaleMoeRunner() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py new file mode 100644 index 000000000000..a6315a2eff9e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_front_router_quant__decode.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's MoE front end in one Triton launch: the router's bf16 logits and the 1x128 e4m3 +quantization of the same rows (bitwise `fp8_quantize_1x128`); the runner routes from the logits +itself (`RoutingMethodType::Renormalize`). +""" + +from __future__ import annotations + +import functools + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber + +#: Columns per activation scale. +QUANT_BLOCK = 128 +#: Largest finite e4m3 magnitude. +E4M3_MAX = 448.0 +#: `fp8_quantize_1x128`'s amax floor, compared in bf16: bfloat16(1e-10). +AMAX_FLOOR = float(torch.tensor(1e-10, dtype=torch.bfloat16)) +#: Largest row count the entry takes (tep4 MTP-3 verify of 4 requests). +MAX_ROWS = 16 +#: The checkpoint's router: 512 experts over hidden 2560. +EXPERTS = 512 +HIDDEN = 2560 + + +@triton.jit +def _scales(amax): + """`(448 / amax, 1 / (448 / amax))` as IEEE divisions, bitwise the CUDA kernel's.""" + quant = tl.math.div_rn(tl.full(amax.shape, 448.0, tl.float32), amax) + dequant = tl.math.div_rn(tl.full(amax.shape, 1.0, tl.float32), quant) + return quant, dequant + + +@triton.jit +def _moe_front_kernel( + x_ptr, + w_ptr, + logits_ptr, + xq_ptr, + xsf_ptr, + rows, + ROWS_PAD: tl.constexpr, + HIDDEN_: tl.constexpr, + CHUNK: tl.constexpr, + EXPERTS_: tl.constexpr, + QBLOCK: tl.constexpr, + AMAX_MIN: tl.constexpr, + USE_PDL: tl.constexpr, +): + # One expert a program: the hidden width as five CHUNK-wide chunks, the weight row's `[1, CHUNK]` + # tiles broadcast over the rows' `[ROWS_PAD, CHUNK]` tiles, reduced once per row at the end. + tl.static_assert(HIDDEN_ == 5 * CHUNK) + pid = tl.program_id(0) + router_program = pid < EXPERTS_ + offs_k = tl.arange(0, CHUNK)[None, :] + offs_r = tl.arange(0, ROWS_PAD)[:, None] + # Nothing in the step writes the router weight, so its row is in flight before the PDL wait. + wmask = (offs_k < CHUNK) & router_program + w_row = w_ptr + pid * HIDDEN_ + offs_k + w0 = tl.load(w_row, mask=wmask, other=0.0) + w1 = tl.load(w_row + CHUNK, mask=wmask, other=0.0) + w2 = tl.load(w_row + 2 * CHUNK, mask=wmask, other=0.0) + w3 = tl.load(w_row + 3 * CHUNK, mask=wmask, other=0.0) + w4 = tl.load(w_row + 4 * CHUNK, mask=wmask, other=0.0) + if USE_PDL: + gdc_wait_with_memory_clobber() + tl.extra.cuda.gdc_launch_dependents() + xmask = (offs_r < rows) & router_program + x_rows = x_ptr + offs_r * HIDDEN_ + offs_k + acc = tl.load(x_rows, mask=xmask, other=0.0).to(tl.float32) * w0.to(tl.float32) + acc += tl.load(x_rows + CHUNK, mask=xmask, other=0.0).to(tl.float32) * w1.to(tl.float32) + acc += tl.load(x_rows + 2 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w2.to(tl.float32) + acc += tl.load(x_rows + 3 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w3.to(tl.float32) + acc += tl.load(x_rows + 4 * CHUNK, mask=xmask, other=0.0).to(tl.float32) * w4.to(tl.float32) + dot = tl.sum(acc, axis=1) + r = tl.arange(0, ROWS_PAD) + tl.store(logits_ptr + r * EXPERTS_ + pid, dot.to(tl.bfloat16), mask=(r < rows) & router_program) + # One 1x128 block of the activation per program, `fp8_quantize_1x128`'s arithmetic. + nblocks: tl.constexpr = HIDDEN_ // QBLOCK + if pid < rows * nblocks: + row = pid // nblocks + blk = pid - row * nblocks + offs_q = blk * QBLOCK + tl.arange(0, QBLOCK) + xb = tl.load(x_ptr + row * HIDDEN_ + offs_q).to(tl.float32) + # The block amax on every lane (the scales are formed lane-wise, the store is one). + amax = tl.zeros([QBLOCK], tl.float32) + tl.max(tl.abs(xb), axis=0) + quant, dequant = _scales(tl.maximum(amax, AMAX_MIN)) + tl.store(xq_ptr + row * HIDDEN_ + offs_q, (xb * quant).to(tl.float8e4nv)) + lane = tl.arange(0, QBLOCK) + tl.store(xsf_ptr + blk * rows + row + lane * 0, dequant, mask=lane == 0) + + +@functools.cache +def _pdl(device_index: int) -> bool: + return get_env_enable_pdl() and torch.cuda.get_device_capability(device_index)[0] >= 9 + + +#: Warps a program runs with, by padded row count (fastest of 2 / 4 / 8 on GB300). +_NUM_WARPS = {1: 4, 2: 4, 4: 4, 8: 4, 16: 2} + + +def _launch(x, router_rows, logits, xq, xsf, num_warps: int, pdl: bool) -> None: + rows = x.shape[0] + # One program per expert, and at least one per 1x128 block of the activation. + grid = (max(EXPERTS, rows * (HIDDEN // QUANT_BLOCK)),) + _moe_front_kernel[grid]( + x, + router_rows, + logits, + xq, + xsf, + rows, + ROWS_PAD=triton.next_power_of_2(rows), + HIDDEN_=HIDDEN, + CHUNK=HIDDEN // 5, + EXPERTS_=EXPERTS, + QBLOCK=QUANT_BLOCK, + AMAX_MIN=AMAX_FLOOR, + USE_PDL=pdl, + num_warps=num_warps, + **(dict(launch_pdl=True) if pdl else {}), + ) + + +def _router_cells(): + rows = (1, 2, 3, 6, 16) + eager = tuple( + Cell( + why=f"decode MoE front end at {r} rows (tep4 / tp1 LL verify and draft steps)", + spec=dict(rows=r, capture=False), + ) + for r in rows + ) + captured = tuple( + Cell( + why=f"captured decode MoE front end at {r} rows, replayed over two steps", + spec=dict(rows=r, capture=True), + ) + for r in (1, 12) + ) + return eager + captured + + +class MoeFrontRouterQuant(OpWrapper): + """`(logits, xq, xsf) = moe_front_router_quant(x, router_rows)` over a decode step's rows. + + `x` is the MoE site's bf16 `[T, 2560]` mixer output, `router_rows` the router weight as + the checkpoint stores it, `[512, 2560]` bf16 row-major (the target's `router` operand is + its `.t()` view; pass `router.t()`). Returns: + + - `logits` `[T, 512]` bf16: `x @ router_rows.T`, a SIMT dot accumulated in fp32 in the + kernel's own order and rounded once (the router projection's value, not its bits); + - `xq` `[T, 2560]` e4m3 and `xsf` `[20, T]` fp32: bitwise `fp8_quantize_1x128(x, + use_ue8m0=False)`, the scales transposed and compact (the layout the FP8 block-scale runner + reads). + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _router_cells() + + note = """ + Source: one Triton kernel (`_moe_front_kernel`), JIT-compiled per padded row count + (1, 2, 4, 8, 16); the engine's warmup forward compiles it before capture. Stateless; runs on + the ambient stream; PDL per `TRTLLM_ENABLE_PDL` on sm_90+. Preconditions, refused by + `is_valid`: `x` contiguous bf16 `[T, 2560]`, `1 <= T <= MAX_ROWS`; `router_rows` the + contiguous `[512, 2560]` checkpoint layout, not the target's `.t()` view. + """ + + def __call__( + self, x: torch.Tensor, router_rows: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + rows = x.shape[0] + logits = torch.empty([rows, EXPERTS], dtype=torch.bfloat16, device=x.device) + xq = torch.empty([rows, HIDDEN], dtype=torch.float8_e4m3fn, device=x.device) + xsf = torch.empty([HIDDEN // QUANT_BLOCK, rows], dtype=torch.float32, device=x.device) + _launch( + x, + router_rows, + logits, + xq, + xsf, + _NUM_WARPS[triton.next_power_of_2(rows)], + _pdl(x.device.index), + ) + return logits, xq, xsf + + def reference( + self, x: torch.Tensor, router_rows: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """fp32 logits rounded once to bf16; the quantization with exact divisions.""" + with true_fp32_matmul(): + logits = (x.float() @ router_rows.float().t()).to(torch.bfloat16) + t, h = x.shape + blocks = x.float().reshape(t, h // QUANT_BLOCK, QUANT_BLOCK) + amax = blocks.abs().amax(dim=2).clamp(min=AMAX_FLOOR) + quant = E4M3_MAX / amax + xq = (blocks * quant.unsqueeze(2)).clamp(-E4M3_MAX, E4M3_MAX).to(torch.float8_e4m3fn) + return logits, xq.reshape(t, h), (1.0 / quant).t().contiguous() + + def is_valid(self, x: torch.Tensor, router_rows: torch.Tensor) -> None: + assert x.dtype == torch.bfloat16 and router_rows.dtype == torch.bfloat16, ( + x.dtype, + router_rows.dtype, + ) + assert x.is_contiguous() and router_rows.is_contiguous(), ( + "x and router_rows are read as dense row-major; pass the checkpoint's [512, 2560] " + "router (the target's `router.t()`), not its transposed view" + ) + assert x.dim() == 2 and x.shape[1] == HIDDEN and 1 <= x.shape[0] <= MAX_ROWS, tuple(x.shape) + assert tuple(router_rows.shape) == (EXPERTS, HIDDEN), tuple(router_rows.shape) + + +moe_front_router_quant = MoeFrontRouterQuant() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py new file mode 100644 index 000000000000..9b848414c84f --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_prefill_grouped__prefill.py @@ -0,0 +1,671 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A prefill chunk's routed experts over a rank's window, pre-routed: the rows permuted into +expert-major slots padded to 128-row M tiles, FC1 and FC2 as contiguous grouped FP8 block-scale +GEMMs (`blockwise_gemm_grouped`, the checkpoint's 1x128 / 128x128 scales as they are), then each +row's routing-weighted sum over its slots (fp32, a fixed tree). + +No host synchronization, so a call is CUDA-graph capturable: buffers are sized for the worst case +of the call's rows and the GEMMs skip tiles past the live count. Slot ranks depend on scheduling, +the results do not.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper + +from .blockwise_gemm_grouped import blockwise_gemm_grouped, blockwise_gemm_grouped_quant +from .fp8_block_scale_moe_runner import fp8_block_scale_moe_runner + +HIDDEN = 2560 +INTER = 640 +FC1_ROWS = 2 * INTER +TOPK = 10 +BLOCK = 128 +#: Rows of a grouped GEMM's M tile; every expert's rows are padded to a multiple of it. +M_TILE = 128 +E4M3_MAX = 448.0 +#: The runner's floor of a 1x128 block's amax before its scale (`fp8_block_scale_moe_runner`). +ACT_AMAX_FLOOR = 1.0e-10 +#: Columns of a finalize chunk (`_finalize_kernel`, one row a program). +FIN_BH = 512 +#: Rows of an activation program (`_act_q_kernel`). +ACT_BM = 16 +#: Counter stride (int32 words) of `_count_kernel`'s atomics: one 128-byte line an expert. +CNT_STRIDE = 32 +#: Whether the finalize keeps two chunks in flight (`_finalize_pipe_kernel`, the same sums). +FIN_PIPE = True + +#: (mma_tiler_mn, cluster_shape_mn) of FC1 / FC2, on 1-CTA instructions. +TACTICS = (((128, 128), (1, 1)), ((128, 128), (1, 1))) +#: FC1's epilogue stores the 1x128 e4m3 round trip (`epi_quant`), not fp32: same bits, 1/4 bytes. +EPI_QUANT = True + + +@triton.jit(do_not_specialize=["rows", "offset", "local"]) +def _count_kernel( + ids_ptr, + cnt_ptr, + rank_ptr, + rows, + offset, + local, + K: tl.constexpr, + KP: tl.constexpr, + RB: tl.constexpr, + CS: tl.constexpr, +): + """rank[r, k] = the next free rank of expert ids[r, k] (atomic), or -1 outside the window.""" + r = tl.program_id(0) * RB + tl.arange(0, RB) + kk = tl.arange(0, KP) + m = (r < rows)[:, None] & (kk < K)[None, :] + e = tl.load(ids_ptr + r[:, None] * K + kk[None, :], mask=m, other=-1) - offset + inw = m & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + pos = tl.atomic_add(cnt_ptr + ec * CS, tl.where(inw, 1, 0), mask=inw, sem="relaxed") + tl.store(rank_ptr + r[:, None] * K + kk[None, :], tl.where(inw, pos, -1), mask=m) + + +@triton.jit(do_not_specialize=["local"]) +def _layout_kernel( + cnt_ptr, + cnt_out_ptr, + roff_ptr, + group_ptr, + ntiles_ptr, + local, + LP: tl.constexpr, + TB: tl.constexpr, + MT: tl.constexpr, + CS: tl.constexpr, +): + """Program e: expert e's rows start at roff[e] (a multiple of MT), its M tiles name it in + `group`; program 0 stores the live tile count; with CS > 1, e's count goes to `cnt_out`.""" + pe = tl.program_id(0) + e = tl.arange(0, LP) + c = tl.load(cnt_ptr + e * CS, mask=e < local, other=0) + if CS > 1: + tl.store(cnt_out_ptr + pe, tl.sum(tl.where(e == pe, c, 0), axis=0)) + t = (c + MT - 1) // MT + tend = tl.cumsum(t, axis=0) + mine = tl.sum(tl.where(e == pe, t, 0), axis=0) + start = tl.sum(tl.where(e == pe, tend, 0), axis=0) - mine + tl.store(roff_ptr + pe, start * MT) + if pe == 0: + tl.store(ntiles_ptr, tl.sum(t, axis=0)) + for j0 in range(0, mine, TB): + j = j0 + tl.arange(0, TB) + tl.store(group_ptr + start + j, tl.full([TB], 0, tl.int32) + pe, mask=j < mine) + + +@triton.jit(do_not_specialize=["xs_kb", "xs_t", "sf_m", "offset", "local"]) +def _permute_kernel( + xq_ptr, + xsf_ptr, + xs_kb, + xs_t, + ids_ptr, + rank_ptr, + roff_ptr, + a_ptr, + sfa_ptr, + sf_m, + offset, + local, + K: tl.constexpr, + HW: tl.constexpr, + HWP: tl.constexpr, + NKB: tl.constexpr, + NKBP: tl.constexpr, +): + """Row t's e4m3 codes and NKB scales to each of its window slots (row roff[e] + rank).""" + t = tl.program_id(0).to(tl.int64) + w = tl.arange(0, HWP) + wm = w < HW + v = tl.load(xq_ptr + t * HW + w, mask=wm, other=0) + kb = tl.arange(0, NKBP) + kbm = kb < NKB + sc = tl.load(xsf_ptr + kb * xs_kb + t * xs_t, mask=kbm, other=0.0) + for k in tl.static_range(K): + e = tl.load(ids_ptr + t * K + k) - offset + if (e >= 0) & (e < local): + dst = (tl.load(roff_ptr + e) + tl.load(rank_ptr + t * K + k)).to(tl.int64) + tl.store(a_ptr + dst * HW + w, v, mask=wm) + tl.store(sfa_ptr + kb.to(tl.int64) * sf_m + dst, sc, mask=kbm) + + +@triton.jit +def _requant(x, AMAX_MIN: tl.constexpr, QMAX: tl.constexpr): + """1x128 e4m3 round trip along axis 1.""" + amax = tl.maximum(tl.max(tl.abs(x), axis=1), AMAX_MIN) + sc = tl.math.div_rn(amax, tl.full(amax.shape, QMAX, tl.float32)) + inv = tl.math.div_rn(tl.full(amax.shape, QMAX, tl.float32), amax) + q = tl.minimum(tl.maximum(x * inv[:, None], -QMAX), QMAX).to(tl.float8e4nv) + return q, sc + + +@triton.jit(do_not_specialize=["sf_m"]) +def _act_kernel( + c1_ptr, + group_ptr, + roff_ptr, + cnt_ptr, + ntiles_ptr, + a2_ptr, + sfa2_ptr, + sf_m, + I_: tl.constexpr, + NB: tl.constexpr, + NBP: tl.constexpr, + QB: tl.constexpr, + BM: tl.constexpr, + MT: tl.constexpr, + AMAX_MIN: tl.constexpr, + QMAX: tl.constexpr, +): + """BM rows of one M tile: fp32 up / gate -> e4m3 round trip -> silu(gate) * up -> e4m3 codes + `a2[r, I_]` and M-major scales `sfa2[b, r]`; rows and tiles past the live ones skipped.""" + r0 = tl.program_id(0) * BM + tile = r0 // MT + if tile < tl.load(ntiles_ptr): + g = tl.load(group_ptr + tile) + lim = tl.load(roff_ptr + g) + tl.load(cnt_ptr + g) + if r0 < lim: + r2 = tl.arange(0, BM * NBP) + r = r0 + r2 // NBP + b = r2 % NBP + ok = (r < lim) & (b < NB) + row = r.to(tl.int64) + cols = tl.arange(0, QB) + off = row[:, None] * (2 * I_) + b[:, None] * QB + cols[None, :] + up = tl.load(c1_ptr + off, mask=ok[:, None], other=0.0) + gate = tl.load(c1_ptr + off + I_, mask=ok[:, None], other=0.0) + uq, us = _requant(up, AMAX_MIN, QMAX) + gq, gs = _requant(gate, AMAX_MIN, QMAX) + u = uq.to(tl.float32) * us[:, None] + gv = gq.to(tl.float32) * gs[:, None] + h = gv * tl.sigmoid(gv) * u + hq, hs = _requant(h, AMAX_MIN, QMAX) + tl.store( + a2_ptr + row[:, None] * I_ + b[:, None] * QB + cols[None, :], + hq.to(tl.uint8, bitcast=True), + mask=ok[:, None], + ) + tl.store(sfa2_ptr + b.to(tl.int64) * sf_m + row, hs, mask=ok) + + +@triton.jit(do_not_specialize=["sf_m"]) +def _act_q_kernel( + q1_ptr, + s1_ptr, + group_ptr, + roff_ptr, + cnt_ptr, + ntiles_ptr, + a2_ptr, + sfa2_ptr, + sf_m, + I_: tl.constexpr, + NB: tl.constexpr, + QB: tl.constexpr, + BM: tl.constexpr, + MT: tl.constexpr, + AMAX_MIN: tl.constexpr, + QMAX: tl.constexpr, +): + """`_act_kernel` over FC1's stored codes `q1[r, 2 I_]` and M-major scales `s1[b, r]`.""" + r0 = tl.program_id(0) * BM + tile = r0 // MT + if tile < tl.load(ntiles_ptr): + g = tl.load(group_ptr + tile) + lim = tl.load(roff_ptr + g) + tl.load(cnt_ptr + g) + if r0 < lim: + r = r0 + tl.arange(0, BM) + ok = r < lim + row = r.to(tl.int64) + cols = tl.arange(0, QB) + for b in tl.static_range(NB): + off = row[:, None] * (2 * I_) + b * QB + cols[None, :] + uq = tl.load(q1_ptr + off, mask=ok[:, None], other=0).to( + tl.float8e4nv, bitcast=True + ) + gq = tl.load(q1_ptr + off + I_, mask=ok[:, None], other=0).to( + tl.float8e4nv, bitcast=True + ) + us = tl.load(s1_ptr + b * sf_m + row, mask=ok, other=0.0) + gs = tl.load(s1_ptr + (b + NB) * sf_m + row, mask=ok, other=0.0) + u = uq.to(tl.float32) * us[:, None] + gv = gq.to(tl.float32) * gs[:, None] + h = gv * tl.sigmoid(gv) * u + hq, hs = _requant(h, AMAX_MIN, QMAX) + tl.store( + a2_ptr + row[:, None] * I_ + b * QB + cols[None, :], + hq.to(tl.uint8, bitcast=True), + mask=ok[:, None], + ) + tl.store(sfa2_ptr + b * sf_m + row, hs, mask=ok) + + +@triton.jit(do_not_specialize=["offset", "local", "row0"]) +def _finalize_kernel( + c2_ptr, + ids_ptr, + w_ptr, + rank_ptr, + roff_ptr, + out_ptr, + offset, + local, + row0, + H: tl.constexpr, + K: tl.constexpr, + KP: tl.constexpr, + BH: tl.constexpr, + NCH: tl.constexpr, +): + """Row t = row0 + program: its window slots' FC2 rows times their routing weights, summed in + fp32 (a fixed tree), stored bf16 (zero with no slot in the window). A row reads only its own + slots, so a launch over any row range writes the whole launch's bits there.""" + t = row0 + tl.program_id(0).to(tl.int64) + kk = tl.arange(0, KP) + km = kk < K + e = tl.load(ids_ptr + t * K + kk, mask=km, other=-1) - offset + inw = km & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + rows = ( + tl.load(roff_ptr + ec, mask=inw, other=0) + + tl.load(rank_ptr + t * K + kk, mask=inw, other=0) + ).to(tl.int64) + w = tl.where(inw, tl.load(w_ptr + t * K + kk, mask=km, other=0.0).to(tl.float32), 0.0) + for ch in tl.static_range(NCH): + cols = ch * BH + tl.arange(0, BH) + cm = cols < H + v = tl.load( + c2_ptr + rows[:, None] * H + cols[None, :], mask=inw[:, None] & cm[None, :], other=0.0 + ) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + cols, acc.to(tl.bfloat16), mask=cm) + + +@triton.jit(do_not_specialize=["offset", "local", "row0"]) +def _finalize_pipe_kernel( + c2_ptr, + ids_ptr, + w_ptr, + rank_ptr, + roff_ptr, + out_ptr, + offset, + local, + row0, + H: tl.constexpr, + K: tl.constexpr, + KP: tl.constexpr, + BH: tl.constexpr, + NCH: tl.constexpr, +): + """`_finalize_kernel` with chunk c + 1's loads issued before chunk c's store; the same sums.""" + t = row0 + tl.program_id(0).to(tl.int64) + kk = tl.arange(0, KP) + km = kk < K + e = tl.load(ids_ptr + t * K + kk, mask=km, other=-1) - offset + inw = km & (e >= 0) & (e < local) + ec = tl.where(inw, e, 0) + rows = ( + tl.load(roff_ptr + ec, mask=inw, other=0) + + tl.load(rank_ptr + t * K + kk, mask=inw, other=0) + ).to(tl.int64) + w = tl.where(inw, tl.load(w_ptr + t * K + kk, mask=km, other=0.0).to(tl.float32), 0.0) + cols = tl.arange(0, BH) + v = tl.load( + c2_ptr + rows[:, None] * H + cols[None, :], + mask=inw[:, None] & (cols < H)[None, :], + other=0.0, + ) + for ch in tl.static_range(1, NCH): + nc = ch * BH + tl.arange(0, BH) + vn = tl.load( + c2_ptr + rows[:, None] * H + nc[None, :], + mask=inw[:, None] & (nc < H)[None, :], + other=0.0, + ) + pc = (ch - 1) * BH + tl.arange(0, BH) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + pc, acc.to(tl.bfloat16), mask=pc < H) + v = vn + lc = (NCH - 1) * BH + tl.arange(0, BH) + acc = tl.sum(v.to(tl.float32) * w[:, None], axis=0) + tl.store(out_ptr + t * H + lc, acc.to(tl.bfloat16), mask=lc < H) + + +def tile_bound(rows: int, local: int, topk: int = TOPK) -> int: + """M tiles that hold any routing of `rows` rows over a `local`-expert window.""" + return -(-rows * min(topk, local) // M_TILE) + local + + +def _plan(xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics=None, fin_first=None): + """One call's buffers and its launches in order as (name, launch) pairs, the last writing `out`; + `fin_first = (r0, r1)` finalizes rows `[r0, r1)` before the rest (the same bits).""" + t1, t2 = tactics or TACTICS + rows = xq.shape[0] + local = fc1.shape[0] + dev = xq.device + assert xq.shape[1] == HIDDEN and xq.is_contiguous() and ids.shape == (rows, TOPK) + ids = ids.contiguous() + wts = wts.contiguous() + out = torch.empty(rows, HIDDEN, dtype=torch.bfloat16, device=dev) + if rows == 0: + return out, [] + tiles = tile_bound(rows, local) + m = tiles * M_TILE + cs = CNT_STRIDE + cntp = torch.empty(local * cs, dtype=torch.int32, device=dev) + cnt = cntp if cs == 1 else torch.empty(local, dtype=torch.int32, device=dev) + rank = torch.empty(rows, TOPK, dtype=torch.int32, device=dev) + roff = torch.empty(local, dtype=torch.int32, device=dev) + group = torch.empty(tiles, dtype=torch.int32, device=dev) + ntiles = torch.empty(1, dtype=torch.int32, device=dev) + # A (later the FC2 operand) and C (FC1's codes or fp32 accumulator, later FC2's bf16) are + # reused in place: each is dead by the time its second use writes it (one stream). + a = torch.empty(m * HIDDEN, dtype=torch.uint8, device=dev) + sfa = torch.empty((HIDDEN // BLOCK) * m, dtype=torch.float32, device=dev) + c = torch.empty(m * FC1_ROWS, dtype=torch.float32, device=dev) + a1 = a.view(torch.float8_e4m3fn).view(m, HIDDEN) + a2 = a[: m * INTER].view(m, INTER) + sfa2 = sfa[: (INTER // BLOCK) * m].view(INTER // BLOCK, m) + c2 = c.view(torch.bfloat16)[: m * HIDDEN].view(m, HIDDEN) + epiq = EPI_QUANT and t1[0][1] == BLOCK + q1 = c.view(torch.uint8)[: m * FC1_ROWS].view(m, FC1_ROWS) + s1q = torch.empty(FC1_ROWS // BLOCK, m, dtype=torch.float32, device=dev) if epiq else None + c1 = c.view(m, FC1_ROWS) + rb = 16 + bm = 4 + + def count(): + cntp.zero_() + _count_kernel[(triton.cdiv(rows, rb),)]( + ids, cntp, rank, rows, offset, local, K=TOPK, KP=16, RB=rb, CS=cs, num_warps=4 + ) + + def layout(): + _layout_kernel[(local,)]( + cntp, + cnt, + roff, + group, + ntiles, + local, + LP=triton.next_power_of_2(local), + TB=128, + MT=M_TILE, + CS=cs, + num_warps=4, + ) + + def permute(): + _permute_kernel[(rows,)]( + xq.view(torch.int32), + xsf, + xsf.stride(0), + xsf.stride(1), + ids, + rank, + roff, + a.view(torch.int32), + sfa, + m, + offset, + local, + K=TOPK, + HW=HIDDEN // 4, + HWP=triton.next_power_of_2(HIDDEN // 4), + NKB=HIDDEN // BLOCK, + NKBP=32, + num_warps=4, + ) + + def fc1g(): + if epiq: + blockwise_gemm_grouped_quant( + a1, + fc1, + sfa.view(HIDDEN // BLOCK, m), + fc1_scale, + q1, + s1q, + group, + ntiles, + tiles, + m, + t1, + ) + else: + blockwise_gemm_grouped( + a1, fc1, sfa.view(HIDDEN // BLOCK, m), fc1_scale, c1, group, ntiles, tiles, m, t1 + ) + + def act(): + if epiq: + _act_q_kernel[(m // ACT_BM,)]( + q1, + s1q, + group, + roff, + cnt, + ntiles, + a2, + sfa2, + m, + I_=INTER, + NB=INTER // BLOCK, + QB=BLOCK, + BM=ACT_BM, + MT=M_TILE, + AMAX_MIN=ACT_AMAX_FLOOR, + QMAX=E4M3_MAX, + num_warps=4, + ) + else: + _act_kernel[(m // bm,)]( + c1, + group, + roff, + cnt, + ntiles, + a2, + sfa2, + m, + I_=INTER, + NB=INTER // BLOCK, + NBP=8, + QB=BLOCK, + BM=bm, + MT=M_TILE, + AMAX_MIN=ACT_AMAX_FLOOR, + QMAX=E4M3_MAX, + num_warps=4, + ) + + def fc2g(): + blockwise_gemm_grouped( + a2.view(torch.float8_e4m3fn), fc2, sfa2, fc2_scale, c2, group, ntiles, tiles, m, t2 + ) + + fin = _finalize_pipe_kernel if FIN_PIPE else _finalize_kernel + + def finalize_rows(r0, r1): + if r1 <= r0: + return + fin[(r1 - r0,)]( + c2, + ids, + wts, + rank, + roff, + out, + offset, + local, + r0, + H=HIDDEN, + K=TOPK, + KP=16, + BH=FIN_BH, + NCH=triton.cdiv(HIDDEN, FIN_BH), + num_warps=4, + ) + + stages = [ + ("count", count), + ("layout", layout), + ("permute", permute), + ("fc1", fc1g), + ("act", act), + ("fc2", fc2g), + ] + if fin_first is None: + return out, stages + [("finalize", lambda: finalize_rows(0, rows))] + r0, r1 = fin_first + assert 0 <= r0 <= r1 <= rows and (r0 == 0 or r1 == rows), (fin_first, rows) + rest = (r1, rows) if r0 == 0 else (0, r0) + return out, stages + [ + ("finalize_first", lambda: finalize_rows(r0, r1)), + ("finalize_rest", lambda: finalize_rows(*rest)), + ] + + +def _moe_prefill_grouped( + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, +) -> torch.Tensor: + """This window's partial of the routed experts, `[rows, HIDDEN]` bf16: `xq` `[rows, HIDDEN]` + e4m3 (contiguous), `xsf` `[HIDDEN / 128, rows]` fp32, `ids` / `wts` `[rows, TOPK]` over the + whole routing space (ids outside `[offset, offset + local)` add nothing), the window's `fc1` + `[local, 2I, H]` / `fc2` `[local, H, I]` e4m3 with fp32 128x128 scales. `fin_first = (r0, r1)` + (rows at either end) finalizes those rows, calls `after_first(out)`, then finalizes the rest.""" + out, stages = _plan( + xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics, fin_first + ) + for name, launch in stages: + launch() + if name == "finalize_first" and after_first is not None: + after_first(out) + return out + + +class MoePrefillGrouped(OpWrapper): + """This window's partial of a prefill chunk's routed experts, pre-routed, `[rows, HIDDEN]` + bf16 (see `_moe_prefill_grouped` for the operands). `fin_first = (r0, r1)` (rows at either + end) finalizes those rows, calls `after_first(out)`, then finalizes the rest.""" + + def __call__( + self, + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, + ) -> torch.Tensor: + return _moe_prefill_grouped( + xq, xsf, ids, wts, fc1, fc1_scale, fc2, fc2_scale, offset, tactics, fin_first, + after_first, + ) # fmt: skip + + def reference( + self, + xq: torch.Tensor, + xsf: torch.Tensor, + ids: torch.Tensor, + wts: torch.Tensor, + fc1: torch.Tensor, + fc1_scale: torch.Tensor, + fc2: torch.Tensor, + fc2_scale: torch.Tensor, + offset: int, + tactics=None, + fin_first=None, + after_first=None, + ) -> torch.Tensor: + """`fp8_block_scale_moe_runner`'s pre-routed reference over the same window. `tactics` + and `fin_first` pick launches, not a result; `after_first` is not called.""" + local = fc1.shape[0] + return fp8_block_scale_moe_runner.reference( + routing_logits=None, + routing_bias=None, + hidden_states=xq, + hidden_states_scale=xsf, + gemm1_weights=fc1, + gemm1_weights_scale=fc1_scale, + gemm2_weights=fc2, + gemm2_weights_scale=fc2_scale, + # The routing space only bounds the runner's `is_valid`; the reference reads the window. + num_experts=offset + local, + top_k=TOPK, + num_fused_shared_experts=None, + n_group=None, + topk_group=None, + intermediate_size=INTER, + local_expert_offset=offset, + local_num_experts=local, + routed_scaling_factor=None, + routing_method_type=1, + topk_weights=wts, + topk_ids=ids, + ) + + +moe_prefill_grouped = MoePrefillGrouped() + + +_READY: set = set() + + +def prefill_grouped_ready(local: int) -> bool: + """Whether `warm_prefill_grouped` compiled this window's kernels (a call can be captured).""" + return (TACTICS, local, EPI_QUANT) in _READY + + +def warm_prefill_grouped(device, local: int) -> None: + """Compile every kernel of `moe_prefill_grouped` for a `local`-expert window with one eager + call whose rows all go to expert 0 (the GEMMs' shapes are dynamic). Idempotent.""" + if (TACTICS, local, EPI_QUANT) in _READY: + return + dev = torch.device("cuda", device) if isinstance(device, int) else torch.device(device) + rows = 2 * M_TILE + fc1 = torch.zeros(1, FC1_ROWS, HIDDEN, dtype=torch.float8_e4m3fn, device=dev).expand( + local, -1, -1 + ) + fc2 = torch.zeros(1, HIDDEN, INTER, dtype=torch.float8_e4m3fn, device=dev).expand(local, -1, -1) + s1 = torch.ones(1, FC1_ROWS // BLOCK, HIDDEN // BLOCK, dtype=torch.float32, device=dev).expand( + local, -1, -1 + ) + s2 = torch.ones(1, HIDDEN // BLOCK, INTER // BLOCK, dtype=torch.float32, device=dev).expand( + local, -1, -1 + ) + xq = torch.zeros(rows, HIDDEN, dtype=torch.float8_e4m3fn, device=dev) + xsf = torch.ones(HIDDEN // BLOCK, rows, dtype=torch.float32, device=dev) + ids = torch.zeros(rows, TOPK, dtype=torch.int32, device=dev) + ids[:, 1:] = -1 + wts = torch.ones(rows, TOPK, dtype=torch.bfloat16, device=dev) + moe_prefill_grouped(xq, xsf, ids, wts, fc1, s1, fc2, s2, 0) + torch.cuda.synchronize(dev) + _READY.add((TACTICS, local, EPI_QUANT)) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py new file mode 100644 index 000000000000..b7759c784273 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/moe_tpb_decode__decode.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A decode step's routed experts over 128-row intermediate blocks, in one launch of 4- or 8-CTA +clusters (`native_ops.fp8_block_moe_tpb`, CUDA C++ in this target's `csrc/fp8BlockMoeTpb.cu`). + +An expert's intermediate (640) is five 128-row blocks and its activation needs nothing outside a +block, so the routed experts are dealt by blocks over the four tep4 ranks: expert e's block b lives +on rank (e + b) % 4, and each rank returns its partial for the TP sum after the layer. A cluster +runs a block's FC1 -> activation -> FC2, sharing the FC1 outputs through distributed shared +memory; partials are summed in cluster order (deterministic). + +Operands (`BlockExperts`), block-major, a rank's 640 blocks as the loader lays them out +(`weights._block_experts`, blocks dealt by `block_ids`): +`w1 [640, 256, 2560]` e4m3 (rows: the block's 128 up rows, then its 128 gate rows), `s1 [640, 2, 20]` +fp32, `w2 [640, 2560, 128]` e4m3 (the block's 128 down_proj columns), `s2 [640, 20]` fp32. +""" + +from __future__ import annotations + +import functools +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( + renorm_moe_routing_op, +) + +from ... import native_ops +from .fp8_block_scale_moe_runner import dequantize_block_weight, requantize_1x128 + +EXPERTS = 512 +TOP_K = 10 +HIDDEN = 2560 +INTER = 640 +#: 128-row intermediate blocks an expert has. +EXPERT_BLOCKS = INTER // 128 +#: Blocks a rank holds (tep4 round robin: 512 * 5 / 4). +BLOCKS = 640 +RANKS = 4 +MAX_ROWS = 8 + + +class BlockExperts(NamedTuple): + """One MoE layer's routed experts at this rank, block-major (see the module docstring).""" + + w1: torch.Tensor + s1: torch.Tensor + w2: torch.Tensor + s2: torch.Tensor + rank: int + + +#: The shared expert's intermediate, and the work items its TP slice is folded in as (`SharedBlocks`). +SHARED_INTER = 640 +SHARED_ITEMS = 5 +SHARED_ITEM_COLS = 32 + + +class SharedBlocks(NamedTuple): + """This rank's TP4 slice of the bf16 shared expert, folded into the launch (3..8 rows): `x` the + MoE site's `[T, 2560]` bf16 rows, `gu` `[5, 64, 2560]` the slice's gate / up rows (item sb = columns + `[160 rank + 32 sb, +32)` as 8 groups of 4 columns, each `[gate 4 ; up 4]`), `gl` the `[2560]` gate-logit row, + `dn` `[5, 2560, 32]` the slice's down_proj columns (the loader's `tpb_sh_gu` / `tpb_sh_dn`).""" + + x: torch.Tensor + gu: torch.Tensor + gl: torch.Tensor + dn: torch.Tensor + + +def block_ids(rank: int) -> tuple[list[int], list[int]]: + """(expert, block) of each of a rank's 640 blocks: j < 512 is expert j's block (rank - j) % 4, + j >= 512 is expert 4 (j - 512) + rank's block 4.""" + experts = [j if j < EXPERTS else 4 * (j - EXPERTS) + rank for j in range(BLOCKS)] + blocks = [(rank - j) % RANKS if j < EXPERTS else EXPERT_BLOCKS - 1 for j in range(BLOCKS)] + return experts, blocks + + +_WORKSPACE: dict = {} + + +def _workspace(device: torch.device) -> torch.Tensor: + index = device.index if device.index is not None else torch.cuda.current_device() + ws = _WORKSPACE.get(index) + if ws is None: + nbytes = native_ops.fp8_block_moe_tpb_workspace_bytes() + ws = torch.zeros(nbytes, dtype=torch.uint8, device=torch.device("cuda", index)) + _WORKSPACE[index] = ws + return ws + + +@functools.cache +def _pdl(device_index: int) -> bool: + return get_env_enable_pdl() and torch.cuda.get_device_capability(device_index)[0] >= 9 + + +def _cells(): + eager = tuple( + Cell( + why=f"decode routed experts at {r} rows, round-robin blocks of tep4 rank {rank}", + spec=dict(rows=r, rank=rank, capture=False), + ) + for r, rank in ((1, 0), (8, 3)) + ) + captured = tuple( + Cell( + why=f"captured decode routed experts at {r} rows, round-robin blocks, replayed twice", + spec=dict(rows=r, rank=2, capture=True), + ) + for r in (3,) + ) + return eager + captured + + +class MoeTpbDecode(OpWrapper): + """`partial = moe_tpb_decode(logits, xq, xsf, blocks, shared=None)`. + + `logits` `[T, 512]` bf16 (the front end's), `xq` `[T, 2560]` e4m3 and `xsf` `[20, >= T]` fp32 its + 1x128 scales transposed (`moe_front_router_quant`'s), `blocks` this rank's `BlockExperts`, `shared` + this rank's slice of the shared expert to fold in (`SharedBlocks`, 3..8 rows). Returns the + `[T, 2560]` bf16 partial of this rank's blocks: the ranks' partials sum to the layer's routed + experts (plus `sigmoid(gate logit) * shared` with `shared`).""" + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: `torch_ext::fp8BlockMoeTpb` (`csrc/fp8BlockMoeTpbOp.cpp`, kernel in + `csrc/fp8BlockMoeTpb.cu`). The clusters' partials meet through counters in the op's + per-device workspace, so calls on one device must not overlap in time. PDL per + `TRTLLM_ENABLE_PDL` on sm_90+. + """ + + def __call__(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + extra = {} + if shared is not None: + extra.update(sh_x=shared.x, sh_gu=shared.gu, sh_gl=shared.gl, sh_dn=shared.dn) + return native_ops.fp8_block_moe_tpb( + logits, + xq, + xsf, + blocks.w1, + blocks.s1, + blocks.w2, + blocks.s2, + int(blocks.rank), + _workspace(logits.device), + _pdl(logits.device.index), + **extra, + ) + + def reference(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + """The runner reference's arithmetic over this rank's blocks of `renorm_moe_routing_op`'s + routing, fp32, rounded once; with `shared`, plus `sigmoid(gate logit) * slice`.""" + routed = self._routed(logits, xq, xsf, blocks) + if shared is None: + return routed.to(torch.bfloat16) + scale, part = shared_reference(shared) + return (routed + scale * part).to(torch.bfloat16) + + @staticmethod + def _routed(logits, xq, xsf, blocks: BlockExperts): + ids, w = renorm_moe_routing_op(logits, topk=TOP_K, output_dtype=torch.bfloat16) + T = xq.shape[0] + nb = HIDDEN // 128 + act_in = ( + xq.float().reshape(T, nb, 128) * xsf[:, :T].t().reshape(T, nb, 1).float() + ).reshape(T, HIDDEN) + out = torch.zeros(T, HIDDEN, dtype=torch.float32, device=xq.device) + experts, _ = block_ids(blocks.rank) + by_expert: dict[int, list[int]] = {} + for j, e in enumerate(experts): + by_expert.setdefault(e, []).append(j) + for e in torch.unique(ids).tolist(): + tok, slot = (ids.long() == e).nonzero(as_tuple=True) + for j in by_expert.get(e, []): + w1 = dequantize_block_weight(blocks.w1[j], blocks.s1[j]) + y1 = requantize_1x128(act_in[tok] @ w1.t()) + act = requantize_1x128(torch.nn.functional.silu(y1[:, 128:]) * y1[:, :128]) + w2 = dequantize_block_weight(blocks.w2[j], blocks.s2[j].view(-1, 1)) + out.index_add_(0, tok, (act @ w2.t()) * w[tok, slot].float().unsqueeze(1)) + return out + + def is_valid(self, logits, xq, xsf, blocks: BlockExperts, shared: SharedBlocks | None = None): + rows = logits.shape[0] + assert logits.dtype == torch.bfloat16 and logits.is_contiguous(), logits.dtype + assert tuple(logits.shape[1:]) == (EXPERTS,) and 1 <= rows <= MAX_ROWS, tuple(logits.shape) + assert xq.dtype == torch.float8_e4m3fn and tuple(xq.shape) == (rows, HIDDEN) + assert xq.is_contiguous() + assert xsf.dtype == torch.float32 and xsf.shape[0] == HIDDEN // 128 and xsf.shape[1] >= rows + assert xsf.stride(1) == 1, "xsf is read as [20, stride] with unit column stride" + assert tuple(blocks.w1.shape) == (BLOCKS, 256, HIDDEN) and blocks.w1.is_contiguous() + assert tuple(blocks.s1.shape) == (BLOCKS, 2, HIDDEN // 128) and blocks.s1.is_contiguous() + assert tuple(blocks.w2.shape) == (BLOCKS, HIDDEN, 128) and blocks.w2.is_contiguous() + assert tuple(blocks.s2.shape) == (BLOCKS, HIDDEN // 128) and blocks.s2.is_contiguous() + assert 0 <= blocks.rank < RANKS, blocks.rank + if shared is not None: + assert 3 <= rows <= MAX_ROWS, rows + assert shared.x.dtype == torch.bfloat16 and tuple(shared.x.shape) == (rows, HIDDEN) + assert shared.x.is_contiguous() + assert shared.gu.dtype == torch.bfloat16 and tuple(shared.gu.shape) == ( + SHARED_ITEMS, + 64, + HIDDEN, + ) + assert shared.gl.dtype == torch.bfloat16 and shared.gl.numel() == HIDDEN + assert shared.dn.dtype == torch.bfloat16 and tuple(shared.dn.shape) == ( + SHARED_ITEMS, + HIDDEN, + SHARED_ITEM_COLS, + ) + assert ( + shared.gu.is_contiguous() + and shared.gl.is_contiguous() + and shared.dn.is_contiguous() + ) + + +def shared_reference(shared: SharedBlocks) -> tuple[torch.Tensor, torch.Tensor]: + """`(sigmoid(gate logit) [T, 1], slice [T, 2560])` in fp32 with the unfused path's bf16 roundings + (the gate|up GEMM's outputs, `silu_and_mul_rows`' activation); the slice summed over the ranks + is the shared expert's output before its bf16 rounding.""" + x = shared.x.float() + gu = shared.gu.float().reshape(SHARED_ITEMS, 8, 2, 4, HIDDEN) + gate = gu[:, :, 0].reshape(-1, HIDDEN) + up = gu[:, :, 1].reshape(-1, HIDDEN) + g = (x @ gate.t()).to(torch.bfloat16).float() + u = (x @ up.t()).to(torch.bfloat16).float() + logit = (x @ shared.gl.float().reshape(HIDDEN, 1)).to(torch.bfloat16).float() + act = (torch.nn.functional.silu(g) * u).to(torch.bfloat16).float() + dn = shared.dn.float().permute(1, 0, 2).reshape(HIDDEN, -1) + return torch.sigmoid(logit), act @ dn.t() + + +moe_tpb_decode = MoeTpbDecode() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py new file mode 100644 index 000000000000..19893a7f2c02 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/moe/mtp_moe_combine_inject__decode.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The draft layer's MoE combination and hyper-connection injection in one Triton launch. + +Bitwise the composed `fused_sigmoid_gate_mul_add` + `qwen4_exp_post_inject` path: one fp32 +`fma.rn` rounded once to bf16 for the combination, and an exact bf16 x bf16 product for the +injection, rounded once on the store as `addcmul` does. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: Columns a program forms. +BLOCK = 512 + + +# The gate logit is one scalar a row: its row stride and address carry no vector width. +@triton.jit(do_not_specialize=["s_gate"], do_not_specialize_on_alignment=["gate"]) +def _combine_inject_kernel( + x, + routed, + shared, + gate, + inject, + out, + s_x, + s_routed, + s_shared, + s_gate, + s_inject, + s_out, + H: tl.constexpr, + HC: tl.constexpr, + BLOCK_: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.program_id(1) * BLOCK_ + tl.arange(0, BLOCK_) + mask = cols < H + s = tl.sigmoid(tl.load(gate + row * s_gate).to(tl.float32)) + f = tl.load(routed + row * s_routed + cols, mask=mask, other=0.0).to(tl.float32) + sh = tl.load(shared + row * s_shared + cols, mask=mask, other=0.0).to(tl.float32) + b = tl.fma(s, sh, f).to(routed.dtype.element_ty).to(tl.float32) + for c in tl.static_range(HC): + w = tl.load(inject + row * s_inject + c).to(tl.float32) + xv = tl.load(x + row * s_x + c * H + cols, mask=mask, other=0.0).to(tl.float32) + tl.store(out + row * s_out + c * H + cols, (xv + b * w).to(out.dtype.element_ty), mask=mask) + + +def _ckpt(rows: int, gate: str = "strided") -> dict: + return dict(rows=rows, hc_count=4, hidden=2560, gate=gate) + + +class MtpMoeCombineInject(OpWrapper): + """`out = mtp_moe_combine_inject(x, routed, gate_logits, shared, inject)`: + `out[t, s * H + i] = x[t, s * H + i] + bf16(routed[t, i] + sigmoid(gate_logits[t]) * + shared[t, i]) * inject[t, s]`, a fresh `[T, hc_count * H]` tensor. + + - `x` `[T, hc_count * H]` bf16, unit column stride (the pre-norm residual, stream-major); + - `routed`, `shared` `[T, H]` bf16, unit column stride; + - `gate_logits` `[T, 1]` / `[T]` (any row stride: the column the shared expert's fused GEMM + leaves); `inject` `[T, hc_count]` bf16, unit column stride. + + No input is written. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep4 draft step 1: 1 row", spec=_ckpt(1)), + Cell(why="batch-4 draft step 0: 12 rows", spec=_ckpt(12)), + Cell(why="contiguous gate vector, 4 rows", spec=_ckpt(4, "contiguous")), + ) + + note = """ + Source: one Triton kernel (`_combine_inject_kernel`), grid `(T, ceil(H / 512))`, JIT-compiled + per (H, hc_count, dtypes); the engine's warmup forward compiles it before capture. + """ + + def __call__(self, x, routed, gate_logits, shared, inject) -> torch.Tensor: + rows, width = x.shape + hidden = routed.shape[1] + hc = inject.shape[1] + out = torch.empty([rows, width], dtype=x.dtype, device=x.device) + if rows: + gate = gate_logits.reshape(rows, -1) if gate_logits.dim() == 1 else gate_logits + _combine_inject_kernel[(rows, triton.cdiv(hidden, BLOCK))]( + x, + routed, + shared, + gate, + inject, + out, + x.stride(0), + routed.stride(0), + shared.stride(0), + gate.stride(0), + inject.stride(0), + out.stride(0), + H=hidden, + HC=hc, + BLOCK_=BLOCK, + num_warps=4, + ) + return out + + def reference(self, x, routed, gate_logits, shared, inject) -> torch.Tensor: + """fp32 combination rounded once to bf16, then the fp32 injection rounded once.""" + rows = x.shape[0] + g = torch.sigmoid(gate_logits.reshape(rows, 1).float()) + b = (routed.float() + g * shared.float()).to(routed.dtype).float() + streams = x.float().unflatten(-1, (inject.shape[1], routed.shape[1])) + return (streams + b.unsqueeze(-2) * inject.float().unsqueeze(-1)).flatten(-2).to(x.dtype) + + def is_valid(self, x, routed, gate_logits, shared, inject) -> None: + rows = x.shape[0] + assert x.dim() == 2 and routed.dim() == 2 and shared.dim() == 2 and inject.dim() == 2 + assert x.dtype == routed.dtype == shared.dtype == inject.dtype == torch.bfloat16 + hidden, hc = routed.shape[1], inject.shape[1] + assert x.shape[1] == hc * hidden, (tuple(x.shape), hc, hidden) + assert routed.shape[0] == shared.shape[0] == inject.shape[0] == rows + assert tuple(shared.shape) == (rows, hidden) + for t in (x, routed, shared, inject): + assert t.stride(1) == 1 or t.shape[1] == 1, (tuple(t.shape), t.stride()) + assert gate_logits.numel() == rows and (gate_logits.dim() == 1 or gate_logits.shape[1] == 1) + assert x.device == routed.device == shared.device == inject.device == gate_logits.device + + +mtp_moe_combine_inject = MtpMoeCombineInject() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py new file mode 100644 index 000000000000..304eb33eade5 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/causal_conv1d_token_major__verify.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A verify step's multi-token causal conv1d written token-major (`[n * p, C]`, the layout the GDN +cached replay reads): bitwise the family `causal_conv1d_triton_update` plus a token-major copy. +With `wait_producer` the kernel reads the slots and the conv pool before `griddepcontrol.wait` +and `x` after it. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_triton_update import ( # noqa: E501 + causal_conv1d_triton_update, +) +from tensorrt_llm._torch.modules.fla.op import gdc_wait_with_memory_clobber +from tensorrt_llm._torch.modules.mamba.causal_conv1d_triton import _causal_conv1d_update_kernel +from tensorrt_llm._utils import get_sm_version + +_BLOCK_N = 128 +_NUM_WARPS = 4 +#: Sequences a `_verify_conv_seqs_kernel` program (taps loaded once): 0 takes `_AUTO_SEQS` from +#: `_AUTO_MIN_SEQUENCES` sequences on and the family kernel below; a positive value forces it. +_SEQS = 0 +_AUTO_SEQS = 2 +_AUTO_MIN_SEQUENCES = 104 + + +def _seqs_a_program(batch: int) -> int: + if _SEQS > 0: + return _SEQS + return _AUTO_SEQS if batch >= _AUTO_MIN_SEQUENCES else 1 + + +@triton.jit(do_not_specialize=["batch"]) +def _verify_conv_seqs_kernel( + x_ptr, + w_ptr, + conv_state_ptr, + slots_ptr, + window_ptr, + window_rows_ptr, + o_ptr, + batch, + dim: tl.constexpr, + stride_x_seq: tl.constexpr, + stride_x_dim: tl.constexpr, + stride_x_token: tl.constexpr, + stride_w_dim: tl.constexpr, + stride_w_width: tl.constexpr, + stride_cs_seq: tl.constexpr, + stride_cs_dim: tl.constexpr, + stride_cs_tok: tl.constexpr, + stride_slots: tl.constexpr, + stride_win_seq: tl.constexpr, + stride_win_step: tl.constexpr, + stride_win_dim: tl.constexpr, + stride_win_win: tl.constexpr, + stride_wrows: tl.constexpr, + stride_o_seq: tl.constexpr, + stride_o_dim: tl.constexpr, + stride_o_token: tl.constexpr, + pad_slot_id: tl.constexpr, + SEQLEN: tl.constexpr, + SEQS: tl.constexpr, + BLOCK_N: tl.constexpr, + LAUNCH_DEPENDENT_KERNELS: tl.constexpr, + WAIT_PRODUCER: tl.constexpr, +): + """`_causal_conv1d_update_kernel` (width 4, SiLU, pad slots, snapshots, `SEQLEN >= 3`) over + `SEQS` sequences a program, per element the same arithmetic and stores.""" + if LAUNCH_DEPENDENT_KERNELS: + tl.extra.cuda.gdc_launch_dependents() + seq0 = tl.program_id(0) * SEQS + idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask_w = idx_feats < dim + w_base = w_ptr + (idx_feats * stride_w_dim) + w_col0 = tl.load(w_base + (0 * stride_w_width), mask_w, other=0.0) + w_col1 = tl.load(w_base + (1 * stride_w_width), mask_w, other=0.0) + w_col2 = tl.load(w_base + (2 * stride_w_width), mask_w, other=0.0) + w_col3 = tl.load(w_base + (3 * stride_w_width), mask_w, other=0.0) + # Slots, snapshot rows and pre-step windows (earlier steps' data) before the dependency wait. + slots = () + wrows = () + lives = () + cols = () + for s in tl.static_range(SEQS): + seq = seq0 + s + in_batch = seq < batch + slot = tl.load(slots_ptr + seq * stride_slots, mask=in_batch, other=pad_slot_id).to( + tl.int64 + ) + wrow = tl.load(window_rows_ptr + seq * stride_wrows, mask=in_batch, other=0).to(tl.int64) + live = in_batch & (slot != pad_slot_id) + m = mask_w & live + prior = conv_state_ptr + (slot * stride_cs_seq) + (idx_feats * stride_cs_dim) + cols = cols + ( + tl.load(prior, m, 0.0), + tl.load(prior + 1 * stride_cs_tok, m, 0.0), + tl.load(prior + 2 * stride_cs_tok, m, 0.0), + ) + slots = slots + (slot,) + wrows = wrows + (wrow,) + lives = lives + (live,) + if WAIT_PRODUCER: + gdc_wait_with_memory_clobber() + xs = () + for s in tl.static_range(SEQS): + x_base = x_ptr + ((seq0 + s) * stride_x_seq) + (idx_feats * stride_x_dim) + for t in tl.static_range(SEQLEN): + xs = xs + (tl.load(x_base + t * stride_x_token, mask=mask_w & lives[s]),) + tl.debug_barrier() + for s in tl.static_range(SEQS): + seq = seq0 + s + m = mask_w & lives[s] + # The advanced pool: the last three inputs. + state_base = conv_state_ptr + (slots[s] * stride_cs_seq) + (idx_feats * stride_cs_dim) + for k in tl.static_range(3): + tl.store(state_base + k * stride_cs_tok, xs[s * SEQLEN + SEQLEN - 3 + k], m) + col0 = cols[3 * s] + col1 = cols[3 * s + 1] + col2 = cols[3 * s + 2] + for t in tl.static_range(SEQLEN): + acc = tl.zeros((BLOCK_N,), dtype=tl.float32) + acc += col0 * w_col0 + acc += col1 * w_col1 + acc += col2 * w_col2 + x_t = xs[s * SEQLEN + t] + acc += x_t * w_col3 + col0 = col1 + col1 = col2 + col2 = x_t + win = ( + window_ptr + + wrows[s] * stride_win_seq + + t * stride_win_step + + idx_feats * stride_win_dim + ) + tl.store(win + 0 * stride_win_win, col0, mask=m) + tl.store(win + 1 * stride_win_win, col1, mask=m) + tl.store(win + 2 * stride_win_win, col2, mask=m) + acc = acc / (1 + tl.exp(-acc)) + tl.store( + o_ptr + seq * stride_o_seq + t * stride_o_token + idx_feats * stride_o_dim, + acc, + mask=m, + ) + + +def _launch_seqs( + x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait, seqs +): + batch, dim, seqlen = x3.shape + _verify_conv_seqs_kernel[(triton.cdiv(batch, seqs), triton.cdiv(dim, _BLOCK_N))]( + x3, + weight, + conv_state, + slots, + window, + window_rows, + out3, + batch, + dim, + *x3.stride(), + *weight.stride(), + *conv_state.stride(), + slots.stride(0), + *window.stride(), + window_rows.stride(0), + *out3.stride(), + pad_slot_id, + SEQLEN=seqlen, + SEQS=seqs, + BLOCK_N=_BLOCK_N, + LAUNCH_DEPENDENT_KERNELS=ldk, + WAIT_PRODUCER=wait, + num_warps=_NUM_WARPS, + **({"launch_pdl": True} if wait else {}), + ) + + +def _launch(x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait): + batch, dim, seqlen = x3.shape + width = weight.shape[1] + seqs = _seqs_a_program(batch) + if seqs > 1 and width == 4 and seqlen >= 3 and conv_state.shape[-1] == 3: + return _launch_seqs( + x3, conv_state, weight, slots, window, window_rows, pad_slot_id, out3, ldk, wait, seqs + ) + state_len = width - 1 + # `HOIST_X` (width 4) loads every token's x row before the stores, which ptxas cannot prove + # do not alias them: one L2 round trip for x a row instead of p. Bitwise. + hoist = width == 4 + _causal_conv1d_update_kernel[(batch, triton.cdiv(dim, _BLOCK_N))]( + x3, + weight, + None, + conv_state, + None, + slots, + None, + window, + window_rows, + None, + None, + None, + out3, + batch, + dim, + seqlen, + state_len, + conv_state.shape[0], + *x3.stride(), + *weight.stride(), + *conv_state.stride(), + slots.stride(0), + *window.stride(), + window_rows.stride(0), + 0, + 0, + 0, + 0, + 0, + 0, + *out3.stride(), + pad_slot_id, + HAS_BIAS=False, + KERNEL_WIDTH=width, + SILU_ACTIVATION=True, + IS_CONTINUOUS_BATCHING=True, + IS_SPEC_DECODING=False, + NP2_STATELEN=triton.next_power_of_2(state_len), + NP2_SEQLEN=triton.next_power_of_2(seqlen), + USE_PAD_SLOT=True, + BLOCK_N=_BLOCK_N, + SAVE_INTERMEDIATE=True, + HAS_EAGLE_TREE_CUSTOM_ATTN_MASK=False, + LAUNCH_DEPENDENT_KERNELS=ldk, + num_warps=_NUM_WARPS, + **({"WAIT_PRODUCER": True, "launch_pdl": True} if wait else {}), + **({"HOIST_X": True} if hoist else {}), + ) + + +class CausalConv1dTokenMajor(OpWrapper): + """`out = causal_conv1d_token_major(x, rows, conv_state, weight, slots, window, + window_rows, pad_slot_id, launch_dependent_kernels, wait_producer)`. + + `x` is the `[rows * p, C]` q|k|v column view of the projection; the step is the family's + `causal_conv1d_triton_update` over its `[rows, C, p]` view (SiLU, no bias, + `conv_state_indices=slots`, snapshots into `window` at `window_rows`). Returns the + contiguous token-major `[rows * p, C]` output. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = ( + Cell(why="tep4 verify, 1 request x 3 tokens (MTP-2 c=1)", spec=dict(rows=1, p=3)), + Cell(why="tep4 verify, 4 requests x 4 tokens (MTP-3)", spec=dict(rows=4, p=4)), + Cell(why="tep4 verify, 1 request x 3 tokens, PDL wait", spec=dict(rows=1, p=3, wait=True)), + ) + + note = """ + Source: the Triton `_causal_conv1d_update_kernel` (`modules/mamba/causal_conv1d_triton.py`) + with token-major output strides, and `_verify_conv_seqs_kernel` above. Certified bitwise + against the family entry plus the copy, pool and snapshots included. + """ + + def __call__( + self, + x: torch.Tensor, + rows: int, + conv_state: torch.Tensor, + weight: torch.Tensor, + slots: torch.Tensor, + window: torch.Tensor, + window_rows: torch.Tensor, + pad_slot_id: int, + launch_dependent_kernels: bool = False, + wait_producer: bool = False, + ) -> torch.Tensor: + tokens, dim = x.shape + p = tokens // rows + out = torch.empty([tokens, dim], dtype=x.dtype, device=x.device) + pdl = get_sm_version() >= 90 + _launch( + torch.transpose(torch.reshape(x, [rows, p, dim]), 1, 2), + conv_state, + weight, + slots, + window, + window_rows, + pad_slot_id, + torch.transpose(torch.reshape(out, [rows, p, dim]), 1, 2), + launch_dependent_kernels and pdl, + wait_producer and pdl, + ) + return out + + def reference( + self, + x: torch.Tensor, + rows: int, + conv_state: torch.Tensor, + weight: torch.Tensor, + slots: torch.Tensor, + window: torch.Tensor, + window_rows: torch.Tensor, + pad_slot_id: int, + launch_dependent_kernels: bool = False, + wait_producer: bool = False, + ) -> torch.Tensor: + """The family entry over `x`'s `[rows, C, p]` view, copied token-major (it advances + `conv_state` and writes `window` as the op does: call it on copies).""" + tokens, dim = x.shape + p = tokens // rows + out = causal_conv1d_triton_update( + x=torch.transpose(torch.reshape(x, [rows, p, dim]), 1, 2), + conv_state=conv_state, + weight=weight, + bias=None, + activation="silu", + conv_state_indices=slots, + pad_slot_id=pad_slot_id, + num_accepted_tokens=None, + intermediate_conv_window=window, + intermediate_state_indices=window_rows, + ) + return torch.reshape(torch.transpose(out, 1, 2), [tokens, dim]) + + def is_valid(self, x, rows, conv_state, weight, slots, window, window_rows, *args, **kwargs): + assert x.dim() == 2 and x.stride(1) == 1 and x.shape[0] % rows == 0, ( + "x is the [rows * p, C] column view of the projection, unit column stride" + ) + # The pool is read through its strides; the taps must be unit-stride along the width. + assert weight.stride(1) == 1, "conv taps [C, W] with a unit width stride" + + +causal_conv1d_token_major = CausalConv1dTokenMajor() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py new file mode 100644 index 000000000000..f02d8d805245 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/flashinfer_chunk_gated_delta_rule.py @@ -0,0 +1,649 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Chunked gated delta rule on FlashInfer's Blackwell kernels: the GDN prefill core. + +The FP8 target's copy of the family entry +`models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule`, with in-place paths. Every call +is the target's form: `has_initial_state` (a fresh sequence starts from zeros and its pool row +is not read, so the caller need not reset it), the in-place indexed state update, and +`flashinfer.chunk_gated_delta_rule` (CuTe-DSL tcgen05 kernels on sm_100/103) on one of the three +paths `__call__` picks. +""" + +from __future__ import annotations + +import functools +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper +from tensorrt_llm._torch.modules.fla.fused_state_io import cast_scatter_fp32_vk_to_vk + +#: The Gated DeltaNet at tp1 and dep4: (H, HV, K, V). +QWEN38_GEOMETRY = (16, 48, 128, 128) +#: Slot count of the dense-pool cell. +ENGINE_SLOTS = 461 + + +#: A tp4/tep4 rank: tp1's heads split four ways. +RANK4_GEOMETRY = (4, 12, 128, 128) +#: Per-rank slot counts the V2 hybrid manager allocates on this checkpoint (bf16 KV and state, +#: KV fraction 0.6). It sizes the pool from the rank's memory budget, so a dep4 rank (every GDN +#: head) holds fewer slots than a tep4 rank. +TEP4_SLOTS = 1874 +DEP4_SLOTS = 850 +#: Not measured: a tp4 rank's state per slot is tep4's. +TP4_SLOTS = TEP4_SLOTS +#: GDN layers whose states share one slot page: a layer's pool view has a slot stride of 36 +#: `[HV, V, K]` states (a tep4 rank's last slot starts 1.3e10 elements past the base). +PAGE_LAYERS = 36 + +#: A tp2/tep2 rank: tp1's heads split two ways. +RANK2_GEOMETRY = (8, 24, 128, 128) +#: Not measured: a tp2/tep2 slot holds half a dep4 state, so twice dep4's count spans dep4's +#: page (2.4e10 elements, the largest certified). +TP2_SLOTS = 2 * DEP4_SLOTS + + +def _engine_slots(num_seqs: int) -> tuple[int, ...]: + """`num_seqs` distinct slots of the engine pool, slot 0 and the last live.""" + return ((0, ENGINE_SLOTS - 1) + tuple(i * 13 + 4 for i in range(num_seqs)))[:num_seqs] + + +def _rank_slots(num_seqs: int, slots: int) -> tuple[int, ...]: + """`num_seqs` distinct slots: the last, the first, then a stride-37 walk.""" + rest = tuple((i * 37 + 4) % (slots - 2) + 1 for i in range(max(num_seqs - 2, 0))) + picked = ((slots - 1, 0) + rest)[:num_seqs] + assert len(set(picked)) == num_seqs + return picked + + +def _rank_cell(why: str, geometry, seqlens, slots: int, initial, fi_path: str) -> Cell: + """The target's call at one rank: the layer's view of the bf16 state page at the member's + slot count, a prefix view of a wider output buffer. `fi_path` is the path it reaches on + sm_103 (`cp`, `chunked` or `inplace`).""" + return Cell( + why=why, + spec=dict( + geometry=geometry, + seqlens=seqlens, + slots=slots, + page_layers=PAGE_LAYERS, + indices=_rank_slots(len(seqlens), slots), + pool_dtype=torch.bfloat16, + output="slice", + initial=initial, + linear_gate=True, + fi_path=fi_path, + ), + ) + + +#: Elements of a `[V, K]` state plane a gather program copies. +_GATHER_BLOCK = 4096 + + +# Slot ids and carry flags are scalar-read views of the step's metadata. +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "init_ptr"]) +def _gather_carried_states_kernel( + pool_ptr, + dst_ptr, + slot_ptr, + init_ptr, + stride_ps, + stride_ph, + stride_pv, + stride_pk, + H: tl.constexpr, + K: tl.constexpr, + PLANE: tl.constexpr, + BLOCK: tl.constexpr, +): + """`BLOCK` elements of sequence `n`'s head `h` plane into the packed `[N, H, V, K]` buffer: + pool row `slot_ptr[n]` when `init_ptr[n]`, else +0.0 (the row is not read).""" + n = tl.program_id(0) + h = tl.program_id(1) + i = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + mask = i < PLANE + dst = dst_ptr + (n.to(tl.int64) * H + h) * PLANE + i + if tl.load(init_ptr + n) != 0: + slot = tl.load(slot_ptr + n).to(tl.int64) + src = ( + pool_ptr + slot * stride_ps + h * stride_ph + (i // K) * stride_pv + (i % K) * stride_pk + ) + tl.store(dst, tl.load(src, mask=mask), mask=mask) + else: + tl.store(dst, tl.zeros([BLOCK], dtype=dst_ptr.dtype.element_ty), mask=mask) + + +def _gather_carried_states( + pool: torch.Tensor, indices: torch.Tensor, has_initial_state: torch.Tensor +) -> torch.Tensor: + """`[N, H, V, K]` in the pool's dtype: `pool[indices[n]]` for a carried sequence, zeros + for a fresh one.""" + num_seqs = indices.shape[0] + _, h, v, k = pool.shape + out = torch.empty(num_seqs, h, v, k, dtype=pool.dtype, device=pool.device) + _gather_carried_states_kernel[(num_seqs, h, triton.cdiv(v * k, _GATHER_BLOCK))]( + pool, out, indices, has_initial_state, *pool.stride(), + H=h, K=k, PLANE=v * k, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + return out + + +#: Context tokens up to which a call runs in place on the chunked kernel: at most one 64-token +#: chunk a sequence, one launch, where the CP pipeline buys nothing. +INPLACE_MAX_TOKENS = 64 + + +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "init_ptr"]) +def _zero_fresh_states_kernel( + pool_ptr, + slot_ptr, + init_ptr, + stride_ps, + stride_ph, + stride_pv, + stride_pk, + K: tl.constexpr, + PLANE: tl.constexpr, + BLOCK: tl.constexpr, +): + """Zero `BLOCK` elements of sequence `n`'s head `h` plane of pool row `slot_ptr[n]` when + `init_ptr[n]` is false; a carried sequence's row is not touched.""" + n = tl.program_id(0) + if tl.load(init_ptr + n) == 0: + h = tl.program_id(1) + i = tl.program_id(2) * BLOCK + tl.arange(0, BLOCK) + slot = tl.load(slot_ptr + n).to(tl.int64) + dst = ( + pool_ptr + slot * stride_ps + h * stride_ph + (i // K) * stride_pv + (i % K) * stride_pk + ) + tl.store(dst, tl.zeros([BLOCK], dtype=pool_ptr.dtype.element_ty), mask=i < PLANE) + + +def _chunk_inplace( + q, k, v, g, beta, scale, pool, indices, has_initial_state, cu_seqlens, output, linear_gate +) -> torch.Tensor: + """The fresh rows zeroed, then the chunked kernel (`use_cp=False`) reading and writing the + named pool rows itself (`state_indices`, the pool as `output_state`): no packed copies.""" + import flashinfer + + num_seqs = indices.shape[0] + _, h, v_dim, k_dim = pool.shape + _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( + pool, indices, has_initial_state, *pool.stride(), + K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=pool, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=pool, + use_cp=False, + state_indices=indices, + ) + return out.unsqueeze(0) + + +#: FlashInfer's sm_10x rule for its context-parallel pipeline (`should_use_cp_host`): CP when +#: `num_seqs x num_v_heads x 4` is below the SM count. +_CP_PARALLELISM_DENOMINATOR = 4 + + +@functools.lru_cache(maxsize=None) +def _sm_count(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +@functools.lru_cache(maxsize=1024) +def _cp_chunk_len_cached(seqlen: int, num_heads: int, device_index: int) -> int: + from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import choose_cp_chunk_len_host + from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + + device = torch.device("cuda", device_index) + return int( + choose_cp_chunk_len_host( + seqlen, + num_heads, + get_device_sm_count(device), + device_capability=get_compute_capability(device), + total_seqlen=seqlen, + num_seqs=1, + device_name=get_device_name(device), + ) + ) + + +def cp_chunk_len_for(seqlen: int, num_heads: int, device: torch.device) -> int: + """The CP chunk length FlashInfer picks by itself for one `seqlen`-token sequence over + `num_heads` (max of q and v) heads. A call over a padded row count passes it + (`_cp_chunk_len`) to cut the sequence as the unpadded call does.""" + index = device.index if device.index is not None else torch.cuda.current_device() + return _cp_chunk_len_cached(seqlen, num_heads, index) + + +def _reaches_cp(num_seqs: int, num_v_heads: int, device: torch.device) -> bool: + """Whether FlashInfer's `use_cp="auto"` routes `num_seqs` sequences to the CP pipeline.""" + return num_seqs * num_v_heads * _CP_PARALLELISM_DENOMINATOR < _sm_count(device.index) + + +def _chunk_cp_inplace( + q, + k, + v, + g, + beta, + scale, + pool, + indices, + has_initial_state, + cu_seqlens, + output, + linear_gate, + fi_kwargs=None, +) -> torch.Tensor: + """The CP pipeline in place over the pool: the fresh rows zeroed, then the pipeline reading + and writing the named rows itself. It keeps the state in its own workspace between its + kernels and writes each final state once, so the result is bitwise `_chunk_carried`'s + without its gather and scatter.""" + import flashinfer + + num_seqs = indices.shape[0] + _, h, v_dim, k_dim = pool.shape + _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( + pool, indices, has_initial_state, *pool.stride(), + K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=pool, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=pool, + use_cp=True, + state_indices=indices, + **(fi_kwargs or {}), + ) + return out.unsqueeze(0) + + +def _chunk_carried( + q, + k, + v, + g, + beta, + scale, + pool, + indices, + has_initial_state, + cu_seqlens, + output, + linear_gate, + keep_rows=None, + keep_slots=None, + use_cp=None, + fi_kwargs=None, +) -> torch.Tensor: + """Packed path: the named states gathered in the pool's dtype (zeros for a fresh sequence), + FlashInfer's kernel (`g` exponentiated unless `linear_gate`), and the scatter of the final + states into the same slots, or with `keep_rows` only those sequences' into `keep_slots`. + `use_cp` None is FlashInfer's heuristic; `fi_kwargs` are passed through (a padded call's + `max_seqlen` and `_cp_chunk_len`).""" + import flashinfer + + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + packed = _gather_carried_states(pool, indices, has_initial_state) + out = ( + output.squeeze(0) + if output is not None + else q3.new_empty(q3.shape[0], v3.shape[1], v3.shape[2]) + ) + final = torch.empty_like(packed) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g.squeeze(0) if linear_gate else torch.exp(g.squeeze(0)), + beta=beta.squeeze(0).to(torch.float32), + scale=scale, + initial_state=packed, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=final, + **({} if use_cp is None else {"use_cp": use_cp}), + **(fi_kwargs or {}), + ) + if keep_rows is None: + cast_scatter_fp32_vk_to_vk(final, pool, indices) + else: + cast_scatter_fp32_vk_to_vk(final.index_select(0, keep_rows), pool, keep_slots) + return out.unsqueeze(0) + + +def chunk_split_first_pass( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + pool: torch.Tensor, + gather_slots: torch.Tensor, + has_initial_state: torch.Tensor, + cu_seqlens: torch.Tensor, + keep_rows: torch.Tensor, + keep_slots: torch.Tensor, + output: torch.Tensor, + num_whole_seqs: int, + whole: Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None, +) -> torch.Tensor: + """First pass of an in-forward snapshot split (`PrefillTarget._gdn_split`): `_chunk_carried` + over `cu_seqlens` with extra sequence boundaries (`linear_gate`), only sequences `keep_rows` + writing their final states, to pool rows `keep_slots`. Bitwise the unsplit two-iteration + schedule: on the chunked kernel every sequence runs on its own; the CP pipeline segments a + sequence by how many sequences the call has, so when FlashInfer picks CP for the + `num_whole_seqs` unsplit sequences the pass runs over the kept rows alone, `whole` = (their + rows, boundaries, carry flags), copied in and out, or on views when `rows` is an int (the + kept rows lead the step).""" + assert q.shape[1] > INPLACE_MAX_TOKENS, ( + f"a split first pass of {q.shape[1]} tokens would leave the carried path" + ) + if whole is not None and _use_cp(num_whole_seqs * pool.shape[1], q.device): + rows, cu, carry = whole + if isinstance(rows, int): + # The kept rows lead: the same operands as their gather, without the copies. + _chunk_carried( + *(x[:, :rows] for x in (q, k, v, g, beta)), + None, pool, keep_slots, carry, cu, output[:, :rows], True, + ) # fmt: skip + return output + o = _chunk_carried( + *(x.index_select(1, rows) for x in (q, k, v, g, beta)), + None, pool, keep_slots, carry, cu, None, True, + ) # fmt: skip + output[0].index_copy_(0, rows, o[0]) + return output + return _chunk_carried( + q, k, v, g, beta, None, pool, gather_slots, has_initial_state, cu_seqlens, output, True, + keep_rows=keep_rows, keep_slots=keep_slots, + use_cp=_use_cp(num_whole_seqs * pool.shape[1], q.device), + ) # fmt: skip + + +@functools.lru_cache(maxsize=64) +def _use_cp_cached(units: int, device_index: int) -> bool: + from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import should_use_cp_host + from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + + device = torch.device("cuda", device_index) + return bool( + should_use_cp_host( + units, + get_device_sm_count(device), + get_device_name(device), + device_capability=get_compute_capability(device), + ) + ) + + +def split_takes_cp(num_seqs: int, v_heads: int, device: torch.device) -> bool: + """Whether `chunk_split_first_pass` runs over the kept rows alone (FlashInfer picks CP).""" + return _use_cp(num_seqs * v_heads, device) + + +def _use_cp(units: int, device: torch.device) -> bool: + """FlashInfer's CP choice (`should_use_cp_host`) for `units` = sequences x value heads.""" + index = device.index if device.index is not None else torch.cuda.current_device() + return _use_cp_cached(units, index) + + +class FlashinferChunkGatedDeltaRule(OpWrapper): + """The gated delta rule over whole sequences, in place over the state pool. + + Per token `t` and value head, `h` the `[V, K]` state, `q_h`/`k_h` the key heads broadcast + (value head `i` reads key head `i // (HV // H)`), fp32 accumulation: + + h *= exp(g[t]); u = (v[t] - h k_h[t]) * beta[t]; h += u k_h[t]^T + o[t] = h (scale * q_h[t]) + + computed 64 tokens at a time on tensor cores (bf16 operands); `g` is in log space, or with + `linear_gate` the factor `exp(g)`; `scale=None` is `K ** -0.5`. One packed row split at + `cu_seqlens`; sequence `n` starts from `initial_state[initial_state_indices[n]]` (the pool + `[S, HV, V, K]`), or from zeros when `has_initial_state[n]` is false, and its final state is + written back into that slot. The conv, q/k/v/g/beta preparation and the gated norm are the + caller's. Returns `(o, None)`, `o` `[1, T, HV, V]` in `q.dtype` (`output` when given). + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: geometry (H, HV, K, V); seqlens; `slots`-row pool with `indices`; pool_dtype; + # output (None | "buffer" | "slice", a prefix view of a wider buffer); page_layers (the + # pool is the layer's view of the state page); initial; linear_gate; fi_path. Each head + # layout on each path `__call__` routes it to: in place (with the 64 / 65-token edge), + # CP and chunked. + CELLS: tuple[Cell, ...] = ( + Cell( + why="target call: bf16 pool, has_initial_state [1, 0, 1], seqlens [300, 1024, 500]", + spec=dict( + geometry=QWEN38_GEOMETRY, + seqlens=(300, 1024, 500), + slots=ENGINE_SLOTS, + indices=_engine_slots(3), + pool_dtype=torch.bfloat16, + output="buffer", + initial=(True, False, True), + linear_gate=True, + ), + ), + _rank_cell( + "tep4 4/12, [200, 37, 91] carried/fresh/carried, CP path, 1874 slots", + RANK4_GEOMETRY, (200, 37, 91), TEP4_SLOTS, (True, False, True), "cp", + ), + _rank_cell( + "tp4 4/12, carried 8192-token chunk, CP path, 1874-slot bf16 page", + RANK4_GEOMETRY, (8192,), TP4_SLOTS, (True,), "cp", + ), + _rank_cell( + "dep4 16/48, carried 512-token prompt, 850-slot bf16 page", + QWEN38_GEOMETRY, (512,), DEP4_SLOTS, (True,), "chunked", + ), + _rank_cell( + "tep4 4/12 envelope, 8192 rows over 128 seqs of 63/65, chunked path", + RANK4_GEOMETRY, (63, 65) * 64, TEP4_SLOTS, (True, False) * 64, "chunked", + ), + _rank_cell( + "tep4 4/12, one carried 64-token chunk (the in-place bound), 1874 slots", + RANK4_GEOMETRY, (64,), TEP4_SLOTS, (True,), "inplace", + ), + _rank_cell( + "tep4 4/12, one carried 65-token chunk, past the in-place bound: CP path", + RANK4_GEOMETRY, (65,), TEP4_SLOTS, (True,), "cp", + ), + _rank_cell( + "dep4 16/48, carried 2 + fresh 30 + carried 1 tokens, in place, 850 slots", + QWEN38_GEOMETRY, (2, 30, 1), DEP4_SLOTS, (True, False, True), "inplace", + ), + # tp2/tep2: 24 v heads reach the CP pipeline for one context sequence only + # (24 x 4 < 152 SMs, 48 x 4 is not). + _rank_cell( + "tp2/tep2 8/24, fresh 8192-token prompt, CP path, 1700-slot bf16 page", + RANK2_GEOMETRY, (8192,), TP2_SLOTS, (False,), "cp", + ), + _rank_cell( + "tp2/tep2 8/24, [200, 37, 91] carried/fresh/carried, chunked path, 1700 slots", + RANK2_GEOMETRY, (200, 37, 91), TP2_SLOTS, (True, False, True), "chunked", + ), + _rank_cell( + "tp2/tep2 8/24, fresh 5 + carried 2 tokens, in place, 1700 slots", + RANK2_GEOMETRY, (5, 2), TP2_SLOTS, (False, True), "inplace", + ), + ) # fmt: skip + + note = """ + Source: `flashinfer.chunk_gated_delta_rule` (`flashinfer.gdn_prefill`), CuTe-DSL kernels + JIT-compiled in-process at the first call per path and device; Triton kernels for the + fresh-row zeroing, the packed gather and the scatter (`cast_scatter_fp32_vk_to_vk`). + + Paths (`__call__`): at most `INPLACE_MAX_TOKENS` context tokens run in place on the chunked + kernel; calls FlashInfer routes to its CP pipeline (`_reaches_cp`: sequences x HV x 4 below + the SM count, so on GB300 one sequence at 8/24, 1-3 at 4/12, never at 16/48) run in place + on it; the rest take the packed path (`_chunk_carried`: a copy of the named states in and + out). The state is fp32 inside and rounded once to the pool's dtype. + + Preconditions: q/k L2-normalized by the caller, `g` fp32, `cu_seqlens` given, slots + distinct and in range (not checked), context rows only (single-token rows belong to the + decode kernel). FlashInfer keeps one TMA workspace per device: two streams must not run + this op concurrently. Slot offsets are int64 (every certified page's last slot is past + int32). + + Certified on GB300 against the fp64 reference at the cells: bf16 pools at 16/48, 8/24 and + 4/12 heads on each path they reach. Not certified: fp32 / fp16 pools, `scale` other than the + default, other head layouts, K or V other than 128, 2 context sequences on CP at 4/12, a + carried sequence on CP at 8/24, the measured slot counts of tp4, tp2 and tep2 (stand-ins + above). The CP in-place path runs inside the breakable prefill graph + (`PrefillTarget._gdn_core_graph`), covered end to end by the `tep4_fp8_kv_breakable` CI + case; a captured call is not certified for 2-sequence CP at 4/12, a carried sequence at + 8/24, or padded rows at those head counts. + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: Optional[float] = None, + initial_state: Optional[torch.Tensor] = None, + initial_state_indices: Optional[torch.Tensor] = None, + inplace_indexed_state_update: bool = False, + output_final_state: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, + output: Optional[torch.Tensor] = None, + state_workspace: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + has_initial_state: Optional[torch.Tensor] = None, + linear_gate: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + assert has_initial_state is not None, ( + "the target's call only: has_initial_state, the in-place indexed update" + ) + assert ( + inplace_indexed_state_update + and initial_state_indices is not None + and not output_final_state + and not head_first + and not use_qk_l2norm_in_kernel + and state_workspace is None + and cu_seqlens is not None + and g.dtype == torch.float32 + ), "has_initial_state: the in-place indexed call only, q/k normalized by the caller" + if q.shape[1] <= INPLACE_MAX_TOKENS: + path = _chunk_inplace + elif _reaches_cp(initial_state_indices.shape[0], v.shape[2], q.device): + path = _chunk_cp_inplace + else: + path = _chunk_carried + o = path( + q, k, v, g, beta, scale, initial_state, initial_state_indices, + has_initial_state, cu_seqlens, output, linear_gate, + ) # fmt: skip + return o, None + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: Optional[float] = None, + initial_state: Optional[torch.Tensor] = None, + initial_state_indices: Optional[torch.Tensor] = None, + inplace_indexed_state_update: bool = False, + output_final_state: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, + output: Optional[torch.Tensor] = None, + state_workspace: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + has_initial_state: Optional[torch.Tensor] = None, + linear_gate: bool = False, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], torch.Tensor]: + """`(o, final_state, initial_state as the call leaves it)`: an fp64 token-at-a-time + recurrence, `o` rounded once to `q.dtype`, the states to the pool's dtype. Mutates + nothing; `output`, `head_first` and `state_workspace` are ignored.""" + dt = torch.float64 + _, _, num_k_heads, head_k = q.shape + num_v_heads = v.shape[2] + repeats = num_v_heads // num_k_heads + scale = head_k**-0.5 if scale is None else scale + bounds = cu_seqlens.tolist() + slots = initial_state_indices.tolist() + carried = [True] * len(slots) if has_initial_state is None else has_initial_state.tolist() + out = torch.zeros(1, q.shape[1], num_v_heads, v.shape[3], dtype=dt, device=q.device) + finals = [] + for n in range(len(bounds) - 1): + state = initial_state[slots[n]].to(dt) + if not carried[n]: + state = torch.zeros_like(state) + for t in range(bounds[n], bounds[n + 1]): + q_t, k_t = q[0, t].to(dt), k[0, t].to(dt) + if use_qk_l2norm_in_kernel: + q_t = q_t / torch.sqrt(q_t.pow(2).sum(-1, keepdim=True) + 1e-6) + k_t = k_t / torch.sqrt(k_t.pow(2).sum(-1, keepdim=True) + 1e-6) + q_t = q_t.repeat_interleave(repeats, dim=0) * scale + k_t = k_t.repeat_interleave(repeats, dim=0) + decay = g[0, t].to(dt) if linear_gate else g[0, t].to(dt).exp() + state = state * decay[:, None, None] + u = (v[0, t].to(dt) - (state * k_t[:, None, :]).sum(-1)) * beta[0, t].to(dt)[ + :, None + ] + state = state + u[:, :, None] * k_t[:, None, :] + out[0, t] = (state * q_t[:, None, :]).sum(-1) + finals.append(state) + pool = initial_state.clone() + final_state = None + if inplace_indexed_state_update: + for slot, state in zip(slots, finals): + pool[slot] = state.to(pool.dtype) + elif output_final_state: + final_state = torch.stack(finals).to(initial_state.dtype) + return out.to(q.dtype), final_state, pool + + +flashinfer_chunk_gated_delta_rule = FlashinferChunkGatedDeltaRule() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py new file mode 100644 index 000000000000..720e2c8bbc16 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_cached_replay_cute__verify.py @@ -0,0 +1,2999 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GDN cached-replay verify step on warp tensor cores (CuTe DSL): the Triton +`_cached_replay_kernel` (`fused_recurrent_gated_delta_rule_cached_replay_update`) at this +checkpoint's geometry (K=V=128, history 16, bf16 state, T=1..8 new tokens) over H q/k heads and +HV = 3 H value heads (`GEOMETRIES`, compile-time parameters of `_Replay`). + +A CTA serves one (request, q/k head, split): the q/k head's 384 state rows (3 v heads x 128, K +innermost) are 24 m-tiles of 16 rows, `tpc` of them a CTA. Per m-tile a warp computes the state +dots D[v, k_t] = h0[v, :] . k_t and D[v, q_t] = h0[v, :] . q_t on mma.sync m16n8k16 (bf16 in, +fp32 out), and the history term as one more MMA with the [16 x 2T] coefficients rounded to bf16 +as the Triton kernel rounds them; the T x T forward substitution runs per state row on CUDA +cores. With `commit`, a row whose history wraps (`p + T > 16`) writes the advanced checkpoint over +the tile it just used (u split into bf16 hi and lo, as the Triton commit does). + +With PDL the state and history (not written by the previous kernel) are read before +`griddepcontrol.wait`, the step's own inputs after it. The history and its bookkeeping are +addressed by a replay slot of their own (`replay_indices`, the checkpoint slot when not given) at +the slot strides of their buffers, so a manager may keep them in pool pages apart from the +checkpoint. +""" + +from __future__ import annotations + +from typing import Callable, Optional + +import cutlass +import cutlass.cute as cute +import torch +import torch.nn.functional as F +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import T as _ty +from cutlass.cutlass_dsl import dsl_user_op + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +K, V, HIST = 128, 128, 16 +#: Value heads a q/k head serves. +GRP = 3 +#: `(q/k heads, value heads)` compiled for: a whole layer (tp1, attention DP), a tp2/tep2 and a +#: tp4/tep4 rank. +GEOMETRIES = ((16, 48), (8, 24), (4, 12)) +TILES = GRP * (V // 16) # m-tiles of one (request, q/k head) +PU = 272 # shared row pitch (bytes) of 128-element bf16 rows read by ldmatrix (conflict-free) +PK = 320 # shared row pitch of the history keys read as permuted 16-byte fragments +WI_REPLAY_SLOT = 4 # a work item's replay-slot field, when it has one (`REPLAY_WORK_REPLAY_SLOT`) +LOG2E = 1.4426950408889634 +LN2 = 0.6931471805599453 + +# -- inline PTX helpers ------------------------------------------------------------------------ + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +def _unpack(res, n, ty, loc, ip): + return [ + ty( + llvm.extractvalue( + _ty.i32() if ty is cutlass.Int32 else _ty.f32(), res, [i], loc=loc, ip=ip + ) + ) + for i in range(n) + ] + + +def _make_ldg4(qual): + @dsl_user_op + def _ldg4(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip)], + f"ld.global{qual}.v4.b32 {{$0, $1, $2, $3}}, [$4];", + "=r,=r,=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + return _ldg4 + + +@dsl_user_op +def _policy_evict_first(*, loc=None, ip=None): + return cutlass.Int64( + _asm(_ty.i64(), [], "createpolicy.fractional.L2::evict_first.b64 $0, 1.0;", "=l", loc, ip) + ) + + +@dsl_user_op +def _ldg4_hint(gaddr, pol, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pol, cutlass.Int64, loc, ip)], + "ld.global.nc.L1::no_allocate.L2::cache_hint.v4.b32 {$0, $1, $2, $3}, [$4], $5;", + "=r,=r,=r,=r,l,l", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +_LDG4 = { + "plain": _make_ldg4(""), + "na": _make_ldg4(".L1::no_allocate"), + "nc": _make_ldg4(".nc.L1::no_allocate"), + "nc256": _make_ldg4(".nc.L1::no_allocate.L2::256B"), + "cs": _make_ldg4(".cs"), + "lu": _make_ldg4(".lu"), +} +_ldg4 = _LDG4["plain"] + + +@dsl_user_op +def _ldg4p(gaddr, pred, *, loc=None, ip=None): + """16 bytes, or zeros where `pred` is 0 (nothing read).""" + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred p; setp.ne.b32 p, $5, 0; mov.b32 $0, 0; mov.b32 $1, 0; mov.b32 $2, 0; " + "mov.b32 $3, 0; @p ld.global.v4.b32 {$0, $1, $2, $3}, [$4]; }", + "=r,=r,=r,=r,l,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _stg4(gaddr, a, b, c, d, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)] + + [_ir(x, cutlass.Int32, loc, ip) for x in (a, b, c, d)], + "{ st.global.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,l,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg_bf16(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _stg_f32(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ st.global.f32 [$1], $2; mov.u32 $0, 0; }", + "=r,l,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _ldg_i32(gaddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.b32 $0, [$1];", + "=r,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_f32(gaddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.f32 $0, [$1];", + "=f,l", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_f32p(gaddr, pred, *, loc=None, ip=None): + """An f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.f32 $0, 0f00000000; @q ld.global.f32 $0, [$1]; }", + "=f,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _ldg_b16p(gaddr, pred, *, loc=None, ip=None): + """16 bits zero-extended (0 where `pred` is 0), converted where used so the load does not stall.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.b32 $0, 0; @q ld.global.u16 $0, [$1]; }", + "=r,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _b16_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b32 t; shl.b32 t, $1, 16; mov.b32 $0, t; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _f32_bits(x, *, loc=None, ip=None): + return cutlass.Int32( + _asm(_ty.i32(), [_ir(x, cutlass.Float32, loc, ip)], "mov.b32 $0, $1;", "=r,f", loc, ip) + ) + + +@dsl_user_op +def _bits_f32(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Int32, loc, ip)], "mov.b32 $0, $1;", "=f,r", loc, ip) + ) + + +@dsl_user_op +def _ldg_bf16p(gaddr, pred, *, loc=None, ip=None): + """A bf16 widened to f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; .reg .b16 h; setp.ne.b32 q, $2, 0; mov.b16 h, 0; @q ld.global.b16 h, [$1]; " + "cvt.f32.bf16 $0, h; }", + "=f,l,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _cp16z(saddr, gaddr, nbytes, *, loc=None, ip=None): + """cp.async of 16 bytes, zero-filled past `nbytes` (0 or 16).""" + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + ], + "{ cp.async.cg.shared.global [$1], [$2], 16, $3; mov.u32 $0, 0; }", + "=r,r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _cp_commit(*, loc=None, ip=None): + _asm(_ty.i32(), [], "{ cp.async.commit_group; mov.u32 $0, 0; }", "=r", loc, ip, side=True) + + +@dsl_user_op +def _cp_wait_all(*, loc=None, ip=None): + _asm(_ty.i32(), [], "{ cp.async.wait_all; mov.u32 $0, 0; }", "=r", loc, ip, side=True) + + +@dsl_user_op +def _sts_i32(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Int32, loc, ip)], + "{ st.shared.b32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts_f32(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Float32, loc, ip)], + "{ st.shared.f32 [$1], $2; mov.u32 $0, 0; }", + "=r,r,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts2_f32(saddr, a, b, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(a, cutlass.Float32, loc, ip), + _ir(b, cutlass.Float32, loc, ip), + ], + "{ st.shared.v2.f32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,r,f,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _sts_bf16(saddr, a, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(a, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.shared.b16 [$1], h; mov.u32 $0, 0; }", + "=r,r,f", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _lds_i32(saddr, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.b32 $0, [$1];", + "=r,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds_f32(saddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.f32 $0, [$1];", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds_bf16(saddr, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(saddr, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; ld.shared.b16 h, [$1]; cvt.f32.bf16 $0, h; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _lds4_i32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.b32 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds4_f32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.f32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", + "=f,=f,=f,=f,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _ldsm4t(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 4), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {$0, $1, $2, $3}, [$4];", + "=r,=r,=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 4, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _mma(a, b, c, *, loc=None, ip=None): + """D = A B + C on m16n8k16 (bf16 A/B, f32 C/D); a: 4 regs, b: 2 regs, c: 4 f32.""" + res = _asm( + llvm.StructType.get_literal([_ty.f32()] * 4), + [_ir(v, cutlass.Int32, loc, ip) for v in a] + + [_ir(v, cutlass.Int32, loc, ip) for v in b] + + [_ir(v, cutlass.Float32, loc, ip) for v in c], + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + loc, + ip, + ) + return _unpack(res, 4, cutlass.Float32, loc, ip) + + +@dsl_user_op +def _bf2_pack(lo, hi, *, loc=None, ip=None): + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(lo, cutlass.Float32, loc, ip), _ir(hi, cutlass.Float32, loc, ip)], + "cvt.rn.bf16x2.f32 $0, $2, $1;", + "=r,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_lo_v(w, *, loc=None, ip=None): + """`_bf2_lo` pinned in program order, so the conversion of a prefetched word is not hoisted + up to its load.""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_hi_v(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + side=True, + ) + ) + + +@dsl_user_op +def _bf2_lo(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, lo; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _bf2_hi(w, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(w, cutlass.Int32, loc, ip)], + "{ .reg .b16 lo, hi; mov.b32 {lo, hi}, $1; cvt.f32.bf16 $0, hi; }", + "=f,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _ex2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "ex2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _lg2(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip)], + "lg2.approx.ftz.f32 $0, $1;", + "=f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _div_approx(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "div.approx.ftz.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _div_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "div.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sqrt_rn(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], "sqrt.rn.f32 $0, $1;", "=f,f", loc, ip) + ) + + +@dsl_user_op +def _mul_bf16(a, b, *, loc=None, ip=None): + """bf16 x bf16 rounded to bf16 (`mul.rn.bf16`), bf16 bits in the low half of an i32.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(a, cutlass.Int32, loc, ip), _ir(b, cutlass.Int32, loc, ip)], + "{ .reg .b16 x, y, z; cvt.u16.u32 x, $1; cvt.u16.u32 y, $2; mul.rn.bf16 z, x, y; " + "cvt.u32.u16 $0, z; }", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _conv4_silu(p0, p1, p2, p3, *, loc=None, ip=None): + """The verify conv's per-element tail in the Triton `_causal_conv1d_update_kernel`'s PTX: + the bf16 tap products summed in fp32 in tap order, SiLU as `acc / (1 + ex2.approx(acc * + -log2e))` (`div.full.f32`), rounded to bf16 (bits in the low half of an i32).""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(x, cutlass.Int32, loc, ip) for x in (p0, p1, p2, p3)], + "{ .reg .b16 h0, h1, h2, h3, o; .reg .f32 a, t, e, d, y; " + "cvt.u16.u32 h0, $1; cvt.u16.u32 h1, $2; cvt.u16.u32 h2, $3; cvt.u16.u32 h3, $4; " + "cvt.f32.bf16 a, h0; add.f32 a, a, 0f00000000; add.rn.f32.bf16 a, h1, a; " + "add.rn.f32.bf16 a, h2, a; add.rn.f32.bf16 a, h3, a; " + "mul.f32 t, a, 0fBFB8AA3B; ex2.approx.f32 e, t; add.f32 d, e, 0f3F800000; " + "div.full.f32 y, a, d; cvt.rn.bf16.f32 o, y; cvt.u32.u16 $0, o; }", + "=r,r,r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _sts4(saddr, a, b, c, d, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip)] + + [_ir(x, cutlass.Int32, loc, ip) for x in (a, b, c, d)], + "{ st.shared.v4.b32 [$1], {$2, $3, $4, $5}; mov.u32 $0, 0; }", + "=r,r,r,r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _pack16(lo, hi, *, loc=None, ip=None): + """Two bf16 bit patterns (low halves of i32) packed into one word, `lo` in the low half.""" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(lo, cutlass.Int32, loc, ip), _ir(hi, cutlass.Int32, loc, ip)], + "prmt.b32 $0, $1, $2, 0x5410;", + "=r,r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _half16(w, hi, *, loc=None, ip=None): + """The low (`hi` False) or high half of a word, zero-extended.""" + sel = "0x4432" if hi else "0x4410" + return cutlass.Int32( + _asm( + _ty.i32(), + [_ir(w, cutlass.Int32, loc, ip)], + f"prmt.b32 $0, $1, 0, {sel};", + "=r,r", + loc, + ip, + ) + ) + + +@dsl_user_op +def _stg_b16(gaddr, x, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Int32, loc, ip)], + "{ .reg .b16 h; cvt.u16.u32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_init(saddr, count, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(saddr, cutlass.Int32, loc, ip), _ir(count, cutlass.Int32, loc, ip)], + "{ mbarrier.init.shared::cta.b64 [$1], $2; mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _fence_mbar_init(*, loc=None, ip=None): + _asm( + _ty.i32(), + [], + "{ fence.mbarrier_init.release.cluster; mov.u32 $0, 0; }", + "=r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s(saddr, gaddr, nbytes, mbar, *, loc=None, ip=None): + """mbarrier.arrive.expect_tx + one 1-D bulk copy global -> shared of `nbytes`.""" + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + ], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$4], $3; " + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1], [$2], $3, [$4]; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _bulk_g2s_hint(saddr, gaddr, nbytes, mbar, pol, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(saddr, cutlass.Int32, loc, ip), + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(nbytes, cutlass.Int32, loc, ip), + _ir(mbar, cutlass.Int32, loc, ip), + _ir(pol, cutlass.Int64, loc, ip), + ], + "{ mbarrier.arrive.expect_tx.shared::cta.b64 _, [$4], $3; " + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$1], [$2], $3, [$4], $5; " + "mov.u32 $0, 0; }", + "=r,r,l,r,r,l", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_wait(mbar, phase, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(mbar, cutlass.Int32, loc, ip), _ir(phase, cutlass.Int32, loc, ip)], + "{ .reg .pred P1; LAB_WAIT: mbarrier.try_wait.parity.shared::cta.b64 P1, [$1], $2; " + "@P1 bra DONE; bra LAB_WAIT; DONE: mov.u32 $0, 0; }", + "=r,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _mbar_arrive(mbar, *, loc=None, ip=None): + _asm( + _ty.i32(), + [_ir(mbar, cutlass.Int32, loc, ip)], + "{ mbarrier.arrive.shared::cta.b64 _, [$1]; mov.u32 $0, 0; }", + "=r,r", + loc, + ip, + side=True, + ) + + +# -- the folded gated RMS norm (`_Replay(norm=True)`): the instructions of the Triton +# `_rms_norm_gated_fwd_multirow_kernel` (one row a warp, four columns a lane), so its bits. + + +@dsl_user_op +def _ldg2_i32(gaddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 2), + [_ir(gaddr, cutlass.Int64, loc, ip)], + "ld.global.v2.b32 {$0, $1}, [$2];", + "=r,=r,l", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _lds2_i32(saddr, *, loc=None, ip=None): + res = _asm( + llvm.StructType.get_literal([_ty.i32()] * 2), + [_ir(saddr, cutlass.Int32, loc, ip)], + "ld.shared.v2.b32 {$0, $1}, [$2];", + "=r,=r,r", + loc, + ip, + side=True, + ) + return _unpack(res, 2, cutlass.Int32, loc, ip) + + +@dsl_user_op +def _stg2_i32(gaddr, a, b, *, loc=None, ip=None): + _asm( + _ty.i32(), + [ + _ir(gaddr, cutlass.Int64, loc, ip), + _ir(a, cutlass.Int32, loc, ip), + _ir(b, cutlass.Int32, loc, ip), + ], + "{ st.global.v2.b32 [$1], {$2, $3}; mov.u32 $0, 0; }", + "=r,l,r,r", + loc, + ip, + side=True, + ) + + +@dsl_user_op +def _gn_sumsq(x0, x1, x2, x3, *, loc=None, ip=None): + """A lane's sum of squares in the Triton kernel's order: (x0 x0 + x2 x2) + (x1 x1 + x3 x3).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(x, cutlass.Float32, loc, ip) for x in (x0, x1, x2, x3)], + "{ .reg .f32 p2, p3, a, b; mul.rn.f32 p2, $3, $3; mul.rn.f32 p3, $4, $4; " + "fma.rn.f32 a, $1, $1, p2; fma.rn.f32 b, $2, $2, p3; add.rn.f32 $0, a, b; }", + "=f,f,f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _add_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "add.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _mul_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "mul.rn.f32 $0, $1, $2;", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _gn_rstd(sumsq, eps, *, loc=None, ip=None): + """1 / sqrt(sum / 128 + eps): `div.full.f32`, `add.f32`, `sqrt.approx.ftz.f32`, `div.full.f32`.""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(sumsq, cutlass.Float32, loc, ip), _ir(eps, cutlass.Float32, loc, ip)], + "{ .reg .f32 n, v, a, q, one; mov.b32 n, 0f43000000; mov.b32 one, 0f3F800000; " + "div.full.f32 v, $1, n; add.rn.f32 a, $2, v; sqrt.approx.ftz.f32 q, a; " + "div.full.f32 $0, one, q; }", + "=f,f,f", + loc, + ip, + ) + ) + + +@dsl_user_op +def _gn_sigmoid(z, *, loc=None, ip=None): + """sigmoid(z) as Triton's `tl.sigmoid`: 1 / (1 + ex2.approx(z * -log2(e))).""" + return cutlass.Float32( + _asm( + _ty.f32(), + [_ir(z, cutlass.Float32, loc, ip)], + "{ .reg .f32 t, e, d, one; mov.b32 one, 0f3F800000; mul.rn.f32 t, $1, 0fBFB8AA3B; " + "ex2.approx.f32 e, t; add.rn.f32 d, e, one; div.full.f32 $0, one, d; }", + "=f,f", + loc, + ip, + ) + ) + + +def _exp(x): + return _ex2(x * LOG2E) + + +def _shfl_xor(x, m): + return cute.arch.shuffle_sync_bfly(x, offset=m) + + +# -- the kernel -------------------------------------------------------------------------------- + + +class _Replay: + def __init__( + self, + *, + T, + tpc, + warps, + commit, + pdl, + ld="nc", + pf=1, + minb=1, + l2norm=True, + gating=True, + g_f32=False, + b_f32=False, + early_trigger=True, + lo_split=True, + state_before_wait=True, + smem_tiles=True, + spec_hist=False, + qk_first=True, + use_wi=False, + wi_w=4, + rsep=False, + evict_first=True, + early_acc=False, + h=16, + hv=48, + fold=False, + norm=False, + norm_eps=1e-6, + ): + assert 1 <= T <= 8 + # the gated RMS norm folded in: a CTA holds one value head's 128 outputs of its T rows + self.norm = bool(norm) + self.norm_eps = float(norm_eps) + assert not self.norm or (tpc == 8 and T <= warps), "the folded norm: one value head a CTA" + assert (h, hv) in GEOMETRIES, (h, hv) + # the verify conv folded in: a CTA computes its head's q / k and its value heads' v rows + # from the projection, the conv pool and the taps; a value head belongs to one CTA + assert not fold or tpc in (8, 24), "the folded conv needs whole value heads a CTA" + self.fold = bool(fold) + if self.fold: + qk_first = False + self.H, self.HV = h, hv + assert tpc in (24, 12, 8, 4, 2, 1) and tpc % warps == 0 + self.T = T + self.TP = 4 if T <= 4 else 8 + self.NT = self.TP // 4 + self.tpc, self.W = tpc, warps + self.tpw = tpc // warps + self.S = TILES // tpc + self.nhv = 3 if tpc == 24 else (2 if tpc == 12 else 1) + self.commit, self.pdl = commit, pdl + # pf = 0: every state tile by bulk copy as soon as the slot is known, none in registers + self.ld, self.pf, self.minb = ld, max(0 if smem_tiles else 1, min(pf, self.tpw)), minb + # the tiles' state dots before the first barrier, while the history is in flight + self.use_early = bool(early_acc) + self.l2norm, self.gating, self.g_f32, self.b_f32 = l2norm, gating, g_f32, b_f32 + self.early_trigger, self.lo_split = early_trigger, lo_split + self.state_before_wait = state_before_wait + # tiles past the first `pf` of a warp: bulk-copied into shared memory at the start + self.nsm = self.tpw - self.pf if smem_tiles else 0 + self.spec_hist = spec_hist + self.qk_first = qk_first + # (row, slot, pending count, half) per item from the engine's replay work items + self.use_wi = use_wi + # int32 fields per work item: 4 is one 16-byte load, anything else scalar loads + self.wi_w = wi_w + # the history and its bookkeeping have replay slots of their own (`replay_indices`) + self.rsep = rsep + self.evict_first = evict_first + nset = 2 if self.spec_hist else 1 + self.nset = nset + self.RP = (2 * self.TP + 4) * 4 # per-warp scratch row pitch (bytes) + o = 0 + self.o_st = o + o += warps * self.nsm * 4096 + self.o_mb = o + o += -(-(warps * self.nsm * 8) // 16) * 16 + self.o_u = o + o += nset * self.nhv * 16 * PU + self.o_kh = o + o += nset * 16 * PK + self.o_kp = o + o += 16 * PU if commit else 0 + self.o_v = o + o += self.nhv * T * 256 + self.o_x = o + o += warps * 16 * self.RP + self.o_kk = o + o += 16 * 8 * 4 + self.o_cf = o + o += self.nhv * 16 * 16 * 2 + self.o_gh = o + o += nset * self.nhv * 16 * 4 + self.o_dec = o + o += self.nhv * 16 * 4 + self.o_g = o + o += self.nhv * 8 * 4 + self.o_eg = o + o += self.nhv * 8 * 4 + self.o_beta = o + o += self.nhv * 8 * 4 + self.o_a = o + o += self.nhv * 64 * 4 + self.o_cq = o + o += self.nhv * 64 * 4 + self.o_egs = o + o += 16 + # the folded conv's q | k rows: T x 256 bf16 + self.o_cv = o + o += T * 512 if self.fold else 0 + # the folded norm's rows: T x 128 bf16 (the outputs, rounded as they would be stored) + self.o_on = o + o += T * 256 if self.norm else 0 + self.smem = -(-o // 128) * 128 + 128 + + @cute.jit + def __call__( + self, + sidx_t: cute.Tensor, + q_ptr: cutlass.Int64, + k_ptr: cutlass.Int64, + v_ptr: cutlass.Int64, + q_sn: cutlass.Int64, + q_st: cutlass.Int64, + k_sn: cutlass.Int64, + k_st: cutlass.Int64, + v_sn: cutlass.Int64, + v_st: cutlass.Int64, + g_ptr: cutlass.Int64, + g_st: cutlass.Int64, + b_ptr: cutlass.Int64, + b_st: cutlass.Int64, + alog_ptr: cutlass.Int64, + dtb_ptr: cutlass.Int64, + o_ptr: cutlass.Int64, + h_ptr: cutlass.Int64, + h_ss: cutlass.Int64, + pnat_ptr: cutlass.Int64, + buf_ptr: cutlass.Int64, + ou_ptr: cutlass.Int64, + ok_ptr: cutlass.Int64, + og_ptr: cutlass.Int64, + wi_ptr: cutlass.Int64, + ridx_ptr: cutlass.Int64, + u_ss: cutlass.Int64, + k_ss: cutlass.Int64, + g_ss: cutlass.Int64, + pnat_ss: cutlass.Int64, + buf_ss: cutlass.Int64, + cx_ptr: cutlass.Int64, + cx_st: cutlass.Int64, + cw_ptr: cutlass.Int64, + cs_ptr: cutlass.Int64, + cs_ss: cutlass.Int64, + win_ptr: cutlass.Int64, + win_sn: cutlass.Int64, + win_st: cutlass.Int64, + wrow_ptr: cutlass.Int64, + zn_ptr: cutlass.Int64, + zn_st: cutlass.Int64, + gw_ptr: cutlass.Int64, + on_ptr: cutlass.Int64, + n_rows: cutlass.Int32, + scale: cutlass.Float32, + stream, + ): + self.kernel( + sidx_t.iterator.toint(), + q_ptr, + k_ptr, + v_ptr, + q_sn, + q_st, + k_sn, + k_st, + v_sn, + v_st, + g_ptr, + g_st, + b_ptr, + b_st, + alog_ptr, + dtb_ptr, + o_ptr, + h_ptr, + h_ss, + pnat_ptr, + buf_ptr, + ou_ptr, + ok_ptr, + og_ptr, + wi_ptr, + ridx_ptr, + u_ss, + k_ss, + g_ss, + pnat_ss, + buf_ss, + cx_ptr, + cx_st, + cw_ptr, + cs_ptr, + cs_ss, + win_ptr, + win_sn, + win_st, + wrow_ptr, + zn_ptr, + zn_st, + gw_ptr, + on_ptr, + scale, + ).launch( + grid=[n_rows * (self.H * self.S), 1, 1], + block=[32 * self.W, 1, 1], + smem=self.smem, + stream=stream, + use_pdl=self.pdl, + min_blocks_per_mp=self.minb, + ) + + # -- straight-line helpers (traced inline) ------------------------------------------------ + + def _tile_addr(self, h_ptr, slot64, h_ss, h, sp, warp, i, g, c): + mt = sp * self.tpc + warp + self.W * i + hv = h * GRP + mt // 8 + v0 = (mt % 8) * 16 + base = h_ptr + (slot64 * h_ss + cutlass.Int64(hv * V + v0) * K) * 2 + return base + cutlass.Int64(g * 256 + c * 16) + + def _qk_raw(self, nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, live): + """Raw bf16 words of key / query column nt * 8 + g at this lane's head dims, the + column's kind and its scale.""" + T, TP = self.T, self.TP + zi = cutlass.Int32(0) + t = g % 4 if TP == 4 else g + valid = (t < T) & live + tc = cutlass.min(t, T - 1) + qa = q_ptr + (n64 * q_sn + cutlass.Int64(tc) * q_st + cutlass.Int64(h * K + c * 8)) * 2 + ka = k_ptr + (n64 * k_sn + cutlass.Int64(tc) * k_st + cutlass.Int64(h * K + c * 8)) * 2 + if TP == 4: + is_q = g >= 4 + src = cutlass.Int64(cutlass.select_(is_q, qa, ka)) + mult = cutlass.Float32(cutlass.select_(is_q, scale, cutlass.Float32(1.0))) + elif nt == 1: + is_q = g >= 0 + src = qa + mult = scale + else: + is_q = g < 0 + src = ka + mult = cutlass.Float32(1.0) + vpred = cutlass.Int32(cutlass.select_(valid, cutlass.Int32(1), zi)) + raw = [] + for jj in range(4): + raw.extend(_ldg4p(src + jj * 64, vpred)) + return raw, is_q, mult + + def _qk_raw_smem(self, nt, s_cv, g, c, scale): + """`_qk_raw` over the folded conv's shared q | k rows (the kernel's fold prologue).""" + T, TP = self.T, self.TP + zi = cutlass.Int32(0) + t = g % 4 if TP == 4 else g + valid = t < T + tc = cutlass.min(t, T - 1) + if TP == 4: + is_q = g >= 4 + elif nt == 1: + is_q = g >= 0 + else: + is_q = g < 0 + mult = cutlass.Float32(cutlass.select_(is_q, scale, cutlass.Float32(1.0))) + src = ( + s_cv + tc * 512 + cutlass.Int32(cutlass.select_(is_q, zi, cutlass.Int32(256))) + c * 16 + ) + raw = [] + for jj in range(4): + w4 = _lds4_i32(src + jj * 64) + for e in range(4): + raw.append(cutlass.Int32(cutlass.select_(valid, w4[e], zi))) + return raw, is_q, mult + + def _qk_frag(self, raw, is_q, mult): + """L2-normalized (queries also scaled) bf16 B fragment from `_qk_raw`.""" + ss = cutlass.Float32(0.0) + fl = [] + for e in range(16): + lo = _bf2_lo_v(raw[e]) + hi = _bf2_hi_v(raw[e]) + fl.append(lo) + fl.append(hi) + ss = ss + lo * lo + hi * hi + if self.l2norm: + ss = ss + _shfl_xor(ss, 1) + ss = ss + _shfl_xor(ss, 2) + inv = _div_rn(mult, _sqrt_rn(ss + 1e-6)) + return [_bf2_pack(fl[2 * e] * inv, fl[2 * e + 1] * inv) for e in range(16)] + return [ + cutlass.Int32( + cutlass.select_(is_q, _bf2_pack(fl[2 * e] * mult, fl[2 * e + 1] * mult), raw[e]) + ) + for e in range(16) + ] + + def _gate_raw(self, pred, hv, n64, g_ptr, g_st, b_ptr, b_st, alog_ptr, dtb_ptr): + """Raw g / beta of the T tokens for v head `hv` and its A_log / dt_bias (predicated).""" + T = self.T + zf = cutlass.Float32(0.0) + pi = cutlass.Int32(cutlass.select_(pred, cutlass.Int32(1), cutlass.Int32(0))) + alv, dtb = zf, zf + if self.gating: + alv = _ldg_f32p(alog_ptr + cutlass.Int64(hv) * 4, pi) + dtb = _ldg_f32p(dtb_ptr + cutlass.Int64(hv) * 4, pi) + graws, braws = [], [] + for t in range(T): + if self.g_f32: + graws.append( + _ldg_f32p(g_ptr + (n64 * T + t) * g_st * 4 + cutlass.Int64(hv) * 4, pi) + ) + else: + graws.append( + _ldg_b16p(g_ptr + (n64 * T + t) * g_st * 2 + cutlass.Int64(hv) * 2, pi) + ) + if self.b_f32: + braws.append( + _ldg_f32p(b_ptr + (n64 * T + t) * b_st * 4 + cutlass.Int64(hv) * 4, pi) + ) + else: + braws.append( + _ldg_b16p(b_ptr + (n64 * T + t) * b_st * 2 + cutlass.Int64(hv) * 2, pi) + ) + return graws, braws, alv, dtb + + def _smem_tile(self, s_st, s_mb, i, g, c, zi): + """This lane's A fragment words of staged tile i: rows g and g + 8, chunk order by row + parity so each quarter-warp's two rows are on disjoint banks.""" + kslot = i - self.pf + _mbar_wait(s_mb + kslot * 8, zi) + tb = s_st + kslot * 4096 + g * 256 + c * 16 + godd = (g % 2) == 1 + raw = [] + for ii in range(4): + jsw = (g % 2) ^ ii + raw.append(_lds4_i32(tb + jsw * 64)) + raw.append(_lds4_i32(tb + 2048 + jsw * 64)) + cur = [] + for jj in range(4): + for rr in range(2): + for e in range(4): + cur.append( + cutlass.Int32( + cutlass.select_(godd, raw[2 * (jj ^ 1) + rr][e], raw[2 * jj + rr][e]) + ) + ) + return cur + + def _tile_dots(self, cur, bq, zf): + """D = h0 [16 x 128] times the T keys / scaled queries, two MMA chains (even / odd k-steps).""" + NT = self.NT + acc0 = [[zf, zf, zf, zf] for _ in range(NT)] + acc1 = [[zf, zf, zf, zf] for _ in range(NT)] + for jj in range(4): + for s in range(2): + a = ( + cur[8 * jj + 2 * s], + cur[8 * jj + 4 + 2 * s], + cur[8 * jj + 2 * s + 1], + cur[8 * jj + 4 + 2 * s + 1], + ) + for nt in range(NT): + bfr = (bq[nt][4 * jj + 2 * s], bq[nt][4 * jj + 2 * s + 1]) + if s == 0: + acc0[nt] = _mma(a, bfr, acc0[nt]) + else: + acc1[nt] = _mma(a, bfr, acc1[nt]) + return [[acc0[nt][e] + acc1[nt][e] for e in range(4)] for nt in range(NT)] + + def _g_f32(self, x): + return x if self.g_f32 else _b16_f32(x) + + def _b_f32(self, x): + return x if self.b_f32 else _b16_f32(x) + + def _work_item(self, wi_ptr, item): + """Work item `item`'s int32 fields: (row, slot, pending count, half), then the replay + slot when the history has slots of its own and the items carry them.""" + base = wi_ptr + cutlass.Int64(item) * (4 * self.wi_w) + if self.wi_w == 4: + return _LDG4["plain"](base) + fields = WI_REPLAY_SLOT + 1 if self.rsep and self.wi_w > WI_REPLAY_SLOT else 4 + return [_ldg_i32(base + 4 * f) for f in range(fields)] + + def _load_tile(self, addr, pol=None): + regs = [] + for jj in range(4): + for off in (jj * 64, 2048 + jj * 64): + if pol is not None: + regs.extend(_ldg4_hint(addr + off, pol)) + else: + regs.extend(_LDG4[self.ld](addr + off)) + return regs + + @cute.kernel + def kernel( + self, + sidx_ptr: cutlass.Int64, + q_ptr: cutlass.Int64, + k_ptr: cutlass.Int64, + v_ptr: cutlass.Int64, + q_sn: cutlass.Int64, + q_st: cutlass.Int64, + k_sn: cutlass.Int64, + k_st: cutlass.Int64, + v_sn: cutlass.Int64, + v_st: cutlass.Int64, + g_ptr: cutlass.Int64, + g_st: cutlass.Int64, + b_ptr: cutlass.Int64, + b_st: cutlass.Int64, + alog_ptr: cutlass.Int64, + dtb_ptr: cutlass.Int64, + o_ptr: cutlass.Int64, + h_ptr: cutlass.Int64, + h_ss: cutlass.Int64, + pnat_ptr: cutlass.Int64, + buf_ptr: cutlass.Int64, + ou_ptr: cutlass.Int64, + ok_ptr: cutlass.Int64, + og_ptr: cutlass.Int64, + wi_ptr: cutlass.Int64, + ridx_ptr: cutlass.Int64, + u_ss: cutlass.Int64, + k_ss: cutlass.Int64, + g_ss: cutlass.Int64, + pnat_ss: cutlass.Int64, + buf_ss: cutlass.Int64, + cx_ptr: cutlass.Int64, + cx_st: cutlass.Int64, + cw_ptr: cutlass.Int64, + cs_ptr: cutlass.Int64, + cs_ss: cutlass.Int64, + win_ptr: cutlass.Int64, + win_sn: cutlass.Int64, + win_st: cutlass.Int64, + wrow_ptr: cutlass.Int64, + zn_ptr: cutlass.Int64, + zn_st: cutlass.Int64, + gw_ptr: cutlass.Int64, + on_ptr: cutlass.Int64, + scale: cutlass.Float32, + ): + T: cutlass.Constexpr = self.T + TP: cutlass.Constexpr = self.TP + NT: cutlass.Constexpr = self.NT + W: cutlass.Constexpr = self.W + TPW: cutlass.Constexpr = self.tpw + TPC: cutlass.Constexpr = self.tpc + NHV: cutlass.Constexpr = self.nhv + S: cutlass.Constexpr = self.S + NTHR: cutlass.Constexpr = 32 * self.W + RP: cutlass.Constexpr = self.RP + H: cutlass.Constexpr = self.H + HV: cutlass.Constexpr = self.HV + + tidx, _, _ = cute.arch.thread_idx() + bidx, _, _ = cute.arch.block_idx() + warp = tidx // 32 + lane = tidx % 32 + g = lane // 4 + c = lane % 4 + sp = bidx % S + nh = bidx // S + h = nh % H + if const_expr(self.use_wi): + wi = self._work_item(wi_ptr, nh // H) + n = wi[0] + else: + n = nh // H + n64 = cutlass.Int64(n) + zf = cutlass.Float32(0.0) + zi = cutlass.Int32(0) + + smem = cutlass.utils.SmemAllocator() + base_t = smem.allocate_tensor( + cutlass.Int8, cute.make_layout((self.smem - 128,), stride=(1,)), byte_alignment=128 + ) + sbase = cutlass.Int32(base_t.iterator.toint()) + s_u = sbase + self.o_u + s_kh = sbase + self.o_kh + s_kp = sbase + self.o_kp + s_v = sbase + self.o_v + s_x = sbase + self.o_x + warp * (16 * RP) + s_kk = sbase + self.o_kk + s_cf = sbase + self.o_cf + s_gh = sbase + self.o_gh + s_dec = sbase + self.o_dec + s_g = sbase + self.o_g + s_eg = sbase + self.o_eg + s_beta = sbase + self.o_beta + s_a = sbase + self.o_a + s_cq = sbase + self.o_cq + s_egs = sbase + self.o_egs + s_cv = sbase + self.o_cv + s_on = sbase + self.o_on + + if const_expr(self.pdl and not self.state_before_wait and not self.qk_first): + cute.arch.griddepcontrol_wait() + + NSM: cutlass.Constexpr = self.nsm + s_st = sbase + self.o_st + warp * (NSM * 4096) + s_mb = sbase + self.o_mb + warp * (NSM * 8) + if const_expr(NSM > 0): + if lane == 0: + for k_ in cutlass.range_constexpr(NSM): + _mbar_init(s_mb + k_ * 8, cutlass.Int32(1)) + _fence_mbar_init() + + hv_first = (sp * TPC) // 8 + hv0 = h * GRP + hv_first + gi = tidx - (32 * (W - 1) if W > 1 else 0) + gpred = (gi >= 0) & (gi < NHV) + if const_expr(self.qk_first): + # the step's own (L2-resident) inputs first, ahead of the state stream + if const_expr(self.pdl): + cute.arch.griddepcontrol_wait() + if const_expr(self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + pre = [ + self._qk_raw( + nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, tidx >= 0 + ) + for nt in range(NT) + ] + gpre = self._gate_raw( + gpred, + hv0 + cutlass.min(cutlass.max(gi, zi), NHV - 1), + n64, + g_ptr, + g_st, + b_ptr, + b_st, + alog_ptr, + dtb_ptr, + ) + n_v0: cutlass.Constexpr = NHV * T * 16 + for it0 in cutlass.range_constexpr(-(-n_v0 // NTHR)): + ev0 = tidx + it0 * NTHR + if ev0 < n_v0: + hl0 = ev0 // (T * 16) + tv0 = (ev0 // 16) % T + qv0 = ev0 % 16 + sv0 = ( + v_ptr + + ( + n64 * v_sn + + cutlass.Int64(tv0) * v_st + + cutlass.Int64((hv0 + hl0) * V + qv0 * 8) + ) + * 2 + ) + _cp16z(s_v + (hl0 * T + tv0) * 256 + qv0 * 16, sv0, cutlass.Int32(16)) + _cp_commit() + + if const_expr(self.use_wi): + slot = wi[1] + else: + slot = _ldg_i32(sidx_ptr + n64 * 4) + # the history and bookkeeping row: the checkpoint's slot unless it has its own + rslot = slot + if const_expr(self.rsep): + if const_expr(self.use_wi and self.wi_w > WI_REPLAY_SLOT): + rslot = wi[WI_REPLAY_SLOT] + else: + rslot = _ldg_i32(ridx_ptr + n64 * 4) + if slot >= 0: + slot64 = cutlass.Int64(slot) + rs64 = cutlass.Int64(rslot) + # ---- 1. the state tiles: bulk copies first, then the register tile ----------------- + pol = _policy_evict_first() if const_expr(self.evict_first) else None + if const_expr(NSM > 0): + if lane == 0: + for k_ in cutlass.range_constexpr(NSM): + if const_expr(self.evict_first): + _bulk_g2s_hint( + s_st + k_ * 4096, + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, self.pf + k_, 0, 0 + ), + cutlass.Int32(4096), + s_mb + k_ * 8, + pol, + ) + else: + _bulk_g2s( + s_st + k_ * 4096, + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, self.pf + k_, 0, 0 + ), + cutlass.Int32(4096), + s_mb + k_ * 8, + ) + tiles = [] + for i in cutlass.range_constexpr(self.pf): + tiles.append( + self._load_tile(self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i, g, c), pol) + ) + + # ---- 2. history bookkeeping and the history rows into shared memory ------------ + if const_expr(self.use_wi): + p = wi[2] + rb = wi[3] + else: + p = _ldg_i32(pnat_ptr + rs64 * pnat_ss) + rb = _ldg_i32(buf_ptr + rs64 * buf_ss) + wrap = (p + T) > HIST + wb = cutlass.Int32(cutlass.select_(wrap, 1 - rb, rb)) + woff = cutlass.Int32(cutlass.select_(wrap, zi, p)) + # a replay slot's two history halves are packed behind its slot stride + u_sl = ou_ptr + rs64 * u_ss + k_sl = ok_ptr + rs64 * k_ss + g_sl = og_ptr + rs64 * g_ss + g_rd = g_sl + cutlass.Int64(rb) * (HV * HIST * 4) + u_rd = u_sl + cutlass.Int64(rb) * (HIST * HV * V * 2) + k_rd = k_sl + cutlass.Int64(rb) * (HIST * H * K * 2) + u_wr = u_sl + cutlass.Int64(wb) * (HIST * HV * V * 2) + k_wr = k_sl + cutlass.Int64(wb) * (HIST * H * K * 2) + g_wr = g_sl + cutlass.Int64(wb) * (HV * HIST * 4) + + n_u: cutlass.Constexpr = NHV * 256 + if const_expr(self.spec_hist): + # both halves, every row, before p / half are known; rows j >= p masked at use + for hb in cutlass.range_constexpr(2): + u_h = u_sl + hb * (HIST * HV * V * 2) + k_h = k_sl + hb * (HIST * H * K * 2) + g_h = g_sl + hb * (HV * HIST * 4) + for it in cutlass.range_constexpr(-(-n_u // NTHR)): + e = tidx + it * NTHR + if e < n_u: + hl = e // 256 + j = (e // 16) % 16 + qq = e % 16 + src = u_h + cutlass.Int64(((j * HV + hv0 + hl) * V + qq * 8) * 2) + _cp16z( + s_u + hb * (NHV * 16 * PU) + hl * (16 * PU) + j * PU + qq * 16, + src, + cutlass.Int32(16), + ) + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + qq = e % 16 + src = k_h + cutlass.Int64(((j * H + h) * K + qq * 8) * 2) + _cp16z(s_kh + hb * (16 * PK) + j * PK + qq * 16, src, cutlass.Int32(16)) + if tidx < NHV * 4: + hl = tidx // 4 + qq = tidx % 4 + src = g_h + cutlass.Int64(((hv0 + hl) * HIST + qq * 4) * 4) + _cp16z(s_gh + hb * (NHV * 64) + hl * 64 + qq * 16, src, cutlass.Int32(16)) + else: + for it in cutlass.range_constexpr(-(-n_u // NTHR)): + e = tidx + it * NTHR + if e < n_u: + hl = e // 256 + j = (e // 16) % 16 + qq = e % 16 + jc = cutlass.min( + j, cutlass.max(p - 1, zi) + ) # an in-bounds row for a zero fill + src = u_rd + cutlass.Int64(((jc * HV + hv0 + hl) * V + qq * 8) * 2) + _cp16z( + s_u + hl * (16 * PU) + j * PU + qq * 16, + src, + cutlass.Int32(cutlass.select_(j < p, cutlass.Int32(16), zi)), + ) + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + qq = e % 16 + jc = cutlass.min(j, cutlass.max(p - 1, zi)) + src = k_rd + cutlass.Int64(((jc * H + h) * K + qq * 8) * 2) + _cp16z( + s_kh + j * PK + qq * 16, + src, + cutlass.Int32(cutlass.select_(j < p, cutlass.Int32(16), zi)), + ) + if tidx < NHV * 4: + hl = tidx // 4 + qq = tidx % 4 + src = g_rd + cutlass.Int64(((hv0 + hl) * HIST + qq * 4) * 4) + _cp16z(s_gh + hl * 64 + qq * 16, src, cutlass.Int32(16)) + _cp_commit() + + if const_expr(self.spec_hist): + s_u = s_u + rb * (NHV * 16 * PU) + s_kh = s_kh + rb * (16 * PK) + s_gh = s_gh + rb * (NHV * 64) + + # ---- 3. the step's own inputs (written by the previous kernel) ------------------ + if const_expr(not self.qk_first): + if const_expr(self.pdl and self.state_before_wait): + cute.arch.griddepcontrol_wait() + if const_expr(self.pdl and self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + if const_expr(self.fold): + # The verify conv of this CTA's channels for the T rows, a thread an 8-channel + # group. The owning CTA writes the per-row windows (q/k: split 0); the pool is + # not advanced (the engine's acceptance promote writes it). + ngr: cutlass.Constexpr = 32 + NHV * 16 + wrow = cutlass.Int64(_ldg_i32(wrow_ptr + n64 * 4)) + for itc in cutlass.range_constexpr(-(-ngr // NTHR)): + gc = tidx + itc * NTHR + if gc < ngr: + ec = gc * 8 + is_v = ec >= 256 + vv = ec - 256 + hlc = vv // 128 + vc = vv - hlc * 128 + ch = cutlass.Int32( + cutlass.select_( + is_v, + 2 * H * K + (hv0 + hlc) * V + vc, + cutlass.select_( + ec < 128, h * K + ec, H * K + h * K + (ec - 128) + ), + ) + ) + dst = cutlass.Int32( + cutlass.select_(is_v, s_v + (hlc * T) * 256 + vc * 2, s_cv + ec * 2) + ) + pitch = cutlass.Int32( + cutlass.select_(is_v, cutlass.Int32(256), cutlass.Int32(512)) + ) + owns = is_v | (sp == 0) + ch64 = cutlass.Int64(ch) + wa = cw_ptr + ch64 * 8 + ww = [] + for q4 in cutlass.range_constexpr(4): + ww.extend(_LDG4["plain"](wa + q4 * 16)) + pa = cs_ptr + (slot64 * cs_ss + ch64 * 3) * 2 + pw = [] + for q4 in cutlass.range_constexpr(3): + pw.extend(_LDG4["plain"](pa + q4 * 16)) + xw = [] + for tt in cutlass.range_constexpr(T): + xw.append( + _LDG4["plain"](cx_ptr + ((n64 * T + tt) * cx_st + ch64) * 2) + ) + # per channel: taps = pool[0..2], x rows; weights w[0..3] + outs = [[] for _ in range(T)] + wins = [[] for _ in range(T)] + for cl in cutlass.range_constexpr(8): + wt = [ + _half16(ww[(cl * 4 + j) // 2], (cl * 4 + j) % 2 == 1) + for j in range(4) + ] + taps = [ + _half16(pw[(cl * 3 + j) // 2], (cl * 3 + j) % 2 == 1) + for j in range(3) + ] + for tt in cutlass.range_constexpr(T): + taps.append(_half16(xw[tt][cl // 2], cl % 2 == 1)) + for tt in cutlass.range_constexpr(T): + outs[tt].append( + _conv4_silu( + _mul_bf16(taps[tt], wt[0]), + _mul_bf16(taps[tt + 1], wt[1]), + _mul_bf16(taps[tt + 2], wt[2]), + _mul_bf16(taps[tt + 3], wt[3]), + ) + ) + for j in cutlass.range_constexpr(3): + wins[tt].append(taps[tt + 1 + j]) + for tt in cutlass.range_constexpr(T): + o = outs[tt] + _sts4( + dst + tt * pitch, + _pack16(o[0], o[1]), + _pack16(o[2], o[3]), + _pack16(o[4], o[5]), + _pack16(o[6], o[7]), + ) + if owns: + wv = wins[tt] + wbase = ( + win_ptr + + (wrow * win_sn + cutlass.Int64(tt) * win_st + ch64 * 3) + * 2 + ) + for q4 in cutlass.range_constexpr(3): + _stg4( + wbase + q4 * 16, + _pack16(wv[q4 * 8], wv[q4 * 8 + 1]), + _pack16(wv[q4 * 8 + 2], wv[q4 * 8 + 3]), + _pack16(wv[q4 * 8 + 4], wv[q4 * 8 + 5]), + _pack16(wv[q4 * 8 + 6], wv[q4 * 8 + 7]), + ) + cute.arch.sync_threads() + n_v: cutlass.Constexpr = 0 if self.fold else NHV * T * 16 + for it in cutlass.range_constexpr(-(-n_v // NTHR)): + e = tidx + it * NTHR + if e < n_v: + hl = e // (T * 16) + t = (e // 16) % T + qq = e % 16 + src = ( + v_ptr + + ( + n64 * v_sn + + cutlass.Int64(t) * v_st + + cutlass.Int64((hv0 + hl) * V + qq * 8) + ) + * 2 + ) + _cp16z(s_v + (hl * T + t) * 256 + qq * 16, src, cutlass.Int32(16)) + _cp_commit() + + # key / query B fragments: column nt * 8 + g; lane quad c holds dims 32 jj + 8 c + 0..7 + bq = [] + for nt in cutlass.range_constexpr(NT): + if const_expr(self.qk_first): + rq = pre[nt] + elif const_expr(self.fold): + rq = self._qk_raw_smem(nt, s_cv, g, c, scale) + else: + rq = self._qk_raw( + nt, q_ptr, k_ptr, q_sn, q_st, k_sn, k_st, n64, h, g, c, scale, tidx >= 0 + ) + bq.append(self._qk_frag(rq[0], rq[1], rq[2])) + + # the new keys into the history (one CTA of the head, warp 0) + if sp == 0: + if warp == 0: + # column g < T of tile 0 is key g (both column layouts) + if g < T: + kfr = bq[0] + dst = k_wr + cutlass.Int64((((woff + g) * H + h) * K + c * 8) * 2) + for jj in cutlass.range_constexpr(4): + _stg4( + dst + jj * 64, + kfr[4 * jj], + kfr[4 * jj + 1], + kfr[4 * jj + 2], + kfr[4 * jj + 3], + ) + + # gating, cumulative decays, the G history (a thread per v head, off warp 0) + if gpred: + hl = gi + hv = hv0 + hl + gs = _ldg_f32p( + g_rd + cutlass.Int64(hv * HIST + p - 1) * 4, + cutlass.Int32(cutlass.select_(p > 0, cutlass.Int32(1), zi)), + ) + if const_expr(self.qk_first): + graws, braws, alv, dtb = gpre + else: + graws, braws, alv, dtb = self._gate_raw( + gpred, hv, n64, g_ptr, g_st, b_ptr, b_st, alog_ptr, dtb_ptr + ) + aexp = _exp(alv) + owns = (hv_first + hl) * 8 >= sp * TPC + gl = zf + for t in cutlass.range_constexpr(T): + graw = self._g_f32(graws[t]) + braw = self._b_f32(braws[t]) + if const_expr(self.gating): + x = graw + dtb + spl = cutlass.Float32( + cutlass.select_(x <= 20.0, LN2 * _lg2(1.0 + _ex2(x * LOG2E)), x) + ) + gt = -aexp * spl + beta = _div_approx(cutlass.Float32(1.0), 1.0 + _ex2(-braw * LOG2E)) + else: + gt = graw + beta = braw + gl = gl + gt + gabs = gs + gl + _sts_f32(s_g + (hl * 8 + t) * 4, gabs) + _sts_f32(s_eg + (hl * 8 + t) * 4, _exp(gabs)) + _sts_f32(s_beta + (hl * 8 + t) * 4, beta) + if owns: + _stg_f32( + g_wr + cutlass.Int64((hv * HIST + woff + t) * 4), + cutlass.Float32(cutlass.select_(wrap, gl, gabs)), + ) + _sts_f32(s_egs + hl * 4, _exp(gs)) + + # new-token dot products: rows (keys | queries) x key columns + if warp == 0: + d = [zf, zf, zf, zf] + for jj in cutlass.range_constexpr(4): + for s in cutlass.range_constexpr(2): + b0 = bq[0][4 * jj + 2 * s] + b1 = bq[0][4 * jj + 2 * s + 1] + if const_expr(NT == 1): + a = (b0, zi, b1, zi) + else: + a = (b0, bq[1][4 * jj + 2 * s], b1, bq[1][4 * jj + 2 * s + 1]) + d = _mma(a, (b0, b1), d) + _sts2_f32(s_kk + (g * 8 + 2 * c) * 4, d[0], d[1]) + _sts2_f32(s_kk + ((g + 8) * 8 + 2 * c) * 4, d[2], d[3]) + + # the state dots need no history: run them while it is in flight + early = [] + if const_expr(self.use_early): + for i in cutlass.range_constexpr(TPW): + if const_expr(NSM == 0): + if const_expr(i + self.pf < TPW): + tiles.append( + self._load_tile( + self._tile_addr( + h_ptr, slot64, h_ss, h, sp, warp, i + self.pf, g, c + ) + ) + ) + if const_expr(i < self.pf or NSM == 0): + early.append(self._tile_dots(tiles[i], bq, zf)) + else: + early.append( + self._tile_dots(self._smem_tile(s_st, s_mb, i, g, c, zi), bq, zf) + ) + + _cp_wait_all() + cute.arch.sync_threads() + + # ---- 4. history-key dots, commit operand, per-head scalars ---------------------- + hk_warp: cutlass.Constexpr = 1 if W > 1 else 0 + if warp == hk_warp: + dh = [[zf, zf, zf, zf] for _ in range(NT)] + for jj in cutlass.range_constexpr(4): + rg = _lds4_i32(s_kh + g * PK + jj * 64 + c * 16) + rh = _lds4_i32(s_kh + (g + 8) * PK + jj * 64 + c * 16) + for s in cutlass.range_constexpr(2): + a = (rg[2 * s], rh[2 * s], rg[2 * s + 1], rh[2 * s + 1]) + for nt in cutlass.range_constexpr(NT): + dh[nt] = _mma( + a, (bq[nt][4 * jj + 2 * s], bq[nt][4 * jj + 2 * s + 1]), dh[nt] + ) + # history coefficients (k_t . k_j or q_t . k_j) exp(G_t - G_j), j < p, bf16 B fragments + for hl in cutlass.range_constexpr(NHV): + gh_g = _lds_f32(s_gh + (hl * 16 + g) * 4) + gh_h = _lds_f32(s_gh + (hl * 16 + g + 8) * 4) + for nt in cutlass.range_constexpr(NT): + for e in cutlass.range_constexpr(2): + col = nt * 8 + 2 * c + e + if const_expr(TP == 4): + tt = (2 * c + e) % 4 + else: + tt = 2 * c + e + tv = tt < T + gtt = _lds_f32(s_g + (hl * 8 + tt) * 4) + vg = cutlass.Float32( + cutlass.select_((g < p) & tv, dh[nt][e] * _exp(gtt - gh_g), zf) + ) + vh = cutlass.Float32( + cutlass.select_( + ((g + 8) < p) & tv, dh[nt][2 + e] * _exp(gtt - gh_h), zf + ) + ) + _sts_bf16(s_cf + ((hl * 16 + col) * 16 + g) * 2, vg) + _sts_bf16(s_cf + ((hl * 16 + col) * 16 + g + 8) * 2, vh) + + if const_expr(self.commit): + if wrap: + for it in cutlass.range_constexpr(-(-256 // NTHR)): + e = tidx + it * NTHR + if e < 256: + j = e // 16 + jj = (e // 4) % 4 + aa = e % 4 + w4 = _lds4_i32(s_kh + j * PK + jj * 64 + aa * 16) + for s4 in cutlass.range_constexpr(4): + wv = w4[s4] + if const_expr(self.spec_hist): + wv = cutlass.Int32(cutlass.select_(j < p, wv, zi)) + _sts_i32(s_kp + j * PU + (16 * jj + 4 * s4 + aa) * 4, wv) + + n_ts: cutlass.Constexpr = NHV * T * T + for it in cutlass.range_constexpr(-(-n_ts // NTHR)): + e = tidx + it * NTHR + if e < n_ts: + hl = e // (T * T) + t = (e // T) % T + s = e % T + gt = _lds_f32(s_g + (hl * 8 + t) * 4) + gsv = _lds_f32(s_g + (hl * 8 + s) * 4) + bt = _lds_f32(s_beta + (hl * 8 + t) * 4) + dec = _exp(gt - gsv) + kk = _lds_f32(s_kk + (t * 8 + s) * 4) + qk = _lds_f32(s_kk + ((TP + t) * 8 + s) * 4) + av = cutlass.Float32(cutlass.select_(s < t, bt * kk * dec, zf)) + cv = cutlass.Float32(cutlass.select_(s <= t, qk * dec, zf)) + _sts_f32(s_a + (hl * 64 + t * 8 + s) * 4, av) + _sts_f32(s_cq + (hl * 64 + t * 8 + s) * 4, cv) + if const_expr(self.commit): + if tidx < NHV * 16: + hl = tidx // 16 + j = tidx % 16 + gsv = cutlass.Float32(0.0) + if p > 0: + gsv = _lds_f32(s_gh + (hl * 16 + p - 1) * 4) + ghj = _lds_f32(s_gh + (hl * 16 + j) * 4) + _sts_f32( + s_dec + (hl * 16 + j) * 4, + cutlass.Float32(cutlass.select_(j < p, _exp(gsv - ghj), zf)), + ) + cute.arch.sync_threads() + + # the folded norm's gate and weight words (after any dependency wait), landing during the tiles + if const_expr(self.norm): + nz0 = zi + nz1 = zi + nw0 = zi + nw1 = zi + if warp < T: + hvp = h * GRP + sp + rowp = n64 * T + cutlass.Int64(warp) + colp = cutlass.Int64(4 * lane) + zpre = _ldg2_i32(zn_ptr + (rowp * zn_st + cutlass.Int64(hvp * V) + colp) * 2) + wpre = _ldg2_i32(gw_ptr + colp * 2) + nz0 = zpre[0] + nz1 = zpre[1] + nw0 = wpre[0] + nw1 = wpre[1] + + # ---- 5. the warp's m-tiles ------------------------------------------------------ + jrow = (lane % 8) + 8 * (lane // 16) + v8 = 8 * ((lane // 8) % 2) + r = lane % 16 + hf = lane // 16 + godd = (g % 2) == 1 + mask01 = cutlass.Int32( + cutlass.select_(2 * c < p, cutlass.Int32(0xFFFF), zi) + ) | cutlass.Int32(cutlass.select_(2 * c + 1 < p, cutlass.Int32(-65536), zi)) + mask89 = cutlass.Int32( + cutlass.select_(2 * c + 8 < p, cutlass.Int32(0xFFFF), zi) + ) | cutlass.Int32(cutlass.select_(2 * c + 9 < p, cutlass.Int32(-65536), zi)) + for i in cutlass.range_constexpr(TPW): + if const_expr(NSM == 0 and not self.use_early): + if const_expr(i + self.pf < TPW): + tiles.append( + self._load_tile( + self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i + self.pf, g, c) + ) + ) + if const_expr(i < self.pf or NSM == 0): + cur = tiles[i] + elif const_expr(self.use_early): + # a staged tile is read again from shared memory only for its commit + cur = None + else: + # rows g and g + 8 of the staged tile, chunk order by row parity (bank-disjoint) + kslot = i - self.pf + _mbar_wait(s_mb + kslot * 8, zi) + tb = s_st + kslot * 4096 + g * 256 + c * 16 + raw = [] + for ii in cutlass.range_constexpr(4): + jsw = (g % 2) ^ ii + raw.append(_lds4_i32(tb + jsw * 64)) + raw.append(_lds4_i32(tb + 2048 + jsw * 64)) + cur = [] + for jj in cutlass.range_constexpr(4): + for rr in cutlass.range_constexpr(2): + for e in cutlass.range_constexpr(4): + cur.append( + cutlass.Int32( + cutlass.select_( + godd, raw[2 * (jj ^ 1) + rr][e], raw[2 * jj + rr][e] + ) + ) + ) + mt = sp * TPC + warp + W * i + hvr = mt // 8 + hv = h * GRP + hvr + hl = hvr - hv_first + v0 = (mt % 8) * 16 + + if const_expr(self.use_early): + acc = early[i] + else: + acc = self._tile_dots(cur, bq, zf) + + au = _ldsm4t(s_u + hl * (16 * PU) + jrow * PU + (v0 + v8) * 2) + if const_expr(self.spec_hist): + au = [cutlass.Int32(au[q4] & (mask01 if q4 < 2 else mask89)) for q4 in range(4)] + hacc = [] + for nt in cutlass.range_constexpr(NT): + cfa = s_cf + ((hl * 16 + nt * 8 + g) * 16 + 2 * c) * 2 + bc = (_lds_i32(cfa), _lds_i32(cfa + 16)) + hacc.append(_mma(au, bc, [zf, zf, zf, zf])) + + for nt in cutlass.range_constexpr(NT): + col0 = nt * 8 + 2 * c + if const_expr(TP == 4): + t0 = (2 * c) % 4 + else: + t0 = 2 * c + eg0 = _lds_f32(s_eg + (hl * 8 + t0) * 4) + eg1 = _lds_f32(s_eg + (hl * 8 + t0 + 1) * 4) + _sts2_f32( + s_x + g * RP + col0 * 4, + eg0 * acc[nt][0] + hacc[nt][0], + eg1 * acc[nt][1] + hacc[nt][1], + ) + _sts2_f32( + s_x + (g + 8) * RP + col0 * 4, + eg0 * acc[nt][2] + hacc[nt][2], + eg1 * acc[nt][3] + hacc[nt][3], + ) + cute.arch.sync_warp() + + # per state row: the forward substitution and the outputs + xs = [] + ys = [] + for q4 in cutlass.range_constexpr(TP // 4): + xs.extend(_lds4_f32(s_x + r * RP + q4 * 16)) + ys.extend(_lds4_f32(s_x + r * RP + (TP + 4 * q4) * 4)) + us = [] + for t in cutlass.range_constexpr(T): + vt = _lds_bf16(s_v + ((hl * T + t) * 128 + v0 + r) * 2) + bt = _lds_f32(s_beta + (hl * 8 + t) * 4) + u = bt * (vt - xs[t]) + for s in cutlass.range_constexpr(t): + u = u - _lds_f32(s_a + (hl * 64 + t * 8 + s) * 4) * us[s] + us.append(u) + if hf == 0: + for t in cutlass.range_constexpr(T): + o = ys[t] + for s in cutlass.range_constexpr(t + 1): + o = o + _lds_f32(s_cq + (hl * 64 + t * 8 + s) * 4) * us[s] + if const_expr(self.norm): + # the CTA's one value head: staged for the norm below + _sts_bf16(s_on + (t * V + v0 + r) * 2, o) + else: + _stg_bf16(o_ptr + (((n64 * T + t) * HV + hv) * V + v0 + r) * 2, o) + else: + for t in cutlass.range_constexpr(T): + _stg_bf16( + u_wr + cutlass.Int64((((woff + t) * HV + hv) * V + v0 + r) * 2), us[t] + ) + cute.arch.sync_warp() + + # the checkpoint commit of a wrapping row, over the tile just used + if const_expr(self.commit): + if wrap: + dc = [_lds_f32(s_dec + (hl * 16 + 2 * c + jx) * 4) for jx in [0, 1, 8, 9]] + egs = _lds_f32(s_egs + hl * 4) + a_hi = [] + a_lo = [] + for q4 in cutlass.range_constexpr(4): + dl = dc[0] if q4 < 2 else dc[2] + dhh = dc[1] if q4 < 2 else dc[3] + x0 = _bf2_lo(au[q4]) * dl + x1 = _bf2_hi(au[q4]) * dhh + hw = _bf2_pack(x0, x1) + a_hi.append(hw) + a_lo.append(_bf2_pack(x0 - _bf2_lo(hw), x1 - _bf2_hi(hw))) + if const_expr(cur is None): + curc = self._smem_tile(s_st, s_mb, i, g, c, zi) + else: + curc = cur + taddr = self._tile_addr(h_ptr, slot64, h_ss, h, sp, warp, i, g, c) + kp_row = ( + s_kp + ((lane % 8) + 8 * ((lane // 8) % 2)) * PU + 16 * (lane // 16) + ) + for jj in cutlass.range_constexpr(4): + bb0 = _ldsm4t(kp_row + (4 * jj) * 16) + bb1 = _ldsm4t(kp_row + (4 * jj + 2) * 16) + ng = [] + nh8 = [] + for s4 in cutlass.range_constexpr(4): + bsrc = bb0 if s4 < 2 else bb1 + bf = (bsrc[2 * (s4 % 2)], bsrc[2 * (s4 % 2) + 1]) + dcm = _mma(a_hi, bf, [zf, zf, zf, zf]) + if const_expr(self.lo_split): + dcm = _mma(a_lo, bf, dcm) + wg = curc[8 * jj + s4] + wh = curc[8 * jj + 4 + s4] + ng.append( + _bf2_pack( + egs * _bf2_lo(wg) + dcm[0], egs * _bf2_hi(wg) + dcm[1] + ) + ) + nh8.append( + _bf2_pack( + egs * _bf2_lo(wh) + dcm[2], egs * _bf2_hi(wh) + dcm[3] + ) + ) + _stg4(taddr + jj * 64, ng[0], ng[1], ng[2], ng[3]) + _stg4(taddr + (2048 + jj * 64), nh8[0], nh8[1], nh8[2], nh8[3]) + + # ---- 6. the folded gated RMS norm: warp t takes row t of the CTA's value head ------ + if const_expr(self.norm): + cute.arch.sync_threads() + if warp < T: + hvn = h * GRP + sp # tpc 8: the CTA's value head + row64 = n64 * T + cutlass.Int64(warp) + col = cutlass.Int64(4 * lane) + xw = _lds2_i32(s_on + (warp * V + 4 * lane) * 2) + zw = [nz0, nz1] + ww = [nw0, nw1] + xs4 = [_bf2_lo(xw[0]), _bf2_hi(xw[0]), _bf2_lo(xw[1]), _bf2_hi(xw[1])] + ssq = _gn_sumsq(xs4[0], xs4[1], xs4[2], xs4[3]) + for m_ in cutlass.range_constexpr(5): + ssq = _add_rn(ssq, _shfl_xor(ssq, 16 >> m_)) + rstd = _gn_rstd(ssq, cutlass.Float32(self.norm_eps)) + ws4 = [_bf2_lo(ww[0]), _bf2_hi(ww[0]), _bf2_lo(ww[1]), _bf2_hi(ww[1])] + zs4 = [_bf2_lo(zw[0]), _bf2_hi(zw[0]), _bf2_lo(zw[1]), _bf2_hi(zw[1])] + ys4 = [ + _mul_rn(_mul_rn(_mul_rn(rstd, xs4[e]), ws4[e]), _gn_sigmoid(zs4[e])) + for e in range(4) + ] + _stg2_i32( + on_ptr + ((row64 * HV + cutlass.Int64(hvn)) * V + col) * 2, + _bf2_pack(ys4[0], ys4[1]), + _bf2_pack(ys4[2], ys4[3]), + ) + + if const_expr(self.pdl and not self.early_trigger): + cute.arch.griddepcontrol_launch_dependents() + + +# -- host side --------------------------------------------------------------------------------- + +_COMPILED: dict = {} + + +def default_config(n: int, t: int, commit: bool = False) -> dict: + """Launch mapping by request count: smaller CTAs for small (latency-bound) batches; from 32 + requests (or 8 with the commit) CTAs of 8 m-tiles x 4 warps, at most 5 (4) an SM.""" + if n <= 2: + return dict(tpc=2, warps=2, pf=0, minb=4, early_acc=True) + if n <= 8: + return dict(tpc=4, warps=4, pf=0, minb=4, early_acc=True) + if commit: + return dict(tpc=8, warps=4, pf=1, minb=4) + if n <= 16: + return dict(tpc=24, warps=8, pf=1, minb=2) + return dict(tpc=8, warps=4, pf=1, minb=5) + + +def _compile(key, kern): + fn = _COMPILED.get(key) + if fn is None: + dummy = torch.zeros(4, dtype=torch.int32, device="cuda") + i64 = cutlass.Int64(0) + fn = cute.compile( + kern, + from_dlpack(dummy, assumed_align=4).mark_layout_dynamic(leading_dim=0), + *([i64] * 43), + cutlass.Int32(1), + cutlass.Float32(1.0), + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + _COMPILED[key] = fn + return fn + + +def _kernel_kw( + n: int, + t: int, + h: int, + hv: int, + *, + commit: bool, + pdl: bool, + l2norm: bool, + gating: bool, + g_f32: bool, + b_f32: bool, + work_items: bool, + config: Optional[dict] = None, + fold: bool = False, + norm_eps: Optional[float] = None, + work_item_width: int = 4, + separate_replay_slots: bool = False, +) -> dict: + """The kernel's compile-time parameters for a call of `n` requests: its key in `_COMPILED`. + `fold` is a `conv_fold` call, `norm_eps` a `gated_norm` call's epsilon, `work_item_width` + the int32 fields per work item, and `separate_replay_slots` says the history is addressed by + replay slots of its own (`replay_indices`).""" + cfg = dict(default_config(n, t, commit)) + if config: + cfg.update(config) + if fold: + if cfg.get("tpc") not in (8, 24): + # small batches' mappings split a value head over CTAs: take the whole-head ones + whole = ( + dict(tpc=8, warps=4, pf=1, minb=5) + if n > 16 + else dict(tpc=24, warps=8, pf=1, minb=2) + ) + cfg.update(whole) + cfg.pop("early_acc", None) + cfg["fold"] = True + if norm_eps is not None: + if cfg.get("tpc") != 8 or cfg.get("warps") != 4: + cfg.update(tpc=8, warps=4) + cfg["pf"] = min(int(cfg.get("pf", 1)), 2) + cfg.setdefault("minb", 5) + cfg["norm"] = True + cfg["norm_eps"] = norm_eps + # PDL: state / history before the wait; without it, the step's own inputs first + cfg.setdefault("qk_first", not pdl) + cfg["use_wi"] = bool(cfg.pop("use_wi", True)) and work_items + cfg["wi_w"] = work_item_width if cfg["use_wi"] else 4 + cfg["rsep"] = separate_replay_slots + return dict( + T=t, + h=h, + hv=hv, + commit=commit, + pdl=pdl, + l2norm=l2norm, + gating=gating, + g_f32=g_f32, + b_f32=b_f32, + **cfg, + ) + + +def precompile_keys( + t: int, + h: int, + hv: int, + max_requests: int, + *, + partition_from: int, + launch: Callable[[int], tuple], + conv_fold: Optional[Callable[[int], bool]] = None, + fold_norm: Optional[Callable[[int], bool]] = None, + norm_eps: Optional[float] = None, + work_item_width: int = 4, + separate_replay_slots: bool = False, +) -> set: + """The `_COMPILED` key of every kernel a verify step of 1..`max_requests` requests launches, + the way a target calls this entry: work items, and with them no in-kernel commit, from + `partition_from` requests on; the q/k L2 norm and the gating in-kernel; bf16 `a` / `b`. + `launch(n)` returns `(pdl, config)` for a batch of `n` requests (config None: the default + mapping), `conv_fold(n)` whether it folds the conv, and `fold_norm(n)` whether a + generation-only batch folds the gated norm (epsilon `norm_eps`); a mixed step's verify rows + never fold it, so such a batch reaches both kernels. `work_item_width` is the int32 fields + of the manager's work items and `separate_replay_slots` whether it keeps the history at + replay slots of its own (every step then passes `replay_indices`).""" + keys = set() + for n in range(1, max_requests + 1): + partitioned = n >= partition_from + n_pdl, config = launch(n) + norms = [None] + ([norm_eps] if fold_norm is not None and fold_norm(n) else []) + for eps in norms: + kw = _kernel_kw( + n, + t, + h, + hv, + commit=not partitioned, + pdl=bool(n_pdl), + l2norm=True, + gating=True, + g_f32=False, + b_f32=False, + work_items=partitioned, + config=config, + fold=conv_fold is not None and conv_fold(n), + norm_eps=eps, + work_item_width=work_item_width, + separate_replay_slots=separate_replay_slots, + ) + keys.add(tuple(sorted(kw.items()))) + return keys + + +def precompile(t: int, h: int, hv: int, max_requests: int, **launches) -> int: + """Compile every kernel of `precompile_keys(t, h, hv, max_requests, **launches)`. A compile + costs about 0.5 s on GB300, so a target runs this at bind, before any step: the captured + decode graphs do not reach every mapping (padded graphs skip 9 to 15 requests, which an + eager mixed step carries). Returns the number of distinct kernels.""" + keys = precompile_keys(t, h, hv, max_requests, **launches) + for key in keys: + _compile(key, _Replay(**dict(key))) + return len(keys) + + +def conv_fold_supported(x, weight, pool, window) -> bool: + """Whether `cached_replay_update(conv_fold=...)` takes these operands: bf16, the layouts it + reads, and 16-byte alignment of every 8-channel group (vector loads / stores).""" + tensors = (x, weight, pool, window) + if any(t.dtype != torch.bfloat16 for t in tensors): + return False + if x.stride(1) != 1 or not weight.is_contiguous() or weight.shape[1] != 4: + return False + if pool.stride()[1:] != (3, 1) or window.stride()[2:] != (3, 1): + return False + strides = (x.stride(0), pool.stride(0), window.stride(0), window.stride(1)) + return all(t.data_ptr() % 16 == 0 for t in tensors) and all( + (2 * st) % 16 == 0 for st in strides + ) + + +def cached_replay_update( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: Optional[torch.Tensor], + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, + **ignored, +) -> torch.Tensor: + """Same contract as `fused_recurrent_gated_delta_rule_cached_replay_update` (bf16 state). + + A wrapping row commits its checkpoint in-kernel unless work items + `use_all_layer_commit` + are given. Work items must list all N rows (`config={"use_wi": False}` reads + `state_indices` instead); with separate replay slots their fifth field + (`REPLAY_WORK_REPLAY_SLOT`) gives each row's replay slot, else `replay_indices` does. The + Triton launch knobs are ignored. + + `replay_indices` (`state_indices` when None) address the history (`old_u` / `old_k` / + `old_G`) and its bookkeeping (`prev_num_accepted_tokens` / `cache_buf_idx`); each replay + slot's rows are packed behind a slot stride of 16-byte multiples, and the bookkeeping may + have any element stride (a manager keeps both in one row per slot). + + `conv_fold` folds the verify conv in, bitwise the conv kernel: `x` (the q|k|v projection + columns), `weight` `[C, 4]`, `pool` `[slots, C, 3]` (read, not advanced: the engine's + acceptance promote rewrites it from `window`), `window` `[rows, T, C, 3]` and `window_rows`; + the q/k/v arguments then give shapes only. Needs `tpc` 8 or 24. + + `gated_norm` folds `rms_norm_gated_token_major` (sigmoid gate) in, bitwise: `z`, `weight`, + `eps`; the un-normed outputs are not stored. Needs `tpc` 8, 4 warps and T <= 4. + """ + del old_beta + N, T, H, Kd = k.shape + HV = v.shape[2] + assert (H, HV) in GEOMETRIES and Kd == K and v.shape[3] == V and history_size == HIST, ( + f"geometry: {H} q/k x {Kd}, {HV} v x {v.shape[3]}, history {history_size}" + ) + qkv_width = 2 * H * K + HV * V + assert 1 <= T <= 8 + assert ssm_states.dtype == torch.bfloat16, "bf16 checkpoint state only" + assert ssm_states.stride()[1:] == (V * K, K, 1) + assert old_u.dtype == torch.bfloat16 and old_k.dtype == torch.bfloat16 + assert old_G.dtype == torch.float32 + for name, x, rows in ( + ("old_u", old_u, (2, HIST, HV, V)), + ("old_k", old_k, (2, HIST, H, K)), + ("old_G", old_G, (2, HV, HIST)), + ): + assert tuple(x.shape[1:]) == rows and x[0].is_contiguous(), ( + f"{name} is {tuple(x.shape)} with strides {x.stride()}; the kernel reads packed " + f"[slots, {', '.join(map(str, rows))}] rows" + ) + assert x.data_ptr() % 16 == 0 and (x.stride(0) * x.element_size()) % 16 == 0, ( + f"{name}: the kernel copies 16-byte chunks, so its base and slot stride must be " + "16-byte aligned" + ) + for name, x in ( + ("prev_num_accepted_tokens", prev_num_accepted_tokens), + ("cache_buf_idx", cache_buf_idx), + ): + assert x.dim() == 1 and x.dtype == torch.int32, f"{name}: one int32 per replay slot" + rsep = replay_indices is not None and replay_indices.data_ptr() != state_indices.data_ptr() + if rsep: + assert ( + replay_indices.dim() == 1 + and replay_indices.shape[0] >= N + and replay_indices.dtype == torch.int32 + and replay_indices.is_contiguous() + ), "replay_indices: one contiguous int32 replay slot per row" + assert g.shape == (N, T, HV) and beta.shape == (N, T, HV) + assert g.stride(2) == 1 and beta.stride(2) == 1 + assert g.stride(0) == T * g.stride(1) and beta.stride(0) == T * beta.stride(1) + assert g.dtype in (torch.bfloat16, torch.float32) and beta.dtype in ( + torch.bfloat16, + torch.float32, + ) + if (replay_work_items is None) != (n_writes is None): + raise ValueError("replay_work_items and n_writes must either both be set or both be None") + if replay_work_items is not None and not use_all_layer_commit: + raise ValueError("Partitioned replay requires the all-layer commit") + if replay_work_items is None and use_all_layer_commit: + raise ValueError("use_all_layer_commit requires replay work items") + commit = replay_work_items is None + if scale is None: + scale = K**-0.5 + if output is None: + output = torch.empty((N, T, HV, V), dtype=q.dtype, device=q.device) + assert output.is_contiguous() and output.dtype == torch.bfloat16 + if N == 0: + return output + if packed_qkv is not None: + assert packed_qkv.shape == (N * T, qkv_width) and packed_qkv.is_contiguous() + base = packed_qkv.data_ptr() + es = packed_qkv.element_size() + qp, kp, vp = base, base + H * K * es, base + 2 * H * K * es + qs = ks = vs = (T * qkv_width, qkv_width) + else: + for x, nh, d in ((q, H, K), (k, H, K), (v, HV, V)): + assert x.stride(3) == 1 and x.stride(2) == d, "head rows must be dense" + qp, kp, vp = q.data_ptr(), k.data_ptr(), v.data_ptr() + qs, ks, vs = q.stride()[:2], k.stride()[:2], v.stride()[:2] + gating = A_log is not None + if gating: + assert dt_bias is not None + A_log = A_log.float().contiguous() + dt_bias = dt_bias.float().contiguous() + ap, dp = A_log.data_ptr(), dt_bias.data_ptr() + else: + ap = dp = 0 + if replay_work_items is not None: + assert ( + replay_work_items.dim() == 2 + and replay_work_items.shape[1] >= 4 + and replay_work_items.dtype == torch.int32 + and replay_work_items.is_contiguous() + ), "replay_work_items: contiguous int32 rows of (row, slot, pending count, half[, ...])" + fold_args = (0,) * 9 + if conv_fold is not None: + cx, cw, cs = conv_fold["x"], conv_fold["weight"], conv_fold["pool"] + win, wrows = conv_fold["window"], conv_fold["window_rows"] + assert cx.shape == (N * T, qkv_width) and cx.stride(1) == 1 and cx.dtype == torch.bfloat16 + assert cw.shape == (qkv_width, 4) and cw.is_contiguous() and cw.dtype == torch.bfloat16 + assert cs.shape[1:] == (qkv_width, 3) and cs.stride()[1:] == (3, 1) + assert cs.dtype == torch.bfloat16 + assert win.shape[1:] == (T, qkv_width, 3) and win.stride()[2:] == (3, 1) + assert win.dtype == torch.bfloat16 + assert wrows.dtype == torch.int32 and wrows.shape[0] >= N and wrows.is_contiguous() + assert conv_fold_supported(cx, cw, cs, win), ( + "conv_fold operands: layout / 16-byte alignment" + ) + fold_args = ( + cx.data_ptr(), cx.stride(0), cw.data_ptr(), cs.data_ptr(), cs.stride(0), + win.data_ptr(), win.stride(0), win.stride(1), wrows.data_ptr(), + ) # fmt: skip + norm_args = (0,) * 4 + norm_eps = None + if gated_norm is not None: + zn, gw = gated_norm["z"], gated_norm["weight"] + assert T <= 4, "the folded norm takes a warp a row" + assert ( + zn.shape == (N * T, HV, V) and zn.stride()[1:] == (V, 1) and zn.dtype == torch.bfloat16 + ) + assert gw.shape == (V,) and gw.is_contiguous() and gw.dtype == torch.bfloat16 + assert zn.data_ptr() % 8 == 0 and zn.stride(0) % 4 == 0 and gw.data_ptr() % 8 == 0 + norm_eps = float(gated_norm["eps"]) + norm_args = (zn.data_ptr(), zn.stride(0), gw.data_ptr(), output.data_ptr()) + kw = _kernel_kw( + N, + T, + H, + HV, + commit=commit, + pdl=bool(launch_with_pdl), + l2norm=bool(use_qk_l2norm_in_kernel), + gating=gating, + g_f32=g.dtype == torch.float32, + b_f32=beta.dtype == torch.float32, + work_items=replay_work_items is not None, + config=config, + fold=conv_fold is not None, + norm_eps=norm_eps, + work_item_width=replay_work_items.shape[1] if replay_work_items is not None else 4, + separate_replay_slots=rsep, + ) + use_wi = kw["use_wi"] + key = tuple(sorted(kw.items())) + fn = _COMPILED.get(key) + if fn is None: + fn = _compile(key, _Replay(**kw)) + sidx = state_indices if state_indices.dtype == torch.int32 else state_indices.to(torch.int32) + fn( + sidx, + qp, + kp, + vp, + qs[0], + qs[1], + ks[0], + ks[1], + vs[0], + vs[1], + g.data_ptr(), + g.stride(1), + beta.data_ptr(), + beta.stride(1), + ap, + dp, + output.data_ptr(), + ssm_states.data_ptr(), + ssm_states.stride(0), + prev_num_accepted_tokens.data_ptr(), + cache_buf_idx.data_ptr(), + old_u.data_ptr(), + old_k.data_ptr(), + old_G.data_ptr(), + replay_work_items.data_ptr() if use_wi else 0, + replay_indices.data_ptr() if rsep else 0, + old_u.stride(0) * old_u.element_size(), + old_k.stride(0) * old_k.element_size(), + old_G.stride(0) * old_G.element_size(), + prev_num_accepted_tokens.stride(0) * prev_num_accepted_tokens.element_size(), + cache_buf_idx.stride(0) * cache_buf_idx.element_size(), + *fold_args, + *norm_args, + N, + float(scale), + ) + return output + + +#: A tp4/tep4 rank's GDN heads. +RANK_K_HEADS, RANK_V_HEADS = 4, 12 +#: A tep4 engine rank's state slots, and the GDN layers sharing one slot page (slot stride 36 +#: layers' states). +TEP4_SLOTS, PAGE_LAYERS = 1874, 36 +#: The same 26.5 GB page at a dep4 rank's (48-head) and a tp2/tep2 rank's (24-head) slot +#: stride: the top slot sits 1.3e10 elements past the base, as tep4's does. +DEP4_SLOTS = 468 +TEP2_SLOTS = 937 + + +def _tep2_spec(path, n, t, **spec) -> dict: + """A tp2/tep2 rank's heads, 8 / 24, over a bf16 stock-manager page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=8, + HV=24, + history_size=HIST, + slots=TEP2_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +def _tep4_spec(path, n, t, **spec) -> dict: + """A tp4/tep4 rank's heads over the stock tep4 engine's bf16 state page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=RANK_K_HEADS, + HV=RANK_V_HEADS, + history_size=HIST, + slots=TEP4_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +def _dep4_spec(path, n, t, **spec) -> dict: + """A dep4 rank's heads, all 16 / 48 under attention DP, over a bf16 stock-manager page.""" + return dict( + path=path, + n=n, + t=t, + state_dtype=torch.bfloat16, + H=16, + HV=48, + history_size=HIST, + slots=DEP4_SLOTS, + page_layers=PAGE_LAYERS, + **spec, + ) + + +class GdnCachedReplayCute(OpWrapper): + """`cached_replay_update`: one verify step of `N` sequences x `T` tokens through one GDN + layer, the contract of `fused_recurrent_gated_delta_rule_cached_replay_update` (bf16 state). + Head counts from the call's shapes (`GEOMETRIES`); `g` / `beta` may be strided column views + of the input projection (unit head stride). + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: the test harness's `_Bench` spec (path, n x t, steps with random acceptance, start, + # accept, dup, H / HV, slots, page_layers) plus `strided_gates` (g / beta as column views of + # the projection). The small (in-kernel commit) and all_layer paths at each geometry. + CELLS: tuple[Cell, ...] = ( + # tep4: 4 / 12 heads. + Cell( + why="tep4 small n=1 T=4: MTP3 verify at con1, in-kernel commit on the page's last slot", + spec=_tep4_spec("small", 1, 4, steps=8, strided_gates=True), + ), + Cell( + why="tep4 small n=1 T=8 full accept: MTP7 appends to the history's end, then wraps", + spec=_tep4_spec( + "small", 1, 8, steps=8, start="fresh", accept="full", strided_gates=True + ), + ), + Cell( + why="tep4 all_layer n=16 T=4: the smallest partitioned batch, all-layer commit", + spec=_tep4_spec("all_layer", 16, 4, steps=6, strided_gates=True), + ), + Cell( + why="tep4 all_layer n=32 T=8: the widest verify below N=128", + spec=_tep4_spec("all_layer", 32, 8, steps=6, strided_gates=True), + ), + Cell( + why="tep4 all_layer n=128 T=4: a large partitioned batch", + spec=_tep4_spec("all_layer", 128, 4, steps=4), + ), + # dep4: 16 / 48 heads. + Cell( + why="dep4 small n=1 T=4: the in-kernel commit of the page's last slot (467)", + spec=_dep4_spec("small", 1, 4, steps=8, strided_gates=True), + ), + Cell( + why="dep4 small n=8 T=4 dup: rows 5..7 share one CUDA-graph padding slot", + spec=_dep4_spec("small", 8, 4, steps=6, dup=(5, 6, 7), strided_gates=True), + ), + # tep2: 8 / 24 heads, in-kernel commit below 16 requests, all-layer from 16. + Cell( + why="tep2 small n=4 T=4: a 16-row verify, in-kernel commit", + spec=_tep2_spec("small", 4, 4, steps=6, strided_gates=True), + ), + Cell( + why="tep2 all_layer n=32 T=4: a 128-row verify, all-layer commit", + spec=_tep2_spec("all_layer", 32, 4, steps=4, strided_gates=True), + ), + Cell( + why="dep4 all_layer n=128 T=4: a rank's largest verify", + spec=_dep4_spec("all_layer", 128, 4, steps=4, strided_gates=True), + ), + ) + + note = """ + Source: this file's CuTe DSL kernel (`_Replay`), one compile per (geometry, T, launch + mapping, commit, PDL, folds) on first use (about 0.5 s): the warm-up before CUDA-graph capture + must cover it. Called by `_gdn_verify` over the bf16 recurrent pool (slot stride 36 layers' + states) when the rank's GDN heads are a `GEOMETRIES` pair, with work items + + `use_all_layer_commit` exactly when the Triton entry gets them (N >= 16). The history is + addressed by the manager's replay slots (`replay_indices`) when it keeps the history in + slots of its own; the work items then carry the replay slot as a fifth field. PDL only at up + to 24 (request x value head) pairs: the conv before it triggers its dependents at its start. + + Numerics: the Triton kernel's rounding points (bf16 L2-normalized q / k and history + coefficients, fp32 recurrence, fp32 commit rounded once to the bf16 pool, bf16 history u / k, + fp32 G); the dot order differs, so outputs match it to a few bf16 ulps, not bitwise. Launch + mappings are bitwise each other, and CUDA-graph replay is bitwise eager. + + Certified on GB300 at the cells (rows at both ends of the page, consecutive steps from warm + histories with random acceptance) within the Triton entry's test bands. Not supported: fp32 + state, T > 8, other geometries, K or V other than 128, history other than 16, pre-gated + g / beta (`A_log=None` compiles but is not certified). + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return cached_replay_update( + q=q, + k=k, + v=v, + g=g, + beta=beta, + ssm_states=ssm_states, + state_indices=state_indices, + old_u=old_u, + old_k=old_k, + old_G=old_G, + old_beta=old_beta, + cache_buf_idx=cache_buf_idx, + prev_num_accepted_tokens=prev_num_accepted_tokens, + history_size=history_size, + scale=scale, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + A_log=A_log, + dt_bias=dt_bias, + launch_with_pdl=launch_with_pdl, + replay_work_items=replay_work_items, + n_writes=n_writes, + packed_qkv=packed_qkv, + use_all_layer_commit=use_all_layer_commit, + output=output, + config=config, + conv_fold=conv_fold, + gated_norm=gated_norm, + replay_indices=replay_indices, + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """`(out, ssm_states, old_u, old_k, old_G)` as the call leaves them; mutates nothing. + The state is rebuilt from the checkpoint and history in fp32 and the new tokens stepped + one at a time in fp32 (no matmul, so no TF32), every store rounded once to its buffer's + dtype. Rows with a negative slot keep `output`'s prior content (NaN without `output`). + The history and its bookkeeping are read and written at `replay_indices` (the checkpoint + slots when None). `replay_work_items` only decides whether a wrap commits here; the + launch knobs, `config`, `old_beta` and `n_writes` are ignored.""" + N, T, H, K = k.shape + HV, V = v.shape[2], v.shape[3] + rep = HV // H + dev = v.device + scale = K**-0.5 if scale is None else scale + if packed_qkv is not None: + rows = packed_qkv.view(N, T, -1) + q = rows[..., : H * K].view(N, T, H, K) + k = rows[..., H * K : 2 * H * K].view(N, T, H, K) + v = rows[..., 2 * H * K :].view(N, T, HV, V) + qf, kf, vf = q.float(), k.float(), v.float() + if use_qk_l2norm_in_kernel: + qf = qf / torch.sqrt(qf.square().sum(-1, keepdim=True) + 1e-6) + kf = kf / torch.sqrt(kf.square().sum(-1, keepdim=True) + 1e-6) + qf = (qf * scale).repeat_interleave(rep, dim=2) # [N, T, HV, K] + kv = kf.repeat_interleave(rep, dim=2) # [N, T, HV, K] + if A_log is not None: + gf = -torch.exp(A_log.float()) * F.softplus(g.float() + dt_bias.float()) + bf = torch.sigmoid(beta.float()) + else: + gf, bf = g.float(), beta.float() + + out_dtype = q.dtype if output is None else output.dtype + out = ( + torch.full((N, T, HV, V), float("nan"), device=dev) + if output is None + else output.float().clone() + ) + pool, u_buf, k_buf, G_buf = (x.clone() for x in (ssm_states, old_u, old_k, old_G)) + live = (state_indices[:N] >= 0).nonzero().flatten() + if live.numel() == 0: + return out.to(out_dtype), pool, u_buf, k_buf, G_buf + slot = state_indices[:N][live].long() + rslot = slot if replay_indices is None else replay_indices[:N][live].long() + p = prev_num_accepted_tokens[rslot].long() + b = cache_buf_idx[rslot].long() + M, depth = live.numel(), old_u.shape[2] + + # 1. the pending state: checkpoint plus the p history entries of half b + pending = torch.arange(depth, device=dev)[None, :] < p[:, None] # [M, depth] + Gh = torch.where(pending[:, None, :], old_G[rslot, b].float(), 0.0) # [M, HV, depth] + Uh = torch.where(pending[:, :, None, None], old_u[rslot, b].float(), 0.0) + Kh = torch.where(pending[:, :, None, None], old_k[rslot, b].float(), 0.0) + Kh = Kh.repeat_interleave(rep, dim=2) # [M, depth, HV, K] + last = (p - 1).clamp(min=0)[:, None, None].expand(M, HV, 1) + g_start = torch.where(p[:, None] > 0, Gh.gather(2, last).squeeze(2), 0.0) # [M, HV] + weight = torch.where(pending[:, None, :], torch.exp(g_start[:, :, None] - Gh), 0.0) + h = torch.exp(g_start)[:, :, None, None] * ssm_states[slot].float() # [M, HV, V, K] + for j in range(depth): + h = h + weight[:, :, j, None, None] * Uh[:, j, :, :, None] * Kh[:, j, :, None, :] + checkpoint = h + + # 2. the new tokens, in order + gl, bl = gf[live], bf[live] # [M, T, HV] + ql, kl, vl = qf[live], kv[live], vf[live] + G_local = torch.zeros(M, HV, device=dev) + u_new, G_new = [], [] + for t in range(T): + h = h * torch.exp(gl[:, t])[:, :, None, None] + u = bl[:, t, :, None] * (vl[:, t] - (h * kl[:, t, :, None, :]).sum(-1)) # [M, HV, V] + h = h + u[:, :, :, None] * kl[:, t, :, None, :] + out[live, t] = (h * ql[:, t, :, None, :]).sum(-1) + G_local = G_local + gl[:, t] + u_new.append(u) + G_new.append(G_local.clone()) + + # 3. the history append, and the checkpoint commit on a wrap + wrap = p + T > history_size + w_buf = torch.where(wrap, 1 - b, b) + w_off = torch.where(wrap, 0, p) + for t in range(T): + pos = w_off + t + u_buf[rslot, w_buf, pos] = u_new[t].to(u_buf.dtype) + k_buf[rslot, w_buf, pos] = kf[live, t].to(k_buf.dtype) + G_store = torch.where(wrap[:, None], G_new[t], g_start + G_new[t]) + G_buf[rslot, w_buf, :, pos] = G_store.to(G_buf.dtype) + commit = replay_work_items is None + if commit and bool(wrap.any()): + pool[slot[wrap]] = checkpoint[wrap].to(pool.dtype) + return out.to(out_dtype), pool, u_buf, k_buf, G_buf + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, + config: Optional[dict] = None, + conv_fold: Optional[dict] = None, + gated_norm: Optional[dict] = None, + replay_indices: Optional[torch.Tensor] = None, + ) -> None: + N, _, H, K = k.shape + HV, V = v.shape[2], v.shape[3] + assert state_indices.dim() == 1 and state_indices.shape[0] >= N, ( + f"state_indices has shape {tuple(state_indices.shape)} for {N} sequences -- the " + "kernel reads one slot per sequence with no bound, so a shorter tensor hands the " + "trailing rows whatever follows it in memory, and they write that slot's history" + ) + # The history is addressed by the replay slots, the state slots when none are given. + if replay_indices is None: + history_slots, slot_space = ssm_states.shape[0], "the recurrent pool" + else: + assert replay_indices.dim() == 1 and replay_indices.shape[0] >= N, ( + f"replay_indices has shape {tuple(replay_indices.shape)} for {N} sequences -- " + "the kernel reads one replay slot per sequence with no bound" + ) + history_slots, slot_space = int(replay_indices[:N].max()) + 1, "the replay slots" + slots = old_u.shape[0] + assert slots >= history_slots, ( + f"the history buffers have {slots} slots for {history_slots} of {slot_space} -- a " + "slot past the history's end is read and written out of bounds" + ) + for name, tensor in ( + ("prev_num_accepted_tokens", prev_num_accepted_tokens), + ("cache_buf_idx", cache_buf_idx), + ): + assert tensor.dim() == 1 and tensor.shape[0] >= history_slots, ( + f"{name} has shape {tuple(tensor.shape)}; the kernel reads one element per " + f"history slot ({history_slots} of {slot_space}), at the slot's index times " + "the element stride" + ) + expected = { + "old_u": (old_u, (slots, 2, history_size, HV, V)), + "old_k": (old_k, (slots, 2, history_size, H, K)), + "old_G": (old_G, (slots, 2, HV, history_size)), + } + for name, (tensor, shape) in expected.items(): + assert tuple(tensor.shape) == shape, ( + f"{name} has shape {tuple(tensor.shape)}; history_size={history_size} and this " + f"geometry need {shape} -- the kernel addresses the history by history_size and " + "the head geometry, never by the buffer's own shape" + ) + + +gdn_cached_replay_cute = GdnCachedReplayCute() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py new file mode 100644 index 000000000000..765a5b882b4e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_in_proj_conv__verify.py @@ -0,0 +1,741 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A verify step's Gated-DeltaNet input projection with the causal conv of its q|k|v columns in +the GEMM's epilogue: `low_m_bf16_mm` then `causal_conv1d_token_major`, in one launch. + +The GEMM is FlashInfer's warp split-K kernel (`dense_bf16_gemm_warp_splitk`), subclassed with +its body unchanged. A CTA owns 16 output columns for every row of the step (one token tile), so +the reduce warp keeps the bf16-rounded tile in shared memory and its first 16 lanes run the conv +of their channel as `_causal_conv1d_update_kernel` does (window snapshots, SiLU, the token-major +`[n * p, C]` output, the pool advanced). No two CTAs touch one channel. +""" + +from __future__ import annotations + +import functools + +import cuda.bindings.driver as _cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass import const_expr +from cutlass._mlir.dialects import llvm +from cutlass.cute import experimental as cute_ext +from cutlass.cute.nvgpu import warp +from cutlass.cutlass_dsl import T as _ty +from cutlass.cutlass_dsl import dsl_user_op +from cutlass.experimental import primitives as prims +from flashinfer.gemm.kernels import dense_bf16_gemm_warp_splitk as _fw + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ..gemm import low_m_bf16_mm__decode as _lowm + +#: The GDN conv (width 4) over a tp4/tep4 rank's q|k|v channels. +CONV_WIDTH = 4 +CONV_DIM = 2560 +#: The projection a tp4/tep4 rank runs: `[q k v | z | a | b | pad]` = 4128 over hidden 2560. +N_OUT, K_IN = 4128, 2560 + + +def _ir(x, cls, loc, ip): + return cls(x).ir_value(loc=loc, ip=ip) + + +def _asm(ret, args, text, cons, loc, ip, side=False): + return llvm.inline_asm( + ret, + args, + text, + cons, + has_side_effects=side, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _ldg_i32p(gaddr, pred, other: int, *, loc=None, ip=None): + """An int32, or `other` where `pred` is 0 (nothing read).""" + return cutlass.Int32( + _asm(_ty.i32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; setp.ne.b32 q, $2, 0; mov.b32 $0, " + str(int(other)) + "; " + "@q ld.global.b32 $0, [$1]; }", "=r,l,r", loc, ip, side=True) + ) # fmt: skip + + +@dsl_user_op +def _ldg_bf16p(gaddr, pred, *, loc=None, ip=None): + """One bf16 widened to f32, or 0 where `pred` is 0 (nothing read).""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(pred, cutlass.Int32, loc, ip)], + "{ .reg .pred q; .reg .b16 h; setp.ne.b32 q, $2, 0; mov.b16 h, 0; " + "@q ld.global.b16 h, [$1]; cvt.f32.bf16 $0, h; }", "=f,l,r", loc, ip, side=True) + ) # fmt: skip + + +@dsl_user_op +def _stg_bf16(gaddr, x, *, loc=None, ip=None): + """One f32 rounded to bf16 (round to nearest even) and stored.""" + _asm(_ty.i32(), [_ir(gaddr, cutlass.Int64, loc, ip), _ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $2; st.global.b16 [$1], h; mov.u32 $0, 0; }", + "=r,l,f", loc, ip, side=True) # fmt: skip + + +@dsl_user_op +def _round_bf16(x, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; cvt.rn.bf16.f32 h, $1; cvt.f32.bf16 $0, h; }", "=f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _mul_bf16(a, b, *, loc=None, ip=None): + """A bf16 x bf16 product as the Triton kernel forms it: exact in f32, rounded to bf16.""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "{ .reg .b16 h; .reg .f32 p; mul.rn.f32 p, $1, $2; cvt.rn.bf16.f32 h, p; " + "cvt.f32.bf16 $0, h; }", "=f,f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _add_rn(a, b, *, loc=None, ip=None): + return cutlass.Float32( + _asm(_ty.f32(), [_ir(a, cutlass.Float32, loc, ip), _ir(b, cutlass.Float32, loc, ip)], + "add.rn.f32 $0, $1, $2;", "=f,f,f", loc, ip) + ) # fmt: skip + + +@dsl_user_op +def _silu(x, *, loc=None, ip=None): + """acc / (1 + ex2(-acc * log2(e))): bitwise the Triton kernel's SiLU.""" + return cutlass.Float32( + _asm(_ty.f32(), [_ir(x, cutlass.Float32, loc, ip)], + "{ .reg .f32 t, e, d; neg.f32 t, $1; mul.rn.f32 t, t, 0f3FB8AA3B; " + "ex2.approx.f32 e, t; add.rn.f32 d, e, 0f3F800000; div.full.f32 $0, $1, d; }", + "=f,f", loc, ip) + ) # fmt: skip + + +class _InProjConvKernel(_fw.CpAsyncWarpSplitKKernel): + """FlashInfer's warp split-K kernel with the conv epilogue (see the module docstring).""" + + def __init__(self, tactic, use_pdl: bool, rows_per_request: int) -> None: + super().__init__(tactic, use_pdl, False) + self.P = rows_per_request + + @cute.experimental.jit + def __call__( + self, + weight, + activation, + bias, + output, + conv_w: cutlass.Int64, + pool: cutlass.Int64, + pool_s_seq: cutlass.Int64, + pool_s_dim: cutlass.Int64, + pool_s_tok: cutlass.Int64, + slots: cutlass.Int64, + win: cutlass.Int64, + win_s_seq: cutlass.Int64, + win_s_step: cutlass.Int64, + win_s_dim: cutlass.Int64, + win_s_win: cutlass.Int64, + win_rows: cutlass.Int64, + conv_out: cutlass.Int64, + pad_slot: cutlass.Int32, + stream: _cuda.CUstream, + ): + sA_layout = _fw._make_smem_layout( + weight.element_type, self.output_tile, self.k_tile, self.stages + ) + sB_layout = _fw._make_smem_layout( + activation.element_type, self.token_tile, self.k_tile, self.stages + ) + tiled_mma = cute.make_tiled_mma( + warp.MmaF16BF16Op(weight.element_type, cutlass.Float32, _fw._MMA_SHAPE), + cute.make_layout((1, 1, _fw._COMPUTE_WARPS)), + permutation_mnk=(self.output_tile, self.token_tile, self.k_tile), + ) + self.kernel( + weight, + activation, + bias, + output, + sA_layout, + sB_layout, + tiled_mma, + conv_w, + pool, + pool_s_seq, + pool_s_dim, + pool_s_tok, + slots, + win, + win_s_seq, + win_s_step, + win_s_dim, + win_s_win, + win_rows, + conv_out, + pad_slot, + ).launch( + grid=( + cute.ceil_div(weight.shape[0], self.output_tile), + cute.ceil_div(activation.shape[0], self.token_tile), + 1, + ), + block=(self.threads, 1, 1), + smem=cute.Int64(self.smem_bytes + self.output_tile * self.token_tile * 4 + 64), + stream=stream, + use_pdl=self.use_pdl, + ) + + @cute.experimental.kernel + def kernel( + self, + weight: cute.Tensor, + activation: cute.Tensor, + bias: cute.Tensor, + output: cute.Tensor, + sA_layout: cute.ComposedLayout, + sB_layout: cute.ComposedLayout, + tiled_mma: cute.TiledMma, + conv_w: cutlass.Int64, + pool: cutlass.Int64, + pool_s_seq: cutlass.Int64, + pool_s_dim: cutlass.Int64, + pool_s_tok: cutlass.Int64, + slots: cutlass.Int64, + win: cutlass.Int64, + win_s_seq: cutlass.Int64, + win_s_step: cutlass.Int64, + win_s_dim: cutlass.Int64, + win_s_win: cutlass.Int64, + win_rows: cutlass.Int64, + conv_out: cutlass.Int64, + pad_slot: cutlass.Int32, + ): + # ---- FlashInfer's kernel body, unchanged (dense_bf16_gemm_warp_splitk.py) ---------- + _COMPUTE_WARPS = _fw._COMPUTE_WARPS + _A_LOADER_WARPS = _fw._A_LOADER_WARPS + _MMA_SHAPE = _fw._MMA_SHAPE + _MAILBOX_GROUP_VALUES = _fw._MAILBOX_GROUP_VALUES + _MAILBOX_PADDED_GROUP_VALUES = _fw._MAILBOX_PADDED_GROUP_VALUES + tidx, _, _ = cute.arch.thread_idx() + output_tile_idx, token_tile_idx, _ = cute.arch.block_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane_idx = tidx % cute.arch.WARP_SIZE + reuse_stages = cute.size(weight, mode=[1]) > self.stages * self.k_tile + + sA = cute_ext.allocate( + weight.element_type, cute.AddressSpace.smem, sA_layout, alignment=1024 + ) + sB = cute_ext.allocate( + activation.element_type, cute.AddressSpace.smem, sB_layout, alignment=1024 + ) + values_per_lane = self.output_tile * self.token_tile // cute.arch.WARP_SIZE + partial_stride = ( + self.output_tile * self.token_tile // _MAILBOX_GROUP_VALUES + ) * _MAILBOX_PADDED_GROUP_VALUES + partials = cute_ext.allocate( + cutlass.Float32, + cute.AddressSpace.smem, + cute.make_layout(_COMPUTE_WARPS * partial_stride), + alignment=16, + ) + bar_full_arr = cute_ext.allocate( + cutlass.Int64, cute.AddressSpace.smem, cute.make_layout(self.stages), alignment=8 + ) + bar_empty_arr = cute_ext.allocate( + cutlass.Int64, cute.AddressSpace.smem, cute.make_layout(self.stages), alignment=8 + ) + # The reduced tile, bf16-rounded, for the conv epilogue. + xtile = cute_ext.allocate( + cutlass.Float32, + cute.AddressSpace.smem, + cute.make_layout(self.output_tile * self.token_tile), + alignment=16, + ) + bar_full = bar_full_arr.iterator + bar_empty = bar_empty_arr.iterator + + k_tile_count = cute.size(weight, mode=[1]) // self.k_tile + + # ---- the conv's operands, read before the GEMM (nothing earlier in the step writes them) + P = self.P + MAX_REQ = self.token_tile // P + n_req = cute.size(output, mode=[0]) // P + conv_lane = cutlass.Int32(0) + if warp_idx == self.compute_warp_base: + if lane_idx < self.output_tile: + if output_tile_idx * self.output_tile < CONV_DIM: + conv_lane = cutlass.Int32(1) + c64 = cutlass.Int64(output_tile_idx * self.output_tile + lane_idx % self.output_tile) + wbase = conv_w + c64 * (CONV_WIDTH * 2) + w0 = _ldg_bf16p(wbase, conv_lane) + w1 = _ldg_bf16p(wbase + 2, conv_lane) + w2 = _ldg_bf16p(wbase + 4, conv_lane) + w3 = _ldg_bf16p(wbase + 6, conv_lane) + req_live = [] + req_cols = [] + req_win = [] + for s in cutlass.range_constexpr(MAX_REQ): + ok = conv_lane + if s >= n_req: + ok = cutlass.Int32(0) + slot = _ldg_i32p(slots + s * 4, ok, -1) + live = ok + if slot == pad_slot: + live = cutlass.Int32(0) + prow = pool + (cutlass.Int64(slot) * pool_s_seq + c64 * pool_s_dim) * 2 + req_cols.append( + ( + _ldg_bf16p(prow, live), + _ldg_bf16p(prow + pool_s_tok * 2, live), + _ldg_bf16p(prow + pool_s_tok * 4, live), + prow, + ) + ) + req_win.append(cutlass.Int64(_ldg_i32p(win_rows + s * 4, live, 0))) + req_live.append(live) + + if warp_idx < _A_LOADER_WARPS: + _fw._issue_weight_tile( + sA, + weight, + tidx, + output_tile_idx, + 0, + 0, + cutlass.Int64(0), + self.output_tile, + self.k_tile, + False, + ) + if warp_idx == self.compute_warp_base: + if lane_idx < self.stages: + cute.arch.mbarrier_init(bar_full + lane_idx, self.load_threads) + if const_expr(reuse_stages): + cute.arch.mbarrier_init(bar_empty + lane_idx, _COMPUTE_WARPS) + cute.arch.mbarrier_init_fence() + cute.arch.barrier() + + if warp_idx < _A_LOADER_WARPS: + weight_policy = _fw._l2_policy_evict_first() + empty_phase = cutlass.Int32(1) + prims.cp_async_mbarrier_arrive(bar_full.llvm_ptr, noinc=True) + for k_tile in cutlass.range(1, k_tile_count, unroll=1): + stage = k_tile % self.stages + if const_expr(reuse_stages): + cute.arch.mbarrier_wait(bar_empty + stage, empty_phase) + _fw._issue_weight_tile( + sA, + weight, + tidx, + output_tile_idx, + k_tile, + stage, + weight_policy, + self.output_tile, + self.k_tile, + True, + ) + prims.cp_async_mbarrier_arrive((bar_full + stage).llvm_ptr, noinc=True) + if stage == self.stages - 1: + empty_phase = empty_phase ^ 1 + elif warp_idx < self.compute_warp_base: + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_wait() + b_loader_threads = self.b_loader_warps * 32 + local_tid = tidx - _A_LOADER_WARPS * 32 + copy_count = self.token_tile * self.k_tile // (8 * b_loader_threads) + empty_phase = cutlass.Int32(1) + for k_tile in cutlass.range(k_tile_count, unroll=1): + stage = k_tile % self.stages + if const_expr(reuse_stages): + cute.arch.mbarrier_wait(bar_empty + stage, empty_phase) + for copy_idx in cutlass.range_constexpr(copy_count): + linear = local_tid * 8 + copy_idx * b_loader_threads * 8 + row = linear // self.k_tile + col = linear % self.k_tile + global_row = token_tile_idx * self.token_tile + row + if global_row < cute.size(activation, mode=[0]): + dst = ( + sB.iterator.raw_ptr() + + stage * self.token_tile * self.k_tile + + row * self.k_tile + + (col ^ ((row % 8) * 8)) + ) + src = activation.iterator.raw_ptr() + activation.layout( + (global_row, k_tile * self.k_tile + col) + ) + prims.cp_async_shared_global(dst, src, 16, "cg") + prims.cp_async_mbarrier_arrive((bar_full + stage).llvm_ptr, noinc=True) + if stage == self.stages - 1: + empty_phase = empty_phase ^ 1 + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_launch_dependents() + else: + mma_tid = tidx - self.compute_warp_base * cute.arch.WARP_SIZE + compute_warp = warp_idx - self.compute_warp_base + thr_mma = tiled_mma.get_slice(mma_tid) + acc = cute.make_rmem_tensor( + thr_mma.partition_shape_C((self.output_tile, self.token_tile)), cutlass.Float32 + ) + acc.fill(0.0) + k_phases = self.k_tile // (_COMPUTE_WARPS * _MMA_SHAPE[2]) + m_iters = self.output_tile // _MMA_SHAPE[0] + n_iters = self.token_tile // _MMA_SHAPE[1] + a_offsets = tuple( + tuple( + _fw._a_fragment_offset(lane_idx, compute_warp, m_iter, phase, self.k_tile) + for phase in range(k_phases) + ) + for m_iter in range(m_iters) + ) + b_offsets = tuple( + tuple( + _fw._b_pair_offset( + lane_idx, compute_warp, n_iter, pair, self.token_tile, self.k_tile + ) + for pair in range(k_phases // 2) + ) + for n_iter in range(n_iters) + ) + a_stage_elems = self.output_tile * self.k_tile + b_stage_elems = self.token_tile * self.k_tile + full_phase = cutlass.Int32(0) + for k_tile in cutlass.range(k_tile_count, unroll_full=True): + stage = k_tile % self.stages + cute.arch.mbarrier_wait(bar_full + stage, full_phase) + a_base = stage * a_stage_elems + b_base = stage * b_stage_elems + a_registers = tuple( + tuple( + _fw._ldmatrix_x4(sA, a_base + a_offsets[m_iter][phase]) + for phase in range(k_phases) + ) + for m_iter in range(m_iters) + ) + b_registers = tuple( + tuple( + _fw._ldmatrix_x4(sB, b_base + b_offsets[n_iter][pair]) + for pair in range(k_phases // 2) + ) + for n_iter in range(n_iters) + ) + for phase in cutlass.range_constexpr(k_phases): + for n_iter in cutlass.range_constexpr(n_iters): + packed_b = b_registers[n_iter][phase // 2] + b_offset = (phase % 2) * 2 + for m_iter in cutlass.range_constexpr(m_iters): + acc_base = (m_iter * n_iters + n_iter) * 4 + result = prims.mma_sync( + llvm.StructType.get_literal([cutlass.Float32.mlir_type] * 4), + shape=_MMA_SHAPE, + layout_a=prims.MMALayout.ROW, + layout_b=prims.MMALayout.COL, + operand_a=[ + a_registers[m_iter][phase][i].ir_value() for i in range(4) + ], + operand_b=[packed_b[b_offset + i].ir_value() for i in range(2)], + operand_c=[acc[acc_base + i].ir_value() for i in range(4)], + multiplicand_a_ptx_type=prims.MMAType.BF16, + multiplicand_b_ptx_type=prims.MMAType.BF16, + ) + for i in cutlass.range_constexpr(4): + acc[acc_base + i] = cutlass.Float32( + llvm.extractvalue(cutlass.Float32.mlir_type, result, [i]) + ) + if const_expr(reuse_stages): + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive(bar_empty + stage) + if stage == self.stages - 1: + full_phase = full_phase ^ 1 + for value in cutlass.range_constexpr(values_per_lane): + atom = value % 4 + fragment = value // 4 + m_iter = fragment // n_iters + n_iter = fragment % n_iters + feature = lane_idx // 4 + 8 * (atom // 2) + m_iter * _MMA_SHAPE[0] + token = 8 * n_iter + 2 * (lane_idx % 4) + atom % 2 + partials[ + compute_warp * partial_stride + + _fw._partial_offset(feature, token, self.token_tile) + ] = acc[value] + prims.barrier_cta_sync(1, thread_count=_COMPUTE_WARPS * 32) + + if warp_idx == self.compute_warp_base: + final_acc = cute.make_rmem_tensor(cute.make_layout(values_per_lane), cutlass.Float32) + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + total = cutlass.Float32(0) + for peer in cutlass.range_constexpr(_COMPUTE_WARPS): + total = ( + total + + partials[ + peer * partial_stride + + _fw._partial_offset(feature, token, self.token_tile) + ] + ) + final_acc[value] = total + + output_base = output_tile_idx * self.output_tile + token_base = token_tile_idx * self.token_tile + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + if token_base + token < cute.size(output, mode=[0]): + output[token_base + token, output_base + feature] = final_acc[value].to( + output.element_type + ) + # ---- the conv epilogue ----------------------------------------------------------- + if output_base < CONV_DIM: + for value in cutlass.range_constexpr(values_per_lane): + linear = value * cute.arch.WARP_SIZE + lane_idx + feature = linear % self.output_tile + token = linear // self.output_tile + xtile[token * self.output_tile + feature] = _round_bf16(final_acc[value]) + cute.arch.sync_warp() + for s in cutlass.range_constexpr(MAX_REQ): + if req_live[s] != 0: + # One channel's conv over request s. + col0, col1, col2, prow = req_cols[s] + wb = win + (req_win[s] * win_s_seq + c64 * win_s_dim) * 2 + for t in cutlass.range_constexpr(P): + x = xtile[(s * P + t) * self.output_tile + lane_idx] + a = _add_rn(cutlass.Float32(0.0), _mul_bf16(col0, w0)) + a = _add_rn(a, _mul_bf16(col1, w1)) + a = _add_rn(a, _mul_bf16(col2, w2)) + a = _add_rn(a, _mul_bf16(x, w3)) + col0 = col1 + col1 = col2 + col2 = x + wt = wb + cutlass.Int64(t) * win_s_step * 2 + _stg_bf16(wt, col0) + _stg_bf16(wt + win_s_win * 2, col1) + _stg_bf16(wt + win_s_win * 4, col2) + _stg_bf16( + conv_out + (cutlass.Int64(s * P + t) * CONV_DIM + c64) * 2, + _silu(a), + ) + # The pool advanced to the last 3 inputs of [window | rows]. + _stg_bf16(prow, col0) + _stg_bf16(prow + pool_s_tok * 2, col1) + _stg_bf16(prow + pool_s_tok * 4, col2) + + +def fused_tactic(rows: int): + """The FlashInfer warp split-K tactic `low_m_bf16_mm` runs this projection with at `rows` + rows, when it is that kernel in one token tile (the fused launch's domain), else None.""" + p = _lowm.plan(rows, N_OUT, K_IN) + if p.kind != "warp": + return None + tactic = _fw.WarpSplitKTactic(*p.tactic) + return tactic if rows <= tactic.token_tile else None + + +def _from_dlpack(tensor: torch.Tensor, *, dynamic_m: bool = False): + return _fw._from_dlpack(tensor, dynamic_m=dynamic_m) + + +@functools.cache +def _compile(device_index: int, tactic, use_pdl: bool, rows_per_request: int): + with torch.cuda.device(device_index): + i64 = cutlass.Int64(0) + return cute_ext.compile( + _InProjConvKernel(tactic, use_pdl, rows_per_request), + *( + _from_dlpack( + torch.empty(shape, device="cuda", dtype=torch.bfloat16), dynamic_m=dynamic_m + ) + for shape, dynamic_m in ( + ((N_OUT, K_IN), False), + ((_fw._MAX_M, K_IN), True), + ((N_OUT,), False), + ((_fw._MAX_M, N_OUT), True), + ) + ), # fmt: skip + *([i64] * 13), + cutlass.Int32(0), + _cuda.CUstream(torch.cuda.current_stream().cuda_stream), + ) + + +def _launch( + mat_a, + mat_b, + out, + conv_out, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, +): + compiled = _compile(mat_a.device.index, tactic, pdl, rows_per_request) + compiled( + _from_dlpack(mat_b.T), + _from_dlpack(mat_a, dynamic_m=True), + _from_dlpack(out[0]), + _from_dlpack(out, dynamic_m=True), + conv_weight.data_ptr(), + conv_state.data_ptr(), + conv_state.stride(0), + conv_state.stride(1), + conv_state.stride(2), + slots.data_ptr(), + window.data_ptr(), + window.stride(0), + window.stride(1), + window.stride(2), + window.stride(3), + window_rows.data_ptr(), + conv_out.data_ptr(), + pad_slot_id, + _cuda.CUstream(torch.cuda.current_stream(mat_a.device).cuda_stream), + ) + + +def _cells(): + return tuple( + Cell( + why=f"tep4 MTP verify, {n} request(s) x {p} rows: in_proj + conv in one launch", + spec=dict(requests=n, p=p, capture=capture), + ) + # One per warp split-K tactic `fused_tactic` picks (3, 6 and 16 rows), the first captured. + for n, p, capture in ((1, 3, True), (2, 3, False), (4, 4, False)) + ) + + +class GdnInProjConv(OpWrapper): + """`(proj, conv) = gdn_in_proj_conv(x, in_proj, conv_weight, conv_state, slots, window, + window_rows, pad_slot_id, rows_per_request, tactic, pdl)` for a verify step's rows. + + `x` `[T, 2560]` bf16 contiguous, `T = n * rows_per_request`; `in_proj` the `[2560, 4128]` + column-major operand; `conv_weight` `[2560, 4]`; `conv_state` the `[slots, 2560, 3]` pool; + `slots` `[n]` int32 (`pad_slot_id` skipped); `window` the `[rows, p, 2560, 3]` snapshot + buffer and `window_rows` `[n]` int32; `tactic` the warp split-K tactic `low_m_bf16_mm` picks + at `T` rows (`fused_tactic`). Returns `proj` `[T, 4128]`, bitwise `low_m_bf16_mm`'s, and + `conv` `[T, 2560]` token-major with `conv_state` and `window` written, bitwise + `causal_conv1d_token_major` over `proj`'s q|k|v columns. + """ + + ARCHS = frozenset({Arch.SM_103}) + CELLS: tuple[Cell, ...] = _cells() + + note = """ + Source: FlashInfer's `CpAsyncWarpSplitKKernel` (CuTe DSL) plus the conv epilogue, compiled + per (tactic, PDL, rows per request) with M dynamic: the warmup forward must compile it + before capture. Preconditions, refused by `is_valid`: the shapes above, `T` a multiple of + `rows_per_request`, `T <= tactic.token_tile`. + """ + + def __call__( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ): + rows = x.shape[0] + out = torch.empty([rows, N_OUT], dtype=torch.bfloat16, device=x.device) + conv = torch.empty([rows, CONV_DIM], dtype=torch.bfloat16, device=x.device) + _launch( + x, in_proj, out, conv, conv_weight, conv_state, slots, window, window_rows, + pad_slot_id, rows_per_request, tactic, pdl, + ) # fmt: skip + return out, conv + + def reference( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ): + """fp32 projection rounded to bf16, then the conv's per-element arithmetic in torch + (writes `conv_state` and `window`: call it on copies).""" + with true_fp32_matmul(): + out = (x.float() @ in_proj.float()).to(torch.bfloat16) + rows = x.shape[0] + p = rows_per_request + conv = torch.zeros([rows, CONV_DIM], dtype=torch.bfloat16, device=x.device) + w = conv_weight.float() + for s in range(rows // p): + slot = int(slots[s]) + if slot == pad_slot_id: + continue + cols = [conv_state[slot, :, j].float() for j in range(CONV_WIDTH - 1)] + for t in range(p): + xv = out[s * p + t, :CONV_DIM].float() + acc = torch.zeros(CONV_DIM, dtype=torch.float32, device=x.device) + for j, v in enumerate(cols + [xv]): + acc = acc + (v * w[:, j]).to(torch.bfloat16).float() + cols = cols[1:] + [xv] + for j in range(CONV_WIDTH - 1): + window[int(window_rows[s]), t, :, j] = cols[j].to(window.dtype) + conv[s * p + t] = (acc / (1 + torch.exp(-acc))).to(torch.bfloat16) + for j in range(CONV_WIDTH - 1): + conv_state[slot, :, j] = cols[j].to(conv_state.dtype) + return out, conv + + def is_valid( + self, + x, + in_proj, + conv_weight, + conv_state, + slots, + window, + window_rows, + pad_slot_id: int, + rows_per_request: int, + tactic, + pdl: bool, + ) -> None: + rows = x.shape[0] + assert x.dtype == in_proj.dtype == torch.bfloat16 and x.is_contiguous() + assert tuple(x.shape) == (rows, K_IN) and tuple(in_proj.shape) == (K_IN, N_OUT) + assert in_proj.stride() == (1, K_IN), "in_proj is the column-major [2560, 4128] operand" + assert rows % rows_per_request == 0 and 1 <= rows <= tactic.token_tile, (rows, tactic) + assert conv_weight.dtype == conv_state.dtype == window.dtype == torch.bfloat16, ( + "the epilogue reads and writes the conv taps, pool and window as bf16" + ) + assert tuple(conv_weight.shape) == (CONV_DIM, CONV_WIDTH) and conv_weight.is_contiguous() + assert conv_state.dim() == 3 and tuple(conv_state.shape[1:]) == (CONV_DIM, CONV_WIDTH - 1) + assert window.dim() == 4 and tuple(window.shape[2:]) == (CONV_DIM, CONV_WIDTH - 1) + assert window.shape[1] >= rows_per_request + assert slots.dtype == window_rows.dtype == torch.int32 + n = rows // rows_per_request + assert slots.shape[0] >= n and window_rows.shape[0] >= n + + +gdn_in_proj_conv = GdnInProjConv() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py new file mode 100644 index 000000000000..5c931a08ada1 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/ssm/gdn_inforward_tail.py @@ -0,0 +1,300 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The tails of an in-forward Mamba snapshot split in one launch: each split request's few tokens of +gated delta rule from its state at P (its live pool slot) into its staging row, read in place. + +Per value head `i` (key head `i // (HV // H)`), fp32 state `h` `[V, K]` stored in bf16, `g` the decay: + + h *= g[t]; u = (v[t] - h k[t]) * beta[t]; h += u k[t]^T; o[t] = h (K^-0.5 q[t]) + +token by token, so it matches FlashInfer's chunked form to rounding, not bitwise. q and k arrive +L2-normalized. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import OpWrapper + + +# The tail's token, sequence, slot and stage-row lists are scalar-read views. +@triton.jit(do_not_specialize_on_alignment=["tok_ptr", "cu_ptr", "slot_ptr", "row_ptr"]) +def _gdn_tail_kernel( + q_ptr, + k_ptr, + v_ptr, + g_ptr, + beta_ptr, + tok_ptr, + cu_ptr, + pool_ptr, + slot_ptr, + stage_ptr, + row_ptr, + out_ptr, + stride_qt, + stride_qh, + stride_kt, + stride_kh, + stride_vt, + stride_vh, + stride_gt, + stride_bt, + stride_ps, + stride_ph, + stride_pv, + stride_ss, + stride_sh, + stride_sv, + stride_ot, + stride_oh, + scale, + GROUP: tl.constexpr, + K: tl.constexpr, + BV: tl.constexpr, +): + """One program: sequence `j`, value head `h`, value rows `[vb * BV, (vb + 1) * BV)` of the state.""" + j = tl.program_id(0) + h = tl.program_id(1) + vb = tl.program_id(2) + kh = h // GROUP + rv = vb * BV + tl.arange(0, BV) + rk = tl.arange(0, K) + slot = tl.load(slot_ptr + j).to(tl.int64) + src = pool_ptr + slot * stride_ps + h * stride_ph + rv[:, None] * stride_pv + rk[None, :] + state = tl.load(src).to(tl.float32) + t0 = tl.load(cu_ptr + j) + t1 = tl.load(cu_ptr + j + 1) + for t in range(t0, t1): + row = tl.load(tok_ptr + t).to(tl.int64) + kv = tl.load(k_ptr + row * stride_kt + kh * stride_kh + rk).to(tl.float32) + qv = tl.load(q_ptr + row * stride_qt + kh * stride_qh + rk).to(tl.float32) * scale + vv = tl.load(v_ptr + row * stride_vt + h * stride_vh + rv).to(tl.float32) + gt = tl.load(g_ptr + row * stride_gt + h).to(tl.float32) + bt = tl.load(beta_ptr + row * stride_bt + h).to(tl.float32) + state = state * gt + u = (vv - tl.sum(state * kv[None, :], axis=1)) * bt + state = state + u[:, None] * kv[None, :] + o = tl.sum(state * qv[None, :], axis=1) + tl.store(out_ptr + row * stride_ot + h * stride_oh + rv, o.to(out_ptr.dtype.element_ty)) + srow = tl.load(row_ptr + j).to(tl.int64) + dst = stage_ptr + srow * stride_ss + h * stride_sh + rv[:, None] * stride_sv + rk[None, :] + tl.store(dst, state.to(stage_ptr.dtype.element_ty)) + + +def _gdn_inforward_tail( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, +) -> None: + """Tail `j` covers rows `tail_tok[cu_tail[j] : cu_tail[j + 1]]` of q / k `[1, T, H, K]`, v, g / beta + (fp32) and starts from `pool[tail_slots[j]]` (any slot stride); its final state goes to + `stage[stage_rows[j]]` and its outputs to rows `tail_tok` of `out` `[1, T, HV, V]`.""" + num = tail_slots.numel() + if num == 0: + return + _, _, heads, kdim = q.shape + _, _, vheads, vdim = v.shape + assert kdim == 128 and vdim % 32 == 0, (kdim, vdim) + assert pool.stride(3) == 1 and stage.stride(3) == 1 and q.stride(3) == 1 and k.stride(3) == 1 + assert v.stride(3) == 1 and out.stride(3) == 1 and g.stride(2) == 1 and beta.stride(2) == 1 + bv = 32 + _gdn_tail_kernel[(num, vheads, vdim // bv)]( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out, + q.stride(1), q.stride(2), k.stride(1), k.stride(2), v.stride(1), v.stride(2), + g.stride(1), beta.stride(1), + pool.stride(0), pool.stride(1), pool.stride(2), + stage.stride(0), stage.stride(1), stage.stride(2), + out.stride(1), out.stride(2), + kdim**-0.5, + GROUP=vheads // heads, K=kdim, BV=bv, num_warps=4, + ) # fmt: skip + + +def gdn_inforward_tail_reference( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out +) -> None: + """The same recurrence in float64 on the host side of torch (tests).""" + heads, vheads = q.shape[2], v.shape[2] + group = vheads // heads + scale = q.shape[3] ** -0.5 + cu = cu_tail.tolist() + toks = tail_tok.tolist() + for j, slot in enumerate(tail_slots.tolist()): + state = pool[slot].double() + for t in range(cu[j], cu[j + 1]): + r = toks[t] + kk = k[0, r].double().repeat_interleave(group, 0) # [HV, K] + qq = q[0, r].double().repeat_interleave(group, 0) * scale + state = state * g[0, r].double()[:, None, None] + u = (v[0, r].double() - torch.einsum("hvk,hk->hv", state, kk)) * beta[0, r].double()[ + :, None + ] + state = state + u[:, :, None] * kk[:, None, :] + out[0, r] = torch.einsum("hvk,hk->hv", state, qq).to(out.dtype) + stage[int(stage_rows[j])] = state.to(stage.dtype) + + +@triton.jit(do_not_specialize_on_alignment=["slot_ptr", "row_ptr", "win_ptr"]) +def _conv_swap_kernel( + conv_ptr, + stage_ptr, + x_ptr, + slot_ptr, + row_ptr, + win_ptr, + C, + stride_cs, + stride_cc, + stride_ss, + stride_sc, + stride_xt, + W: tl.constexpr, + BW: tl.constexpr, + BLOCK: tl.constexpr, +): + """One program: split request `j`, channels `[cb * BLOCK, (cb + 1) * BLOCK)`.""" + j = tl.program_id(0) + cb = tl.program_id(1) + c = cb * BLOCK + tl.arange(0, BLOCK) + w = tl.arange(0, BW) + m = (c[:, None] < C) & (w[None, :] < W) + slot = tl.load(slot_ptr + j).to(tl.int64) + srow = tl.load(row_ptr + j).to(tl.int64) + live = conv_ptr + slot * stride_cs + c[:, None] * stride_cc + w[None, :] + tl.store( + stage_ptr + srow * stride_ss + c[:, None] * stride_sc + w[None, :], + tl.load(live, mask=m), + mask=m, + ) + rows = tl.load(win_ptr + j * W + w, mask=w < W, other=0).to(tl.int64) + tl.store(live, tl.load(x_ptr + rows[None, :] * stride_xt + c[:, None], mask=m), mask=m) + + +def _gdn_inforward_conv_swap( + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, +) -> None: + """For split request `j`: `stage[stage_rows[j]] = conv[tail_slots[j]]` (the window ending at the prompt end the + conv pass left), then `conv[tail_slots[j]][:, w] = x[conv_win[j * W + w]]` (the raw rows before P), with `conv` + `[S, C, W]` (any slot stride, rows dense), `stage` `[rows, C, W]` and `x` the `[T, C]` projection columns.""" + num = tail_slots.numel() + if num == 0: + return + _, chans, width = conv.shape + assert ( + conv.stride(2) == 1 + and conv.stride(1) == width + and stage.stride(2) == 1 + and x.stride(1) == 1 + ) + assert x.dtype == conv.dtype == stage.dtype + block = 256 + _conv_swap_kernel[(num, triton.cdiv(chans, block))]( + conv, stage, x, tail_slots, stage_rows, conv_win, chans, + conv.stride(0), conv.stride(1), stage.stride(0), stage.stride(1), x.stride(0), + W=width, BW=triton.next_power_of_2(width), BLOCK=block, num_warps=4, + ) # fmt: skip + + +class GdnInforwardTail(OpWrapper): + """The split requests' tails of gated delta rule, in place (see `_gdn_inforward_tail`).""" + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, + ) -> None: + _gdn_inforward_tail( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, stage, stage_rows, out + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + tail_tok: torch.Tensor, + cu_tail: torch.Tensor, + pool: torch.Tensor, + tail_slots: torch.Tensor, + stage: torch.Tensor, + stage_rows: torch.Tensor, + out: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(stage, out)` after `gdn_inforward_tail_reference` writes copies of them; the inputs + are not touched.""" + expected_stage, expected_out = stage.clone(), out.clone() + gdn_inforward_tail_reference( + q, k, v, g, beta, tail_tok, cu_tail, pool, tail_slots, expected_stage, stage_rows, + expected_out, + ) # fmt: skip + return expected_stage, expected_out + + +gdn_inforward_tail = GdnInforwardTail() + + +class GdnInforwardConvSwap(OpWrapper): + """The split requests' conv windows staged and replaced, in place (see + `_gdn_inforward_conv_swap`).""" + + def __call__( + self, + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, + ) -> None: + _gdn_inforward_conv_swap(conv, stage, x, tail_slots, stage_rows, conv_win) + + def reference( + self, + conv: torch.Tensor, + stage: torch.Tensor, + x: torch.Tensor, + tail_slots: torch.Tensor, + stage_rows: torch.Tensor, + conv_win: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`(conv, stage)` after the swap, as fresh tensors; the inputs are not touched.""" + expected_conv, expected_stage = conv.clone(), stage.clone() + _, chans, width = conv.shape + for j, (slot, row) in enumerate(zip(tail_slots.tolist(), stage_rows.tolist())): + expected_stage[row] = conv[slot] + rows = conv_win[j * width : (j + 1) * width].long() + expected_conv[slot] = x[rows, :chans].t() + return expected_conv, expected_stage + + +gdn_inforward_conv_swap = GdnInforwardConvSwap() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py new file mode 100644 index 000000000000..fa83b33e0ef4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/ops/vision/triton_pos_embed_interpolate.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Vision position-embedding bilinear interpolation onto one image's patch +grid, emitted in spatial-merge order, by the stock Qwen3-VL tower's fused +Triton kernel. + +`__call__` allocates the output and makes one launch of the Triton JIT kernel +`_bilinear_pos_embed_kernel`: the four corner gathers, the bilinear weights and +the merge-order row permutation in one pass. The kernel and its launch are the +ones `tensorrt_llm/_torch/models/modeling_qwen3vl.py` runs for the stock vision +towers (`Qwen3VisionModel.fast_pos_embed_interpolate`, one call per image), with +the grid sides left unspecialized so one compile serves every image. +Concatenating several images' rows is the caller's. The FP8 unified target's entry; the +NVFP4 default target keeps its own copy (the two checkpoints share the bf16 vision tower). +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +#: This checkpoint: num_position_embeddings 2304 (a 48 x 48 grid), hidden 1152, +#: spatial_merge_size 2. +NUM_GRID = 48 +HIDDEN = 1152 +MERGE = 2 + + +def _ckpt(t: int, h: int, w: int) -> dict: + """The checkpoint's bf16 table at one `(t, h, w)` grid.""" + return dict( + t=t, + h=h, + w=w, + num_grid_per_side=NUM_GRID, + m_size=MERGE, + dtype=torch.bfloat16, + hidden=HIDDEN, + ) + + +# The grid sides vary per image: not specialized, so one compile serves every grid. +@triton.jit(do_not_specialize=["H", "W"]) +def _bilinear_pos_embed_kernel( + embed_ptr, + output_ptr, + H, + W, + h_scale, + w_scale, + NUM_GRID: tl.constexpr, + M_SIZE: tl.constexpr, + HIDDEN_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """One output row: the bilinear blend of the table's four corner rows around the + row's patch, the patch taken in spatial-merge order.""" + pid = tl.program_id(0) + total_spatial = H * W + spatial_idx = pid % total_spatial + + num_blocks_w = W // M_SIZE + block_idx = spatial_idx // (M_SIZE * M_SIZE) + local_idx = spatial_idx % (M_SIZE * M_SIZE) + br = block_idx // num_blocks_w + bc = block_idx % num_blocks_w + lr = local_idx // M_SIZE + lc = local_idx % M_SIZE + row = br * M_SIZE + lr + col = bc * M_SIZE + lc + + h_frac = row.to(tl.float32) * h_scale + w_frac = col.to(tl.float32) * w_scale + + hf = tl.math.floor(h_frac).to(tl.int32) + wf = tl.math.floor(w_frac).to(tl.int32) + hc = tl.minimum(hf + 1, NUM_GRID - 1) + wc = tl.minimum(wf + 1, NUM_GRID - 1) + + dh = h_frac - hf.to(tl.float32) + dw = w_frac - wf.to(tl.float32) + w11 = dh * dw + w10 = dh - w11 + w01 = dw - w11 + w00 = 1.0 - dh - w01 + + off00 = (hf * NUM_GRID + wf) * HIDDEN_DIM + off01 = (hf * NUM_GRID + wc) * HIDDEN_DIM + off10 = (hc * NUM_GRID + wf) * HIDDEN_DIM + off11 = (hc * NUM_GRID + wc) * HIDDEN_DIM + out_off = pid * HIDDEN_DIM + + # The corner weights rounded to the output dtype; the multiply-adds run in it. + out_dtype = output_ptr.dtype.element_ty + w00_c = w00.to(out_dtype) + w01_c = w01.to(out_dtype) + w10_c = w10.to(out_dtype) + w11_c = w11.to(out_dtype) + + for d in tl.range(0, HIDDEN_DIM, BLOCK_D): + cols = d + tl.arange(0, BLOCK_D) + mask = cols < HIDDEN_DIM + + e00 = tl.load(embed_ptr + off00 + cols, mask=mask) + e01 = tl.load(embed_ptr + off01 + cols, mask=mask) + e10 = tl.load(embed_ptr + off10 + cols, mask=mask) + e11 = tl.load(embed_ptr + off11 + cols, mask=mask) + + val = w00_c * e00 + w01_c * e01 + w10_c * e10 + w11_c * e11 + + tl.store(output_ptr + out_off + cols, val, mask=mask) + + +def _pos_embed_interpolate( + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, +) -> torch.Tensor: + """One launch over a `(t, h, w)` grid: `[t * h * w, hidden]` in spatial-merge order.""" + hidden_dim = embed_weight.shape[1] + total_out = t * h * w + output = torch.empty(total_out, hidden_dim, device=embed_weight.device, dtype=dtype) + h_scale = (num_grid_per_side - 1) / (h - 1) if h > 1 else 0.0 + w_scale = (num_grid_per_side - 1) / (w - 1) if w > 1 else 0.0 + _bilinear_pos_embed_kernel[(total_out,)]( + embed_weight, + output, + h, + w, + h_scale, + w_scale, + num_grid_per_side, + m_size, + hidden_dim, + triton.next_power_of_2(hidden_dim), + ) + return output + + +class TritonPosEmbedInterpolate(OpWrapper): + """Resample a square row-major `[N*N, hidden]` table onto one `(t, h, w)` + patch grid; return a fresh `[t*h*w, hidden]` tensor in `dtype`. + + Corner-aligned bilinear (`align_corners=True`): row `i` of `h` samples the + table at `i * (N - 1) / (h - 1)` (0 when `h == 1`), `lo = floor`, + `hi = min(lo + 1, N - 1)`; `h` indexes table rows (stride `N`), `w` + columns. Rows come out in spatial-merge order -- whole `m_size x m_size` + blocks row-major, row-major inside a block -- which is what the patch + merger's 4-to-1 reshape consumes. `t` repeats the frame's rows. + + Arithmetic is the HF reference's (`fast_pos_embed_interpolate`): the four + corner weights are computed in fp32, rounded to `dtype`, and the + multiply-adds run in `dtype`. Not an fp32 accumulation; see `note`. + + Fusion boundary: one image's position embeddings. Adding them to the patch + embeddings, the 2-D rope, and joining images are the caller's. + """ + + ARCHS = frozenset({Arch.SM_103}) + + # spec: t, h, w, num_grid_per_side, m_size, dtype, hidden. The tower is + # replicated on every rank, so these are every member's shapes. + CELLS: tuple[Cell, ...] = ( + Cell(why="one 4x8 image: 32 rows, both sides below the table's 48", spec=_ckpt(1, 4, 8)), + Cell(why="the 62x90 serving grid: 5580 rows, one image per call", spec=_ckpt(1, 62, 90)), + Cell(why="a 3-frame 6x10 video: t repeats one frame's 60 rows", spec=_ckpt(3, 6, 10)), + Cell( + why="128x256: 32768 rows, the tower's per-call cap and the envelope's largest", + spec=_ckpt(1, 128, 256), + ), + ) + + note = """ + Source: a Triton JIT kernel, compiled at first call for the device it runs + on: the stock Qwen3-VL towers' kernel (`modeling_qwen3vl`), kept here so its + grid sides can stay unspecialized. `H` and `W` are int32 and not specialized, + so every grid shares one compile (the tower's `warm_up` makes it at load; the + test holds one variant); the scales are fp32, and `NUM_GRID`, `M_SIZE`, + `HIDDEN_DIM` (1152) and `BLOCK_D` (2048) are constexprs, so another table + geometry compiles another kernel. Not a registered `torch.ops.trtllm` op. + + Accuracy, against the exact (fp64) corner-aligned bilinear value, as + row-scale element error / relative RMS in bf16 ulp (`assert_within_ulp`): + at most 3.522 / 0.752 over 50 seeds of the test's smooth table (a bicubic + upsampled normal field, neighbour cosine 0.911 against the checkpoint + table's 0.907), and at most 3.349 / 0.745 on the checkpoint's own + `model.visual.pos_embed.weight`. The test gates 4 / 1. The closest wrong + variant (align_corners=False) lands at 41.3 / 15.2 or more; raster row + order and swapped table axes above 400 / 300. + + Not certified on tables whose neighbouring rows are uncorrelated: a normal + table with independent rows reaches 4.182 / 0.879 over 20 seeds. + Averaging uncorrelated corners shrinks the output's row maximum below the + corners' magnitude, and the bf16 rounding error scales with the corners. A + learned 2-D position table is not such a table. + + Silent, refused by `is_valid` (each measured on GB300): + - a strided table: the kernel indexes `embed_weight` as dense rows of + `hidden`. A column slice of a wider buffer returned, differing from the + same values made contiguous by up to 5.05. + - `num_grid_per_side ** 2 != embed_weight.shape[0]`: the grid side is an + argument, not read off the table. 47 on the 2304-row table returned a + correctly shaped result 564 / 340 ulp off. + - an empty side (`t`, `h` or `w` of 0): returns `[0, hidden]`, no error. + - `h` or `w` not divisible by `m_size`: a 6x9 grid returned a finite + `[54, 1152]` tensor, its rows read from positions outside the grid and the + table. + + Hard limits: the output offset is `program_id * hidden` in int32, so + `t*h*w*hidden` must stay below 2^31 (1,864,135 rows at hidden 1152; the + tower's per-call cap is 32768). Positions are `row * scale` in fp32, + exact up to 2^24 rows per side. + + Not certified: fp16 or fp32 tables, a `dtype` other than the table's, + `m_size` other than 2, a hidden size other than 1152, a CPU table, and + CUDA-graph capture (the call site, the vision tower, is eager only; the + grid is a host argument, frozen at capture). + """ + + def __call__( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> torch.Tensor: + return _pos_embed_interpolate(embed_weight, t, h, w, num_grid_per_side, m_size, dtype) + + def reference( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """The explicit four-corner weighted sum in fp64, returned as fp32. + + Sample positions are the exact `i * (N - 1) / (size - 1)` in fp64. + `dtype` is not applied: the result is the exact value rounded once to + fp32, so a test's band in `dtype` ulp measures the op's error alone + rather than adding the reference's own rounding to `dtype`. + """ + n = num_grid_per_side + hidden = embed_weight.shape[1] + device = embed_weight.device + grid = embed_weight.double().reshape(n, n, hidden) + + def positions(size: int) -> torch.Tensor: + index = torch.arange(size, dtype=torch.float64, device=device) + return index * ((n - 1) / (size - 1)) if size > 1 else torch.zeros_like(index) + + h_pos, w_pos = positions(h), positions(w) + h_lo, w_lo = h_pos.floor().long(), w_pos.floor().long() + h_hi = (h_lo + 1).clamp(max=n - 1) + w_hi = (w_lo + 1).clamp(max=n - 1) + d_h = (h_pos - h_lo)[:, None, None] + d_w = (w_pos - w_lo)[None, :, None] + raster = ( + grid[h_lo][:, w_lo] * ((1 - d_h) * (1 - d_w)) + + grid[h_lo][:, w_hi] * ((1 - d_h) * d_w) + + grid[h_hi][:, w_lo] * (d_h * (1 - d_w)) + + grid[h_hi][:, w_hi] * (d_h * d_w) + ) + rows = ( + raster.reshape(h // m_size, m_size, w // m_size, m_size, hidden) + .permute(0, 2, 1, 3, 4) + .reshape(h * w, hidden) + ) + return (rows.repeat(t, 1) if t > 1 else rows).float() + + def is_valid( + self, + embed_weight: torch.Tensor, + t: int, + h: int, + w: int, + num_grid_per_side: int, + m_size: int, + dtype: torch.dtype, + ) -> None: + assert embed_weight.is_contiguous(), ( + "embed_weight must be contiguous; the kernel indexes it as dense rows " + "and reads a strided table wrongly without raising" + ) + assert num_grid_per_side * num_grid_per_side == embed_weight.shape[0], ( + f"num_grid_per_side {num_grid_per_side} does not describe a " + f"{embed_weight.shape[0]}-row table; the kernel resamples the grid it is " + f"told, in or out of the table" + ) + assert t > 0 and h > 0 and w > 0, ( + f"empty grid side (t, h, w) = ({t}, {h}, {w}); the kernel returns no rows " + f"rather than raising" + ) + assert h % m_size == 0 and w % m_size == 0, ( + f"grid sides h={h}, w={w} are not whole m_size={m_size} merge blocks; " + f"the kernel would read rows outside the grid and the table" + ) + + +triton_pos_embed_interpolate = TritonPosEmbedInterpolate() diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py new file mode 100644 index 000000000000..0d2e18814609 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/pool.py @@ -0,0 +1,578 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The KV cache manager's pools as one rank of this target sees them, and their contract. + +Every rank builds the stock ``Qwen4ExpHybridCacheManager``, a KV cache manager V2 (which QSA +requires). `pool_geometry` derives the per-rank shapes it allocates: + +=================== ============================================ ============================= +pool shape after the slot dim split over ranks by +=================== ============================================ ============================= +paged K/V (HND) ``[2, H_kv, tokens_per_block, kv_width]`` attention TP (a) +NVFP4 block scales ``[2, H_kv, tokens_per_block, head_dim/16]`` attention TP (a) +QSA index plane ``[tokens_per_block, 1, index_head_dim]`` replicated (b) +QSA position plane ``[tokens_per_block, 3]`` int32 replicated (b) +GDN conv state ``[2 * H_k * D_k + H_v * D_v, kernel - 1]`` recurrent-state TP (c) +GDN recurrent state ``[H_v, D_v, D_k]`` recurrent-state TP (c) +PLE conv state ``[hc_count * hidden, (ple_kernel-1) * n]`` replicated (d) +PLE n-gram history ``[n - 1]`` int64 replicated (d) +=================== ============================================ ============================= + +``H_kv`` is ``num_key_value_heads``, ``H_k``/``D_k`` and ``H_v``/``D_v`` the GDN key and value +heads, ``kernel`` ``linear_conv_kernel_dim``, ``ple_kernel`` ``ple_conv_kernel_size`` and ``n`` +``ngram_size``; head counts are the rank's share. + +(a) Over ``tp_size`` (1 under attention DP), rounding up: with more ranks than KV heads, rank + ``r`` holds global KV head ``r * H / tp``, the order ``duplicate_kv_weight`` repeats them in. +(b) No mapping divides the indexer. When an index or position page could equal a K/V or + block-scale page, which V2 would coalesce into that pool, both are packed into one per-slot + ``QSA_SIDE_PLANES`` page (`PoolGeometry.qsa_side_planes`), as strided views of these shapes. +(c) Over ``mamba_effective_tp_size`` (1 under attention DP). Conv channels are ``[q | k | v]`` of + the rank's own heads; rank ``r`` holds key heads from ``r * H_k / tp`` and value heads from + ``r * H_v / tp``, the contiguous ranges a disaggregated transfer between layouts moves. The + recurrent state is served in bf16 only (`SSM_STATE_DTYPES`), not the checkpoint's fp32. +(d) The PLE layer reads the whole hyper-connection stream on every rank. + +With one-model MTP the draft layer (global layer ``num_hidden_layers``) is one more QSA attention +layer at the trunk's per-rank shapes (`draft_kv_cache_is_separate`). + +The ``check_*`` functions read only ``.shape``, ``.dtype`` and ``.stride()``, so they also run on +meta tensors. Each rejects a pool its consuming kernel would accept and mis-address. +""" + +from __future__ import annotations + +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.pyexecutor.config_utils import mamba_effective_tp_size +from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache +from tensorrt_llm.quantization.mode import QuantAlgo + +#: Coordinates per token in the QSA position plane (the mRoPE layout's three rows). +QSA_POSITION_AXES = 3 + +#: Values per e4m3 block scale of an NVFP4 K/V pool. +NVFP4_BLOCK = 16 + +#: Recurrent-state pool dtypes served (`kv_cache_config.mamba_ssm_cache_dtype`). +SSM_STATE_DTYPES = (torch.bfloat16,) + +#: The GDN cached replay's history length the replay entry is certified at. +GDN_REPLAY_HISTORY = 16 + +#: Most query rows per generation request in a verify step (`max_draft_len + 1`): the replay +#: kernel asserts `T <= 8` and `thop_attention`'s `predicted_tokens_per_seq` is certified to it. +MAX_VERIFY_WIDTH = 8 + +#: CUDA-graph padding dummy slots: one per draft length, and routing admits one fixed length. +CUDA_GRAPH_PADDING_DUMMY_SLOTS = 1 + +#: Bytes per element of the QSA index (bf16) and position (int32) planes. +_INDEX_ELEMENT_BYTES = 2 +_POSITION_ELEMENT_BYTES = 4 + + +class PoolGeometry(NamedTuple): + """The pool shapes one rank binds. Head counts and channel widths are this rank's share.""" + + tokens_per_block: int + heads_kv: int # KV heads in this rank's K/V pages + kv_head_start: int # the global KV head the rank's first page head holds + head_dim: int + index_kv_heads: int + index_head_dim: int + conv_dim: int # this rank's GDN conv channels, [q | k | v] of its own heads + conv_kernel: int + num_k_heads: int # this rank's GDN key heads + k_head_start: int # the global key head the rank's first one is + num_v_heads: int # this rank's GDN value heads + v_head_start: int # the global value head the rank's first one is + head_v_dim: int + head_k_dim: int + stream: int # the hyper-connection stream width, the PLE conv channels + ple_state_len: int + ngram_context_len: int + dtype: torch.dtype # the activation dtype: GDN conv, QSA index and PLE conv pools + kv_dtype: torch.dtype # the paged K/V pool's dtype: bf16, e4m3 (FP8) or uint8 (NVFP4) + kv_width: int # its innermost extent: head_dim, or head_dim // 2 for NVFP4's e2m1 pairs + qsa_side_planes: bool # the index and position planes share one packed page per slot + + @property + def conv_sections(self) -> tuple[int, int, int]: + """The q, k and v channel widths of this rank's conv state, in that order.""" + key = self.num_k_heads * self.head_k_dim + return key, key, self.num_v_heads * self.head_v_dim + + @property + def kv_block_scales(self) -> bool: + """Whether the K/V pool is NVFP4, whose e4m3 block scales sit in a second pool.""" + return self.kv_dtype == torch.uint8 + + @property + def index_page_bytes(self) -> int: + per_token = self.index_kv_heads * self.index_head_dim * _INDEX_ELEMENT_BYTES + return self.tokens_per_block * per_token + + @property + def position_page_bytes(self) -> int: + return self.tokens_per_block * QSA_POSITION_AXES * _POSITION_ELEMENT_BYTES + + +def attention_tp_size(mapping) -> int: + """The ranks the KV heads are split over (`KVCacheManagerV2.__init__`).""" + return 1 if mapping.enable_attention_dp else mapping.tp_size + + +def reserved_dummy_slots(mapping) -> int: + """State slots beyond one per request: the graph padding dummy and the attention-DP dummy.""" + return CUDA_GRAPH_PADDING_DUMMY_SLOTS + int(mapping.enable_attention_dp) + + +def draft_kv_cache_is_separate(mapping, spec_config=None) -> bool: + """Whether the one-model MTP draft layer has a KV cache manager of its own. + + Only under attention TP, unless `spec_config` disallows it (disaggregated serving does).""" + if mapping.enable_attention_dp: + return False + return spec_config is None or should_use_separate_draft_kv_cache(spec_config) + + +def _share(total: int, ranks: int, rank: int) -> tuple[int, int]: + """(heads on `rank`, its first global head): an even split, or each head on several ranks.""" + if total % ranks and ranks % total: + raise ValueError(f"{total} heads do not split over {ranks} ranks") + return -(-total // ranks), rank * total // ranks + + +#: K/V pool storage per `kv_cache_quant_algo`: (dtype or None for activation, values per element). +_KV_STORAGE = { + None: (None, 1), + QuantAlgo.FP8: (torch.float8_e4m3fn, 1), + QuantAlgo.NVFP4: (torch.uint8, 2), +} + + +def _side_planes_packed( + num_kv_heads: int, + kv_page_bytes_per_head: int, + scale_page_bytes_per_head: int, + index_page: int, + position_page: int, +) -> bool: + """Whether `_extra_buffers_per_layer` packs the QSA index and position pages into one page.""" + pages = [p for p in (kv_page_bytes_per_head, scale_page_bytes_per_head) if p] + main = {page * heads for page in pages for heads in range(1, num_kv_heads + 1)} + return index_page in main or position_page in main + + +def pool_geometry( + cfg, mapping, *, tokens_per_block: int, kv_cache_quant_algo, dtype: torch.dtype +) -> PoolGeometry: + """The pools rank `mapping.rank` binds, from the engine-normalized `Qwen4ExpTextConfig`.""" + if kv_cache_quant_algo not in _KV_STORAGE: + raise ValueError( + f"kv_cache_quant_algo {kv_cache_quant_algo!r}; the K/V pool stores the checkpoint's " + "bf16, FP8 or NVFP4" + ) + if (mapping.pp_size, mapping.cp_size) != (1, 1): + raise ValueError( + f"pp_size={mapping.pp_size}, cp_size={mapping.cp_size}: the members split heads over " + "the TP group alone, and the manager shards pipeline and helix layouts otherwise" + ) + attn_tp = attention_tp_size(mapping) + heads_kv, kv_head_start = _share( + cfg.num_key_value_heads, attn_tp, mapping.tp_rank if attn_tp > 1 else 0 + ) + state_tp = mamba_effective_tp_size(mapping) + state_rank = mapping.tp_rank if state_tp > 1 else 0 + num_k_heads, k_head_start = _share(cfg.linear_num_key_heads, state_tp, state_rank) + num_v_heads, v_head_start = _share(cfg.linear_num_value_heads, state_tp, state_rank) + kv_dtype, per_element = _KV_STORAGE[kv_cache_quant_algo] + kv_dtype = kv_dtype or dtype + kv_width = cfg.head_dim // per_element + block_scales = kv_dtype == torch.uint8 + index_page = ( + tokens_per_block * cfg.indexer_kv_heads * cfg.indexer_head_dim * _INDEX_ELEMENT_BYTES + ) + position_page = tokens_per_block * QSA_POSITION_AXES * _POSITION_ELEMENT_BYTES + side_planes = _side_planes_packed( + cfg.num_key_value_heads, + tokens_per_block * kv_width * kv_dtype.itemsize, + tokens_per_block * (cfg.head_dim // NVFP4_BLOCK) if block_scales else 0, + index_page, + position_page, + ) + return PoolGeometry( + tokens_per_block=tokens_per_block, + heads_kv=heads_kv, + kv_head_start=kv_head_start, + head_dim=cfg.head_dim, + index_kv_heads=cfg.indexer_kv_heads, + index_head_dim=cfg.indexer_head_dim, + conv_dim=2 * num_k_heads * cfg.linear_key_head_dim + + num_v_heads * cfg.linear_value_head_dim, + conv_kernel=cfg.linear_conv_kernel_dim, + num_k_heads=num_k_heads, + k_head_start=k_head_start, + num_v_heads=num_v_heads, + v_head_start=v_head_start, + head_v_dim=cfg.linear_value_head_dim, + head_k_dim=cfg.linear_key_head_dim, + stream=cfg.hc_count * cfg.hidden_size, + ple_state_len=(cfg.ple_conv_kernel_size - 1) * cfg.ngram_size, + ngram_context_len=cfg.ngram_size - 1, + dtype=dtype, + kv_dtype=kv_dtype, + kv_width=kv_width, + qsa_side_planes=side_planes, + ) + + +def require(cm, names, what: str) -> None: + for name in names: + assert hasattr(cm, name), ( + f"the KV cache manager {type(cm).__name__} does not expose {name}; this target {what}" + ) + + +def _check_pool(t, what, lead, tail, dtype=None) -> None: + """`t` has `lead` leading dims of any size, then exactly `tail`, at `dtype`.""" + assert t is not None, f"{what}: the runtime did not allocate it" + expected = ["*"] * lead + [str(n) for n in tail] + assert t.dim() == lead + len(tail) and tuple(t.shape[lead:]) == tuple(tail), ( + f"{what} is {tuple(t.shape)}, expected [{', '.join(expected)}]" + ) + assert dtype is None or t.dtype == dtype, f"{what} is {t.dtype}, expected {dtype}" + + +def check_position_plane(g: PoolGeometry, position) -> None: + shape = (g.tokens_per_block, QSA_POSITION_AXES) + _check_pool(position, "QSA position plane", 1, shape, torch.int32) + + +def check_attention_pools(g: PoolGeometry, layer: int, index_cache, pool) -> None: + index_shape = (g.tokens_per_block, g.index_kv_heads, g.index_head_dim) + _check_pool(index_cache, f"layer {layer}: QSA index plane", 1, index_shape, g.dtype) + # The sparse path needs adjacent HND K/V role planes: [pages, 2, H, block, D]. + kv_shape = (g.heads_kv, g.tokens_per_block, g.kv_width) + _check_pool(pool, f"layer {layer}: paged K/V", 2, kv_shape, g.kv_dtype) + assert pool.shape[1] == 2, f"layer {layer}: paged K/V has {pool.shape[1]} role planes, not 2" + + +def check_kv_scale_planes(g: PoolGeometry, layer: int, pool, scales) -> None: + """NVFP4 block scales: `[slots, 2, H, block, D / 16]`, as many pages per slot as the data.""" + scale_shape = (g.heads_kv, g.tokens_per_block, g.head_dim // NVFP4_BLOCK) + _check_pool(scales, f"layer {layer}: NVFP4 block scales", 2, scale_shape, torch.uint8) + assert scales.shape[1] == 2 and scales.shape[0] == pool.shape[0], ( + f"layer {layer}: the block scales are {tuple(scales.shape)} against paged K/V " + f"{tuple(pool.shape)}; they must hold the same slots, K then V" + ) + pages = [] + for what, t in (("paged K/V", pool), ("block scales", scales)): + page = t[0, 0].numel() + assert t.stride(0) % page == 0, ( + f"layer {layer}: the {what} slot stride {t.stride(0)} is not a whole number of " + f"{page}-element pages" + ) + pages.append(t.stride(0) // page) + assert pages[0] == pages[1], ( + f"layer {layer}: the paged K/V pool holds {pages[0]} pages per slot and the block-scale " + f"pool {pages[1]}; the attention op would address the scales of other slots" + ) + + +def check_side_planes(g: PoolGeometry, index_caches, position) -> None: + """Packed QSA planes (`g.qsa_side_planes`) all advance by one slot stride covering them.""" + if not g.qsa_side_planes: + return + stride = position.stride(0) * position.element_size() + for layer, index in index_caches.items(): + index_stride = index.stride(0) * index.element_size() + assert index_stride == stride, ( + f"layer {layer}: the QSA index plane advances {index_stride} bytes a slot and the " + f"position plane {stride}; packed side planes share one per-slot page" + ) + need = len(index_caches) * g.index_page_bytes + g.position_page_bytes + assert stride >= need, ( + f"the packed QSA side page is {stride} bytes a slot, short of the {len(index_caches)} " + f"index pages and the position page it holds ({need} bytes)" + ) + + +def check_recurrent_pools(g: PoolGeometry, conv, ssm) -> None: + _check_pool(conv, "conv state pool", 1, (g.conv_dim, g.conv_kernel - 1), g.dtype) + # Key dim innermost. + _check_pool(ssm, "recurrent state pool", 1, (g.num_v_heads, g.head_v_dim, g.head_k_dim)) + assert ssm.dtype in SSM_STATE_DTYPES, ( + f"the recurrent state pool is {ssm.dtype}; this target serves " + f"{' or '.join(str(d) for d in SSM_STATE_DTYPES)} (kv_cache_config.mamba_ssm_cache_dtype)" + ) + + +def check_conv_layout(g: PoolGeometry, section_dims) -> None: + assert tuple(section_dims) == g.conv_sections, ( + f"the manager lays a rank's conv state out as sections {tuple(section_dims)}; this target " + f"writes [q | k | v] = {g.conv_sections} (conv_state_layout 'q_k_v')" + ) + + +def check_ple_pools(g: PoolGeometry, ple_conv, ple_ngram, conv) -> None: + _check_pool(ple_conv, "PLE conv state", 1, (g.stream, g.ple_state_len), g.dtype) + # The id-history entry writes int64 into it; an int32 pool raises. + _check_pool(ple_ngram, "PLE n-gram context pool", 1, (g.ngram_context_len,), torch.int64) + # The forward addresses all three with one scheduler slot vector. + assert ple_conv.shape[0] == conv.shape[0] == ple_ngram.shape[0], ( + "the PLE and Gated-DeltaNet pools must share one slot space" + ) + + +def check_state_slots(mapping, max_batch_size: int, slots: int) -> None: + need = max_batch_size + reserved_dummy_slots(mapping) + assert slots >= need, ( + f"the recurrent-state pools hold {slots} slots; max_batch_size {max_batch_size} plus " + f"{reserved_dummy_slots(mapping)} reserved dummies (attention DP " + f"{mapping.enable_attention_dp}) need {need}" + ) + + +def check_step_buffers(g: PoolGeometry, int_conv, conv, int_idx) -> None: + """MTP verify buffers: per-step conv windows at the pool dtype (promotion does not convert).""" + assert int_conv is not None, ( + "the runtime allocated no per-step conv buffer; a partially accepted draft " + "cannot be rolled back" + ) + _check_pool(int_conv, "per-step conv buffer", 2, (g.conv_dim, g.conv_kernel - 1), conv.dtype) + assert int_idx is not None and int_idx.dtype == torch.int32, ( + "the cache manager exposes no int32 intermediate_state_indices" + ) + + +def check_replay_state(history, width, prev_accepted, cache_buf_idx, ssm, replay_slots) -> None: + """Cached replay, manager-wide: the certified geometry and the per-slot bookkeeping, which + must cover every replay slot. The manager may address the history by replay slots of its + own, which then differ from the recurrent pool's.""" + assert ssm.dtype in SSM_STATE_DTYPES, ( + f"the recurrent pool is {ssm.dtype}; this target runs the replay over a bf16 checkpoint " + "state (kv_cache_config.mamba_ssm_cache_dtype: bfloat16)" + ) + assert history == GDN_REPLAY_HISTORY, ( + f"replay_history_size {history}; the replay entry is certified at {GDN_REPLAY_HISTORY}" + ) + assert 2 <= width <= min(MAX_VERIFY_WIDTH, history), ( + f"replay_step_width {width}; the replay entry is certified at 2..{MAX_VERIFY_WIDTH} " + f"verify rows and never above the history ({history})" + ) + for name, t in (("prev_num_accepted_tokens", prev_accepted), ("cache_buf_idx", cache_buf_idx)): + assert t.dim() == 1 and t.shape[0] >= replay_slots and t.dtype == torch.int32, ( + f"{name} is {tuple(t.shape)} {t.dtype}; the kernel reads one int32 per replay slot " + f"({replay_slots})" + ) + + +def check_replay_history( + g: PoolGeometry, layer: int, history, replay_slots, old_x, old_B, old_dt +) -> None: + """One GDN layer's replay history: `[replay slots, 2 halves, ...]` over this rank's heads, + addressed by `history` and the head geometry, never by the buffer's own shape.""" + for name, t in (("old_x", old_x), ("old_B", old_B), ("old_dt", old_dt)): + assert t is not None, f"layer {layer}: the manager's layer cache carries no {name}" + for name, t in (("old_x", old_x), ("old_B", old_B), ("old_dt", old_dt)): + assert t.shape[0] >= replay_slots, ( + f"layer {layer}: the replay history {name} has {t.shape[0]} slots for " + f"{replay_slots} replay slots" + ) + bf16, fp32 = torch.bfloat16, torch.float32 + x_tail, b_tail = (history, g.num_v_heads, g.head_v_dim), (history, g.num_k_heads, g.head_k_dim) + _check_pool(old_x, f"layer {layer}: old_x", 2, x_tail, bf16) + _check_pool(old_B, f"layer {layer}: old_B", 2, b_tail, bf16) + _check_pool(old_dt, f"layer {layer}: old_dt", 2, (g.num_v_heads, history), fp32) + assert old_x.shape[1] == old_B.shape[1] == old_dt.shape[1] == 2, ( + f"layer {layer}: the replay history is not double-buffered" + ) + + +def check_verify_width(rows: int) -> None: + assert 1 <= rows <= MAX_VERIFY_WIDTH, ( + f"predicted_tokens_per_seq {rows} is outside the 1..{MAX_VERIFY_WIDTH} range " + "attention/thop_attention is certified at" + ) + + +def check_replay_width(replay, rows: int) -> None: + """A cached-replay step carries exactly `replay.width` rows per sequence. + + The pool row is a checkpoint whose pending history only the replay kernel applies.""" + assert replay is None or rows == replay.width, ( + f"the batch carries {rows} query rows per generation sequence and the GDN cached " + f"replay runs at step width {replay.width}; a runtime_draft_len of {rows - 1} (a " + "draft-length schedule, or skipped drafting) cannot run under it. Export " + "TRTLLM_USE_GDN_REPLAY=0 for variable draft lengths" + ) + + +def check_shared_selection(shared, slots, layer: int, block_topk: int) -> None: + assert shared is not None, "IndexShare is on but the QSA metadata allocated no shared buffers" + assert layer in slots and slots[layer] < shared.shape[0], ( + f"the KV cache manager gives layer {layer} no row of the {shared.shape[0]}-row shared " + f"selection buffers (qsa_shared_topk_slots={slots})" + ) + assert shared.shape[2] == block_topk, ( + f"the shared selection rows are {shared.shape[2]} wide and this target selects " + f"{block_topk} blocks per row" + ) + + +class KvPlanes(NamedTuple): + """One layer's paged K/V views, and under an NVFP4 pool its e4m3 block-scale views: + `[slots, H_kv, tokens_per_block, kv_width]` and `[slots, H_kv, tokens_per_block, D / 16]`.""" + + k: torch.Tensor + v: torch.Tensor + k_scale: torch.Tensor | None = None + v_scale: torch.Tensor | None = None + + +class RankPools(NamedTuple): + """The trunk's pool views on one rank, from one manager. A view is a `TensorWrapper` over a + raw device address that keeps no memory alive, so it is valid only for that manager.""" + + pool_idx: dict # attention layer -> pool offset (what thop_attention addresses) + index_cache: dict # attention layer -> QSA index plane + position_cache: torch.Tensor # the shared QSA position plane + kv: dict # attention layer -> `KvPlanes` + conv: dict # GDN layer -> conv state pool + ssm: dict # GDN layer -> recurrent state pool + ple_state: tuple # (PLE conv state, PLE n-gram id history) + + +class DraftPools(NamedTuple): + """The MTP draft layer's views in the manager its draft step is handed.""" + + pool_idx: int # the draft layer's pool offset in that manager + index_cache: torch.Tensor # its QSA index plane + position_cache: torch.Tensor # that manager's position plane + kv: KvPlanes + + +def kv_planes(cm, g: PoolGeometry, layer: int, index_cache) -> KvPlanes: + """One layer's checked paged K/V views from `cm`, plus the block-scale views under NVFP4.""" + pool = cm.get_buffers(layer, kv_layout="HND") + check_attention_pools(g, layer, index_cache, pool) + if not g.kv_block_scales: + return KvPlanes(pool[:, 0], pool[:, 1]) + require(cm, ("get_block_scale_buffers",), "reads an NVFP4 pool's e4m3 block-scale planes") + scales = cm.get_block_scale_buffers(layer, kv_layout="HND") + check_kv_scale_planes(g, layer, pool, scales) + return KvPlanes(pool[:, 0], pool[:, 1], scales[:, 0], scales[:, 1]) + + +def take_pools(cm, g: PoolGeometry, attn_layers, gdn_layers, ple_layer: int) -> RankPools: + """Every trunk pool view of `cm` this rank binds, each checked against `g`. + + Step-dependent state (pool ids, windows, the QSA selection width) is the binder's to check.""" + require( + cm, + ("get_index_k_buffer", "get_qsa_position_buffer", "get_buffers", "layer_offsets"), + "reads the QSA index plane, the shared position plane and the paged K/V", + ) + # Only the V2 hybrid manager has `ple_layer_cache`; under V1 the PLE state is silently absent. + require( + cm, + ("get_conv_states", "get_ssm_states", "conv_section_dims", "ple_layer_cache"), + "reads the recurrent, conv and PLE pools", + ) + require(cm, ("max_batch_size",), "sizes the recurrent-state slots it addresses") + # thop_attention addresses a layer by its pool offset, not its ordinal among attention layers. + pool_idx = {i: cm.layer_offsets[i] for i in attn_layers} + position_cache = cm.get_qsa_position_buffer() + check_position_plane(g, position_cache) + index_cache, kv = {}, {} + for i in attn_layers: + index_cache[i] = cm.get_index_k_buffer(i) + kv[i] = kv_planes(cm, g, i, index_cache[i]) + check_side_planes(g, index_cache, position_cache) + + check_conv_layout(g, cm.conv_section_dims) + conv = {i: cm.get_conv_states(i) for i in gdn_layers} + ssm = {i: cm.get_ssm_states(i) for i in gdn_layers} + for i in gdn_layers: + check_recurrent_pools(g, conv[i], ssm[i]) + + ple_cache = cm.ple_layer_cache(ple_layer) + assert ple_cache is not None, ( + f"no PLE lifecycle state for decoder layer {ple_layer}: the runtime allocated neither " + "the dilated conv state nor the n-gram id history" + ) + ple_conv, ple_ngram = ple_cache + check_ple_pools(g, ple_conv, ple_ngram, conv[gdn_layers[0]]) + check_state_slots(cm.mapping, cm.max_batch_size, ple_conv.shape[0]) + return RankPools(pool_idx, index_cache, position_cache, kv, conv, ssm, (ple_conv, ple_ngram)) + + +def take_draft_pools(cm, g: PoolGeometry, layer: int, *, separate: bool) -> DraftPools: + """The draft layer's checked views in `cm`, the manager its draft step is handed.""" + require( + cm, + ("get_index_k_buffer", "get_qsa_position_buffer", "get_buffers", "layer_offsets"), + "reads the draft layer's QSA index plane, position plane and paged K/V", + ) + layers = set(cm.layer_offsets) + assert layer in layers, ( + f"the MTP module's attention is global layer {layer}, which the cache manager " + f"{type(cm).__name__} does not map: the engine allocated no KV layer for the draft module" + ) + if separate: + assert layers == {layer}, ( + f"the separate draft KV cache maps layers {sorted(layers)}; it holds the draft layer " + f"{layer} alone" + ) + else: + assert len(layers) > 1, ( + f"without a separate draft KV cache the draft layer {layer} is a layer of the trunk's " + "manager, which maps the trunk's layers too" + ) + position_cache = cm.get_qsa_position_buffer() + check_position_plane(g, position_cache) + index_cache = cm.get_index_k_buffer(layer) + kv = kv_planes(cm, g, layer, index_cache) + if separate: + check_side_planes(g, {layer: index_cache}, position_cache) + return DraftPools(cm.layer_offsets[layer], index_cache, position_cache, kv) + + +__all__ = [ + "CUDA_GRAPH_PADDING_DUMMY_SLOTS", + "GDN_REPLAY_HISTORY", + "MAX_VERIFY_WIDTH", + "NVFP4_BLOCK", + "QSA_POSITION_AXES", + "SSM_STATE_DTYPES", + "DraftPools", + "KvPlanes", + "PoolGeometry", + "RankPools", + "attention_tp_size", + "check_attention_pools", + "check_conv_layout", + "check_kv_scale_planes", + "check_ple_pools", + "check_position_plane", + "check_recurrent_pools", + "check_replay_history", + "check_replay_state", + "check_replay_width", + "check_shared_selection", + "check_side_planes", + "check_state_slots", + "check_step_buffers", + "check_verify_width", + "draft_kv_cache_is_separate", + "kv_planes", + "pool_geometry", + "require", + "reserved_dummy_slots", + "take_draft_pools", + "take_pools", +] diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py new file mode 100644 index 000000000000..9066d448bf26 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/vision.py @@ -0,0 +1,704 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The vision tower and the multimodal splice's host bookkeeping. + +The checkpoint's 27-block bf16 ViT and its 2x2 patch merger turn a request's media into one +`[rows, hidden]` row per placeholder token, which the prefill target's `_splice` writes over the +token embedding. Every rank holds the whole tower and runs it eagerly, outside any captured graph, +on the media of the requests it serves (under attention TP the whole batch's, with identical rows +on every rank). The math is HuggingFace's `Qwen3_5VisionModel`. +""" + +from __future__ import annotations + +import math +from typing import NamedTuple + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_erf import gelu_erf +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.gelu_tanh import gelu_tanh +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.layer_norm import layer_norm +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.vision.patch_embed_conv3d import ( + patch_embed_conv3d, +) +from tensorrt_llm.inputs.multimodal import get_multimodal_item_metadata + +from .ops.attention.vision_rope_2d_packed_qk import vision_rope_2d_packed_qk +from .ops.vision.triton_pos_embed_interpolate import triton_pos_embed_interpolate + +#: Tower rope base (`vision_config` has no rope field); unrelated to the text 1e7. +VISION_ROPE_THETA = 10000.0 + +#: LayerNorm epsilon of `norm1`, `norm2`, `merger.norm`: a literal in the reference. +VISION_NORM_EPS = 1e-6 + +#: Most patch rows per tower call (8192 placeholder tokens x `spatial_merge_size`**2), the vision +#: ops' certified ceiling. Calls are cut between frames, so a request is bounded per frame. +VISION_CALL_MAX_PATCHES = 32768 + +#: The grids `VisionTower.warm_up` runs: one patch count that is a multiple of 16 (4x4) and one +#: that is not (2x2). +WARM_UP_GRIDS = (((1, 4, 4),), ((1, 2, 2),)) + +#: `mask_type` 0: bidirectional within a batch segment (causal would depend on raster order). +_VISION_MASK_PADDING = 0 + +# `thop_attention`'s encoder-only mode, selected by omitting the KV cache pool arguments. +# `update_kv_cache=False` is rejected, so it stays True. +_VISION_CALL_CONSTANTS = dict( + k=None, + v=None, + output_sf=None, + kv_cache_block_offsets=None, + host_kv_cache_pool_pointers=None, + host_kv_cache_pool_mapping=None, + tokens_per_block=None, + use_paged_context_fmha=False, + is_fused_qkv=True, + update_kv_cache=True, + predicted_tokens_per_seq=1, + local_layer_idx=0, + max_context_q_len_override=None, + cache_indirection=None, + kv_scale_orig_quant=None, + kv_scale_quant_orig=None, + out_scale=None, + quant_mode=0, + beam_width=1, + mask_type=_VISION_MASK_PADDING, + q_scaling=1.0, + # Rotation is applied before the op (`vision_rope_2d_packed_qk`); rope_* inert. + position_embedding_type=0, + rotary_inv_freq=None, + rotary_cos_sin=None, + rope_dim=0, + rope_base=10000.0, + rope_scale_type=0, + rope_scale=1.0, + rope_short_m_scale=1.0, + rope_long_m_scale=1.0, + rope_max_positions=1024, + rope_original_max_positions=1024, + attention_input_type=0, + is_mla_enable=False, + chunked_prefill_buffer_batch_size=1, + attention_sinks=None, + block_ids_per_seq=None, + latent_cache=None, + q_pe=None, + q_lora_rank=None, + kv_lora_rank=None, + qk_nope_head_dim=None, + qk_rope_head_dim=None, + v_head_dim=None, + rope_append=None, + attention_chunk_size=None, + softmax_stats_tensor=None, + is_spec_decoding_enabled=False, + use_spec_decoding=False, + is_spec_dec_tree=False, + spec_decoding_generation_lengths=None, + spec_decoding_position_offsets=None, + spec_decoding_packed_mask=None, + spec_decoding_bl_tree_mask_offset=None, + spec_decoding_bl_tree_mask=None, + spec_bl_tree_first_sparse_mask_offset_kv=None, + sparse_kv_indices=None, + sparse_kv_offsets=None, + sparse_attn_indices=None, + sparse_attn_offsets=None, + sparse_attn_indices_block_size=0, + mrope_rotary_cos_sin=None, + mrope_position_deltas=None, + helix_position_offsets=None, + helix_is_inactive_rank=None, + is_cross=False, + cross_kv=None, + relative_attention_bias=None, + relative_attention_max_distance=0, + kv_norm_weight=None, + kv_norm_eps=1e-6, + skip_correction_threshold=0.0, +) + +#: Per modality: the `multimodal_data` keys of the flat patch rows and the host `(t, h, w)` grid. +MEDIA_KEYS = { + "image": ("pixel_values", "image_grid_thw"), + "video": ("pixel_values_videos", "video_grid_thw"), +} + +#: `multimodal_data` tensors moved to the device; the grids stay on the host, read sync-free. +DEVICE_PATHS = ( + "image.pixel_values", + "video.pixel_values_videos", + "mrope_config.mrope_position_ids", + "mrope_config.mrope_position_deltas", +) + +#: The position-ids axes of the mRoPE layout a media request carries. +_POSITION_AXES = 3 + +#: The `multimodal_data` key of a chunked request's last tower call that later chunks still read: +#: `(first placeholder row, rows)`. The executor hands every chunk the request's own +#: `py_multimodal_data` dict, so the entry lives on the request; the chunk that consumes its last +#: row drops it, and the executor's post-prefill and termination release clears the dict. +_TOWER_ROWS_KEY = "qwen4_exp_tower_rows" + + +class _VisionBlockWeights(NamedTuple): + """One vision block's weights (`.t()` GEMM views); its LayerNorms have biases.""" + + n1_w: torch.Tensor + n1_b: torch.Tensor + qkv: torch.Tensor + qkv_b: torch.Tensor + proj: torch.Tensor + proj_b: torch.Tensor + n2_w: torch.Tensor + n2_b: torch.Tensor + fc1: torch.Tensor + fc1_b: torch.Tensor + fc2: torch.Tensor + fc2_b: torch.Tensor + + +class _VisionWeights(NamedTuple): + """The tower's weights: patch embedding, position table, blocks and patch merger.""" + + patch_w: torch.Tensor + patch_b: torch.Tensor + pos: torch.Tensor + blocks: list + m_n_w: torch.Tensor + m_n_b: torch.Tensor + m_fc1: torch.Tensor + m_fc1_b: torch.Tensor + m_fc2: torch.Tensor + m_fc2_b: torch.Tensor + + +def _vision_views(w: dict, depth: int) -> _VisionWeights: + """The weight table's `vis_*` entries (`weights.py`) as the views the ops read.""" + return _VisionWeights( + # Not transposed: the entry takes the checkpoint's `[out, in, kt, kh, kw]`. + patch_w=w["vis_patch_w"], + patch_b=w["vis_patch_b"], + pos=w["vis_pos"], + blocks=[ + _VisionBlockWeights( + n1_w=w[f"l{b}_vis_n1_w"], + n1_b=w[f"l{b}_vis_n1_b"], + # Row layout (q heads, k, v) is already thop_attention's packed form. + qkv=w[f"l{b}_vis_qkv_w"].t(), + qkv_b=w[f"l{b}_vis_qkv_b"], + proj=w[f"l{b}_vis_proj_w"].t(), + proj_b=w[f"l{b}_vis_proj_b"], + n2_w=w[f"l{b}_vis_n2_w"], + n2_b=w[f"l{b}_vis_n2_b"], + fc1=w[f"l{b}_vis_fc1_w"].t(), + fc1_b=w[f"l{b}_vis_fc1_b"], + fc2=w[f"l{b}_vis_fc2_w"].t(), + fc2_b=w[f"l{b}_vis_fc2_b"], + ) + for b in range(depth) + ], + m_n_w=w["vis_m_n_w"], + m_n_b=w["vis_m_n_b"], + m_fc1=w["vis_m_fc1_w"].t(), + m_fc1_b=w["vis_m_fc1_b"], + m_fc2=w["vis_m_fc2_w"].t(), + m_fc2_b=w["vis_m_fc2_b"], + ) + + +def _frame_calls( + grid: list[tuple[int, int, int]], max_rows: int +) -> list[list[tuple[int, int, int]]]: + """Split `grid` into calls of whole frames, at most `max_rows` rows each, as grid rows.""" + calls: list[list[tuple[int, int, int]]] = [[]] + room = max_rows + for frames, height, width in grid: + rows = height * width + assert rows <= max_rows, ( + f"one frame is {height}x{width} = {rows} patch rows; the vision ops are " + f"certified up to {max_rows} rows in one call" + ) + while frames: + take = min(frames, room // rows) + if take == 0: + calls.append([]) + room = max_rows + continue + calls[-1].append((take, height, width)) + room -= take * rows + frames -= take + return calls + + +def _merge_order_positions(grid: list[tuple[int, int, int]], merge: int) -> torch.Tensor: + """Each patch row's `(row, col)` in its frame, int64 `[rows, 2]` on the host, in + spatial-merge order: whole `merge x merge` blocks row-major, row-major inside a block, + the order the patch merger's 4-to-1 reshape consumes. A frame group repeats its frame.""" + intra = torch.arange(merge) + parts = [] + for frames, height, width in grid: + blocks_h, blocks_w = height // merge, width // merge + shape = (blocks_h, blocks_w, merge, merge) + rows = torch.arange(blocks_h)[:, None, None, None] * merge + intra[None, None, :, None] + cols = torch.arange(blocks_w)[None, :, None, None] * merge + intra[None, None, None, :] + coords = torch.stack((rows.expand(shape).reshape(-1), cols.expand(shape).reshape(-1)), 1) + parts.append(coords.repeat(frames, 1) if frames > 1 else coords) + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def _rope_tables( + grid: list[tuple[int, int, int]], merge: int, theta: float, head_dim: int, device +) -> tuple[torch.Tensor, torch.Tensor]: + """Per-row fp32 2-D rotary `cos` / `sin`, contiguous `[rows, head_dim // 2]` each. + + Height angles, then width angles; formed in fp64 and rounded once (fp32 math rounds twice).""" + rope_dim = head_dim // 2 + inv_freq = 1.0 / (theta ** (torch.arange(0, rope_dim, 2, dtype=torch.float64) / rope_dim)) + pos = _merge_order_positions(grid, merge).to(torch.float64) + angles = (pos[:, :, None] * inv_freq).flatten(1).to(device=device, dtype=torch.float32) + return angles.cos().contiguous(), angles.sin().contiguous() + + +class VisionTower: + """One request's images or videos through the 27-block ViT and its patch merger. + + Attention is bidirectional per frame (a video's frame group is a frame): one context + "sequence" per frame for `thop_attention`. The position embedding and the 2-D rope emit in + spatial-merge order, which the merger's 4-to-1 reshape needs; raster order gives a correctly + shaped, silently wrong tower. Calls of at most `VISION_CALL_MAX_PATCHES` rows are cut only + between frames: every op is per frame or per row, and merger blocks stay whole.""" + + def __init__(self, w: dict, vision_config) -> None: + """`w` is the core's loaded weight table; `vision_config` the vision half of the config.""" + vcfg = vision_config + self.depth = vcfg.depth + self.hidden = vcfg.hidden_size + self.heads = vcfg.num_heads + self.head_dim = self.hidden // self.heads + self.merge = vcfg.spatial_merge_size + self.patch = vcfg.patch_size + # One flat pixel row per patch, `(in_channels, kt, kh, kw)` channel-slowest. + self.pixel_width = vcfg.in_channels * vcfg.temporal_patch_size * self.patch * self.patch + # `spatial_merge_size ** 2` patches become one token at the merger. + self.merge_dim = self.hidden * self.merge * self.merge + self.grid_side = math.isqrt(vcfg.num_position_embeddings) + self.dtype = w["embed"].dtype + self._w = _vision_views(w, self.depth) + # Persistent scratch `thop_attention` grows in place; not the engine's own workspace. + self._ws: torch.Tensor | None = None + + def warm_up(self) -> None: + """Run the tower once per `WARM_UP_GRIDS` grid on zero pixels, so every JIT kernel it + launches is compiled at load rather than on the first media request. The rope and + position-embedding kernels take the grid as unspecialized ints; the layer norm's row + count is specialized only on divisibility by 16, and the two grids cover both.""" + device = self._w.pos.device + for grid in WARM_UP_GRIDS: + rows = sum(t * h * w for t, h, w in grid) + self._run(torch.zeros(rows, self.pixel_width, dtype=self.dtype, device=device), grid) + + def __call__(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """Return the `[sum(t*h*w) / merge**2, out_hidden_size]` placeholder rows, in grid order. + + `pixel_values` is `[sum(t*h*w), in_channels*kt*kh*kw]`; `grid_thw` stays on the host.""" + assert pixel_values.dtype == self.dtype, ( + f"the tower runs at the activation dtype {self.dtype}; the runtime handed " + f"pixel_values as {pixel_values.dtype}. The input processor's `dtype` property is " + "what decides this, and no cast happens here" + ) + assert pixel_values.dim() == 2 and pixel_values.shape[1] == self.pixel_width, ( + f"pixel_values must be [patches, {self.pixel_width}] flat patch rows; got " + f"{tuple(pixel_values.shape)}" + ) + grid: list[tuple[int, int, int]] = [] + for row in grid_thw.tolist(): + assert len(row) == 3, f"grid_thw rows are (t, h, w); got {row}" + frames, height, width = (int(v) for v in row) + assert frames >= 1 and height >= 1 and width >= 1, row + assert height % self.merge == 0 and width % self.merge == 0, ( + f"grid sides must divide by spatial_merge_size {self.merge}; got " + f"height={height}, width={width}" + ) + grid.append((frames, height, width)) + assert grid, "grid_thw describes no image or video" + described = sum(t * h * w for t, h, w in grid) + assert described == pixel_values.shape[0], ( + f"grid_thw describes {described} patch rows and pixel_values carries " + f"{pixel_values.shape[0]}" + ) + + outputs = [] + start = 0 + for call in _frame_calls(grid, VISION_CALL_MAX_PATCHES): + rows = sum(t * h * w for t, h, w in call) + outputs.append(self._run(pixel_values[start : start + rows], call)) + start += rows + return outputs[0] if len(outputs) == 1 else torch.cat(outputs, 0) + + def _run(self, pixel_values: torch.Tensor, grid: list[tuple[int, int, int]]) -> torch.Tensor: + """One tower call over whole frames; `grid` describes exactly the rows of `pixel_values`.""" + dt, dev = pixel_values.dtype, pixel_values.device + assert pixel_values.is_cuda, "pixel_values must reach the tower on the device" + + # One attention segment per frame: a video's frames do not attend to each other. + seg_lens: list[int] = [] + for frames, height, width in grid: + seg_lens.extend([height * width] * frames) + num_patches = pixel_values.shape[0] + num_segments = len(seg_lens) + max_seg = max(seg_lens) + + host_lens = torch.tensor(seg_lens, dtype=torch.int32) + host_types = torch.zeros([num_segments], dtype=torch.int32) + host_total = torch.tensor([num_patches, 0], dtype=torch.int32) + dev_lens = host_lens.to(dev) + if self._ws is None: + self._ws = torch.empty([0], dtype=torch.int8, device=dev) + cos, sin = _rope_tables(grid, self.merge, VISION_ROPE_THETA, self.head_dim, dev) + + V = self._w + pos = [ + triton_pos_embed_interpolate(V.pos, t, h, w, self.grid_side, self.merge, dt) + for t, h, w in grid + ] + h = torch.add( + patch_embed_conv3d(pixel_values, V.patch_w, V.patch_b), + pos[0] if len(pos) == 1 else torch.cat(pos, 0), + ) + attn_out = torch.empty([num_patches, self.hidden], dtype=dt, device=dev) + + for B in V.blocks: + qkv = cublas_mm(layer_norm(h, B.n1_w, B.n1_b, VISION_NORM_EPS), B.qkv, B.qkv_b) + # The q|k prefix of each packed row, rotated in place; v is never addressed. + qkv_heads = torch.reshape(qkv, [num_patches, 3 * self.heads, self.head_dim]) + vision_rope_2d_packed_qk(qkv_heads[:, : 2 * self.heads], cos, sin) + attn_out = thop_attention( + q=qkv, + output=attn_out, + workspace=self._ws, + sequence_length=dev_lens, + host_past_key_value_lengths=host_lens, + host_total_kv_lens=host_total, + context_lengths=dev_lens, + host_context_lengths=host_lens, + host_request_types=host_types, + max_num_requests=num_segments, + max_context_length=max_seg, + max_seq_len=max_seg, + attention_window_size=max_seg, + num_contexts=num_segments, + num_ctx_tokens=num_patches, + num_heads=self.heads, + num_kv_heads=self.heads, + head_size=self.head_dim, + **_VISION_CALL_CONSTANTS, + ) + h = torch.add(h, cublas_mm(attn_out, B.proj, B.proj_b)) + mlp = cublas_mm(layer_norm(h, B.n2_w, B.n2_b, VISION_NORM_EPS), B.fc1, B.fc1_b) + # Non-gated MLP with the tanh GELU (not the merger's erf). + h = torch.add(h, cublas_mm(gelu_tanh(mlp), B.fc2, B.fc2_b)) + + # Patch merger: norm over `hidden`, then the 4-to-1 reshape (rows in spatial-merge order). + merged = torch.reshape( + layer_norm(h, V.m_n_w, V.m_n_b, VISION_NORM_EPS), [-1, self.merge_dim] + ) + # The erf GELU (`nn.GELU()`); within one bf16 ulp of tanh, hence a separate entry. + return cublas_mm(gelu_erf(cublas_mm(merged, V.m_fc1, V.m_fc1_b)), V.m_fc2, V.m_fc2_b) + + +def multimodal_embeds(tower: VisionTower | None, kwargs: dict): + """Run the tower on this batch's media: `(rows, mm_token_indices)`, or None without media. + + `rows` is `[R, hidden]` in packed-batch order, `mm_token_indices` the `[R]` rows they replace. + The shell calls it: the inherited `DecoderModelForCausalLM.forward` drops `multimodal_params`, + which covers a subset of the context requests (never zip it positionally). + `multimodal_runtime` counts rows over a request's media together, in prompt order; a chunk + runs the tower on only the frames that hold its rows (`_tower_rows`), once per frame across + the request's chunks (`_chunk_rows`).""" + mm_params = kwargs.get("multimodal_params") or [] + if not mm_params: + return None + rows = [] + for params in mm_params: + data = getattr(params, "multimodal_data", None) or {} + if data.get("image") is None and data.get("video") is None: + continue + mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") + assert ( + mrope_ids is not None and mrope_ids.dim() == 3 and mrope_ids.shape[0] == _POSITION_AXES + ), ( + f"a media request needs mrope_config.mrope_position_ids of shape " + f"[{_POSITION_AXES}, 1, L]; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" + ) + runtime = getattr(params, "multimodal_runtime", None) + if runtime is not None and int(runtime.num_mm_tokens_in_chunk) == 0: + # Every placeholder row of this request lies in another chunk. + continue + assert tower is not None, "post_load_weights builds the vision tower before any step" + items = _items_in_prompt_order(params, data) + merge_unit = tower.merge * tower.merge + total = sum(t * h * w for _, _, (t, h, w) in items) // merge_unit + if runtime is None: + rows.append(_tower_rows(tower, items, 0, total)[1]) + continue + # Chunked prefill or reused prefix: only this chunk's slice. + start = int(runtime.num_cached_mm_tokens) + count = int(runtime.num_mm_tokens_in_chunk) + assert int(runtime.total_embeds_in_request) == total, ( + f"the executor counted {runtime.total_embeds_in_request} placeholder rows for " + f"this request and its grids describe {total}; the two are derived from the " + "same grids and a disagreement means the merger's row count is wrong" + ) + assert 0 <= start <= start + count <= total, (start, count, total) + rows.append(_chunk_rows(tower, data, items, start, count)) + if not rows: + return None + vision = rows[0] if len(rows) == 1 else torch.cat(rows, 0) + mm_token_indices = kwargs.get("mm_token_indices") + assert mm_token_indices is not None, ( + "the executor shipped multimodal params but no mm_token_indices; that pairing is what " + "the shell's `multimodal_token_ids` buys, and without it there is no sync-free way to " + "find the placeholder rows" + ) + assert mm_token_indices.shape[0] == vision.shape[0], ( + f"the batch has {mm_token_indices.shape[0]} placeholder rows and the tower produced " + f"{vision.shape[0]} embedding rows" + ) + return vision, mm_token_indices + + +def _items_in_prompt_order(params, data: dict) -> list[tuple[torch.Tensor, int, tuple]]: + """One request's media items in placeholder order: `(pixel_values, first patch row, (t, h, w))`. + + Two modalities interleave by `item_metadata.item_refs`, the only host-side record of order.""" + media = {} + for modality, (pixel_key, grid_key) in MEDIA_KEYS.items(): + payload = data.get(modality) + if payload is None: + continue + pixel_values, grid_thw = payload.get(pixel_key), payload.get(grid_key) + assert pixel_values is not None and grid_thw is not None, ( + f"the multimodal payload carries {sorted(payload)}; this target reads {pixel_key!r} " + f"and {grid_key!r} from it" + ) + assert not grid_thw.is_cuda, f"{grid_key} must stay on the host; see DEVICE_PATHS" + grid = [tuple(int(v) for v in row) for row in grid_thw.tolist()] + firsts, patches = [], 0 + for frames, height, width in grid: + firsts.append(patches) + patches += frames * height * width + media[modality] = (pixel_values, grid, firsts) + if len(media) == 1: + modality, (_, grid, _) = next(iter(media.items())) + order = [(modality, index) for index in range(len(grid))] + else: + metadata = params.item_metadata + order = metadata.item_refs if metadata is not None else None + assert order, ( + f"a request carrying {sorted(media)} needs item metadata to splice them in prompt " + "order; the payload holds each modality's rows apart" + ) + listed = {modality: [] for modality in media} + for modality, index in order: + assert modality in media, ( + f"item_refs lists a {modality!r} item; this request carries {sorted(media)}" + ) + listed[modality].append(int(index)) + for modality, (_, grid, _) in media.items(): + assert sorted(listed[modality]) == list(range(len(grid))), ( + f"item_refs lists {modality} items {listed[modality]}; the payload carries " + f"{len(grid)}" + ) + return [ + (media[modality][0], media[modality][2][int(index)], media[modality][1][int(index)]) + for modality, index in order + ] + + +def _tower_rows( + tower: VisionTower, items: list, start: int, count: int +) -> tuple[int, torch.Tensor]: + """`(first row, rows)`: one tower call over the frames of one request's `items` (prompt + order) that hold its placeholder rows `[start, start + count)`, from the first of those + frames' first row on. + + A frame's rows depend on its own patches alone (attention is per frame, every other op per + row), so a chunk of a long video does not run the request's other frames. Adjacent frames of + one pixel tensor stay one view; items of two modalities are copied into one tensor.""" + merge_unit = tower.merge * tower.merge + end = start + count + spans: list[list] = [] # [pixel_values, first patch row, end patch row] + grid: list[tuple[int, int, int]] = [] + first_row = None + row = 0 # the item's first placeholder row + for pixel_values, first_patch, (frames, height, width) in items: + frame_rows = height * width // merge_unit + lo = max(start - row, 0) // frame_rows + hi = min(-(-(end - row) // frame_rows), frames) + if lo < hi: + if first_row is None: + first_row = row + lo * frame_rows + patch_lo = first_patch + lo * height * width + patch_hi = first_patch + hi * height * width + if spans and spans[-1][0] is pixel_values and spans[-1][2] == patch_lo: + spans[-1][2] = patch_hi + else: + spans.append([pixel_values, patch_lo, patch_hi]) + grid.append((hi - lo, height, width)) + row += frames * frame_rows + assert first_row is not None, (start, count, row) + parts = [pixel_values[lo:hi] for pixel_values, lo, hi in spans] + pixels = parts[0] if len(parts) == 1 else torch.cat(parts, 0) + return first_row, tower(pixels, torch.tensor(grid)) + + +def _chunk_rows(tower: VisionTower, data: dict, items: list, start: int, count: int): + """Placeholder rows `[start, start + count)` of one chunk of a request's prefill. + + Rows the request's previous tower call already produced (`_TOWER_ROWS_KEY` in `data`, the + request's `multimodal_data`) are read from it; the tower runs only on the frames holding the + rest. So an image or frame that several chunks cut runs once: an 8192-row image under + 2048-token chunks runs in its first chunk and keeps its `[8192, out_hidden_size]` rows (40 MiB + at 2560 bf16) until the chunk with its last row. The rows are bitwise the one-pass rows.""" + end = start + count + kept = data.pop(_TOWER_ROWS_KEY, None) + parts = [] + if kept is not None: + first, out = kept + if first <= start < first + out.shape[0]: + take = min(end, first + out.shape[0]) + parts.append(out[start - first : take - first]) + start = take + if start < end: + kept = _tower_rows(tower, items, start, end - start) + first, out = kept + parts.append(out[start - first : end - first]) + if kept is not None and end < kept[0] + kept[1].shape[0]: + data[_TOWER_ROWS_KEY] = kept + return parts[0] if len(parts) == 1 else torch.cat(parts, 0) + + +def validate_request(tower, request) -> None: + """Admission check the executor runs on every new request, from its host-side payload only. + + It refuses what the forward would assert on: an assert inside the step ends the executor. + It also refuses precomputed image embeddings, which the tower cannot splice.""" + data = request.py_multimodal_data + if not data: + return + if data.get("multimodal_embedding") is not None: + raise ValueError( + "this target does not splice precomputed image embeddings " + "(multimodal_data['multimodal_embedding']); send the image/video pixels " + "(multimodal_data['image'/'video']) so the vision tower runs" + ) + grids = {} + for modality, (pixel_key, grid_key) in MEDIA_KEYS.items(): + media = data.get(modality) + if media is not None: + grids[modality] = _media_grid(tower, modality, media, pixel_key, grid_key) + if not grids: + return + + mrope_ids = (data.get("mrope_config") or {}).get("mrope_position_ids") + if ( + mrope_ids is None + or mrope_ids.dim() != 3 + or mrope_ids.shape[0] != _POSITION_AXES + or mrope_ids.shape[2] != request.prompt_len + ): + raise ValueError( + f"a media request needs the [{_POSITION_AXES}, 1, {request.prompt_len}] " + "mrope_config.mrope_position_ids from the input processor, one position per prompt " + f"token; got {None if mrope_ids is None else tuple(mrope_ids.shape)}" + ) + + if len(grids) > 1: + metadata = get_multimodal_item_metadata(data) + _check_item_order(grids, metadata.item_refs if metadata is not None else None) + + cumsum = data.get("multimodal_embed_mask_cumsum") + if cumsum is not None: + merge_unit = tower.merge * tower.merge + rows = sum(t * h * w for grid in grids.values() for t, h, w in grid) // merge_unit + placeholders = int(cumsum[-1]) + if placeholders != rows: + raise ValueError( + f"the prompt holds {placeholders} image/video placeholder tokens and its " + f"{' and '.join(sorted(grids))} payload produces {rows} embedding rows" + ) + + +def _media_grid(tower, modality, media, pixel_key, grid_key): + """One modality's `(frames, height, width)` rows, checked on the host.""" + pixel_values, grid_thw = media.get(pixel_key), media.get(grid_key) + if pixel_values is None or grid_thw is None: + raise ValueError( + f"the {modality} payload carries {sorted(media)}; this target reads {pixel_key!r} " + f"and {grid_key!r} from it" + ) + if grid_thw.dim() != 2 or grid_thw.shape[0] == 0 or grid_thw.shape[1] != 3: + raise ValueError( + f"{grid_key} must hold one (t, h, w) row per {modality}; got shape " + f"{tuple(grid_thw.shape)}" + ) + merge = tower.merge + grid = [tuple(int(v) for v in row) for row in grid_thw.tolist()] + for item, (frames, height, width) in enumerate(grid): + if min(frames, height, width) < 1 or height % merge or width % merge: + raise ValueError( + f"{modality} {item} has grid {(frames, height, width)}; every entry must be " + f"positive and the sides divisible by spatial_merge_size {merge}" + ) + if height * width > VISION_CALL_MAX_PATCHES: + raise ValueError( + f"{modality} {item} has frames of {height}x{width} = {height * width} patches " + f"({height * tower.patch}x{width * tower.patch} pixels); the vision tower " + f"serves frames of at most {VISION_CALL_MAX_PATCHES} patches, so send it at a " + "lower resolution" + ) + patches = sum(t * h * w for t, h, w in grid) + if tuple(pixel_values.shape) != (patches, tower.pixel_width): + raise ValueError( + f"{pixel_key} is {tuple(pixel_values.shape)}; its grids describe {patches} patch " + f"rows of {tower.pixel_width}" + ) + if pixel_values.dtype != tower.dtype: + raise ValueError( + f"{pixel_key} is {pixel_values.dtype}; the vision tower runs at {tower.dtype} " + "(multimodal_config.vision_gpu_normalize ships uint8)" + ) + return grid + + +def _check_item_order(grids, order) -> None: + """A request with several modalities lists its items in prompt order.""" + if not order: + raise ValueError( + f"a request carrying {sorted(grids)} needs item metadata to splice them in prompt " + "order; the payload holds each modality's rows apart" + ) + listed = {modality: [] for modality in grids} + for entry in order: + try: + modality, index = entry + listed[modality].append(int(index)) + except (KeyError, TypeError, ValueError) as error: + raise ValueError( + f"item_refs entry {entry!r} is not an item of this request, which carries " + f"{sorted(grids)}; entries are (modality, index) pairs" + ) from error + for modality, grid in grids.items(): + if sorted(listed[modality]) != list(range(len(grid))): + raise ValueError( + f"item_refs lists {modality} items {listed[modality]}; the payload carries " + f"{len(grid)}" + ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py new file mode 100644 index 000000000000..de3990bc40d6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/weights.py @@ -0,0 +1,1638 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Weight table and loader: Qwen3.8-Flash-Next-FP8 (hf-236dfdf_orig) / sm_103 / unified. + +One `W` per weight role (shape, checkpoint source, the rank's cut, dtype, load-time +transform); `Qwen4ExpFp8Weights` unrolls the table into declaration, manifest and load. +Storage is HF `[out, in]`; GEMM views, grouped-norm slices and fp32 gating params are +derived in `modeling.py` after load. + +**Checkpoint.** Block FP8 (`weight_block_size: [128, 128]`, dynamic activations; the engine's +`FP8_BLOCK_SCALES`). Only the routed experts (trunk 48 x 512, MTP 512) are quantized: per +expert and unfused, e4m3 `gate_proj` / `up_proj` / `down_proj` with one bf16 +`weight_scale_inv` per 128x128 block and no per-tensor scalars. Everything else is bf16, +except the PLE n-gram table (e4m3, one bf16 per-tensor `weight_scale`) and its three int64 +id-space tensors. There is no `model.language_model.norm.weight`: the stack-final +hyper-connection mixer is the final norm. `mtp.*` is loaded only with a `speculative_config`. + +**Shares** (`rank_dims`, from the engine's `Mapping`: tp1, tp2, tep2, dep2, tp4, tep4, dep4). +Rank `r` of `t` holds: + +* attention (the attention TP group; one rank under attention DP): q heads + `[24r / t, 24(r + 1) / t)`, i.e. their `q_proj` rows (q and output gate interleave per + head) and `o_proj` input columns, and KV head `r * 2 // t` (split at two ranks, repeated at + four, as `duplicate_kv_weight` and the K/V pool do). The QSA indexer and q/k norms are + replicated; +* Gated-DeltaNet (`mamba_effective_tp_size`, as the recurrent pools): key heads + `[16r / t, 16(r + 1) / t)` and value heads `[48r / t, 48(r + 1) / t)`; +* routed experts, trunk and draft: experts `[512r / t, 512(r + 1) / t)` over the whole TP + group, each at its whole 640-row intermediate. The cut ignores `moe_tp_size` / `moe_ep_size`: + 640 / 2 and 640 / 4 rows do not tile the 128-row FP8 block, so nothing is requantized. An + off-window slot adds nothing, so the windows' partial outputs sum to the layer's. + +The MTP layer is cut like a trunk attention layer. Everything else is replicated; `lm_head` +is the shell's (vocab-parallel under attention TP, whole under attention DP and at tp1). + +A cut is applied to the checkpoint entry before it is materialized, so a rank reads only its +share (a lazy safetensors slice reads only those bytes) and never an off-window expert. + +Parameters (`core.w`, an `nn.ParameterDict`) are keyed `l{i}_` for decoder layer `i`, +`l{num_hidden_layers}_mtp_` for the MTP module, `l{b}_vis_` for vision block `b`, +and bare names otherwise. + +`load` asserts coverage both ways: every parameter written once, and every checkpoint key +consumed or predicted by `expected_unconsumed`. +""" + +import mmap +import os +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Callable, NamedTuple + +import torch +from cuda.bindings import runtime as cudart +from torch import nn + +from tensorrt_llm._torch.pyexecutor.config_utils import mamba_effective_tp_size +from tensorrt_llm._utils import CUASSERT +from tensorrt_llm.logger import logger + +from .. import ple_host_table as _ple_host + +ALL_LAYERS = object() + +Cut = tuple[slice, ...] + +#: The spellings of a `TRTLLM_MODELING_V2_*` switch: these turn it on, these off. +SWITCH_ON = frozenset({"1", "true", "yes", "on"}) +SWITCH_OFF = frozenset({"0", "false", "no", "off"}) + + +def switch(env: str, on: frozenset = SWITCH_ON) -> bool: + """A `TRTLLM_MODELING_V2_*` switch, default off: unset, empty or a `SWITCH_OFF` spelling is + off, an `on` spelling on; any other value raises.""" + raw = os.environ.get(env, "") + value = raw.strip().lower() + if not value or value in SWITCH_OFF: + return False + if value in on: + return True + raise ValueError( + f"{env}={raw!r}: expected {' / '.join(sorted(on))} (on) or " + f"{' / '.join(sorted(SWITCH_OFF))} (off)" + ) + + +#: Opt-in (`rr`; default off) block-major copy of every MoE layer's routed experts for +#: `ops/moe/moe_tpb_decode` (tp4 and tep4 only, `moe_tpb_enabled`; `check_deployment` declines +#: it at another layout): expert e's 128-row intermediate block b on rank (e + b) % 4, plus this +#: rank's slice of the shared expert. The whole-expert window stays for prefill and the runner, +#: so the copy costs one more window per layer (~31 GB / rank at tep4). +MOE_TPB_ENV = "TRTLLM_MODELING_V2_DECODE_MOE_TPB" + + +def moe_tpb_requested() -> bool: + """Whether `MOE_TPB_ENV` asks for the block-major copy (`rr`); another value raises.""" + return switch(MOE_TPB_ENV, on=frozenset({"rr"})) + + +def moe_tpb_enabled(mapping) -> bool: + return moe_tpb_requested() and mapping.tp_size == 4 and not mapping.enable_attention_dp + + +@dataclass(frozen=True) +class W: + """One weight role; `d` below is the `rank_dims` bundle. + + `shape(d)` is the declared shape at this rank's share. `src` is a key template (`{p}` is + the layer prefix) or `(d, i) -> [(key, index[, cut]), ...]`: `index` addresses the + parameter (None for all of it), a tuple `key` hands its tensors to `transform` together. + `cut(d)` is this rank's rows or columns of the checkpoint tensor (one per key for a tuple + key; None reads it whole). `layers` is `ALL_LAYERS`, None (not per-layer) or + `d -> iterable[int]`. `transform(t, d)` relayouts the cut tensor(s) on the target device. + """ + + name: str + shape: Callable[[Any], tuple[int, ...]] + src: str | Callable + dtype: torch.dtype | None = None + cut: Callable[[Any], Any] | None = None + transform: Callable | None = None + layers: object | None = ALL_LAYERS + + def layer_indices(self, d: SimpleNamespace) -> list[int] | None: + if self.layers is None: + return None + if self.layers is ALL_LAYERS: + return list(range(d.num_layers)) + return list(self.layers(d)) + + def sources(self, d: SimpleNamespace, i: int | None, prefix: str) -> list[tuple]: + """`[(checkpoint key or key tuple, index into the param | None, cut | None), ...]`.""" + rows = self.src(d, i) if callable(self.src) else [(self.src.format(p=prefix), None)] + cut = self.cut(d) if self.cut is not None else None + out = [] + for row in rows: + if len(row) == 3: + assert self.cut is None, f"{self.name}: a source with its own cut takes no W.cut" + out.append(tuple(row)) + else: + out.append((*row, cut)) + return out + + +# ----------------------------------------------------------- host residency -- +# The PLE n-gram table is a `cuda` tensor over registered, device-mapped host pages (not a +# host-side gather): the gather's row indices are produced on device inside the decode graph. + +#: `cudaMemoryType` of host memory; the core asserts on it after load. +CUDA_MEMORY_TYPE_HOST = 1 + + +def memory_kind(ptr: int) -> int: + (attributes,) = CUASSERT(cudart.cudaPointerGetAttributes(ptr)) + return int(attributes.type) + + +#: `cudaHostRegister` flags: mapped (kernels may dereference the pages), portable (all contexts). +_CUDA_HOST_REGISTER_PORTABLE = 0x01 +_CUDA_HOST_REGISTER_MAPPED = 0x02 + + +def _local_cpus(device) -> set | None: + """CPUs local to `device`'s PCI function that this process may run on, or None.""" + try: + props = torch.cuda.get_device_properties(device) + path = ( + f"/sys/bus/pci/devices/{props.pci_domain_id:04x}:{props.pci_bus_id:02x}:" + f"{props.pci_device_id:02x}.0/local_cpulist" + ) + with open(path) as fh: + text = fh.read().strip() + except (OSError, AttributeError, RuntimeError): + return None + cpus = set() + for part in filter(None, text.split(",")): + lo, _, hi = part.partition("-") + cpus.update(range(int(lo), int(hi or lo) + 1)) + return (cpus & os.sched_getaffinity(0)) or None + + +class _RegisteredHostBuffer: + """A pageable CPU mapping of its own, registered (pinned, mapped) for the device. + + Unregistered before its pages are unmapped: freeing registered pages would leave the + driver mapping memory the OS may hand out again. The mapping is advised `MADV_HUGEPAGE` + before the registration faults it in, so the table sits on transparent huge pages whatever + torch's CPU allocator does (TensorRT LLM imports torch with mimalloc's THP advice off): on + GB300 the device's random row gathers from 64 KiB pages run 2.5x (64 tokens) to 23x + (1024 tokens) slower. Not torch's pinned allocator, which rounds the 47.7 GiB table up to + 64 GiB. + """ + + def __init__(self, nbytes: int, device=None) -> None: + # Registration faults the pages in on this thread, so they land on its NUMA node: + # run on `device`'s local CPUs meanwhile to put the table next to its GPU. + local = _local_cpus(device) if device is not None else None + previous = os.sched_getaffinity(0) if local else None + if local: + os.sched_setaffinity(0, local) + try: + self._mapping = mmap.mmap(-1, nbytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) + try: + self._mapping.madvise(mmap.MADV_HUGEPAGE) + except OSError as error: # a kernel without transparent huge pages + logger.warning( + f"madvise(MADV_HUGEPAGE) failed for the host-resident PLE table ({error}); " + "the device's row gathers from its base pages run several times slower" + ) + # Shares the mapping's pages, and keeps the mapping alive while it lives. + self._storage = torch.frombuffer(self._mapping, dtype=torch.uint8) + self.pointer = self._storage.data_ptr() + CUASSERT( + cudart.cudaHostRegister( + self.pointer, nbytes, _CUDA_HOST_REGISTER_PORTABLE | _CUDA_HOST_REGISTER_MAPPED + ) + ) + finally: + if previous: + os.sched_setaffinity(0, previous) + # Bound now: at interpreter shutdown the module may already be torn down. + self._unregister = cudart.cudaHostUnregister + + def __del__(self) -> None: + pointer, self.pointer = self.pointer, 0 + if pointer: + self._unregister(pointer) + + +def _host_resident_empty( + rows: int, cols: int, dtype, device +) -> tuple[torch.Tensor, _RegisteredHostBuffer]: + """`(table, owner)`: `[rows, cols]` on `device` in host memory; keep `owner` alive.""" + itemsize = torch.empty(0, dtype=dtype).element_size() + assert itemsize == 1, ( + f"the host-resident allocation reinterprets a `|u1` buffer, so its " + f"element must be one byte; {dtype} is {itemsize}" + ) + assert device.type == "cuda", f"host offload needs a cuda device to map into; got {device}" + + host = _RegisteredHostBuffer(rows * cols, device) + return _device_view(host.pointer, rows, cols, dtype, device), host + + +def _device_view(host_pointer: int, rows: int, cols: int, dtype, device) -> torch.Tensor: + """`[rows, cols]` one-byte `dtype` view of mapped host pages at `host_pointer`.""" + (device_pointer,) = CUASSERT(cudart.cudaHostGetDevicePointer(host_pointer, 0)) + + # No public torch API wraps a pointer the caching allocator did not hand out. `|u1` + # because `__cuda_array_interface__` has no float8 typestr; reinterpreted below. + interface = SimpleNamespace( + __cuda_array_interface__={ + "shape": (rows, cols), + "typestr": "|u1", + "data": (int(device_pointer), False), + "version": 3, + "strides": None, + } + ) + table = torch.as_tensor(interface, device=device).view(dtype) + # A copy here (`as_tensor` may copy instead of view) would be a silent 48 GiB allocation. + assert table.data_ptr() == int(device_pointer), ( + "torch.as_tensor copied the host-mapped buffer instead of viewing it, " + "so the table would be device-resident after all" + ) + assert table.shape == (rows, cols) and table.dtype == dtype, (table.shape, table.dtype) + return table + + +# ------------------------------------------------------------------ the table -- + +_LM = "model.language_model" +_VISUAL = "model.visual" +_MTP = "mtp" +_MTP_LAYER = "mtp.layers.0" + +#: FP8 block edge: 128x128 expert weight-scale blocks, 1x128 activation quantization. +FP8_BLOCK = 128 + +#: The n-gram table is kept as stored (per-tensor FP8); the gather dequantizes the rows it reads. +PLE_TABLE_DTYPE = torch.float8_e4m3fn + +#: Expert block-scale dtype the runner takes; the checkpoint's bf16 converts exactly. +EXPERT_SCALE_DTYPE = torch.float32 + +# Parameter-key suffixes after which `load` empties the CUDA cache, so peak load memory +# stays near one parameter (sources are staged one expert or shard at a time). +_RELEASE_AFTER = ( + "_fc1_w", + "_fc1_s", + "_fc2_w", + "_fc2_s", + "ple_table", +) + + +def _read(entry, cut: Cut | None) -> torch.Tensor: + """Materialize `entry`, reading only `cut` of it (a 0-dim entry rejects `[:]`).""" + if cut is None: + return entry if getattr(entry, "ndim", 1) == 0 else entry[:] + return entry[cut[0]] if len(cut) == 1 else entry[cut] + + +# --- cuts: this rank's block of rows or columns of a checkpoint tensor --- + + +def _span(start: int, width: int) -> slice: + return slice(start, start + width) + + +def _q_rows(d) -> Cut: + """`q_proj` rows of this rank's q heads: each head's q rows, then its gate rows.""" + return (_span(d.q_head_start * 2 * d.head_dim, 2 * d.q_width),) + + +def _kv_rows(d) -> Cut: + return (_span(d.kv_head_start * d.head_dim, d.kv_width),) + + +def _o_cols(d) -> Cut: + return (slice(None), _span(d.q_head_start * d.head_dim, d.q_width)) + + +def _key_rows(d, section: int) -> Cut: + """Rows of this rank's key heads in section 0 (q) or 1 (k) of a `[q | k | v]` GDN tensor.""" + return (_span(section * d.key_dim_total + d.k_head_start * d.head_k_dim, d.key_dim),) + + +def _value_rows(d, base: int = 0) -> Cut: + return (_span(base + d.v_head_start * d.head_v_dim, d.value_dim),) + + +def _value_head_rows(d) -> Cut: + """One row per value head (`in_proj_a`, `in_proj_b`, `A_log`, `dt_bias`).""" + return (_span(d.v_head_start, d.num_v_heads),) + + +def _value_cols(d) -> Cut: + return (slice(None), _span(d.v_head_start * d.head_v_dim, d.value_dim)) + + +def _qkv_blocks(d) -> tuple[tuple[slice, Cut], ...]: + """(destination rows, checkpoint cut) of this rank's q, k and v blocks, in conv-pool order.""" + return ( + (slice(0, d.key_dim), _key_rows(d, 0)), + (slice(d.key_dim, 2 * d.key_dim), _key_rows(d, 1)), + (slice(2 * d.key_dim, d.conv_dim), _value_rows(d, 2 * d.key_dim_total)), + ) + + +# --- relayouts, on the rank's cut --- + + +#: Rows after the shared expert's gate|up rows in its fused operand: the per-token gate's +#: one row, then zeros, so the GEMM's output row stays a whole number of 16-byte vectors. +SH_GATE_ROWS = 8 + +#: `in_proj_w` rows are padded to a multiple of this (`d.in_proj_rows`): FlashInfer's low-M warp +#: split-K GEMM needs 16 | N (tp4/tep4: 4120 -> 4128). The checkpoint fills +#: `sum(d.in_proj_split)` rows; the core zeroes the rest after load. +IN_PROJ_ROW_ALIGN = 16 + + +def _cat_rows(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(gate, up, gate logit) -> `[gate ; up ; logit ; zeros]`, one GEMM for gate|up and the logit. + + Gate first: the half order `flashinfer_silu_and_mul` reads. + """ + gate, up, sh_gate = t + pad = sh_gate.new_zeros(SH_GATE_ROWS - 1, sh_gate.shape[-1]) + return torch.cat((gate, up, sh_gate, pad), dim=0).contiguous() + + +def _attn_in_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(q_proj, k_proj, v_proj, index_qk_proj) at this rank's cut -> `[q|k|v|gate|index|pad]`. + + `q_proj` rows are `[heads, 2, head_dim]` (q, then gate). q goes first so the GEMM output + leads with the `[q | k | v]` the fused norm+rope rewrites in place. + """ + q, k, v, idx = t + heads, head_dim, hidden = d.heads_q, d.head_dim, q.shape[-1] + qg = q.reshape(heads, 2, head_dim, hidden) + pad = q.new_zeros(d.index_head_dim, hidden) + rows = (qg[:, 0].reshape(heads * head_dim, hidden), k, v) + rows += (qg[:, 1].reshape(heads * head_dim, hidden), idx, pad) + return torch.cat(rows, dim=0).contiguous() + + +def _expert_fc1_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) e4m3 codes -> one expert's FC1 operand `[up ; gate]`, plain row-major.""" + up, gate = t + return torch.cat([up, gate], dim=0).contiguous() + + +def _expert_fc1_s(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) bf16 block scales -> one expert's `[up ; gate]` FC1 scales in fp32.""" + up, gate = t + return torch.cat([up, gate], dim=0).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _expert_fc2_s(t: torch.Tensor, d) -> torch.Tensor: + """`down_proj` bf16 block scales -> fp32; FC2's e4m3 codes are used as stored.""" + return t.to(EXPERT_SCALE_DTYPE).contiguous() + + +# --- which layers carry a weight --- + + +def _attn_layers(d): + """The full-attention layers (`i % 4 == 3` on this checkpoint).""" + return [i for i, a in enumerate(d.is_attn) if a] + + +def _gdn_layers(d): + return [i for i, a in enumerate(d.is_attn) if not a] + + +def _vision_blocks(d): + """The vision tower's blocks. Keyed `l{b}_vis_*`: block `b`, not decoder layer `b`.""" + return range(d.v_depth) + + +def _mtp_only(d): + """The MTP module, at its own layer index `num_hidden_layers`, only when it is live.""" + return [d.num_layers] if d.mtp_enabled else [] + + +# --- where the bytes come from --- + + +def _ple(suffix): + return lambda d, i: [(f"{_LM}.layers.{d.ple_layer}.ple.{suffix}", None)] + + +def _ngram_prefix(ple_layer: int) -> str: + """The quantized n-gram submodule, holding the shards and their per-tensor scale.""" + return f"{_LM}.layers.{ple_layer}.ple.ple_embedding.ngram_embedding" + + +def _visual(suffix): + return lambda d, i: [(f"{_VISUAL}.{suffix}", None)] + + +def _vision_block(suffix): + return lambda d, i: [(f"{_VISUAL}.blocks.{i}.{suffix}", None)] + + +def _window(d) -> tuple[int, int]: + """(first expert, count) of this rank's routed-expert window, trunk and draft alike.""" + return d.expert_offset, d.local_experts + + +def _experts(prefix, *suffixes, window): + """One source per expert of `window(d)`, at its local index.""" + + def build(d, i): + start, count = window(d) + base = prefix(i) + out = [] + for local in range(count): + keys = tuple(f"{base}.experts.{start + local}.{s}" for s in suffixes) + out.append((keys if len(keys) > 1 else keys[0], (local,))) + return out + + return build + + +def tpb_blocks(d) -> list[tuple[int, int]]: + """(expert, intermediate block) of the 640 blocks this rank's block-major copy holds + (`MOE_TPB_ENV`): j < 512 is expert j's block (rank - j) % 4, j >= 512 is expert + 4 (j - 512) + rank's block 4.""" + blocks = d.moe_inter // FP8_BLOCK + r, n = d.tp_rank, d.num_experts + return [ + (j, (r - j) % 4) if j < n else (4 * (j - n) + r, blocks - 1) for j in range(n * blocks // 4) + ] + + +def _tpb_experts(prefix, suffixes, cut): + """One source per block of `tpb_blocks(d)`, at its block index, read through `cut(b)`.""" + + def build(d, i): + base = prefix(i) + out = [] + for j, (e, b) in enumerate(tpb_blocks(d)): + keys = tuple(f"{base}.experts.{e}.{s}" for s in suffixes) + c = cut(b) + out.append((keys, (j,), (c,) * len(keys)) if len(keys) > 1 else (keys[0], (j,), c)) + return out + + return build + + +def _tpb_fc1_w(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + up, gate = t + return torch.cat([up, gate], dim=0).contiguous() + + +def _tpb_fc1_s(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(up, gate) block-scale rows of one block -> `[2, hidden / 128]` fp32.""" + up, gate = t + return torch.cat([up, gate], dim=0).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _tpb_fc2_s(t: torch.Tensor, d) -> torch.Tensor: + """down_proj's block-scale column of one block -> `[hidden / 128]` fp32.""" + return t.reshape(-1).to(EXPERT_SCALE_DTYPE).contiguous() + + +def _tpb_sh_gu(t: tuple[torch.Tensor, ...], d) -> torch.Tensor: + """(gate, up) rows of one 32-column shared-expert item -> `[64, hidden]`. + + 8 groups of 4 columns, each `[gate 4 ; up 4]` (`moe_tpb_decode.SharedBlocks`). + """ + gate, up = t + return ( + torch.stack([gate.reshape(8, 4, -1), up.reshape(8, 4, -1)], dim=1) + .reshape(64, -1) + .contiguous() + ) + + +def _block_shared(tag, prefix, layers): + """This rank's TP slice of a MoE layer's bf16 shared expert (`moe_tpb_decode.SharedBlocks`). + + Columns `[rank * w, (rank + 1) * w)` (w = shared_inter / 4) in 5 items of 32: gate / up + rows interleaved by 4 (`tpb_sh_gu [5, 64, hidden]`) and down_proj columns + (`tpb_sh_dn [5, hidden, 32]`). + """ + items, cols = 5, 32 + + def lo(d, sb): + return d.tp_rank * (d.shared_inter // 4) + cols * sb + + return ( + W( + f"{tag}tpb_sh_gu", + shape=lambda d: (items, 64, d.hidden), + src=lambda d, i: [ + ( + ( + f"{prefix(i)}.shared_expert.gate_proj.weight", + f"{prefix(i)}.shared_expert.up_proj.weight", + ), + (sb,), + ((slice(lo(d, sb), lo(d, sb) + cols),), (slice(lo(d, sb), lo(d, sb) + cols),)), + ) + for sb in range(items) + ], + transform=_tpb_sh_gu, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_sh_dn", + shape=lambda d: (items, d.hidden, cols), + src=lambda d, i: [ + ( + f"{prefix(i)}.shared_expert.down_proj.weight", + (sb,), + (slice(None), slice(lo(d, sb), lo(d, sb) + cols)), + ) + for sb in range(items) + ], + layers=_tpb_layers(layers), + ), + ) + + +def _tpb_layers(layers): + """`layers` when the block-major copy is on (`d.moe_tpb`), else none.""" + + def pick(d): + if not getattr(d, "moe_tpb", False): + return [] + return range(d.num_layers) if layers is ALL_LAYERS else list(layers(d)) + + return pick + + +def _block_experts(tag, prefix, layers): + """One MoE layer's routed experts block-major (`MOE_TPB_ENV`), one entry per `tpb_blocks(d)`.""" + rows = lambda b: (slice(FP8_BLOCK * b, FP8_BLOCK * (b + 1)),) # noqa: E731 + srow = lambda b: (slice(b, b + 1),) # noqa: E731 + cols = lambda b: (slice(None), slice(FP8_BLOCK * b, FP8_BLOCK * (b + 1))) # noqa: E731 + scol = lambda b: (slice(None), slice(b, b + 1)) # noqa: E731 + n = lambda d: len(tpb_blocks(d)) # noqa: E731 + return ( + W( + f"{tag}tpb_w1", + shape=lambda d: (n(d), 2 * FP8_BLOCK, d.hidden), + dtype=f8, + src=_tpb_experts(prefix, ("up_proj.weight", "gate_proj.weight"), rows), + transform=_tpb_fc1_w, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_s1", + shape=lambda d: (n(d), 2, d.hidden // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_tpb_experts( + prefix, ("up_proj.weight_scale_inv", "gate_proj.weight_scale_inv"), srow + ), + transform=_tpb_fc1_s, + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_w2", + shape=lambda d: (n(d), d.hidden, FP8_BLOCK), + dtype=f8, + src=_tpb_experts(prefix, ("down_proj.weight",), cols), + layers=_tpb_layers(layers), + ), + W( + f"{tag}tpb_s2", + shape=lambda d: (n(d), d.hidden // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_tpb_experts(prefix, ("down_proj.weight_scale_inv",), scol), + transform=_tpb_fc2_s, + layers=_tpb_layers(layers), + ), + ) + + +def _trunk_mlp(i): + return f"{_LM}.layers.{i}.mlp" + + +def _mtp_mlp(i): + return f"{_MTP_LAYER}.mlp" + + +def _in_proj(d, i): + """GDN input projections at this rank's heads, in `in_proj_w`'s row order + `[q | k | v (conv_dim) | z (value_dim) | a | b]` (`d.in_proj_split`).""" + p = f"{_LM}.layers.{i}.linear_attn" + out = [(f"{p}.in_proj_qkv.weight", (rows,), cut) for rows, cut in _qkv_blocks(d)] + row = d.conv_dim + out.append((f"{p}.in_proj_z.weight", (_span(row, d.value_dim),), _value_rows(d))) + row += d.value_dim + for name in ("in_proj_a", "in_proj_b"): + out.append((f"{p}.{name}.weight", (_span(row, d.num_v_heads),), _value_head_rows(d))) + row += d.num_v_heads + return out + + +def _conv(d, i): + """The depthwise conv's `[q | k | v]` channels of this rank's heads.""" + key = f"{_LM}.layers.{i}.linear_attn.conv1d.weight" + return [(key, (rows,), cut) for rows, cut in _qkv_blocks(d)] + + +#: One routed expert's checkpoint tensors, left unread by a rank outside its window. +_PROJECTIONS = ("gate_proj", "up_proj", "down_proj") +_EXPERT_TENSORS = tuple(f"{p}.{s}" for p in _PROJECTIONS for s in ("weight", "weight_scale_inv")) + + +#: (parameter tag, checkpoint module) of the two per-layer hyper-connection gates. +_HC_MODULES = (("ahc", "attn_hyper_connection"), ("mhc", "mlp_hyper_connection")) + + +def _hyper_connection(tag, src, layers): + """The four weights of one gated-residual hyper-connection; `src(name)` names the source.""" + return ( + W(f"{tag}_norm", shape=lambda d: (d.stream,), src=src("hc_norm"), layers=layers), + W( + f"{tag}_down", + shape=lambda d: (d.hc_lowrank, d.stream), + src=src("input_mix_weight_down"), + layers=layers, + ), + W( + f"{tag}_up", + shape=lambda d: (d.stream, d.hc_lowrank), + src=src("input_mix_weight_up"), + layers=layers, + ), + W( + f"{tag}_inject", + shape=lambda d: (d.hc_count, d.stream), + src=src("block_inject_weight"), + layers=layers, + ), + ) + + +def _mixer(tag, prefix, layers): + """A stack-final hyper-connection mixer: a layer's gate minus the injection weight.""" + return ( + W( + f"{tag}_norm", + shape=lambda d: (d.stream,), + src=f"{prefix}.hc_norm.weight", + layers=layers, + ), + W( + f"{tag}_down", + shape=lambda d: (d.hc_lowrank, d.stream), + src=f"{prefix}.input_mix_weight_down.weight", + layers=layers, + ), + W( + f"{tag}_up", + shape=lambda d: (d.stream, d.hc_lowrank), + src=f"{prefix}.input_mix_weight_up.weight", + layers=layers, + ), + ) + + +def _attn_in_fused(d, i): + """Trunk attention input sources: q/k/v_proj at this rank's heads, replicated index_qk_proj.""" + p = f"{_LM}.layers.{i}.self_attn" + keys = ( + f"{p}.q_proj.weight", + f"{p}.k_proj.weight", + f"{p}.v_proj.weight", + f"{p}.indexer.index_qk_proj.weight", + ) + return [(keys, None, (_q_rows(d), _kv_rows(d), _kv_rows(d), None))] + + +def _mtp_attn_in_fused(d, i): + """`_attn_in_fused` for the draft layer.""" + p = f"{_MTP_LAYER}.self_attn" + keys = ( + f"{p}.q_proj.weight", + f"{p}.k_proj.weight", + f"{p}.v_proj.weight", + f"{p}.indexer.index_qk_proj.weight", + ) + return [(keys, None, (_q_rows(d), _kv_rows(d), _kv_rows(d), None))] + + +def _attn_out_and_norms(tag, prefix, layers): + """o_proj at this rank's q heads, and the replicated per-head q/k norms.""" + return ( + W( + f"{tag}o_w", + shape=lambda d: (d.hidden, d.q_width), + src=f"{prefix}.o_proj.weight", + cut=_o_cols, + layers=layers, + ), + W( + f"{tag}q_norm", + shape=lambda d: (d.head_dim,), + src=f"{prefix}.q_norm.weight", + layers=layers, + ), + W( + f"{tag}k_norm", + shape=lambda d: (d.head_dim,), + src=f"{prefix}.k_norm.weight", + layers=layers, + ), + ) + + +def _index_branch(tag, prefix, layers): + """The QSA index branch's two per-head norms; its projection is part of `attn_in_w`.""" + return ( + W( + f"{tag}idx_qn", + shape=lambda d: (d.index_head_dim,), + src=f"{prefix}.q_layernorm.weight", + layers=layers, + ), + W( + f"{tag}idx_kn", + shape=lambda d: (d.index_head_dim,), + src=f"{prefix}.k_layernorm.weight", + layers=layers, + ), + ) + + +def _shared_expert(tag, prefix, layers): + """The replicated bf16 shared expert; its gate logit shares the gate|up operand.""" + return ( + W( + f"{tag}sh_gu", + shape=lambda d: (2 * d.shared_inter + SH_GATE_ROWS, d.hidden), + src=lambda d, i: [ + ( + ( + f"{prefix(i)}.shared_expert.gate_proj.weight", + f"{prefix(i)}.shared_expert.up_proj.weight", + f"{prefix(i)}.shared_expert_gate.weight", + ), + None, + ) + ], + transform=_cat_rows, + layers=layers, + ), + W( + f"{tag}sh_dn", + shape=lambda d: (d.hidden, d.shared_inter), + src=lambda d, i: [(f"{prefix(i)}.shared_expert.down_proj.weight", None)], + layers=layers, + ), + ) + + +def _routed_experts(tag, prefix, layers): + """One MoE layer's routed experts at this rank's window, as `fp8_block_scale_moe_runner` + takes them. + + Plain row-major e4m3 (no shuffle, swizzle or padding) with one fp32 scale per 128x128 + block; whole experts only, no cut reaches the intermediate. + """ + return ( + W( + f"{tag}fc1_w", + shape=lambda d: (d.local_experts, 2 * d.moe_inter, d.hidden), + dtype=f8, + src=_experts(prefix, "up_proj.weight", "gate_proj.weight", window=_window), + transform=_expert_fc1_w, + layers=layers, + ), + W( + f"{tag}fc1_s", + shape=lambda d: ( + d.local_experts, + (2 * d.moe_inter) // FP8_BLOCK, + d.hidden // FP8_BLOCK, + ), + dtype=EXPERT_SCALE_DTYPE, + src=_experts( + prefix, "up_proj.weight_scale_inv", "gate_proj.weight_scale_inv", window=_window + ), + transform=_expert_fc1_s, + layers=layers, + ), + W( + f"{tag}fc2_w", + shape=lambda d: (d.local_experts, d.hidden, d.moe_inter), + dtype=f8, + src=_experts(prefix, "down_proj.weight", window=_window), + layers=layers, + ), + W( + f"{tag}fc2_s", + shape=lambda d: (d.local_experts, d.hidden // FP8_BLOCK, d.moe_inter // FP8_BLOCK), + dtype=EXPERT_SCALE_DTYPE, + src=_experts(prefix, "down_proj.weight_scale_inv", window=_window), + transform=_expert_fc2_s, + layers=layers, + ), + ) + + +f8, i64 = torch.float8_e4m3fn, torch.int64 + + +class Qwen4ExpFp8Weights: + """`declare`, `manifest` and `load` over `WEIGHTS`, at one rank's share. + + `declare` and `manifest` share `_targets`, so they agree on which parameters exist. + """ + + WEIGHTS: tuple[W, ...] = ( + # --- the two per-layer hyper-connection gates: every trunk layer, replicated --- + *( + w + for tag, module in _HC_MODULES + for w in _hyper_connection( + tag, lambda name, m=module: f"{{p}}.{m}.{name}.weight", ALL_LAYERS + ) + ), + # --- the gated GQA layers and their QSA index branch --- + W( + "attn_in_w", + shape=lambda d: (d.attn_in_rows, d.hidden), + src=_attn_in_fused, + transform=_attn_in_w, + layers=_attn_layers, + ), + *_attn_out_and_norms("", "{p}.self_attn", _attn_layers), + *_index_branch("", "{p}.self_attn.indexer", _attn_layers), + # --- the Gated-DeltaNet layers, at this rank's key and value heads --- + W( + "in_proj_w", + shape=lambda d: (d.in_proj_rows, d.hidden), + src=_in_proj, + layers=_gdn_layers, + ), + W( + "A_log", + shape=lambda d: (d.num_v_heads,), + src="{p}.linear_attn.A_log", + cut=_value_head_rows, + layers=_gdn_layers, + ), + W( + "dt_bias", + shape=lambda d: (d.num_v_heads,), + src="{p}.linear_attn.dt_bias", + cut=_value_head_rows, + layers=_gdn_layers, + ), + # Stored `[conv_dim, 1, width]`; the core takes the `[conv_dim, width]` view after load. + W( + "conv_w", + shape=lambda d: (d.conv_dim, 1, d.conv_kernel), + src=_conv, + layers=_gdn_layers, + ), + W( + "gnorm", + shape=lambda d: (d.head_v_dim,), + src="{p}.linear_attn.norm.weight", + layers=_gdn_layers, + ), + W( + "out_w", + shape=lambda d: (d.hidden, d.value_dim), + src="{p}.linear_attn.out_proj.weight", + cut=_value_cols, + layers=_gdn_layers, + ), + # --- MoE on every layer: routed-expert window, replicated router and shared expert --- + W("router", shape=lambda d: (d.num_experts, d.hidden), src="{p}.mlp.gate.weight"), + *_routed_experts("", _trunk_mlp, ALL_LAYERS), + *_block_experts("", _trunk_mlp, ALL_LAYERS), + *_block_shared("", _trunk_mlp, ALL_LAYERS), + *_shared_expert("", _trunk_mlp, ALL_LAYERS), + # --- not per-layer: the stack-final mixer, which is also the final norm --- + *_mixer("fhc", f"{_LM}.hyper_connection_mixer", None), + W( + "embed", + shape=lambda d: (d.vocab, d.hidden), + src=f"{_LM}.embed_tokens.weight", + layers=None, + ), + # --- PLE, on its one decoder layer (the n-gram table itself is declared at load) --- + W( + "ple_key_w", + shape=lambda d: (d.stream, d.hidden), + src=_ple("key_proj.weight"), + layers=None, + ), + W( + "ple_val_w", + shape=lambda d: (d.ple_embed_dim, d.hidden), + src=_ple("value_proj.weight"), + layers=None, + ), + W("ple_nk", shape=lambda d: (d.stream,), src=_ple("norm_key.weight"), layers=None), + W("ple_nq", shape=lambda d: (d.stream,), src=_ple("norm_query.weight"), layers=None), + W("ple_nc", shape=lambda d: (d.stream,), src=_ple("norm_conv.weight"), layers=None), + W( + "ple_conv_w", + shape=lambda d: (d.stream, 1, d.ple_conv_kernel), + src=_ple("conv1d.weight"), + layers=None, + ), + W( + "ple_mult", + shape=lambda d: (d.ngram_size,), + dtype=i64, + src=_ple("ple_embedding.layer_multipliers"), + layers=None, + ), + W( + "ple_vocab", + shape=lambda d: (d.ngram_heads,), + dtype=i64, + src=_ple("ple_embedding.ngram_heads_vocab_sizes"), + layers=None, + ), + W( + "ple_offs", + shape=lambda d: (d.ngram_heads,), + dtype=i64, + src=_ple("ple_embedding.ngram_heads_offsets"), + layers=None, + ), + # The table's scale is on `ngram_embedding` with the shards, not on `ple_embedding`. + W( + "ple_tscale", + shape=lambda d: (1,), + src=lambda d, i: [(f"{_ngram_prefix(d.ple_layer)}.weight_scale", None)], + layers=None, + ), + # --- the vision tower: all bf16, replicated, one-to-one, no transform --- + W( + "vis_patch_w", + shape=lambda d: (d.v_hidden, d.v_channels, d.v_tpatch, d.v_patch, d.v_patch), + src=_visual("patch_embed.proj.weight"), + layers=None, + ), + W( + "vis_patch_b", + shape=lambda d: (d.v_hidden,), + src=_visual("patch_embed.proj.bias"), + layers=None, + ), + W( + "vis_pos", + shape=lambda d: (d.v_pos_rows, d.v_hidden), + src=_visual("pos_embed.weight"), + layers=None, + ), + W( + "vis_n1_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.weight"), + layers=_vision_blocks, + ), + W( + "vis_n1_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm1.bias"), + layers=_vision_blocks, + ), + W( + "vis_qkv_w", + shape=lambda d: (3 * d.v_hidden, d.v_hidden), + src=_vision_block("attn.qkv.weight"), + layers=_vision_blocks, + ), + W( + "vis_qkv_b", + shape=lambda d: (3 * d.v_hidden,), + src=_vision_block("attn.qkv.bias"), + layers=_vision_blocks, + ), + W( + "vis_proj_w", + shape=lambda d: (d.v_hidden, d.v_hidden), + src=_vision_block("attn.proj.weight"), + layers=_vision_blocks, + ), + W( + "vis_proj_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("attn.proj.bias"), + layers=_vision_blocks, + ), + W( + "vis_n2_w", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.weight"), + layers=_vision_blocks, + ), + W( + "vis_n2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("norm2.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc1_w", + shape=lambda d: (d.v_inter, d.v_hidden), + src=_vision_block("mlp.linear_fc1.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc1_b", + shape=lambda d: (d.v_inter,), + src=_vision_block("mlp.linear_fc1.bias"), + layers=_vision_blocks, + ), + W( + "vis_fc2_w", + shape=lambda d: (d.v_hidden, d.v_inter), + src=_vision_block("mlp.linear_fc2.weight"), + layers=_vision_blocks, + ), + W( + "vis_fc2_b", + shape=lambda d: (d.v_hidden,), + src=_vision_block("mlp.linear_fc2.bias"), + layers=_vision_blocks, + ), + # The merger's norm runs before the 4-to-1 shuffle, so it is `hidden` wide. + W( + "vis_m_n_w", + shape=lambda d: (d.v_hidden,), + src=_visual("merger.norm.weight"), + layers=None, + ), + W("vis_m_n_b", shape=lambda d: (d.v_hidden,), src=_visual("merger.norm.bias"), layers=None), + W( + "vis_m_fc1_w", + shape=lambda d: (d.v_merge_dim, d.v_merge_dim), + src=_visual("merger.linear_fc1.weight"), + layers=None, + ), + W( + "vis_m_fc1_b", + shape=lambda d: (d.v_merge_dim,), + src=_visual("merger.linear_fc1.bias"), + layers=None, + ), + W( + "vis_m_fc2_w", + shape=lambda d: (d.hidden, d.v_merge_dim), + src=_visual("merger.linear_fc2.weight"), + layers=None, + ), + W( + "vis_m_fc2_b", + shape=lambda d: (d.hidden,), + src=_visual("merger.linear_fc2.bias"), + layers=None, + ), + # --- the MTP draft module, at layer index num_hidden_layers, only when live --- + # A trunk full-attention layer plus a two-branch front end; keys carry no `{p}`. + W( + "mtp_pre_e_norm", + shape=lambda d: (d.hidden,), + src=f"{_MTP}.pre_fc_norm_embedding.weight", + layers=_mtp_only, + ), + # `[hc_count * hidden]`: the module's input is the trunk's four-stream residual. + W( + "mtp_pre_h_norm", + shape=lambda d: (d.stream,), + src=f"{_MTP}.pre_fc_norm_hidden.weight", + layers=_mtp_only, + ), + W( + "mtp_fc_e", + shape=lambda d: (d.hidden, d.hidden), + src=f"{_MTP}.fc_embedding.weight", + layers=_mtp_only, + ), + W( + "mtp_fc_h", + shape=lambda d: (d.hidden, d.hidden), + src=f"{_MTP}.fc_hidden.weight", + layers=_mtp_only, + ), + *( + w + for tag, module in _HC_MODULES + for w in _hyper_connection( + f"mtp_{tag}", + lambda name, m=module: f"{_MTP_LAYER}.{m}.{name}.weight", + _mtp_only, + ) + ), + W( + "mtp_attn_in_w", + shape=lambda d: (d.attn_in_rows, d.hidden), + src=_mtp_attn_in_fused, + transform=_attn_in_w, + layers=_mtp_only, + ), + *_attn_out_and_norms("mtp_", f"{_MTP_LAYER}.self_attn", _mtp_only), + *_index_branch("mtp_", f"{_MTP_LAYER}.self_attn.indexer", _mtp_only), + W( + "mtp_router", + shape=lambda d: (d.num_experts, d.hidden), + src=f"{_MTP_LAYER}.mlp.gate.weight", + layers=_mtp_only, + ), + *_routed_experts("mtp_", _mtp_mlp, _mtp_only), + *_block_experts("mtp_", _mtp_mlp, _mtp_only), + *_block_shared("mtp_", _mtp_mlp, _mtp_only), + *_shared_expert("mtp_", _mtp_mlp, _mtp_only), + # The module's own stack-final mixer, which is its `shared_head`. + *_mixer("mtp_fhc", f"{_MTP}.hyper_connection_mixer", _mtp_only), + ) + + def dims(self, core) -> SimpleNamespace: + """This rank's `rank_dims` bundle; `core.vision_config` must be set before `declare`.""" + return rank_dims( + core.model_config.pretrained_config, + core.vision_config, + core.model_config.mapping, + mtp_enabled=getattr(core.model_config, "spec_config", None) is not None, + dtype=core.model_config.torch_dtype, + ) + + def expected_unconsumed(self, d: SimpleNamespace, keys) -> set: + """Keys this rank leaves unread: off-window experts, and `mtp.*` without MTP. + + Computed from `d`, not the checkpoint, so a real weight's key can never land here. + """ + start, count = _window(d) + + def off_window(i): + if getattr(d, "moe_tpb", False): + return [] # the block-major copy reads a block of every expert + return [e for e in range(d.num_experts) if not start <= e < start + count] + + out = { + f"{_trunk_mlp(i)}.experts.{e}.{t}" + for i in range(d.num_layers) + for e in off_window(i) + for t in _EXPERT_TENSORS + } + if not d.mtp_enabled: + return out | {k for k in keys if k.startswith(f"{_MTP}.")} + out.update( + f"{_mtp_mlp(None)}.experts.{e}.{t}" + for e in off_window(d.num_layers) + for t in _EXPERT_TENSORS + ) + return out + + def declare(self, d: SimpleNamespace) -> nn.ParameterDict: + """Allocate every weight at this rank's share (meta-init intercepts `torch.empty`).""" + w = nn.ParameterDict() + for entry in self.WEIGHTS: + dtype = entry.dtype or d.dtype + for key, _, _ in self._targets(entry, d): + w[key] = nn.Parameter( + torch.empty(*entry.shape(d), dtype=dtype), requires_grad=False + ) + return w + + def _targets(self, entry: W, d: SimpleNamespace) -> list[tuple[str, int | None, str]]: + """`[(param key, layer index | None, checkpoint layer prefix), ...]` of one entry.""" + layers = entry.layer_indices(d) + if layers is None: + return [(entry.name, None, "")] + return [(f"l{i}_{entry.name}", i, f"{_LM}.layers.{i}") for i in layers] + + def manifest(self, d: SimpleNamespace) -> dict[str, list[tuple]]: + """Param key -> `[(checkpoint key, index | None, cut | None, transform | None), ...]`.""" + rows: dict[str, list[tuple]] = {} + for entry in self.WEIGHTS: + for key, i, prefix in self._targets(entry, d): + rows[key] = [ + (ckpt, index, cut, entry.transform) + for ckpt, index, cut in entry.sources(d, i, prefix) + ] + return rows + + def read(self, weights, ckpt_key, cut, transform, d, device=None) -> torch.Tensor: + """One manifest source -> its destination's tensor: keys read through cuts, transformed.""" + multi = isinstance(ckpt_key, tuple) + keys = ckpt_key if multi else (ckpt_key,) + cuts = (cut if cut is not None else (None,) * len(keys)) if multi else (cut,) + assert len(cuts) == len(keys), (ckpt_key, cut) + for key in keys: + assert key in weights, f"checkpoint key missing: {key}" + src_all = tuple(_read(weights[key], c) for key, c in zip(keys, cuts)) + if device is not None: + src_all = tuple(t.to(device, non_blocking=True) for t in src_all) + if transform is not None: + return transform(src_all if multi else src_all[0], d) + assert len(src_all) == 1, "a multi-key row needs a transform" + return src_all[0] + + def fill(self, param: nn.Parameter, sources, weights, d: SimpleNamespace) -> set: + """Fill `param` from its manifest rows on the parameter's device; return the keys read.""" + consumed = set() + for ckpt_key, index, cut, transform in sources: + dst = param.data if index is None else param.data[index] + src = self.read(weights, ckpt_key, cut, transform, d, dst.device) + assert dst.shape == src.shape, (ckpt_key, tuple(dst.shape), tuple(src.shape)) + assert src.dtype == dst.dtype, (ckpt_key, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + consumed.update(ckpt_key if isinstance(ckpt_key, tuple) else (ckpt_key,)) + return consumed + + def load(self, model, weights) -> None: + """Fill every declared parameter and assert checkpoint coverage both ways.""" + core = model.model + d = self.dims(core) + # Declared before the manifest: its height comes from the shards. + rows_per_shard = _declare_ngram_table(core, d, weights) + try: + self._fill_declared(model, d, weights, rows_per_shard) + except BaseException: + _close_ngram_table(core) + raise + + def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int) -> None: + core = model.model + manifest = self.manifest(d) + manifest["ple_table"] = ngram_table_sources(d, rows_per_shard) + consumed: set = set() + + assert set(manifest.keys()) == set(core.w.keys()), ( + "manifest/parameter drift", + set(manifest.keys()) ^ set(core.w.keys()), + ) + for param_key, sources in manifest.items(): + if param_key == "ple_table": + consumed |= _fill_ngram_table(core, d, weights, rows_per_shard) + continue + consumed |= self.fill(core.w[param_key], sources, weights, d) + if param_key.endswith(_RELEASE_AFTER): + torch.cuda.empty_cache() + + # The shell owns lm_head (untied, bf16); read only the rows it built. + cut, pad = lm_head_share(model.lm_head.weight.shape[0], d.vocab, d.tp_rank) + assert tuple(model.lm_head.weight.shape[1:]) == (d.hidden,), ( + f"the shell built lm_head {tuple(model.lm_head.weight.shape)}; this checkpoint's " + f"is a dense [{d.vocab}, {d.hidden}] weight" + ) + consumed |= self.fill( + model.lm_head.weight, [("lm_head.weight", None, cut, pad)], weights, d + ) + # Every parameter outside `core.w` must be lm_head. + shell = {n for n, _ in model.named_parameters() if not n.startswith("model.w.")} + assert shell == {"lm_head.weight"}, f"unfed shell parameters: {sorted(shell)}" + + torch.cuda.synchronize() + leftover = set(weights.keys()) - consumed + expected = self.expected_unconsumed(d, weights.keys()) + assert leftover == expected, ( + "checkpoint coverage: leftover keys are not exactly this rank's off-window " + f"experts{'' if d.mtp_enabled else ' and the MTP module'}; unexpected " + f"{sorted(leftover - expected)[:8]}, missed {sorted(expected - leftover)[:8]}" + ) + + +MODEL_WEIGHTS = Qwen4ExpFp8Weights() + + +class ExpertWindow(NamedTuple): + """`fp8_block_scale_moe_runner` operands of one MoE layer at this rank's window. + + `offset` / `count` are `local_expert_offset` / `local_num_experts`. + """ + + fc1_w: torch.Tensor # [count, 2 * moe_inter, hidden] e4m3, rows `[up ; gate]` + fc1_s: torch.Tensor # [count, 2 * moe_inter / 128, hidden / 128] fp32 + fc2_w: torch.Tensor # [count, hidden, moe_inter] e4m3 + fc2_s: torch.Tensor # [count, hidden / 128, moe_inter / 128] fp32 + offset: int + count: int + #: the layer's block-major copy (`MOE_TPB_ENV`), or None + blocks: "BlockWindow | None" = None + + +class BlockWindow(NamedTuple): + """One MoE layer's block-major routed experts and shared-expert slice at this rank.""" + + w1: torch.Tensor # [640, 256, hidden] e4m3, a block's rows `[up ; gate]` + s1: torch.Tensor # [640, 2, hidden / 128] fp32 + w2: torch.Tensor # [640, hidden, 128] e4m3 + s2: torch.Tensor # [640, hidden / 128] fp32 + rank: int + sh_gu: torch.Tensor # [5, 64, hidden] bf16: the slice's gate / up items + sh_gl: torch.Tensor # [hidden] bf16: the gate-logit row + sh_dn: torch.Tensor # [5, hidden, 32] bf16: the slice's down items + + +def _block_window(w, d: SimpleNamespace, layer: int) -> BlockWindow | None: + if not getattr(d, "moe_tpb", False): + return None + p = f"l{layer}_mtp_" if layer == d.num_layers else f"l{layer}_" + return BlockWindow( + w[f"{p}tpb_w1"], + w[f"{p}tpb_s1"], + w[f"{p}tpb_w2"], + w[f"{p}tpb_s2"], + d.tp_rank, + w[f"{p}tpb_sh_gu"], + w[f"{p}sh_gu"][2 * d.shared_inter], + w[f"{p}tpb_sh_dn"], + ) + + +def expert_window(w, d: SimpleNamespace, layer: int) -> ExpertWindow: + """Routed-expert operands of decoder layer `layer`, or of the MTP module at `d.num_layers`.""" + assert 0 <= layer < d.num_layers or (layer == d.num_layers and d.mtp_enabled), ( + f"layer {layer}: the trunk has {d.num_layers} MoE layers" + f"{' and the MTP module is not loaded' if layer == d.num_layers else ''}" + ) + p = f"l{layer}_mtp_" if layer == d.num_layers else f"l{layer}_" + start, count = _window(d) + return ExpertWindow( + w[f"{p}fc1_w"], + w[f"{p}fc1_s"], + w[f"{p}fc2_w"], + w[f"{p}fc2_s"], + start, + count, + _block_window(w, d, layer), + ) + + +def attention_tp_size(mapping) -> int: + """Ranks attention and K/V pool heads split over: the TP group, or 1 under attention DP.""" + return 1 if mapping.enable_attention_dp else mapping.tp_size + + +def _heads(total: int, ranks: int, rank: int) -> tuple[int, int]: + """(heads on `rank`, first head); with more ranks than heads, neighbours repeat a head.""" + assert total % ranks == 0 or ranks % total == 0, (total, ranks) + return -(-total // ranks), rank * total // ranks + + +def rank_dims(cfg, vcfg, mapping, *, mtp_enabled: bool, dtype: torch.dtype) -> SimpleNamespace: + """Every width the table declares at rank `mapping.rank`, and the shares behind them.""" + hidden = cfg.hidden_size + head_dim = cfg.head_dim + index_heads, index_kv_heads = cfg.indexer_n_heads, cfg.indexer_kv_heads + + # Every member spans its whole world in one TP group (routing admits no other). + assert mapping.tp_size == mapping.world_size, (mapping.tp_size, mapping.world_size) + + attn_tp = attention_tp_size(mapping) + attn_rank = mapping.tp_rank if attn_tp > 1 else 0 + heads_q, q_head_start = _heads(cfg.num_attention_heads, attn_tp, attn_rank) + heads_kv, kv_head_start = _heads(cfg.num_key_value_heads, attn_tp, attn_rank) + + state_tp = mamba_effective_tp_size(mapping) + state_rank = mapping.tp_rank if state_tp > 1 else 0 + num_k_heads, k_head_start = _heads(cfg.linear_num_key_heads, state_tp, state_rank) + num_v_heads, v_head_start = _heads(cfg.linear_num_value_heads, state_tp, state_rank) + key_dim = num_k_heads * cfg.linear_key_head_dim + value_dim = num_v_heads * cfg.linear_value_head_dim + + # Routed experts: whole-expert windows over the TP group, whatever the MoE TP / EP split. + num_experts = cfg.num_experts + moe_inter = cfg.moe_intermediate_size + assert mapping.moe_tp_size * mapping.moe_ep_size == mapping.tp_size, ( + mapping.moe_tp_size, + mapping.moe_ep_size, + mapping.tp_size, + ) + assert num_experts % mapping.tp_size == 0, (num_experts, mapping.tp_size) + assert moe_inter % FP8_BLOCK == 0 and hidden % FP8_BLOCK == 0, (moe_inter, hidden) + local_experts = num_experts // mapping.tp_size + + ngram_heads = cfg.heads_per_ngram * (cfg.ngram_size - 1) + v_hidden = vcfg.hidden_size + return SimpleNamespace( + num_layers=cfg.num_hidden_layers, + is_attn=[t == "full_attention" for t in cfg.layer_types], + hidden=hidden, + hc_count=cfg.hc_count, + hc_lowrank=cfg.hc_lowrank, + stream=cfg.hc_count * hidden, + tp_rank=mapping.tp_rank, + # --- attention, at this rank's heads --- + head_dim=head_dim, + heads_q=heads_q, + q_head_start=q_head_start, + heads_kv=heads_kv, + kv_head_start=kv_head_start, + q_width=heads_q * head_dim, + kv_width=heads_kv * head_dim, + index_head_dim=cfg.indexer_head_dim, + # [q | k | v | gate | index q, k | pad of one index head] + attn_in_rows=2 * heads_q * head_dim + + 2 * heads_kv * head_dim + + (index_heads + index_kv_heads + 1) * cfg.indexer_head_dim, + # --- Gated-DeltaNet, at this rank's key and value heads --- + num_k_heads=num_k_heads, + k_head_start=k_head_start, + num_v_heads=num_v_heads, + v_head_start=v_head_start, + head_k_dim=cfg.linear_key_head_dim, + head_v_dim=cfg.linear_value_head_dim, + key_dim=key_dim, + key_dim_total=cfg.linear_num_key_heads * cfg.linear_key_head_dim, + value_dim=value_dim, + conv_dim=2 * key_dim + value_dim, + in_proj_split=(2 * key_dim + value_dim, value_dim, num_v_heads, num_v_heads), + in_proj_rows=-(-(2 * key_dim + 2 * value_dim + 2 * num_v_heads) // IN_PROJ_ROW_ALIGN) + * IN_PROJ_ROW_ALIGN, + conv_kernel=cfg.linear_conv_kernel_dim, + # --- MoE: window `[expert_offset, expert_offset + local_experts)`, whole `moe_inter` --- + num_experts=num_experts, + local_experts=local_experts, + expert_offset=local_experts * mapping.tp_rank, + moe_tpb=moe_tpb_enabled(mapping), + moe_inter=moe_inter, + shared_inter=cfg.shared_expert_intermediate_size, + vocab=cfg.vocab_size, + ple_layer=list(cfg.ple_layer_ids)[0] - 1, # one-based in the config + ple_embed_dim=cfg.ple_embed_dim, + ple_conv_kernel=cfg.ple_conv_kernel_size, + ngram_size=cfg.ngram_size, + ngram_heads=ngram_heads, + ple_head_dim=cfg.ple_embed_dim // ngram_heads, + ple_shards=cfg.split_ngram_parts, + ngram_vocab_divisor=cfg.make_ngram_vocab_size_divisible_by, + ple_table_dtype=PLE_TABLE_DTYPE, + v_depth=vcfg.depth, + v_hidden=v_hidden, + v_inter=vcfg.intermediate_size, + v_channels=vcfg.in_channels, + v_tpatch=vcfg.temporal_patch_size, + v_patch=vcfg.patch_size, + v_pos_rows=vcfg.num_position_embeddings, + v_merge_dim=v_hidden * vcfg.spatial_merge_size * vcfg.spatial_merge_size, + mtp_enabled=mtp_enabled, + dtype=dtype, + ) + + +def lm_head_share(rows: int, vocab: int, tp_rank: int) -> tuple[Cut, Callable]: + """The cut and the pad that fill an lm_head the shell built with `rows` rows. + + Whole under attention DP (`rows == vocab`), else a vocab-parallel shard; an uneven vocab + leaves the last shard short, zero-padded as the built-in LMHead does. + """ + start = 0 if rows == vocab else tp_rank * rows + stop = min(start + rows, vocab) + assert start < stop, (rows, vocab, tp_rank) + + def pad(t: torch.Tensor, d) -> torch.Tensor: + if t.shape[0] == rows: + return t.contiguous() + out = torch.zeros(rows, t.shape[1], dtype=t.dtype, device=t.device) + out[: t.shape[0]] = t + return out + + return (slice(start, stop),), pad + + +def ngram_shard_keys(d: SimpleNamespace) -> list[str]: + """The n-gram table's shard keys, in concatenation order.""" + prefix = _ngram_prefix(d.ple_layer) + return [f"{prefix}.shard_{s}.weight" for s in range(d.ple_shards)] + + +def ngram_table_sources(d: SimpleNamespace, rows_per_shard: int) -> list[tuple]: + """`ple_table` manifest rows: shard `s`, read whole, fills rows from `s * rows_per_shard`.""" + return [ + (key, (_span(s * rows_per_shard, rows_per_shard),), None, None) + for s, key in enumerate(ngram_shard_keys(d)) + ] + + +def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: + """Register the n-gram table at the height its shards concatenate to; return rows per shard. + + The shards are contiguous row chunks (global row `r` is shard `r // rows_per_shard`), so + the flat buffer takes the hash's global rows. On CUDA it is host memory the device + addresses, stored as `core.ple_host_mode` says (`ple_host_table`); otherwise an ordinary + allocation on the weights' device. + """ + keys = ngram_shard_keys(d) + for key in keys: + assert key in weights, f"checkpoint key missing: {key}" + shapes = {tuple(weights[key].shape) for key in keys} + assert len(shapes) == 1, ( + f"the n-gram table's {len(keys)} shards do not share one shape: " + f"{sorted(shapes)}; they are concatenated as equal contiguous row " + f"chunks and an uneven one would shift every row above it" + ) + rows_per_shard, head_dim = shapes.pop() + assert head_dim == d.ple_head_dim, ( + f"the n-gram table is {head_dim} wide but {d.ngram_heads} heads must " + f"tile ple_embed_dim {d.ple_embed_dim}, i.e. {d.ple_head_dim}" + ) + dtype = weights[keys[0]].dtype + assert dtype == d.ple_table_dtype, ( + f"the n-gram table is stored {dtype}; this target declares " + f"{d.ple_table_dtype} and does not convert it" + ) + device = core.w["embed"].device + rows = d.ple_shards * rows_per_shard + _check_ngram_id_space(d, weights, rows) + mode = getattr(core, "ple_host_mode", _ple_host.RANK) + identity = None + if mode == _ple_host.NODE: + _, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + identity = _ple_host.table_identity(read, rows, head_dim, dtype, *keys) + mapping = getattr(core, "mapping", None) + core._ple_table_host = _ple_host.PleHostTable( + mode, + rows, + head_dim, + dtype, + device, + rank=mapping.tp_rank if mapping is not None else 0, + world=mapping.tp_size if mapping is not None else 1, + dp=getattr(core, "dp_size", 1), + private=_host_resident_empty, + identity=identity, + ) + _bind_ngram_table(core) + return rows_per_shard + + +def _check_ngram_id_space(d: SimpleNamespace, weights, rows: int) -> None: + """Raise `ValueError` unless a table of `rows` rows holds the id space the checkpoint's + per-head offsets and vocabulary sizes address, padded to a multiple of + `ngram_vocab_divisor` by less than one divisor. + + An id past the end is a device-side assert that poisons the process. Shards padded one by + one carry more padding than that, and do not concatenate to the padded id space: every id + past the first shard would read a shifted row.""" + prefix = f"{_LM}.layers.{d.ple_layer}.ple.ple_embedding" + offsets = _read(weights[f"{prefix}.ngram_heads_offsets"], None) + vocab = _read(weights[f"{prefix}.ngram_heads_vocab_sizes"], None) + used = int(offsets[-1]) + int(vocab[-1]) + divisor = d.ngram_vocab_divisor + if used > rows: + raise ValueError( + f"the n-gram table has {rows} rows but its per-head offsets and vocabulary sizes " + f"address {used}" + ) + if rows % divisor or rows - used >= divisor: + raise ValueError( + f"the n-gram table has {rows} rows for an id space of {used}: not padded to a " + f"multiple of make_ngram_vocab_size_divisible_by {divisor} by less than one " + "divisor. Shards padded one by one do not concatenate to the padded id space, and " + "every id past the first shard would read a shifted row" + ) + + +def _bind_ngram_table(core) -> None: + """`core.w["ple_table"]` as the host table's current view (a placeholder until a `node` + table is mapped).""" + host = core._ple_table_host + table = host.table + if table is None: + table = torch.empty(0, host.cols, dtype=host.dtype, device="meta") + core.w["ple_table"] = nn.Parameter(table, requires_grad=False) + # Host offload relies on `nn.Parameter` sharing storage rather than copying. + assert core.w["ple_table"].data_ptr() == table.data_ptr(), ( + "wrapping the n-gram table in nn.Parameter moved its storage" + ) + + +def _close_ngram_table(core) -> None: + """Release the n-gram table (in `node`, its chunks and the broker names other processes of + the host may wait on); `ple_table` becomes the placeholder.""" + host = core._ple_table_host + if host is None: + return + host.close() + _bind_ngram_table(core) + core._ple_table_host = None + + +def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: + """Fill this rank's rows of the n-gram table (`ple_host_table`); return the shard keys.""" + keys = ngram_shard_keys(d) + copy, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + core._ple_table_host.fill(copy, read) + _bind_ngram_table(core) + return set(keys) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 220f46095266..e6437bb637c1 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -431,6 +431,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo +from .. import ple_host_table as _ple_host from . import pool as _pool from . import vision as _vision from . import weights as _weights @@ -905,6 +906,18 @@ def _moe_prefill_copy(one_rank: bool, trunk_moe_backend: str, max_num_tokens: in ) +def _ple_window(core): + """The n-gram gather's `row_window` for this rank's table (None: the whole table).""" + host = core._ple_table_host + return None if host is None else host.window + + +def _ple_needs_sum(core) -> bool: + """Whether the gathered n-gram rows are this rank's share, summed over attention TP.""" + host = core._ple_table_host + return host is not None and host.needs_sum + + def _ple_host_offload_enabled() -> bool: """Whether the n-gram table is declared in host memory (the default).""" return _switch(_PLE_HOST_OFFLOAD_ENV, default=True) @@ -1194,6 +1207,13 @@ def __init__(self, model_config: ModelConfig, composite_config: PretrainedConfig self.eos_token_id = int(eos[0] if isinstance(eos, (list, tuple)) else eos) self.ple_table_dtype = _weights.PLE_TABLE_DTYPE self.ple_host_offload = _ple_host_offload_enabled() + # How the host-resident table is stored (`ple_host_table`). + self.ple_host_mode = _ple_host.host_table_mode() + if self.ple_host_mode != _ple_host.RANK and not self.ple_host_offload: + logger.warning( + f"{_ple_host.PLE_HOST_TABLE_ENV}={self.ple_host_mode} applies to the host-resident " + f"n-gram table; {_PLE_HOST_OFFLOAD_ENV}=0 keeps one device copy per rank" + ) _check_switches(self.one_rank) # CPU allocation behind the offloaded table; the `ple_table` parameter is a # device-address view that owns none of those pages. @@ -1354,7 +1374,9 @@ def post_load_weights(self) -> None: } # Reconcile the table height against the checkpoint's id-space tensors: an index past # the end is a device-side assert that poisons the process. - table_rows = w["ple_table"].shape[0] + # Rows of the whole table: under `server` storage `ple_table` holds only this rank's. + host = self._ple_table_host + table_rows = host.rows if host is not None else w["ple_table"].shape[0] used_rows = int(w["ple_offs"][-1]) + int(w["ple_vocab"][-1]) assert used_rows <= table_rows, ( f"the n-gram table has {table_rows} rows but its per-head offsets and vocabulary " @@ -1576,23 +1598,24 @@ def _report_ple_residency(self) -> None: output-identical, so the pointer's memory kind read back from the driver is the signal that cannot be faked.""" table = self.w["ple_table"] - kind = _weights.memory_kind(table.data_ptr()) - expected = ( - _weights.CUDA_MEMORY_TYPE_HOST - if self.ple_host_offload - else _weights.CUDA_MEMORY_TYPE_DEVICE - ) - assert kind == expected, ( - f"the n-gram table was declared with {_PLE_HOST_OFFLOAD_ENV}=" - f"{'1' if self.ple_host_offload else '0'}, but the driver reports cudaMemoryType " - f"{kind} for the pointer the forward gathers from" - ) + host = self._ple_table_host + if host is not None: + assert host.resident_on_host(), ( + f"the n-gram table ({host.describe()}) was declared in host memory, but the " + "driver reports device memory for the pointer the forward gathers from" + ) + else: + kind = _weights.memory_kind(table.data_ptr()) + assert kind == _weights.CUDA_MEMORY_TYPE_DEVICE, ( + f"the n-gram table was declared with {_PLE_HOST_OFFLOAD_ENV}=0, but the driver " + f"reports cudaMemoryType {kind} for the pointer the forward gathers from" + ) mib = 1024.0 * 1024.0 free, total = torch.cuda.mem_get_info() print( f"ple n-gram table: {tuple(table.shape)} {table.dtype}, " f"{table.numel() * table.element_size() / mib:.1f} MiB, " - f"{'HOST-resident (device-addressable)' if self.ple_host_offload else 'DEVICE-resident'}" + f"{f'HOST-resident ({host.describe()})' if host is not None else 'DEVICE-resident'}" f"; after load: allocated {torch.cuda.memory_allocated() / mib:.1f} MiB, " f"driver-used {(total - free) / mib:.1f} MiB of {total / mib:.1f} MiB", flush=True, @@ -4179,7 +4202,10 @@ def _ple_block(self, hc, hc_out, normed_out) -> None: S.ple_offs, core.eos_token_id, num_tokens, + row_window=_ple_window(core), ) + if _ple_needs_sum(core): + ngram = core.tp_sum(ngram) ple_gate_conv_inject_norm( hc[:num_tokens], cublas_mm(ngram, S.ple_key), @@ -5863,7 +5889,10 @@ def _ple_block(self, hc, input_ids, ple_state, next_norm, stage) -> tuple: S.ple_offs, core.eos_token_id, rows, + row_window=_ple_window(core), ) + if _ple_needs_sum(core): + ngram = core.tp_sum(ngram) conv_input = torch.empty_like(hc) if stage else None out = ple_gate_conv_inject_norm( hc, @@ -6828,6 +6857,8 @@ def check_deployment(model_config: ModelConfig, llm_args: TorchLlmArgs, layout: f"target implements { ({k: _TEXT_SETTINGS[k] for k in differs}) }" ) + # Read here so a misspelled value fails before the build. + _ple_host.host_table_mode() mapping = model_config.mapping one_rank = mapping.world_size == 1 spec = model_config.spec_config diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py index f94e9a0abe81..6b0a48b44ab8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/weights.py @@ -70,8 +70,8 @@ checkpoint row order. Under trtllm-gen a second copy of the window in the CuteDSL layout (`fc*_cd`) is declared when the core keeps it (`moe_prefill_copy`); * n-gram table: 128 row chunks concatenated into one `[rows, 160]` buffer (the hash - emits global row indices). It is replicated, and host-resident by default - (`_host_resident_empty`): one copy per rank. + emits global row indices). It is host-resident by default, one copy per rank, per server + or per node (`qwen4_exp/ple_host_table.py`). The n-gram table is not in the table: its height is a checkpoint fact, so `load` registers it with `torch.empty` (`_declare_ngram_table`) and adds its rows to the @@ -103,6 +103,7 @@ from tensorrt_llm.llmapi.utils import get_numa_aware_cpu_affinity from tensorrt_llm.logger import logger +from .. import ple_host_table as _ple_host from .pool import attention_tp_size #: `layers=ALL_LAYERS`, the default: the weight is on every layer. @@ -1572,6 +1573,14 @@ def load(self, model, weights) -> None: # The one load-time declaration, before the manifest is built: the table's # height comes from the shards, and the manifest addresses it by row range. rows_per_shard = _declare_ngram_table(core, d, weights) + try: + self._fill_declared(model, d, weights, rows_per_shard) + except BaseException: + _close_ngram_table(core) + raise + + def _fill_declared(self, model, d: SimpleNamespace, weights, rows_per_shard: int) -> None: + core = model.model manifest = self.manifest(d) manifest["ple_table"] = ngram_table_sources(d, rows_per_shard) consumed: set = set() @@ -1591,6 +1600,9 @@ def fill(param: nn.Parameter, ckpt_key, index, cut, transform) -> None: set(manifest.keys()) ^ set(core.w.keys()), ) for param_key, sources in manifest.items(): + if param_key == "ple_table" and core._ple_table_host is not None: + consumed |= _fill_ngram_table(core, d, weights, rows_per_shard) + continue for ckpt_key, index, cut, transform in sources: fill(core.w[param_key], ckpt_key, index, cut, transform) if param_key.endswith(_RELEASE_AFTER): @@ -1845,15 +1857,63 @@ def _declare_ngram_table(core, d: SimpleNamespace, weights) -> int: device = core.w["embed"].device rows = d.ple_shards * rows_per_shard if core.ple_host_offload: - table, core._ple_table_host = _host_resident_empty(rows, head_dim, dtype, device) + mode = getattr(core, "ple_host_mode", _ple_host.RANK) + identity = None + if mode == _ple_host.NODE: + _, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + identity = _ple_host.table_identity(read, rows, head_dim, dtype, *keys) + mapping = getattr(core, "mapping", None) + core._ple_table_host = _ple_host.PleHostTable( + mode, + rows, + head_dim, + dtype, + device, + rank=mapping.tp_rank if mapping is not None else 0, + world=mapping.tp_size if mapping is not None else 1, + dp=getattr(core, "dp_size", 1), + private=_host_resident_empty, + identity=identity, + ) + _bind_ngram_table(core) else: table = torch.empty(rows, head_dim, dtype=dtype, device=device) core._ple_table_host = None + core.w["ple_table"] = nn.Parameter(table, requires_grad=False) + core.ple_rows_per_shard = rows_per_shard + return rows_per_shard + + +def _bind_ngram_table(core) -> None: + """`core.w["ple_table"]` as the host table's current view (a placeholder until a `node` + table is mapped). `nn.Parameter` (`_make_subclass`) shares storage rather than copying; + host offload depends on that, so it is asserted.""" + host = core._ple_table_host + table = host.table + if table is None: + table = torch.empty(0, host.cols, dtype=host.dtype, device="meta") core.w["ple_table"] = nn.Parameter(table, requires_grad=False) - # `nn.Parameter` (`_make_subclass`) shares storage rather than copying; host offload - # depends on that, so it is asserted. assert core.w["ple_table"].data_ptr() == table.data_ptr(), ( "wrapping the n-gram table in nn.Parameter moved its storage" ) - core.ple_rows_per_shard = rows_per_shard - return rows_per_shard + + +def _close_ngram_table(core) -> None: + """Release the host-resident n-gram table (in `node`, its chunks and the broker names other + processes of the host may wait on); `ple_table` becomes the placeholder.""" + host = core._ple_table_host + if host is None: + return + host.close() + _bind_ngram_table(core) + core._ple_table_host = None + + +def _fill_ngram_table(core, d: SimpleNamespace, weights, rows_per_shard: int) -> set: + """Fill this rank's rows of the host-resident n-gram table (`ple_host_table`); return the + shard keys.""" + keys = ngram_shard_keys(d) + copy, read = _ple_host.shard_rows(weights, keys, rows_per_shard, _read) + core._ple_table_host.fill(copy, read) + _bind_ngram_table(core) + return set(keys) diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py index f76bf6a20204..2eff81e160ce 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/routing.py @@ -3,10 +3,11 @@ """Routing for Qwen4ExpForConditionalGeneration. Qwen3.8-Flash-Next-NVFP4 on sm103 lands on `qwen3_8_flash_next_nvfp4__sm_103__default` at one -of the layouts in `_LAYOUTS`, with one-model MTP-Eagle (not at dep2) or no speculative -decoding. The target's `check_deployment` then declines the checkpoints and engine choices it -does not run, so `auto` falls back to the built-in implementation instead of failing in the -build. +of the layouts in `_LAYOUTS`, and Qwen3.8-Flash-Next-FP8 (FP8_BLOCK_SCALES) on +`qwen3_8_flash_next_fp8__sm_103__unified` at one of `_FP8_LAYOUTS`, each with one-model +MTP-Eagle (not at dep2) or no speculative decoding. The target's `check_deployment` then +declines the checkpoints and engine choices it does not run, so `auto` falls back to the +built-in implementation instead of failing in the build. """ from __future__ import annotations @@ -14,6 +15,7 @@ from typing import TYPE_CHECKING, Optional, Tuple from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig +from tensorrt_llm.quantization.mode import QuantAlgo from ..._router_index import NoTarget, routes @@ -22,7 +24,7 @@ from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from tensorrt_llm.mapping import Mapping -# The GPU the target is certified on. +# The GPU the targets are certified on. _SM = (10, 3) # (world size, attention DP, MoE TP, MoE EP) -> the target's layout, each in one node. tp1 holds @@ -37,9 +39,12 @@ (2, True, 1, 2): "dep2", } +# The FP8 target also runs tp2 and tep2. +_FP8_LAYOUTS = {**_LAYOUTS, (2, False, 2, 1): "tp2", (2, False, 1, 2): "tep2"} -def layout(mapping: Mapping) -> Optional[str]: - """The layout ``mapping`` describes, or None. + +def layout(mapping: Mapping, layouts: dict = _LAYOUTS) -> Optional[str]: + """The layout ``mapping`` describes in ``layouts``, or None. Attention runs TP over the whole world, or DP with the LM head whole. Pipeline and context parallelism, MoE clusters and a world spanning nodes are other forwards. @@ -55,7 +60,7 @@ def layout(mapping: Mapping) -> Optional[str]: or mapping.is_multi_node() ): return None - return _LAYOUTS.get( + return layouts.get( (mapping.world_size, mapping.enable_attention_dp, mapping.moe_tp_size, mapping.moe_ep_size) ) @@ -68,7 +73,8 @@ def route( ) -> type: """Return the target class for this deployment, or raise ``NoTarget`` saying why none fits.""" mapping, spec = model_config.mapping, model_config.spec_config - member = layout(mapping) + fp8 = getattr(model_config.quant_config, "quant_algo", None) == QuantAlgo.FP8_BLOCK_SCALES + member = layout(mapping, _FP8_LAYOUTS if fp8 else _LAYOUTS) if compute_capability != _SM or member is None: raise NoTarget( f"compute capability {compute_capability}, world_size={mapping.world_size} " @@ -76,7 +82,8 @@ def route( f"attention_dp={mapping.enable_attention_dp}, moe_tp={mapping.moe_tp_size}, " f"moe_ep={mapping.moe_ep_size}); the qwen4_exp target is Qwen3.8-Flash-Next-NVFP4 " "on sm103 in one node: tp1, tp4 (experts split by intermediate), tep4, dep4 " - "(attention DP) or dep2 (attention DP, experts EP2)" + "(attention DP) or dep2 (attention DP, experts EP2); Qwen3.8-Flash-Next-FP8's also " + "tp2 or tep2" ) if spec is not None and not ( isinstance(spec, MTPDecodingConfig) and spec.spec_dec_mode.is_mtp_eagle_one_model() @@ -90,6 +97,13 @@ def route( "speculative decoding with attention DP (dep2): no draft acceptance was measured at " "this layout; MTP under attention DP is admitted at dep4" ) + if fp8: + from .qwen3_8_flash_next_fp8__sm_103__unified.modeling import ( + ModelingV2Qwen38FlashNextFp8Sm103Unified, + ) + + ModelingV2Qwen38FlashNextFp8Sm103Unified.check_deployment(model_config, llm_args, member) + return ModelingV2Qwen38FlashNextFp8Sm103Unified from .qwen3_8_flash_next_nvfp4__sm_103__default.modeling import ( ModelingV2Qwen38FlashNextNvfp4Sm103Default, ) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index e15d9838903d..1f1665113d82 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -25,6 +25,8 @@ "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp4::test_gsm8k": 931.6819444444445, "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp8::test_eagle3_gsm8k_and_acceptance": 791.973, "accuracy/modeling_v2/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_accuracy.py::TestModelingV2MiniMaxM3Nvfp4Sm103Tp8::test_gsm8k": 659.1681428571428, + "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2": 360.976, + "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4]": 666.068, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2": 314.12083333333334, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_and_long_context": 585.9465129533679, "accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[dep4_fp8_kv]": 600.46915, diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..467079831e16 --- /dev/null +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py new file mode 100644 index 000000000000..3edb23b070d8 --- /dev/null +++ b/tests/integration/defs/accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py @@ -0,0 +1,358 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Whole-model gates for the qwen3_8_flash_next_fp8__sm_103__unified target on sm_103. + +GSM8K at tp1, tep4, dep4 and dep2 disaggregated; long-context pass keys at tep4. +""" + +import pytest +from transformers import AutoTokenizer + +from tensorrt_llm import LLM +from tensorrt_llm._torch.modeling_v2 import MODELING_V2_VALIDATE_ENV +from tensorrt_llm._utils import get_sm_version +from tensorrt_llm.llmapi import ( + CudaGraphConfig, + KvCacheConfig, + MambaStateConfig, + MoeConfig, + MTPDecodingConfig, + QSASparseAttentionConfig, + SamplingParams, +) +from tensorrt_llm.quantization import QuantAlgo + +from .....conftest import llm_models_root +from ....accuracy_core import ( + GSM8K, + LlmapiAccuracyTestHarness, + assert_acceptance_length, + compute_acceptance_length, +) +from ....test_disaggregated_serving import launch_disaggregated_llm, run_accuracy_test +from ..._env import scoped_env_overrides +from ..qwen3_8_flash_next_nvfp4__sm_103__default.test_accuracy import ( + _PASSKEY_LENGTHS, + _PASSKEY_MAX_TOKENS, + _acceptance_length_from_requests, + _passkey_prompts, +) + +skip_not_sm103 = pytest.mark.skipif( + get_sm_version() != 103, reason="modeling_v2 targets in this batch are certified on sm_103 only" +) + +# Check the step contract on the first forward; the flag has to reach the worker ranks. +_VALIDATE = {MODELING_V2_VALIDATE_ENV: "1"} + +# Workers spawn their second rank over MPI in a root container (docker/Dockerfile.multi). +_DISAGG_WORKER_ENV = { + **_VALIDATE, + "PRTE_ALLOW_RUN_AS_ROOT": "1", + "PRTE_ALLOW_RUN_AS_ROOT_CONFIRM": "1", +} + + +def _disagg_server_config() -> dict: + """One context and one generation server on this host.""" + return { + "hostname": "localhost", + "port": 8000, + "backend": "pytorch", + "context_servers": {"num_instances": 1, "urls": ["localhost:8001"]}, + "generation_servers": {"num_instances": 1, "urls": ["localhost:8002"]}, + } + + +def _passkey_acceptance(llm, prompts, label: str, draft_len: int) -> float: + """Pass-key retrieval on ``llm`` and the requests' mean acceptance length. + + Every key must be stated back, and two identical requests far apart in one batch (different + slots) must give the same tokens, which fails if a request's selection leaks into another + slot. Needs no iteration statistics: the requests carry their own speculative metrics. + """ + texts = [prompt for prompt, _ in prompts] + [prompts[len(prompts) // 2][0]] + outputs = llm.generate( + texts, + SamplingParams(max_tokens=_PASSKEY_MAX_TOKENS, temperature=0, return_perf_metrics=True), + use_tqdm=False, + ) + missed = [ + key + for (_, key), output in zip(prompts, outputs) + if key not in output.outputs[0].text[:_PASSKEY_MAX_TOKENS] + ] + assert not missed, ( + f"{label}: pass keys not stated back: {missed} of {[key for _, key in prompts]}" + ) + twins = outputs[len(prompts) // 2], outputs[-1] + assert twins[0].outputs[0].token_ids == twins[1].outputs[0].token_ids, ( + f"{label}: the same prompt in two slots gave different tokens" + ) + acceptance_length = _acceptance_length_from_requests(outputs, draft_len) + print(f"[AL] {label} acceptance_length = {acceptance_length:.3f}") + return acceptance_length + + +_TEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=False) +_DEP4 = dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=True) + +# Only tep4 runs the breakable prefill graph: routing admits it under attention TP alone. +_MTP3_ENGINES = [ + pytest.param(_TEP4, "fp8", True, {}, id="tep4_fp8_kv_breakable"), + pytest.param(_DEP4, "auto", False, {}, id="dep4"), + pytest.param( + _DEP4, "auto", False, {"TRTLLM_FORCE_COMM_METHOD": "NVLINK_ONE_SIDED"}, id="dep4_alltoall" + ), +] + + +class TestModelingV2Qwen38FlashNextFp8Sm103Unified(LlmapiAccuracyTestHarness): + """Every test requires modeling_v2, so a missed route fails rather than scoring the built-in.""" + + # References are keyed by this name plus the resolved quantization (FP8_BLOCK_SCALES). + MODEL_NAME = "Qwen/Qwen3.8-Flash-Next" + MODEL_PATH = f"{llm_models_root()}/Qwen3.8-Flash-Next-FP8" + + # The anchors' protocol: the chat template thinks, so the system prompt caps the reasoning + # and asks for the bare answer, which then fits GSM8K_MAX_OUTPUT_LEN. + EXTRA_EVALUATOR_KWARGS = dict( + apply_chat_template=True, + fewshot_as_multiturn=True, + system_prompt=( + "Use at most three short reasoning sentences, then end with `#### NUMBER`. " + "Do not restate the problem." + ), + chat_template_kwargs=dict(enable_thinking=True, reasoning_effort="xhigh"), + ) + + # The anchor's `max_tokens`; the class default (256) truncates this checkpoint's answers. + GSM8K_MAX_OUTPUT_LEN = 512 + + # The checkpoint's bf16 K/V and a bf16 recurrent state, the one the target serves. + KV_CACHE = KvCacheConfig( + free_gpu_memory_fraction=0.6, + enable_block_reuse=False, + mamba_ssm_cache_dtype="bfloat16", + ) + + # The trtllm-gen FP8 block-scale runner is the one MoE backend the target implements. + MOE = MoeConfig(backend="TRTLLM") + + ENGINE = dict( + max_seq_len=9216, + max_batch_size=128, + max_num_tokens=8192, + enable_chunked_prefill=False, + kv_cache_config=KV_CACHE, + cuda_graph_config=CudaGraphConfig(enable_padding=True, max_batch_size=128), + moe_config=MOE, + ) + + # One-model MTP-Eagle with the checkpoint's single MTP module replayed three times. + MTP3 = MTPDecodingConfig(max_draft_len=3) + + # The acceptance-length floors in references/acceptance_length.yaml: gsm8k over bf16 KV + # (`mtp3`) and over an FP8 pool (`mtp3_fp8_kv`), and the long-context pass keys. + ACCEPTANCE_KEY_MTP3 = "ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3" + ACCEPTANCE_KEY_LONG_CONTEXT = "ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_long_context" + + # The breakable prefill CUDA graph, up to max_num_tokens. + BREAKABLE_PREFILL = dict( + prefill_cuda_graph_backend="breakable", + prefill_capture_num_tokens=[512, 1024, 2048, 4096, 8192], + ) + + def _gsm8k(self, mocker, llm, label: str, key: str | None = None) -> None: + """gsm8k against the engine's FP8_BLOCK_SCALES row of references/gsm8k.yaml. + + Under speculation also the acceptance length, held to the floor under `key` in + references/acceptance_length.yaml: rejection sampling keeps the score of a miscomputed + draft layer, so only the acceptance length sees it. It comes from the engine's + iteration statistics, so the engine needs ``enable_iter_perf_stats`` and + ``max_stats_len=-1`` (the default queue keeps only the last 1000 iterations). + """ + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES, ( + f"{label}: the engine resolved quant_algo {llm.args.quant_config.quant_algo}; the " + "references are this checkpoint's FP8_BLOCK_SCALES rows" + ) + mocker.patch.object(GSM8K, "MAX_OUTPUT_LEN", self.GSM8K_MAX_OUTPUT_LEN) + GSM8K(self.MODEL_NAME).evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + if llm.args.speculative_config is None: + return + acceptance_length = compute_acceptance_length(llm) + print(f"[AL] {label} acceptance_length = {acceptance_length:.3f}") + assert_acceptance_length(key, acceptance_length) + + @skip_not_sm103 + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + @pytest.mark.parametrize("parallel, kv_dtype, breakable, env", _MTP3_ENGINES) + def test_mtp3_gsm8k(self, mocker, request, parallel, kv_dtype, breakable, env): + """Three drafts with block reuse and chunked prefill, against the FP8_BLOCK_SCALES MTP row. + + Block reuse needs the Gated-DeltaNet and PLE snapshots (every 256 tokens); without them the + runtime turns reuse off. An FP8 pool reads the FP8_BLOCK_SCALES row with an FP8 KV cache + and its own acceptance-length floor (`mtp3_fp8_kv`). + """ + if env.get("TRTLLM_FORCE_COMM_METHOD") == "NVLINK_ONE_SIDED": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + NVLinkOneSided, + ) + + # The target declines the all-to-all where it cannot run it. + if not NVLinkOneSided.is_platform_supported(): + pytest.skip("no MNNVL on this node: the all-to-all MoE round trip cannot run") + kv_cache_config = KvCacheConfig( + dtype=kv_dtype, + free_gpu_memory_fraction=0.6, + enable_block_reuse=True, + mamba_ssm_cache_dtype="bfloat16", + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + ) + engine = dict(self.ENGINE, enable_chunked_prefill=True, kv_cache_config=kv_cache_config) + key = ( + self.ACCEPTANCE_KEY_MTP3 + if kv_dtype == "auto" + else f"{self.ACCEPTANCE_KEY_MTP3}_{kv_dtype}_kv" + ) + if breakable: + engine.update(self.BREAKABLE_PREFILL) + with ( + scoped_env_overrides({**_VALIDATE, **env}) as rank_env, + LLM( + self.MODEL_PATH, + speculative_config=self.MTP3, + enable_iter_perf_stats=True, + max_stats_len=-1, + **engine, + **parallel, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + if kv_dtype != "auto": + assert llm.args.quant_config.kv_cache_quant_algo == QuantAlgo(kv_dtype.upper()), ( + f"kv_cache_config.dtype={kv_dtype} resolved kv_cache_quant_algo " + f"{llm.args.quant_config.kv_cache_quant_algo}" + ) + self._gsm8k(mocker, llm, request.node.callspec.id, key) + + @skip_not_sm103 + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + def test_mtp3_long_context(self): + """tep4 with three drafts and IndexShare: pass keys over 3k-6k tokens. + + The prompts are past the QSA indexer's 2048-token budget, so the target's decode steps and + its draft layer's take the sparse attention path, which gsm8k never leaves, and IndexShare + makes the draft loop reuse the selection of its first step. A miscomputed draft path keeps + the text correct and only lowers the acceptance length, which is held to its floor in + references/acceptance_length.yaml. + """ + tokenizer = AutoTokenizer.from_pretrained(self.MODEL_PATH) + prompts = _passkey_prompts(tokenizer, _PASSKEY_LENGTHS) + with ( + scoped_env_overrides(_VALIDATE) as rank_env, + LLM( + self.MODEL_PATH, + speculative_config=self.MTP3, + sparse_attention_config=QSASparseAttentionConfig( + index_share_for_mtp_iteration=True + ), + **self.ENGINE, + **_TEP4, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + acceptance_length = _passkey_acceptance( + llm, prompts, "tep4 mtp3 long context", self.MTP3.max_draft_len + ) + assert_acceptance_length(self.ACCEPTANCE_KEY_LONG_CONTEXT, acceptance_length) + + @skip_not_sm103 + @pytest.mark.timeout(5400) + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(145000) + @pytest.mark.skip_less_host_memory(204800) + def test_disaggregated_dep2(self, mocker): + """dep2 on both engines, one context and one generation server of two GPUs each. + + Both engines run one layout, so every pool matches on both sides of the NIXL transceiver. + """ + common_config = { + "modeling_v2": "require", + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_attention_dp": True, + "max_seq_len": 9216, + "max_batch_size": 128, + "max_num_tokens": 8192, + "enable_chunked_prefill": False, + "moe_config": {"backend": "TRTLLM"}, + "kv_cache_config": { + "enable_block_reuse": False, + "mamba_ssm_cache_dtype": "bfloat16", + "free_gpu_memory_fraction": 0.6, + }, + # The V2 hybrid manager's pools travel over the Python transceiver. + "cache_transceiver_config": {"backend": "NIXL", "transceiver_runtime": "PYTHON"}, + } + ctx_server_config = { + **common_config, + "disable_overlap_scheduler": True, + "cuda_graph_config": None, + } + gen_server_config = { + **common_config, + "disable_overlap_scheduler": False, + "cuda_graph_config": {"enable_padding": True, "max_batch_size": 128}, + } + + mocker.patch.object(GSM8K, "MAX_OUTPUT_LEN", self.GSM8K_MAX_OUTPUT_LEN) + with launch_disaggregated_llm( + _disagg_server_config(), + ctx_server_config, + gen_server_config, + self.MODEL_PATH, + max_workers=128, + extra_env=_DISAGG_WORKER_ENV, + # What every rank of both engines prints once it has bound the manager's pools. + assert_worker_log_contains="modeling_v2 qwen4_exp fp8 unified target: rank", + ) as llm: + # The reference is keyed by this quantization; the launcher reads none off the name. + llm.args.quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES + run_accuracy_test( + llm, + self.MODEL_NAME, + ["GSM8K"], + extra_evaluator_kwargs={GSM8K: self.EXTRA_EVALUATOR_KWARGS}, + timeout=3000, + ) + + @skip_not_sm103 + def test_tp1_gsm8k(self, mocker): + """One rank without speculation, against the FP8_BLOCK_SCALES row.""" + with ( + scoped_env_overrides(_VALIDATE) as rank_env, + LLM( + self.MODEL_PATH, + **self.ENGINE, + modeling_v2="require", + env_overrides=rank_env, + ) as llm, + ): + self._gsm8k(mocker, llm, "tp1") diff --git a/tests/integration/defs/accuracy/references/acceptance_length.yaml b/tests/integration/defs/accuracy/references/acceptance_length.yaml index f30517a4599f..97b798463943 100644 --- a/tests/integration/defs/accuracy/references/acceptance_length.yaml +++ b/tests/integration/defs/accuracy/references/acceptance_length.yaml @@ -283,3 +283,22 @@ ModelingV2Qwen38FlashNextNvfp4Sm103Default::mtp3_long_context: # 2.5% below ref_al, as for tp1_mtp3_long_context, whose deliberately wrong # reused selection scores 3.760. min_al: 3.9 +# Read by TestModelingV2Qwen38FlashNextFp8Sm103Unified (the qwen4_exp FP8 target's +# test_accuracy.py): three drafts of the checkpoint's single MTP module, so acceptance length +# lives in [1.0, 4.0]. modeling_v2's own readings on GB300 (TRTLLM_POPULATE_ACCEPTANCE_LENGTH=1), +# min_al 95% of each. +# test_mtp3_gsm8k[dep4] and [dep4_alltoall], over gsm8k: bf16 KV under attention DP; dep4 read +# 3.394 and dep4_alltoall 3.375. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3: + ref_al: 3.394 + min_al: 3.224 +# test_mtp3_gsm8k[tep4_fp8_kv_breakable], over gsm8k: an FP8 KV pool with the breakable prefill +# graph. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_fp8_kv: + ref_al: 3.274 + min_al: 3.110 +# test_mtp3_long_context, over the 3k-6k-token pass keys at tep4 (the sparse attention path): +# IndexShare on, bf16 KV. Every draft of a pass-key answer is accepted. +ModelingV2Qwen38FlashNextFp8Sm103Unified::mtp3_long_context: + ref_al: 4.000 + min_al: 3.800 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index fc51a8ccd73a..0e4d94084381 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -310,9 +310,20 @@ Qwen/Qwen3.8-2.4T-A95B: accuracy: 93.80 Qwen/Qwen3.8-Flash-Next: - accuracy: 95.72 + # Block FP8 without speculative decoding, under the protocol of the FP8 MTP row (chat + # template, thinking with a short-reasoning system prompt, 512 output tokens), measured on + # the built-in implementation: TEP4 on GB300, TRTLLM MoE, bf16 Mamba SSM cache. + - quant_algo: FP8_BLOCK_SCALES + accuracy: 95.34 - quant_algo: FP8_BLOCK_SCALES spec_dec_algo: MTP accuracy: 95.11 + # FP8 KV cache over the checkpoint's bf16 KV, measured on the modeling_v2 FP8 target: TEP4 on + # GB300, MTP draft length 3, the breakable prefill graph, bf16 Mamba SSM cache. + - quant_algo: FP8_BLOCK_SCALES + kv_cache_quant_algo: FP8 + spec_dec_algo: MTP + accuracy: 95.15 - quant_algo: MIXED_PRECISION accuracy: 96.36 - quant_algo: MIXED_PRECISION diff --git a/tests/integration/test_lists/qa/functional/llm_function_core.txt b/tests/integration/test_lists/qa/functional/llm_function_core.txt index 18bdb4c2d696..3be578a7dfe4 100644 --- a/tests/integration/test_lists/qa/functional/llm_function_core.txt +++ b/tests/integration/test_lists/qa/functional/llm_function_core.txt @@ -813,6 +813,15 @@ accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_ac accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_mmmu TIMEOUT (90) accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2 TIMEOUT (90) +# Qwen3.8-Flash-Next-FP8, modeling_v2 qwen4_exp FP8 target (sm_103 only): every accuracy test of +# the target, the CI ones included. +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_tp1_gsm8k TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_long_context TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4_alltoall] TIMEOUT (60) +accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) + # Dense FP4 MLA accuracy accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_mla[unchunked] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_mla[chunked] diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 1bcefa55770a..911bc9308ff7 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -179,6 +179,9 @@ l0_b300: - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default TIMEOUT (30) + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified TIMEOUT (30) + # qwen4_exp FP8 target at tp1, pre-merge: gsm8k without speculation. + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_tp1_gsm8k TIMEOUT (60) - unittest/_torch/modeling_v2/models/minimax_m3/test_shared_kernel_contracts.py --import-mode=importlib TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) - unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8 --import-mode=importlib --ignore-glob=unittest/_torch/modeling_v2/models/minimax_m3/**/comm/* TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 158acee33ce7..d338b53b88a8 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -104,6 +104,8 @@ l0_cpu: # The qwen4_exp default target's cpu_only tests (its routing among them); this stage runs # -m cpu_only, so the directory's GPU tests are left to l0_b300's modeling_v2 entry. - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default + # The qwen4_exp FP8 target's cpu_only tests, likewise. + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified - unittest/_torch/modeling_v2/models/glm5_next/test_glm5_next_routing.py - unittest/_torch/pyexecutor - unittest/_torch/executor/test_handle_responses_fast_path.py diff --git a/tests/integration/test_lists/test-db/l0_gb300.yml b/tests/integration/test_lists/test-db/l0_gb300.yml index 2795556365d1..8f4587b96a04 100644 --- a/tests/integration/test_lists/test-db/l0_gb300.yml +++ b/tests/integration/test_lists/test-db/l0_gb300.yml @@ -165,6 +165,13 @@ l0_gb300: - unittest/_torch/attention/multi_gpu/test_gqa_kv_replica.py - unittest/_torch/multi_gpu/test_mnnvl_fused_allreduce_norm.py - unittest/_torch/multi_gpu/test_ll_exchange.py + # qwen4_exp FP8 target, pre-merge: its 2- and 4-rank collective matrices (l0_b300 skips them), + # tep4 MTP3 over an FP8 KV pool, and tep4 MTP3 with IndexShare over 3k-6k-token pass keys (the + # sparse attention path) with their acceptance-length floor. dep4 and dep2 disaggregated run + # post-merge. + - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[tep4_fp8_kv_breakable] TIMEOUT (60) + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_long_context TIMEOUT (60) # K3 collectives and decode-path exchanges, four ranks in one NVLink domain. - condition: diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml index 2972172dcbdb..6f6ef09a4304 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml @@ -43,6 +43,9 @@ l0_gb300_multi_gpus: - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_fp8_kv_flashinfer_ring] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_mtp3_gsm8k_quantized_kv[tep4_nvfp4_kv_no_reuse] TIMEOUT (60) - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_accuracy.py::TestModelingV2Qwen38FlashNextNvfp4Sm103Default::test_disaggregated_dep2 TIMEOUT (90) + # qwen4_exp FP8 target, post-merge: dep4 MTP3, dep2 disaggregated. + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_mtp3_gsm8k[dep4] TIMEOUT (60) + - accuracy/modeling_v2/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_accuracy.py::TestModelingV2Qwen38FlashNextFp8Sm103Unified::test_disaggregated_dep2 TIMEOUT (90) # qwen4_exp target at tp1, post-merge: its AgentX engines (NVFP4 KV, MTP2 with rejection # sampling, the breakable prefill graph, every opt-in projection) on one context and one # generation GPU. diff --git a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py index 83ffa8f68833..948de78ced99 100644 --- a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py +++ b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py @@ -160,6 +160,11 @@ def _deviation(out: torch.Tensor, ref: torch.Tensor) -> str: _QWEN_INDEX_HEADS = dict(num_heads_q=4, num_heads_k=1, num_heads_v=1, head_dim=128) _QWEN_INDEX_MROPE = dict(**_QWEN_INDEX_HEADS, **_QWEN_MROPE) _QWEN_INDEX_PLAIN = dict(**_QWEN_INDEX_HEADS, **_QWEN_ROPE) +# One rank's main heads under attention TP2 / TP4 on the fused input projection row (the heads +# after k pass as v and only set the row stride), and the MTP draft layer's compact TP4 row. +_QWEN_TP2_ROW = dict(_QWEN_MAIN, num_heads_q=12, num_heads_k=1, num_heads_v=16) +_QWEN_TP4_ROW = dict(_QWEN_MAIN, num_heads_q=6, num_heads_k=1, num_heads_v=10) +_QWEN_TP4_MTP = dict(_QWEN_MAIN, num_heads_q=6, num_heads_k=1, num_heads_v=1) # The shapes each shipped target calls this entry with, by target directory # under `models/`. The entry's own `CELLS` are the op's whoever calls it. @@ -258,6 +263,29 @@ def _deviation(out: torch.Tensor, ref: torch.Tensor) -> str: spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_INDEX_PLAIN), ), ), + # Its tp1 / dep2 / dep4 calls are the NVFP4 target's shapes above. + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="qwen4_exp tp2 fused-row main rope, 1 token: 12 q / 1 k, 16 tail heads", + spec=dict(tokens=1, **_QWEN_TP2_ROW), + ), + Cell( + why="qwen4_exp tp2 fused-row main rope, 8192-token prefill, under GEMMA_PREFILL_ATOL", + spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_TP2_ROW), + ), + Cell( + why="qwen4_exp tp4 fused-row main rope, 1 token: 6 q / 1 k, 10 tail heads", + spec=dict(tokens=1, **_QWEN_TP4_ROW), + ), + Cell( + why="qwen4_exp tp4 fused-row main rope, 8192-token prefill, under GEMMA_PREFILL_ATOL", + spec=dict(tokens=8192, gemma_prefill_band=True, **_QWEN_TP4_ROW), + ), + Cell( + why="qwen4_exp tep4 MTP draft compact qkv 6 q / 1 k / 1 v, 1024 rows (128 x 8 verify)", + spec=dict(tokens=1024, **_QWEN_TP4_MTP), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py index 171e3358a88c..3552749064d4 100644 --- a/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py +++ b/tests/unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py @@ -7303,6 +7303,8 @@ def _cell_paged(cell, arm) -> None: _seed(cell) fp8 = bool(spec["quant_mode"] & QUANT_MODE_FP8_KV_CACHE) nvfp4 = bool(spec["quant_mode"] & QUANT_MODE_NVFP4_KV_CACHE) + cached, new, nc = spec["cached"], spec["new"], spec["num_contexts"] + longest = max(c + n for c, n in zip(cached, new)) env = _PagedAttnEnv( num_heads=spec["num_heads"], num_kv_heads=spec["num_kv_heads"], @@ -7310,21 +7312,27 @@ def _cell_paged(cell, arm) -> None: tokens_per_block=spec["tokens_per_block"], num_blocks=spec["num_blocks"], max_batch=spec["max_num_requests"], - max_blocks_per_seq=spec.get("max_blocks_per_seq", 16), + max_blocks_per_seq=spec.get( + "max_blocks_per_seq", max(16, -(-longest // spec["tokens_per_block"])) + ), pool_dtype=torch.float8_e4m3fn if fp8 else torch.bfloat16, quant_mode=spec["quant_mode"], kv_scaling_factor=spec["kv_scale"] if fp8 or nvfp4 else None, attention_window_size=spec["window"], ) - cached, new, nc = spec["cached"], spec["new"], spec["num_contexts"] p, paged = spec["predicted_tokens_per_seq"], spec["use_paged_context_fmha"] assert all(n == p for n in new[nc:]), "a generation row carries P q rows" assert paged or not any(cached[:nc]), "a cached prefix needs the paged-context path" rids = list(range(len(new))) for rid in rids: env.add_request(rid, cached[rid] + (new[rid] if rid < nc else 0)) - prior = [rid for rid in rids if cached[rid]] - if prior: + # The histories go in through context calls of at most `_FILL_TOKENS` rows each. + groups: List[List[int]] = [] + for rid in (rid for rid in rids if cached[rid]): + if not groups or sum(cached[r] for r in groups[-1]) + cached[rid] > _FILL_TOKENS: + groups.append([]) + groups[-1].append(rid) + for prior in groups: hist = [cached[rid] for rid in prior] env.call_op( env.random_qkv(sum(hist)), @@ -7488,6 +7496,40 @@ def _spread(batch: int, longest: int) -> tuple[int, ...]: return tuple(1 + (longest - 1) * i // (batch - 1) for i in range(batch)) +#: The most rows one uncertified history-filling context call of `_cell_paged` takes. +_FILL_TOKENS = 8192 +#: One rank of an attention-TP member of qwen4_exp: 12 q / 1 kv heads (TP2), 6 / 1 (TP4). +_QWEN4_TP2 = dict(QWEN4_DENSE, num_heads=12, num_kv_heads=1) +_QWEN4_TP4 = dict(QWEN4_DENSE, num_heads=6, num_kv_heads=1) + + +def _qwen4_batch(base: dict, requests: int, longest: int, p: int) -> dict: + """A generation step of `requests` `_spread` histories at P = `p`, plus 32 unwritten pages.""" + cached = _spread(requests, longest) + pages = sum(-(-(c + p) // base["tokens_per_block"]) for c in cached) + return dict( + base, + max_num_requests=requests, + num_blocks=pages + 32, + cached=cached, + new=(p,) * requests, + num_contexts=0, + predicted_tokens_per_seq=p, + ) + + +def _qwen4_prefill_chunk(base: dict) -> dict: + """max_num_tokens of fresh prefill: four 2048-token contexts, 8192 q rows.""" + return dict( + base, + num_blocks=4 * 2048 // base["tokens_per_block"] + 32, + cached=(0,) * 4, + new=(2048,) * 4, + num_contexts=4, + predicted_tokens_per_seq=1, + ) + + # The shapes each shipped target calls this entry with, by target directory # under `models/`. The entry's own `CELLS` are the op's whoever calls it. TARGET_CELLS: dict[str, tuple[Cell, ...]] = { @@ -7970,6 +8012,56 @@ def _spread(batch: int, longest: int) -> tuple[int, ...]: ), ), ), + # One rank of each attention layout at the target's largest batches; its other shapes are + # the NVFP4 target's above. + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="tp2 decode 12q/1kv at the dense ceiling: one row over a 2047-token history", + spec=dict( + _QWEN4_TP2, + cached=(2047,), + new=(1,), + num_contexts=0, + predicted_tokens_per_seq=1, + ), + ), + Cell( + why="tp2 decode 12q/1kv at 128 requests over histories 1..2047", + spec=_qwen4_batch(_QWEN4_TP2, 128, 2047, 1), + ), + Cell( + why="tp2 prefill chunk 12q/1kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(_QWEN4_TP2), + ), + Cell( + why="tp4 decode 6q/1kv at 128 requests over histories 1..2047", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2047, 1), + ), + Cell( + why="tep4 MTP3 verify P=4 6q/1kv: 128 blocks over histories 1..2040, 512 rows", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2040, 4), + ), + Cell( + why="tep4 MTP7 verify P=8 6q/1kv: 128 blocks over histories 1..2040, 1024 rows", + spec=_qwen4_batch(_QWEN4_TP4, 128, 2040, 8), + ), + Cell( + why="tp4 prefill chunk 6q/1kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(_QWEN4_TP4), + ), + Cell( + why="dep4 decode 24q/2kv at 128 requests per rank over histories 1..2047", + spec=_qwen4_batch(QWEN4_DENSE, 128, 2047, 1), + ), + Cell( + why="dep4 MTP3 verify P=4 24q/2kv: 128 blocks over histories 1..2040, 512 rows", + spec=_qwen4_batch(QWEN4_DENSE, 128, 2040, 4), + ), + Cell( + why="tp1 prefill chunk 24q/2kv: four 2048-token contexts, 8192 rows", + spec=_qwen4_prefill_chunk(QWEN4_DENSE), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py b/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py index 2d8d25a3f402..e5d4d9bbe225 100644 --- a/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py +++ b/tests/unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py @@ -48,12 +48,17 @@ def _build(spec, seed): `mat_b` is built as [N, K] and transposed, because that is the one thing a spec cannot say: the op reads it column-major, and a stored weight's `.t()` - already is that, which is why no transpose kernel runs in a target. + already is that, which is why no transpose kernel runs in a target. With + `rows`, the weight is `[rows, K]` and `mat_b` the first N columns of its + `.t()` (column stride still K); rows from `live` on are zero. """ g = torch.Generator(device="cuda").manual_seed(seed) m, k, n, dt = spec["m"], spec["k"], spec["n"], spec["dtype"] mat_a = torch.randn(m, k, generator=g, device="cuda").to(dt) - mat_b = torch.randn(n, k, generator=g, device="cuda").to(dt).t() + w = torch.randn(spec.get("rows", n), k, generator=g, device="cuda").to(dt) + if "live" in spec: + w[spec["live"] :] = 0 + mat_b = w.t()[:, :n] out_dtype = spec["out_dtype"] bias = None if spec["bias"]: @@ -114,6 +119,45 @@ def _build(spec, seed): spec=dict(m=16, k=2880, n=5120, dtype=torch.bfloat16, bias=True, out_dtype=None), ), ), + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="qwen fp8 attn in-proj tp2 7424 wide, the largest captured decode batch 128", + spec=dict(m=128, k=2560, n=7424, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 o_proj / GDN out_proj tp1, K 6144 -> hidden 2560, 128 rows", + spec=dict(m=128, k=6144, n=2560, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 GDN in_proj tp4 4120 wide, MTP-7 verify of 128 requests: 1024 rows", + spec=dict(m=1024, k=2560, n=4120, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 attn in-proj tp2 7424 wide, a max_num_tokens prefill chunk: 8192 rows", + spec=dict(m=8192, k=2560, n=7424, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 draft fc_h over 4 streams of an 8192-row chunk: 32768 rows", + spec=dict(m=32768, k=2560, n=2560, dtype=torch.bfloat16, bias=False, out_dtype=None), + ), + Cell( + why="qwen fp8 HC first fp32 logits: 352 columns of a padded [384, 10240] weight", + spec=dict( + m=128, + k=10240, + n=352, + dtype=torch.bfloat16, + bias=False, + out_dtype=torch.float32, + rows=384, + live=324, + ), + ), + Cell( + why="qwen fp8 vision qkv 1152 -> 3456 with bias, 5580 patches", + spec=dict(m=5580, k=1152, n=3456, dtype=torch.bfloat16, bias=True, out_dtype=None), + ), + ), } CELLS = certified_cells(op, TARGET_CELLS) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py index 70c6bec2a95b..41f06f21925a 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/activation/test_modeling_v2_fused_sigmoid_mul_inplace.py @@ -2,19 +2,25 @@ # SPDX-License-Identifier: Apache-2.0 """GPU test for the fused_sigmoid_mul_inplace catalog entry: every cell against the entry's reference with nothing written past `attention_output`'s last row, -graph capture of the default target's fused-row and MTP forms, the band refusing -a wrong gate read, and the loud layout refusal.""" +the direct launch bitwise equal to the registered op, graph capture of the +default target's fused-row and MTP forms, the band refusing a wrong gate read, +and the loud layout refusal.""" import zlib import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation import ( + fused_sigmoid_mul_inplace as op_module, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.activation.fused_sigmoid_mul_inplace import ( # noqa: E501 fused_sigmoid_mul_inplace as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "fused_sigmoid_mul_inplace requires a CUDA device" @@ -68,15 +74,48 @@ def _mtp_spec(tokens: int) -> dict: return dict(spec, tokens=tokens) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The FP8 target calls with `direct_launch=True`: 12 heads off its tp2/tep2 7424-wide row (gate +#: at column 3584), and its dep4 draft layer's 24-head `[T, 24, 512]` q_proj gate half. +_FP8_TP2_ROW = dict( + heads=12, head_dim=256, dtype=torch.bfloat16, gate="fused_row", row_stride=7424, gate_col=3584 +) +_FP8_MTP = dict(heads=24, head_dim=256, dtype=torch.bfloat16, gate="interleaved") + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 decode, one token: 12 heads off the 7424-wide row, direct launch", + spec=dict(tokens=1, direct_launch=True, **_FP8_TP2_ROW), + ), + Cell( + why="fp8 tp2/tep2 prefill chunk, 8192 rows, 12 heads, masked lanes, direct launch", + spec=dict(tokens=8192, direct_launch=True, **_FP8_TP2_ROW), + ), + Cell( + why="fp8 dep4 drafter at 128 requests: 24-head q_proj gate half, direct launch", + spec=dict(tokens=128, direct_launch=True, **_FP8_MTP), + ), + Cell( + why="fp8 dep4 draft layer over an 8192-row prefill chunk, 24 heads, direct launch", + spec=dict(tokens=8192, direct_launch=True, **_FP8_MTP), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name certifying(op) out, out_buffer, gate_buffer, gate = _build(cell.spec, seed=_seed(cell.why)) gate_before = gate_buffer.clone() tail_before = out_buffer[out.numel() :].clone() - expected = op.reference(out, gate) + direct = cell.spec.get("direct_launch", False) + expected = op.reference(out, gate, direct) - result = op(out, gate) + result = op(out, gate, direct) assert result.data_ptr() == out.data_ptr() and result.shape == out.shape torch.testing.assert_close(result, expected) @@ -84,7 +123,41 @@ def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves th assert torch.equal(out_buffer[out.numel() :], tail_before), "a masked lane stored" -def test_cuda_graph_capture_and_replay() -> None: +#: The direct-launch cells and more shapes for the pin test: 0 rows, both block widths +#: either side of 1024 rows, a compact 2-D gate, the default target's fused-row and +#: 6-head forms. +_PIN_SPECS = tuple(c.spec for c in CELLS if c.spec.get("direct_launch")) + ( + dict(tokens=0, **_FP8_TP2_ROW), + dict(tokens=1023, **_FP8_TP2_ROW), + dict(tokens=1024, **_FP8_TP2_ROW), + dict(tokens=1, **_FP8_MTP), + dict(_FP8_TP2_ROW, tokens=64, row_stride=3072, gate_col=0), + dict(next(c.spec for c in op.CELLS if c.spec.get("row_stride") == 4352), tokens=8), + _mtp_spec(512), +) + + +@pytest.mark.parametrize( + "spec", + _PIN_SPECS, + ids=[f"{s['tokens']}x{s['heads']}h-{s['gate']}-{s.get('row_stride', '')}" for s in _PIN_SPECS], +) +def test_direct_launch_is_bitwise_the_registered_op(spec) -> None: + """`_launch` is a copy of `torch.ops.trtllm.fused_sigmoid_mul_inplace`'s launch: it + launches at every shape here and writes the op's bytes, so a change to the op's + launch (kernel, grid, block) fails here.""" + out, _, _, gate = _build(spec, seed=_seed(f"pin {spec}")) + expected = out.clone() + torch.ops.trtllm.fused_sigmoid_mul_inplace(expected, gate) + direct = out.clone() + + assert op_module._launch(direct, gate), "the direct launch declined" + assert torch.equal(direct, expected) + assert torch.equal(op(out, gate, True), expected) + + +@pytest.mark.parametrize("direct_launch", [False, True], ids=["op", "direct"]) +def test_cuda_graph_capture_and_replay(direct_launch) -> None: """The decode call sites run captured: the tp4 / tep4 form (6 heads off the 4352-wide row, two 1024-wide blocks a row) replays on refilled inputs.""" spec = next(c.spec for c in op.CELLS if c.spec.get("row_stride") == 4352) @@ -93,11 +166,11 @@ def test_cuda_graph_capture_and_replay() -> None: side = torch.cuda.Stream() side.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side): - op(out, gate) # JIT compile outside capture + op(out, gate, direct_launch) # JIT compile outside capture torch.cuda.current_stream().wait_stream(side) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): - op(out, gate) + op(out, gate, direct_launch) for seed in (11, 12): fresh_out, _, fresh_gate_buffer, _ = _build(spec, seed=seed) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py index 6606eab87e21..6bd0734141e0 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_paged_index_scores.py @@ -500,3 +500,148 @@ def test_rejects_metadata_shorter_than_the_rows(only_visible_blocks) -> None: args["query_positions"] = args["query_positions"][:4] with pytest.raises(ValueError, match="one entry per row"): op(**args) + + +@pytest.mark.parametrize( + "bad, match", + [ + (dict(tokens_per_block=0), "positive pages"), + (dict(compress_ratio=1), "compression ratio"), + ("float_positions", "integer storage"), + ("cpu_table", "one CUDA device"), + ("empty_table", "non-empty block table"), + ], + ids=["tokens_per_block-0", "compress_ratio-1", "float-positions", "cpu-table", "empty-table"], +) +def test_prefill_query_tile_rejects_what_the_engine_rejects(bad, match) -> None: + """The query-tiled launch refuses the operands the engine's launcher refuses.""" + gen = torch.Generator(device=DEV).manual_seed(1703) + rows, tpb, cr = _entry._DECODE_ROW_LIMIT + 44, 32, 4 + cache = (torch.randn(16, tpb, 1, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + q = (torch.randn(rows, 4, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + args = dict( + q=q, + index_cache=cache, + block_table=torch.arange(16, dtype=torch.int32, device=DEV).view(1, 16), + query_positions=torch.arange(rows, dtype=torch.int64, device=DEV), + request_indices=torch.zeros(rows, dtype=torch.int32, device=DEV), + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=True, + ) + assert _entry._takes_prefill_query_tile(q, True, True, False) + if bad == "float_positions": + args["query_positions"] = args["query_positions"].float() + elif bad == "cpu_table": + args["block_table"] = args["block_table"].cpu() + elif bad == "empty_table": + args["block_table"] = args["block_table"][:, :0] + else: + args.update(bad) + with pytest.raises(ValueError, match=match): + _entry._qsa_index_scores_prefill_query_tile( + **{k: v for k, v in args.items() if k not in ("only_visible_blocks", "context_rows")} + ) + + +@pytest.mark.parametrize( + "chunks", + [((67584, 8190),), ((5000, 300), (20000, 1700)), ((131072, 4093),)], + ids=["one-request-8190-at-67k", "two-requests", "ragged-4093-at-128k"], +) +def test_prefill_query_tile_is_bitwise_the_engine_launch(chunks) -> None: + """`prefill_query_tile` takes the deeper query-tiled launch, bitwise the engine's launch on + every visible column. `chunks`: `(position of the first row, rows)` per request.""" + gen = torch.Generator(device=DEV).manual_seed(len(chunks) * 7919 + chunks[0][0]) + tpb, cr, width = 32, 4, 8196 + pages = [math.ceil((p + n) / tpb) for p, n in chunks] + pool = sum(pages) + 8 + cache = (torch.randn(pool, tpb, 1, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + table = torch.full((16, width), -1, dtype=torch.int32, device=DEV) + perm = 8 + torch.randperm(sum(pages), generator=gen, device=DEV).to(torch.int32) + slots = [3 + 5 * r for r in range(len(chunks))] + start = 0 + for slot, n_pages in zip(slots, pages): + table[slot, :n_pages] = perm[start : start + n_pages] + start += n_pages + rows = sum(n for _, n in chunks) + q = (torch.randn(rows, 6, 128, generator=gen, device=DEV) * 0.5).to(torch.bfloat16)[:, :4] + positions = torch.cat( + [torch.arange(p, p + n, dtype=torch.int32, device=DEV) for p, n in chunks] + ) + requests = torch.cat( + [torch.full((n,), s, dtype=torch.int32, device=DEV) for s, (_, n) in zip(slots, chunks)] + ) + args = dict( + q=q, + index_cache=cache, + block_table=table, + query_positions=positions, + request_indices=requests, + tokens_per_block=tpb, + compress_ratio=cr, + only_visible_blocks=True, + context_rows=True, + ) + assert _entry._takes_prefill_query_tile(q, True, True, False) + with _watch_launches() as watcher: + out = op(**args, prefill_query_tile=True) + torch.cuda.synchronize() + assert watcher.branch == "query_tile" + assert watcher.grid[0] == math.ceil(rows / _entry._PREFILL_QUERY_TILE) + with _watch_launches() as watcher: + default = op(**args) + torch.cuda.synchronize() + assert watcher.branch == "query_tile" + assert watcher.grid[0] != math.ceil(rows / _entry._PREFILL_QUERY_TILE) + ref = _engine_scores(**args) + vis = _visible(args) + assert torch.equal(torch.where(vis, out, 0.0), torch.where(vis, ref, 0.0)) + assert torch.equal(torch.where(vis, default, 0.0), torch.where(vis, ref, 0.0)) + + +def test_visible_split_compiles_one_variant_up_to_a_row_tile() -> None: + """Launches of up to one row tile (`16 // heads` rows) share one compiled variant.""" + dev = torch.cuda.current_device() + kernel = _entry._qsa_index_scores_visible_split_kernel + heads, dim, tpb, cr = 4, 128, 32, 4 + g = torch.Generator(device=DEV).manual_seed(7) + cache = torch.randn(64, tpb, 1, dim, generator=g, device=DEV).to(torch.bfloat16) + table = torch.randint(0, 64, (8, 40), generator=g, device=DEV, dtype=torch.int32) + seen = [] + for rows in (4, 1, 2, 3, 5, 8): + q = torch.randn(rows, heads, dim, generator=g, device=DEV).to(torch.bfloat16) + positions = torch.randint(100, 1279, (rows,), generator=g, device=DEV, dtype=torch.int64) + requests = torch.randint(0, 8, (rows,), generator=g, device=DEV, dtype=torch.int32) + args = (q, cache, table, positions, requests, tpb, cr, True, False) + _compare(op(*args), op.reference(*args)) + seen.append(len(kernel.device_caches[dev][0])) + assert len(set(seen)) == 1, seen + + +@pytest.mark.parametrize("rows", [32, 64, 96]) +def test_visible_split_wide_batch_programs_are_bitwise_the_narrow_split(rows: int) -> None: + """`wide_split` scores every visible column bitwise as the narrow split, over ragged windows.""" + g = torch.Generator(device=DEV).manual_seed(rows) + tpb, cr, heads, dim = 32, 4, 4, 128 + ctx = torch.randint(64, 200000, (rows,), generator=g, device=DEV) + pages = (ctx + tpb - 1) // tpb + max_pages = int(pages.max()) + total = int(pages.sum()) + cache = torch.randn(total, tpb, 1, dim, generator=g, device=DEV).to(torch.bfloat16) + perm = torch.randperm(total, generator=g, device=DEV).to(torch.int32) + bt = torch.zeros(rows, max_pages, dtype=torch.int32, device=DEV) + off = 0 + for r in range(rows): + bt[r, : int(pages[r])] = perm[off : off + int(pages[r])] + off += int(pages[r]) + q = torch.randn(rows, heads, dim, generator=g, device=DEV).to(torch.bfloat16) + pos = (ctx - 1).to(torch.int64) + req = torch.arange(rows, dtype=torch.int64, device=DEV) + wide = _entry._qsa_index_scores_visible_split(q, cache, bt, pos, req, tpb, cr, False, wide=True) + narrow = _entry._qsa_index_scores_visible_split(q, cache, bt, pos, req, tpb, cr, False) + torch.cuda.synchronize() + visible = torch.clamp((pos + 1) // cr, max=wide.shape[1]) + mask = torch.arange(wide.shape[1], device=DEV)[None, :] < visible[:, None] + assert torch.equal(torch.where(mask, wide, 0.0), torch.where(mask, narrow, 0.0)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py index 5b8bebbb857b..023a8bb6400f 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/attention/test_modeling_v2_qsa_topk_expand_decode.py @@ -139,6 +139,35 @@ def test_wide_rows_equal_the_canonical_composition(width: int, distribution: str assert torch.equal(fused, composed), "fused token expansion" +@pytest.mark.parametrize("rows", [149, 192, 288, 576, 1500]) +@pytest.mark.parametrize("distribution", ["relu_sum", "relu_sparse"]) +def test_wide_rows_past_one_wave_in_row_order(rows: int, distribution: str) -> None: + """A `row_order` past one wave of rows (longest window first, and an arbitrary permutation): + block lists and token selections equal the default launch's and the reference's, over + ragged windows with ties.""" + g = torch.Generator(device=DEV).manual_seed(rows) + pos = torch.randint(1, 262143, (rows,), generator=g, device=DEV) + pos[::17] = 99999 # ties of window length + pos[3] = 262143 + spec = dict(rows=rows, pos=pos.tolist(), width=65568, **{distribution: True}) + ends = _build(spec, seed=rows + 1)[1] + orders = ( + None, + torch.argsort(ends, descending=True, stable=True).to(torch.int32), + torch.randperm(rows, generator=g, device=DEV).to(torch.int32), + ) + outs = [] + for row_order in orders: + args = _build(spec, seed=rows + 1) + sel = op(*args, row_order=row_order) + torch.cuda.synchronize() + outs.append((args[4].clone(), sel.clone())) + want_topk, want_sel = op.reference(*_build(spec, seed=rows + 1)) + for topk, sel in outs: + assert torch.equal(topk, want_topk), "block list vs reference" + assert torch.equal(sel, want_sel.to(torch.int32)), "token selection vs reference" + + # The decode-path shapes, captured in the decode CUDA graph: tp1's one row at 2560 columns, # the default target's relu-sparse decode and verify cells (1 and 128 rows at core's width, 1024 # at MTP's), and the wide tables' decode cells (`graph` in their spec). @@ -180,6 +209,45 @@ def step(): assert torch.equal(graph_sel, want_sel), f"replay {trial}: token selection" +def test_row_order_sorted_in_the_graph_follows_the_replayed_metadata() -> None: + """The decode step's ordered launch: the row order sorted inside the graph from the row + ends, then read by the selection after the wait. Each replay refills the scores and the + rows' positions, lengths and ends (288 ragged rows at the 65568-wide table).""" + rows = 288 + g = torch.Generator(device=DEV).manual_seed(rows) + + def spec(): + pos = torch.randint(1, 262143, (rows,), generator=g, device=DEV) + return dict(rows=rows, pos=pos.tolist(), width=65568, relu_sum=True) + + args = _build(spec(), seed=21) + src = args[0].clone() + + def step(): + args[0].copy_(src * 1.0) + order = torch.argsort(args[1], descending=True, stable=True).to(torch.int32) + return op(*args, row_order=order) + + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + step() + torch.cuda.current_stream().wait_stream(warmup) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_sel = step() + for trial in range(3): + fresh = _build(spec(), seed=100 + trial) + src.copy_(fresh[0]) + for i in (1, 2, 3): + args[i].copy_(fresh[i]) + graph.replay() + torch.cuda.synchronize() + want_topk, want_sel = op.reference(src, *args[1:]) + assert torch.equal(args[4], want_topk), f"replay {trial}: block list" + assert torch.equal(graph_sel, want_sel), f"replay {trial}: token selection" + + def test_guard_refuses_a_token_budget_that_is_not_k_blocks() -> None: """`token_topk` sizes the selection's rows while `k * compress_ratio` places the selected tokens: a smaller budget writes past each row, into the next row and, for the last one, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py index 3ea63f29bd8e..19235c5748e4 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_decode_rows_tc_sync.py @@ -1,17 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Source-level check of the decode hyper-connection kernel's two mailbox waits. - -``_First2`` receives two kinds of data from the peer CTAs of its cluster through shared-memory -mailboxes credited with ``st.async``: the K half's block rows (``bbar``, folded sites only) and -the cluster's row partials (``rbar``). ``st.async``'s complete-tx releases at cluster scope, so -the receiver's mailbox reads must be ordered by a cluster-scope acquire; ``cute.arch.mbarrier_wait`` -acquires at CTA scope only. The kernel uses a relaxed cluster-scope try-wait followed by -``fence.acquire.sync_restrict::shared::cluster.cluster`` at both waits, which orders the reads -without the L1 invalidation an acquire-qualified try-wait incurs. An accuracy test cannot pin -this down (the payload and its completion are not reordered on current hardware), so this test -scans the kernel's source and fails if either ordering is changed back. No GPU, no CUDA and no -cutlass import are needed. +"""Source-level check of the decode hyper-connection kernels' mailbox waits. + +``_First2`` receives data from the peer CTAs of its cluster through shared-memory mailboxes +credited with ``st.async``: the family kernel receives the K half's block rows (``bbar``, folded +sites only) and the cluster's row partials (``rbar``); the FP8 target's copy receives the row +partials (``rbar``). ``st.async``'s complete-tx releases at cluster scope, so the receiver's +mailbox reads must be ordered by a cluster-scope acquire; ``cute.arch.mbarrier_wait`` and an +inline ``mbarrier.try_wait`` without a scope acquire at CTA scope only. Both kernels use a relaxed +cluster-scope try-wait followed by ``fence.acquire.sync_restrict::shared::cluster.cluster`` at +every mailbox wait, which orders the reads without the L1 invalidation an acquire-qualified +try-wait incurs. An accuracy test cannot pin this down (the payload and its completion are not +reordered on current hardware), so this test scans the kernels' sources and fails if either +ordering is changed back. No GPU, no CUDA and no cutlass import are needed. """ import ast @@ -21,30 +22,45 @@ import tensorrt_llm -_KERNEL = ( - Path(tensorrt_llm.__file__).parent - / "_torch" - / "modeling_v2" - / "models" - / "qwen4_exp" +_QWEN4_EXP = Path(tensorrt_llm.__file__).parent.joinpath( + "_torch", "modeling_v2", "models", "qwen4_exp" +) +_FAMILY = _QWEN4_EXP / "ops" / "hc" / "qwen4_exp_hc_decode_rows_tc.py" +_FP8_TARGET = ( + _QWEN4_EXP + / "qwen3_8_flash_next_fp8__sm_103__unified" / "ops" / "hc" - / "qwen4_exp_hc_decode_rows_tc.py" + / "qwen4_exp_hc_decode_rows_tc__decode.py" ) _WAIT = "prims.mbarrier_try_wait_parity" _FENCE = "prims.fence_sync_restrict" # (barrier pointer, barrier address, scatter helper, mailbox tensor read after the wait) -_MAILBOXES = [ - ("bbar_ptr", "bbar", "_st_async_v4", "sblk_vec"), - ("rbar_ptr", "rbar", "_st_async_f32", "sRed"), +_RBAR = ("rbar_ptr", "rbar", "_st_async_f32", "sRed") +_BBAR = ("bbar_ptr", "bbar", "_st_async_v4", "sblk_vec") + +# kernel -> (source file, its mailboxes, the barriers an inline CTA-scope wait may take). The FP8 +# target's weight-row barrier (``wbar``) completes on this CTA's own bulk copies from global +# memory, for which a CTA-scope acquire is enough. +_KERNELS = { + "family": (_FAMILY, [_BBAR, _RBAR], set()), + "fp8_target": (_FP8_TARGET, [_RBAR], {"wbar"}), +} +_CASES = [ + pytest.param(name, mailbox, id=f"{name}-{mailbox[1]}") + for name, (_, mailboxes, _) in _KERNELS.items() + for mailbox in mailboxes ] @pytest.fixture(scope="module") -def module() -> ast.Module: - """The kernel module's AST, parsed once from the installed source file.""" - return ast.parse(_KERNEL.read_text(), filename=str(_KERNEL)) +def modules() -> dict[str, ast.Module]: + """Each kernel module's AST, parsed once from the installed source file.""" + return { + name: ast.parse(path.read_text(), filename=str(path)) + for name, (path, _, _) in _KERNELS.items() + } def _method(module: ast.Module, cls: str, name: str) -> ast.FunctionDef: @@ -54,7 +70,7 @@ def _method(module: ast.Module, cls: str, name: str) -> ast.FunctionDef: for item in node.body: if isinstance(item, ast.FunctionDef) and item.name == name: return item - raise AssertionError(f"{cls}.{name} not found in {_KERNEL.name}") + raise AssertionError(f"{cls}.{name} not found") def _calls(node: ast.AST) -> list[ast.Call]: @@ -79,10 +95,28 @@ def _only(calls: list[ast.Call], what: str) -> ast.Call: return calls[0] -def test_no_cta_scope_wait_on_the_mailbox_barriers(module) -> None: +def _inline_waits(module: ast.Module) -> set[str]: + """The module-level helpers whose inline PTX is an ``mbarrier.try_wait``.""" + return { + node.name + for node in module.body + if isinstance(node, ast.FunctionDef) + and any( + isinstance(s, ast.Constant) + and isinstance(s.value, str) + and "mbarrier.try_wait" in s.value + for s in ast.walk(node) + ) + } + + +@pytest.mark.parametrize("name", list(_KERNELS)) +def test_no_cta_scope_wait_on_the_mailbox_barriers(modules, name: str) -> None: """Neither the DSL's CTA-scope waits nor an inline CTA-scope try-wait touch the mailbox barriers.""" + _, mailboxes, inline_ok = _KERNELS[name] + module = modules[name] kernel = _method(module, "_First2", "kernel") - barriers = {name for mailbox in _MAILBOXES for name in mailbox[:2]} + barriers = {n for mailbox in mailboxes for n in mailbox[:2]} cta_scope_waits = [ c for c in _calls(kernel) @@ -92,22 +126,20 @@ def test_no_cta_scope_wait_on_the_mailbox_barriers(module) -> None: assert not cta_scope_waits, "a mailbox wait must not acquire at CTA scope:\n" + "\n".join( f" line {c.lineno}: {ast.unparse(c)}" for c in cta_scope_waits ) - inline_waits = [ - s.value - for s in ast.walk(module) - if isinstance(s, ast.Constant) - and isinstance(s.value, str) - and "mbarrier.try_wait" in s.value - ] - assert not inline_waits, f"no inline-PTX mbarrier wait is expected any more: {inline_waits}" + helpers = _inline_waits(module) + inline_calls = [c for c in _calls(module) if _callee(c) in helpers] + unexpected = [c for c in inline_calls if _first_arg(c) not in inline_ok] + assert not unexpected, ( + f"an inline CTA-scope mbarrier wait may take only {sorted(inline_ok)}:\n" + + "\n".join(f" line {c.lineno}: {ast.unparse(c)}" for c in unexpected) + ) -@pytest.mark.parametrize(("barrier", "address", "scatter", "mailbox"), _MAILBOXES) -def test_mailbox_wait_acquires_at_cluster_scope( - module, barrier: str, address: str, scatter: str, mailbox: str -) -> None: +@pytest.mark.parametrize(("name", "mailbox_case"), _CASES) +def test_mailbox_wait_acquires_at_cluster_scope(modules, name: str, mailbox_case: tuple) -> None: """A relaxed cluster-scope wait plus a cluster acquire fence, after the pushes and before the reads.""" - kernel = _method(module, "_First2", "kernel") + barrier, address, scatter, mailbox = mailbox_case + kernel = _method(modules[name], "_First2", "kernel") calls = _calls(kernel) wait = _only( diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py index 1cfb886f15a2..7cec23234708 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_lowrank_act.py @@ -48,13 +48,52 @@ def _build(spec, seed): return logits, RANK, hc_count, spec.get("inject", True), spec.get("dtype", torch.bfloat16) +def _capture(args): + """Capture one call over a private copy of the step's logits after a side-stream warmup: + `(graph, static logits, static outputs)`.""" + static = args[0].clone() + rest = args[1:] + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, *rest) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, *rest) + return graph, static, out + + +def _equal(a, b) -> bool: + """Both outputs bitwise equal (`inject` may be None on both sides).""" + return all((x is None and y is None) or torch.equal(x, y) for x, y in zip(a, b)) + + @pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) - args = _build(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) + seed = zlib.crc32(cell.why.encode()) % (2**31) + args = _build(cell.spec, seed=seed) low, inject = op(*args) assert low.shape == (cell.spec["rows"], RANK) _compare((low, inject), op.reference(*args)) + if not cell.spec.get("capture"): + return + + # The replay is bitwise the eager call; a second step's logits replay bitwise their own. + graph, static, replayed = _capture(args) + graph.replay() + torch.cuda.synchronize() + assert _equal(replayed, (low, inject)), "the replay differs from the eager call it captured" + second = _build(cell.spec, seed=(seed + 1) % (2**31)) + assert not torch.equal(second[0], args[0]), "the second step brings other logits" + static.copy_(second[0]) + graph.replay() + torch.cuda.synchronize() + step2 = tuple(None if t is None else t.clone() for t in replayed) + assert _equal(step2, op(*second)), "the second replay differs from its eager call" + assert not _equal(step2, (low, inject)), "the second replay repeated the first step" + _compare(step2, op.reference(*second)) def test_guard_refuses_bf16_logits() -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py index 6c7c33fd817f..da6c074f8d63 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/hc/test_modeling_v2_qwen4_exp_hc_up_gate_mean.py @@ -64,7 +64,7 @@ def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by mixed = op(*args) assert mixed.shape == (cell.spec["rows"], cell.spec.get("hidden", 2560)) # The answer came from the CuTe DSL kernel, JIT-compiled for this device. - assert (_BLOCK_H, args[3], mixed.shape[1], args[0].shape[1], False) in _COMPILED + assert (_BLOCK_H, args[3], mixed.shape[1], args[0].shape[1], False, False) in _COMPILED _compare(mixed, op.reference(*args)) @@ -90,3 +90,22 @@ def test_cuda_graph_capture_and_replay() -> None: graph.replay() torch.cuda.synchronize() assert torch.equal(g_mixed, op(fresh[0], up, fresh[2], c)) + + +@pytest.mark.parametrize("rows", [1, 17, 96, 129, 192, 288, 384, 480]) +def test_wide_epilogue_is_bitwise_the_four_warp_one(rows) -> None: + """`wide=True` gives the four-warp epilogue's bits, eagerly and replayed from a CUDA graph.""" + low, up, normed, hc = _build(dict(rows=rows), seed=4100 + rows) + four = op(low, up, normed, hc) + eight = op(low, up, normed, hc, wide=True) + torch.cuda.synchronize() + assert torch.equal(four, eight) + static = low.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, up, normed, hc, wide=True) + fresh, _, _, _ = _build(dict(rows=rows), seed=4200 + rows) + static.copy_(fresh) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, op(fresh, up, normed, hc)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py index 332fd3e44f42..5c83bc809235 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/moe/test_modeling_v2_renorm_moe_routing_op.py @@ -69,24 +69,35 @@ def _check(logits, topk, output_dtype=None): return ids, weights -# ── the certified cells ─────────────────────────────────────────────────── +def _capture(logits, topk, output_dtype): + """Capture one call over a private copy of the step's logits after a side-stream warmup: + `(graph, static logits, static outputs)`.""" + static = logits.clone() + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, topk, output_dtype) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, topk, output_dtype) + return graph, static, out -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) -def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name - certifying(op) - s = cell.spec - logits = _logits( - s["tokens"], - s["num_experts"], - s["logits_dtype"], - s["dist"], - s.get("magnitude", 1.0), - seed=zlib.crc32(cell.why.encode()), +def _same(a, b) -> bool: + """Ids and weights bitwise equal.""" + return torch.equal(a[0], b[0]) and torch.equal(a[1], b[1]) + + +def _cell_logits(s, seed): + return _logits( + s["tokens"], s["num_experts"], s["logits_dtype"], s["dist"], s.get("magnitude", 1.0), seed ) - ids, weights = _check(logits, s["topk"], s["output_dtype"]) + + +def _check_sums(weights, s) -> None: + """Renormalized: every row sums to 1.""" assert weights.dtype == s["output_dtype"] - # renormalized: every row sums to 1 torch.testing.assert_close( weights.float().sum(-1), torch.ones(s["tokens"], device=DEV), @@ -95,6 +106,38 @@ def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves th ) +# ── the certified cells ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name + certifying(op) + s = cell.spec + seed = zlib.crc32(cell.why.encode()) + logits = _cell_logits(s, seed) + ids, weights = _check(logits, s["topk"], s["output_dtype"]) + _check_sums(weights, s) + if not s.get("capture"): + return + + # The replay is bitwise the eager call; a second step's logits replay bitwise their own. + graph, static, replayed = _capture(logits, s["topk"], s["output_dtype"]) + graph.replay() + torch.cuda.synchronize() + assert _same(replayed, (ids, weights)), "the replay differs from the eager call it captured" + second = _cell_logits(s, seed + 1) + static.copy_(second) + graph.replay() + torch.cuda.synchronize() + step2 = (replayed[0].clone(), replayed[1].clone()) + assert not torch.equal(step2[0], ids), "the second step reroutes" + assert _same(step2, op(second, s["topk"], s["output_dtype"])), ( + "the second replay differs from its eager call" + ) + _compare(step2, op.reference(second, s["topk"], s["output_dtype"])) + _check_sums(step2[1], s) + + # ── what a cell cannot carry ────────────────────────────────────────────── diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py index 74f2bc8bac1d..12da4ea0ae9d 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/norm/test_modeling_v2_qwen4_exp_rms_norm_gated_token_major.py @@ -1,17 +1,24 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""GPU test for the rms_norm_gated_token_major entry.""" +"""GPU test for the rms_norm_gated_token_major entry: every cell against the +entry's reference, the direct launch bitwise equal to the registered op, and the +weight-length guard.""" import zlib import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm import ( + rms_norm_gated_token_major as op_module, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( # noqa: E501 rms_norm_gated_token_major as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "rms_norm_gated_token_major requires a CUDA device" @@ -58,10 +65,52 @@ def randn(*shape): z = randn(t, heads, n) weight = randn(n) - return x, z, weight, spec["eps"], None, spec["gate_activation"] - + return ( + x, + z, + weight, + spec["eps"], + None, + spec["gate_activation"], + spec.get("direct_launch", False), + ) + + +#: The FP8 target's tp2/tep2 rank, called with `direct_launch=True`: 24 heads, `z` in its +#: 8240-wide projection. +_FP8_TP2 = dict( + heads=24, + width=128, + dtype=torch.bfloat16, + eps=1e-6, + gate_activation="sigmoid", + z="in_proj", + z_row_stride=8240, + direct_launch=True, +) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2, one token: 24 heads, z at token stride 8240; 4-row kernel, PDL", + spec=dict(tokens=1, **_FP8_TP2), + ), + Cell( + why="fp8 tp2/tep2, 683 tokens (16392 rows): 4-row kernel, grid past the SM count", + spec=dict(tokens=683, **_FP8_TP2), + ), + Cell( + why="fp8 tp2/tep2 prefill at max_num_tokens 8192: 196608 rows on the 8-row kernel", + spec=dict(tokens=8192, **_FP8_TP2), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args = _build(cell.spec, seed=_seed(cell.why)) @@ -82,9 +131,47 @@ def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest res torch.testing.assert_close(out, op.reference(*args)) +#: The direct-launch cells below the 8-row threshold and more shapes for the pin test: the +#: last row count before it, a PDL grid at each width, the silu gate over a packed `z`. +_PIN_SPECS = tuple( + c.spec + for c in CELLS + if c.spec.get("direct_launch") + and c.spec["tokens"] * c.spec["heads"] < op_module._PREFILL_MIN_ROWS +) + ( + dict(_FP8_TP2, tokens=4), + dict(_FP8_TP2, tokens=128), + dict(_FP8_TP2, tokens=1365), + dict(op.CELLS[3].spec, tokens=4, direct_launch=True), + dict(op.CELLS[3].spec, tokens=2730, direct_launch=True), + dict(op.CELLS[-1].spec, direct_launch=True), + dict(_PACKED, tokens=16, gate_activation="silu", direct_launch=True), +) + + +@pytest.mark.parametrize( + "spec", + _PIN_SPECS, + ids=[ + f"{s['tokens']}x{s['heads']}h-{s.get('z_row_stride', s['z'])}-{s['gate_activation']}" + for s in _PIN_SPECS + ], +) +def test_direct_launch_is_bitwise_the_registered_op(spec) -> None: + """`_direct_launch` is a copy of `torch.ops.trtllm.rms_norm_gated_token_major`'s + multi-row launch: it is taken at every shape here and returns the op's bytes, so a + change to the op's launch (kernel, rows a program, warps) fails here.""" + x, z, weight, eps, _, act, _ = _build(spec, seed=_seed(f"pin {spec}")) + assert x.shape[0] < op_module._PREFILL_MIN_ROWS and op_module._multirow(x, z, act) + expected = torch.ops.trtllm.rms_norm_gated_token_major(x, z, weight, eps, None, act) + + assert torch.equal(op_module._direct_launch(x, z, weight, eps, act), expected) + assert torch.equal(op(x, z, weight, eps, None, act, True), expected) + + def test_guard_refuses_a_weight_of_the_wrong_length() -> None: """Unguarded, a length-192 weight is silently truncated to weight[:128].""" - x, z, weight, eps, _, act = _build(_PACKED, seed=12) + x, z, weight, eps, _, act, _ = _build(_PACKED, seed=12) long_weight = torch.cat([weight, torch.randn(64, dtype=weight.dtype, device="cuda")]) assert torch.equal( torch.ops.trtllm.rms_norm_gated_token_major(x, z, long_weight, eps, None, act), diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py index d9602c215253..fcb6c32779f2 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_gate_conv_inject_norm.py @@ -248,3 +248,90 @@ def test_cuda_graph_capture_and_replay(spec) -> None: assert torch.equal(args[7], eager[2]) if xc is not None: assert torch.equal(xc, eager_xc) + + +@pytest.mark.parametrize("rows_per_seq", (1, 3, 4)) +def test_inject_block_is_the_injection_then_the_op(rows_per_seq) -> None: + """`inject_block` is bitwise `qwen4_exp_hc_inject_norm` then the op over its `hc_out`, the + pool too; the gate is a strided column, as the target hands it.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, + ) + + args = _build(dict(lengths=(rows_per_seq,) * 2), seed=20 + rows_per_seq) + hc = args[0] + rows, width = hc.shape + g = torch.Generator(device=DEV).manual_seed(5) + routed = (0.5 * torch.randn(rows, CKPT_HIDDEN, device=DEV, generator=g)).to(hc.dtype) + shared = (0.5 * torch.randn(rows, CKPT_HIDDEN, device=DEV, generator=g)).to(hc.dtype) + gug = (2.0 * torch.randn(rows, 1288, device=DEV, generator=g)).to(hc.dtype) + gate = gug[:, 1280:1281] + inject = (2.0 * torch.sigmoid(torch.randn(rows, CKPT_HC_COUNT, device=DEV, generator=g))).to( + hc.dtype + ) + hc_inj, _ = qwen4_exp_hc_inject_norm( + hc, routed, inject, args[10], EPS, CKPT_HC_COUNT, shared, gate + ) + want = _run([hc_inj] + list(args[1:])) + args[7] = args[7].clone() + hc_out, normed = op(*args, inject_block=(routed, inject, shared, gate)) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + # Without a shared expert (a fresh pool: the call above advanced this one). + args = _build(dict(lengths=(rows_per_seq,) * 2), seed=20 + rows_per_seq) + hc_inj, _ = qwen4_exp_hc_inject_norm(args[0], routed, inject, args[10], EPS, CKPT_HC_COUNT) + want = _run([hc_inj] + list(args[1:])) + args[7] = args[7].clone() + hc_out, normed = op(*args, inject_block=(routed, inject, None, None)) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + + +@pytest.mark.parametrize("rows_per_seq", (1, 3)) +def test_pdl_launches_are_the_plain_launches(rows_per_seq) -> None: + """`pdl=True` is bitwise the plain launches, eagerly and replayed from a CUDA graph.""" + args = _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq) + want = _run(args) + args[7] = args[7].clone() + hc_out, normed = op(*args, pdl=True) + assert torch.equal(hc_out, want[0]) and torch.equal(normed, want[1]) + assert torch.equal(args[7], want[2]) + # Capture over other inputs, then refill every operand with `want`'s before the replay. + fresh = _build(dict(lengths=(rows_per_seq,) * 4), seed=40 + rows_per_seq) + pool = fresh[7].clone() + fresh[7] = pool + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + g_out, g_normed = op(*fresh, pdl=True) + for dst, src in zip(fresh, _build(dict(lengths=(rows_per_seq,) * 4), seed=30 + rows_per_seq)): + if isinstance(dst, torch.Tensor): + dst.copy_(src) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(g_out, want[0]) and torch.equal(g_normed, want[1]) + assert torch.equal(pool, want[2]) + + +def test_rows_per_sequence_variants_compile_at_the_first_call() -> None: + """Every `MAX_P` variant and the rows kernel compile at a geometry's first call: later row + counts add no variant, and outputs match the reference.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple import ( + ple_gate_conv_inject_norm as mod, + ) + + def variants(): + dev = torch.cuda.current_device() + return [ + len(k.device_caches[dev][0]) for k in (mod._ple_conv_kernel, mod._ple_conv_rows_kernel) + ] + + for carries in (None, True): + first = _build(dict(lengths=[3], carries=None if carries is None else [True]), seed=1) + _run(first) + warmed = variants() + for lengths in ([1], [2], [4], [1] * 5, [2, 4], [17], [40, 3]): + spec = dict(lengths=lengths, carries=None if carries is None else [True] * len(lengths)) + args = _build(spec, seed=len(lengths)) + out = _run(args) + _compare(out, op.reference(*args)) + assert variants() == warmed, (carries, warmed, variants()) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py index 8ed23185a6f9..7a2c44d4fd62 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ple/test_modeling_v2_ple_ngram_embed.py @@ -9,6 +9,7 @@ and graph capture at the note's replay cells. """ +import importlib import zlib from types import SimpleNamespace @@ -38,6 +39,10 @@ ) from tensorrt_llm._utils import CUASSERT +_embed_module = importlib.import_module( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_ngram_embed" +) + __extra_import_path__ = ["../../../.."] from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name @@ -217,3 +222,50 @@ def test_cuda_graph_capture_and_replay(lengths, pool_slots) -> None: assert torch.equal(graph_out, want) assert torch.equal(args[1], want_pool) assert not torch.equal(args[1], pool_before) + + +def test_per_sequence_rows_compile_at_first_call() -> None: + """1 to 4 rows a sequence match the reference, and every `BLOCK_P` variant compiles at the + first call per carry flag: later row counts add no compiled variant.""" + kernel = _embed_module._ple_ngram_embed_kernel + dev = torch.cuda.current_device() + seen = {} + for carries in (None, [True, False, True, True]): + for rows in (1, 2, 3, 4): + spec = dict(lengths=(rows,) * 4, pool_slots=8, carries=carries, eos_density=0.1) + args = _build(spec, seed=20 + rows) + ref, ref_pool = op.reference(*args) + out = op(*args) + torch.cuda.synchronize() + assert torch.equal(out, ref) and torch.equal(args[1], ref_pool), (rows, carries) + seen[(carries is None, rows)] = len(kernel.device_caches[dev][0]) + for flag in (True, False): + counts = {seen[(flag, rows)] for rows in (1, 2, 3, 4)} + assert len(counts) == 1, f"carry={not flag}: variants per row count {seen}" + + +@pytest.mark.parametrize("world", (2, 4)) +@pytest.mark.parametrize("lengths", ((3, 1, 4), (300, 7)), ids=("per-sequence", "rows")) +def test_row_windows_sum_to_the_whole_table(world, lengths) -> None: + """Each rank's `row_window` share matches the reference, from a slice of the table (its own + rows, `base = lo`) or from the whole table (`base = 0`); the shares sum to the whole output.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table + + args = _build(dict(lengths=lengths, pool_slots=8), seed=31 + world) + table = args[5] + rows, cols = table.shape + whole = op(*([a.clone() if isinstance(a, torch.Tensor) else a for a in args])) + total = torch.zeros_like(whole, dtype=torch.float32) + for rank in range(world): + lo, hi = ple_host_table.owned_rows(rank, world, rows, cols, align=cols) + for view, base in ((table[lo:hi], lo), (table, 0)): + # Fresh arguments for each call: the op advances the context pool in place. + call, ref_call = ( + [a.clone() if isinstance(a, torch.Tensor) else a for a in args] for _ in range(2) + ) + call[5] = ref_call[5] = view + share = op(*call, row_window=(lo, hi, base)) + ref, _ = op.reference(*ref_call, row_window=(lo, hi, base)) + assert torch.equal(share, ref), (rank, base) + total += share.float() + assert torch.equal(total.to(whole.dtype), whole) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py index 88ac87ad09de..9abd46e0e6ce 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_bf16_to_fp32.py @@ -8,11 +8,16 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( # noqa: E501 + VOCAB, +) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.bf16_to_fp32 import ( # noqa: E501 bf16_to_fp32 as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "bf16_to_fp32 requires a CUDA device" @@ -35,7 +40,20 @@ def _bitwise(a: torch.Tensor, b: torch.Tensor) -> bool: return a.shape == b.shape and torch.equal(a.view(torch.int32), b.view(torch.int32)) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tep2 MTP-2 verify logits, 128 requests x 3 rows", + spec=dict(shape=(384, VOCAB)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) x = _build(cell.spec, zlib.crc32(cell.why.encode())) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py index 7ef1a5d0b0d2..14cc376f9650 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/quantization/test_modeling_v2_fp8_quantize_1x128.py @@ -106,6 +106,42 @@ def _assert_misses_are_ties(x: torch.Tensor, data: torch.Tensor, ref_data: torch return rel +def _capture(x: torch.Tensor, use_ue8m0: bool): + """Capture one call over a private copy of the step's activations after a side-stream + warmup: `(graph, static input, static outputs)`.""" + static = x.clone() + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + op(static, use_ue8m0) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = op(static, use_ue8m0) + return graph, static, out + + +def _same(a: tuple[torch.Tensor, torch.Tensor], b: tuple[torch.Tensor, torch.Tensor]) -> bool: + """Codes and scales bitwise equal (codes compared as bytes).""" + return torch.equal(a[0].view(torch.uint8), b[0].view(torch.uint8)) and torch.equal(a[1], b[1]) + + +def _check_cell(label, x, use_ue8m0, data, scales) -> None: + """Shapes and layout, the gate against the entry's reference, and every miss a tie.""" + nb = (x.shape[1] + 127) // 128 + assert data.shape == x.shape and data.is_contiguous() and data.device == x.device + assert scales.shape == (nb, x.shape[0]) and scales.is_contiguous() + ref = op.reference(x, use_ue8m0) + _compare((data, scales), ref) + rel = _assert_misses_are_ties(x, data, ref[0]) if not use_ue8m0 else 0.0 + exact = (data.view(torch.uint8) == ref[0].view(torch.uint8)).float().mean().item() + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + print( + f" {label}: {100 * exact:.4f}% bit-exact, worst tie {rel:.3e}, " + f"{x.shape[0] * nb} groups over a {sms * 8 * 8 * 2}-group grid sweep" + ) + + # ── the certified cells ─────────────────────────────────────────────────── @@ -113,20 +149,29 @@ def _assert_misses_are_ties(x: torch.Tensor, data: torch.Tensor, ref_data: torch def test_certified_cells(cell, certifying): # noqa: F811 — pytest resolves the fixture by this name certifying(op) spec = cell.spec - x = _input(spec["tokens"], spec["hidden"], seed=zlib.crc32(cell.why.encode())) + seed = zlib.crc32(cell.why.encode()) + x = _input(spec["tokens"], spec["hidden"], seed=seed) data, scales = op(x, spec["use_ue8m0"]) - nb = (spec["hidden"] + 127) // 128 - assert data.shape == x.shape and data.is_contiguous() and data.device == x.device - assert scales.shape == (nb, spec["tokens"]) and scales.is_contiguous() - ref = op.reference(x, spec["use_ue8m0"]) - _compare((data, scales), ref) - rel = _assert_misses_are_ties(x, data, ref[0]) if not spec["use_ue8m0"] else 0.0 - exact = (data.view(torch.uint8) == ref[0].view(torch.uint8)).float().mean().item() - sms = torch.cuda.get_device_properties(x.device).multi_processor_count - print( - f" {cell.why[:44]}: {100 * exact:.4f}% bit-exact, worst tie {rel:.3e}, " - f"{spec['tokens'] * nb} groups over a {sms * 8 * 8 * 2}-group grid sweep" + _check_cell(cell.why[:44], x, spec["use_ue8m0"], data, scales) + if not spec.get("capture"): + return + + # The replay is bitwise the eager call; a second step's activations replay bitwise their own. + graph, static, replayed = _capture(x, spec["use_ue8m0"]) + graph.replay() + torch.cuda.synchronize() + assert _same(replayed, (data, scales)), "the replay differs from the eager call it captured" + second = _input(spec["tokens"], spec["hidden"], seed=seed + 1) + assert not torch.equal(second, x), "the second step brings other activations" + static.copy_(second) + graph.replay() + torch.cuda.synchronize() + step2 = (replayed[0].clone(), replayed[1].clone()) + assert _same(step2, op(second, spec["use_ue8m0"])), ( + "the second replay differs from its eager call" ) + assert not _same(step2, (data, scales)), "the second replay repeated the first step" + _check_cell(f"{cell.why[:44]} (second replay)", second, spec["use_ue8m0"], *step2) # ── what a cell cannot carry ────────────────────────────────────────────── diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py index ae508d639dd9..f6b4a3395db9 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_causal_conv1d_update.py @@ -7,18 +7,22 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( # noqa: E501 CKPT_DIM, CKPT_X_STRIDE, + GDN_PAGE_LAYERS, RANK_DIM, RANK_X_STRIDE, _decode_conv_ok, + _spread, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.causal_conv1d_update import ( # noqa: E501 causal_conv1d_update as op, ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "causal_conv1d_update requires a CUDA device" @@ -94,7 +98,29 @@ def _run(args, extras): return returned, got[1] -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: A tp2/tep2 rank of the FP8 target: 5120 channels (the TRT-LLM kernel below 64 rows, the CuTe +#: one from 64), in_proj row stride 8240, over the largest pool the TRT-LLM kernel addresses (its +#: `int slot * uint32_t stride` slot offset wraps from slot 7768 at this page's slot stride). +_TP2 = dict(dim=5120, x_row_stride=8240, pool_slots=7768, pool_layers=GDN_PAGE_LAYERS) + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 decode, 1 row at 5120 / stride 8240: the top addressable slot", + spec=dict(_TP2, batch=1, slots=_spread(1, 7768)), + ), + Cell( + why="fp8 tp2/tep2, 128 rows at 5120 / stride 8240, pad rows (-1) on the CuTe kernel", + spec=dict(_TP2, batch=128, slots=_spread(128, 7768, 21)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args, extras = _build(cell.spec, seed=zlib.crc32(cell.why.encode())) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py index 638a42342065..2ab51ad4a955 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_decode_rows_gated_norm.py @@ -28,6 +28,7 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm import ( gdn_decode_rows_gated_norm as rows_mod, ) @@ -39,6 +40,7 @@ ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "gdn_decode_rows_gated_norm requires a CUDA device" @@ -269,7 +271,32 @@ def _stats(out, ref_out, pool, ref_pool) -> str: ) -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +#: A tp2/tep2 rank of the FP8 target: 8 k / 24 v heads on its 8240-wide projection, bf16 state; +#: the register variant up to 12 rows, the bulk copy from 13 (24 v heads x 13 rows > 2 x 152 SMs). +_FP8_TP2 = dict(k_heads=8, v_heads=24, proj_width=8240, state="bf16") +_FP8 = "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified" + +#: The shapes each target calls `gdn_decode_rows_gated_norm` with beyond its own cells. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + _FP8: ( + Cell( + why="fp8 tp2/tep2 decode, 1 row at 8 / 24 heads, slot 7193 of a 7194-slot bf16 page", + spec=dict(_FP8_TP2, rows=1, pool="coalesced", pool_slots=7194, member="tp2"), + ), + Cell( + why="fp8 tp2/tep2, 128 rows (the bulk copy) at 8 / 24 heads on the 7194-slot page, " + "row 1 a pad slot (-1)", + spec=dict( + _FP8_TP2, rows=128, pad=True, pool="coalesced", pool_slots=7194, member="tp2" + ), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(op) args, engine = _build(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) @@ -467,7 +494,24 @@ def _rows_args(spec, seed): return [proj, pool, slots, A_log, dt_bias, h, k, out], buf -@pytest.mark.parametrize("cell", rows_op.CELLS, ids=[c.why[:44] for c in rows_op.CELLS]) +#: The shapes each target calls `gdn_decode_rows` with beyond its own cells. +ROWS_TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + _FP8: ( + Cell( + why="fp8 tp2/tep2 mixed step: 3 generation rows at 8 / 24 heads into a row range", + spec=dict(_FP8_TP2, rows=3, out="row_range"), + ), + Cell( + why="fp8 tp2/tep2 mixed step: 127 generation rows (the bulk copy) at 8 / 24 heads", + spec=dict(_FP8_TP2, rows=127, out="row_range"), + ), + ), +} + +ROWS_CELLS = certified_cells(rows_op, ROWS_TARGET_CELLS) + + +@pytest.mark.parametrize("cell", ROWS_CELLS, ids=[c.why[:44] for c in ROWS_CELLS]) def test_rows_certified_cells(cell, certifying) -> None: # noqa: F811 — fixture by name certifying(rows_op) args, buf = _rows_args(cell.spec, seed=zlib.crc32(cell.why.encode()) % (2**31)) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py index 9973bbd6f50e..7bb2485ef6d6 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_gdn_prefill_conv_post.py @@ -20,6 +20,7 @@ import pytest import torch +from tensorrt_llm._torch.modeling_v2.catalog._op import Cell from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( # noqa: E501 A_LOG_RANGE, CONV_WIDTH, @@ -29,6 +30,7 @@ IN_PROJ_WIDTH, NUM_K_HEADS, NUM_V_HEADS, + PAGE_GDN_LAYERS, QKV_DIM, SOFTPLUS_THRESHOLD, _engine_slots, @@ -38,6 +40,7 @@ ) __extra_import_path__ = ["../../../.."] +from _cells import certified_cells # noqa: E402 — needs the path declared above from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name assert torch.cuda.is_available(), "gdn_prefill_conv_post requires a CUDA device" @@ -146,7 +149,43 @@ def _compare(out, pool, want, what=""): assert torch.equal(pool, pool_r), "conv pool differs" -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:44] for c in op.CELLS]) +def _rank2_cell(lengths, initial, slots) -> dict: + """A tp2/tep2 rank of the FP8 target: 8 k / 24 v heads over its 8240-wide projection, on the + stock conv page at 8192 slots, a bound on every pool such a rank binds (its last slot past + 2^32 elements, its middle past 2^31).""" + hk, hv = NUM_K_HEADS // 2, NUM_V_HEADS // 2 + return dict( + num_k_heads=hk, + num_v_heads=hv, + head_dim=HEAD_K_DIM, + x_row_stride=(2 * hk + 2 * hv) * HEAD_K_DIM + 2 * hv, + lengths=lengths, + initial=initial, + pool_slots=8192, + page_layers=PAGE_GDN_LAYERS, + slots=slots, + linear_gate=True, + ) + + +#: The shapes each target calls this entry with beyond its own cells, by target directory. +TARGET_CELLS: dict[str, tuple[Cell, ...]] = { + "qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified": ( + Cell( + why="fp8 tp2/tep2 rank: 37+1+128 tokens carry/fresh/carry at slots 8191, 0, 4096", + spec=_rank2_cell((37, 1, 128), (True, False, True), (8191, 0, 4096)), + ), + Cell( + why="fp8 tp2/tep2 rank at max_num_tokens: one carrying 8192-token chunk at slot 8191", + spec=_rank2_cell((8192,), (True,), (8191,)), + ), + ), +} + +CELLS = certified_cells(op, TARGET_CELLS) + + +@pytest.mark.parametrize("cell", CELLS, ids=[c.why[:44] for c in CELLS]) def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name certifying(op) args = _build(cell.spec, seed=zlib.crc32(cell.why.encode())) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py new file mode 100644 index 000000000000..8d0ea72120d1 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Package marker: without it pytest imports this `test_vision.py` and the NVFP4 target's under +one module name and refuses the second.""" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py new file mode 100644 index 000000000000..cd818269f696 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_ce_exchange__sp_pair_op_matrix.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's `ce_exchange__sp_pair` copy-engine exchange at the +tep2 / tp2 pair under mpirun: setup failing on one rank yields None on both, then eager exchanges +and graph replays with new inputs, byte-exact. +Run as `CUDA_VISIBLE_DEVICES=0,1 python _ce_exchange__sp_pair_op_matrix.py --world-size 2`.""" + +import argparse +import sys + +DEADLINE_S = 600 +CERTIFIED_WORLDS = (2,) +_WORKER_FLAG = "--rank-worker" +HIDDEN = 2560 +CHAIN = 6 +_PKG = ( + "tensorrt_llm._torch.modeling_v2.models.qwen4_exp." + "qwen3_8_flash_next_fp8__sm_103__unified.ops.comm." +) + + +def _worker() -> None: + import torch + from mpi4py import MPI + + from tensorrt_llm.mapping import Mapping + + comm = MPI.COMM_WORLD + rank, world = comm.Get_rank(), comm.Get_size() + assert world == 2 + torch.cuda.set_device(rank % torch.cuda.device_count()) + dev = torch.device("cuda") + ce_mod = __import__(_PKG + "ce_exchange__sp_pair", fromlist=["x"]) + mapping = Mapping(world_size=2, rank=rank, gpus_per_node=2, tp_size=2) + peer = 1 - rank + + def fill(t, r, it): + g = torch.Generator(device=dev).manual_seed(1000 * it + r) + if t.dtype == torch.int32: + t.copy_( + torch.randint( + -(2**31), 2**31 - 1, t.shape, generator=g, device=dev, dtype=torch.int32 + ) + ) + else: + t.copy_(torch.randn(t.shape, generator=g, device=dev).to(t.dtype)) + + # check_failed_setup: one rank's allocation, then one rank's open of the peer's handle, + # fails; both ranks get None (no hang), and each frees what it allocated. + from cuda.bindings import runtime + + def fail(*_): + return (runtime.cudaError_t.cudaErrorMemoryAllocation, 0) + + for name, failing in (("cudaMalloc", 1), ("cudaIpcOpenMemHandle", 0)): + torch.cuda.synchronize() + free_before = torch.cuda.mem_get_info()[0] + real = getattr(runtime, name) + if rank == failing: + setattr(runtime, name, fail) + try: + got = ce_mod.CopyEngineExchange.create(mapping, 512 << 20) + finally: + setattr(runtime, name, real) + assert got is None, f"rank {rank}: built an exchange although {name} failed on {failing}" + torch.cuda.synchronize() + leaked = free_before - torch.cuda.mem_get_info()[0] + assert leaked < 64 << 20, f"rank {rank}: {leaked} bytes kept after {name} failed" + comm.Barrier() + + # check_create: the buffer holds the largest cell, 8192 int32 rows of 2051. + capacity = 72 << 20 + ce = ce_mod.CopyEngineExchange.create(mapping, capacity) + assert ce is not None, f"rank {rank}: the exchange was not built" + + # check_eager: every certified cell, three rounds interleaved. + cells = [c.spec for c in ce_mod.ce_exchange.CELLS if c.spec["world"] == world] + assert cells, f"no cell is certified at world size {world}" + for it in range(3 * len(cells)): + spec = cells[it % len(cells)] + x = torch.empty(spec["rows"], spec["width"], dtype=spec["dtype"], device=dev) + fill(x, rank, it) + got = ce_mod.ce_exchange(ce, x).clone() + want = torch.empty_like(x) + fill(want, peer, it) + torch.cuda.synchronize() + assert torch.equal(got, want), f"rank {rank}: eager exchange {it} ({spec}) differs" + comm.Barrier() + + # check_graph + xs = [torch.empty(4096, HIDDEN, dtype=torch.bfloat16, device=dev) for _ in range(CHAIN)] + outs = [torch.empty_like(x) for x in xs] + + def body(): + for k in range(CHAIN): + outs[k].copy_(ce.exchange(xs[k])) + + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + body() + torch.cuda.current_stream().wait_stream(side) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + body() + for rep in range(30): + for k in range(CHAIN): + fill(xs[k], rank, 100 + rep * CHAIN + k) + graph.replay() + torch.cuda.synchronize() + for k in range(CHAIN): + want = torch.empty_like(xs[k]) + fill(want, peer, 100 + rep * CHAIN + k) + assert torch.equal(outs[k], want), f"rank {rank}: replay {rep} exchange {k} differs" + comm.Barrier() + + if rank == 0: + print("ce_exchange__sp_pair: all checks passed", flush=True) + MPI.Finalize() + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--world-size", type=int, default=2) + a, _ = p.parse_known_args() + if _WORKER_FLAG in sys.argv: + _worker() + return + assert a.world_size in CERTIFIED_WORLDS, a.world_size + import _launch # beside this file + + _launch.mpirun_ranks(__file__, a.world_size, [_WORKER_FLAG], DEADLINE_S) + + +if __name__ == "__main__": + main() diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py new file mode 100644 index 000000000000..74ffa0569ccc --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_hc_ar_inject_norm_op_matrix.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's fused TP sum + injection at 2 and 4 ranks: `hc_out` +and `normed` bitwise the IPC all-reduce then the Triton injection, with and without the MoE +combine, interleaved with plain all-reduces, and in a CUDA graph chain.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import torch + +assert torch.cuda.is_available(), "the fused TP sum + injection requires CUDA devices" + +DEADLINE_S = 1500 +HC, HIDDEN = 4, 2560 +WIDTH = HC * HIDDEN +EPS = 1e-6 +#: Decode-site row counts on both sides of the switch to 2-warp programs at 512 rows: the entry's +#: CELLS at this world, set by `_run_one_rank`. +ROWS: tuple = () +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +modeling = fused_mod = tp_sum = MPI_RANK = WS = None +RANK = WORLD = 0 + + +def _gen(seed: int) -> torch.Generator: + return torch.Generator(device="cuda").manual_seed(seed) + + +def _inputs(rows: int, seed: int, combine: bool): + g = _gen(seed) + hc = (torch.randn(rows, WIDTH, generator=g, device="cuda") * 2.0).to(torch.bfloat16) + # The injection weights as the mixer leaves them: a column slice of a wider row. + inject = (torch.randn(rows, 336, generator=g, device="cuda") * 0.5 + 1.0).to(torch.bfloat16)[ + :, 320:324 + ] + norm = (torch.randn(WIDTH, generator=g, device="cuda") * 0.477).to(torch.bfloat16) + shared = gate = None + if combine: + shared = (torch.randn(rows, HIDDEN, generator=g, device="cuda") * 0.5).to(torch.bfloat16) + gate = torch.randn(rows, 1288, generator=g, device="cuda").to(torch.bfloat16)[:, 1280:1281] + pg = _gen(seed * 977 + RANK + 1) + partial = (torch.randn(rows, HIDDEN, generator=pg, device="cuda") * 0.25).to(torch.bfloat16) + # Signed zeros: a (+0, -0) bf16 pair is the -0.0f word, the not-arrived marker; every rank + # -0.0 in columns 7..16 (the sanitized push makes their sum +0.0, as the all-reduce does). + partial[:, :7] = 0.0 + partial[:, 7:17] = -0.0 + # A few large rows (residual-stream outliers) and an all-zero residual row (rstd of eps). + if rows > 40: + hc[3] *= 300.0 + hc[5].zero_() + return hc, inject, norm, shared, gate, partial + + +def _warps(rows): + return 2 if rows >= modeling._HC_INJECT_FEW_WARPS_MIN_ROWS else modeling._HC_INJECT_WARPS + + +def _plain(hc, inject, norm, shared, gate, partial): + blk = tp_sum(partial) + return modeling._hc_inject_norm_launch( + hc, blk, inject, norm, EPS, HC, shared, gate, _warps(hc.shape[0]) + ) + + +def _fused(hc, inject, norm, shared, gate, partial): + assert fused_mod.fits(hc, partial, inject, shared, gate) + return fused_mod.hc_allreduce_inject_norm( + partial, WS, RANK, WORLD, hc, inject, norm, EPS, shared, gate, _warps(hc.shape[0]) + ) + + +def _same(a, b, what: str) -> None: + for name, x, y in zip(("hc_out", "normed"), a, b): + if not torch.equal(x, y): + idx = (x != y).nonzero()[:8].tolist() + vals = [(i, float(x[i[0], i[1]]), float(y[i[0], i[1]])) for i in idx] + raise AssertionError( + f"[rank {RANK}] {what}: {name} differs ({(x != y).sum().item()} elements), first {vals}" + ) + + +def check_fused_is_bitwise_the_all_reduce_then_the_injection() -> None: + n = 0 + for rows in ROWS: + for combine in (False, True): + for rep in range(2): + args = _inputs(rows, 7000 + 10 * rows + 2 * combine + rep, combine) + want = _plain(*args) + got = _fused(*args) + again = _plain(*args) + torch.cuda.synchronize() + what = f"{rows} rows combine {combine} rep {rep}" + _same(got, want, what) + _same(again, want, what + " (plain after fused)") + n += 1 + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] {n} fused calls bitwise the all-reduce + injection", flush=True) + + +def check_cuda_graph_chain_replays_bitwise() -> None: + """A captured chain of mixed plain and fused sites (an odd call count, so replays start on + every slot), replayed with fresh partials, is bitwise the plain chain run eagerly.""" + for rows in (96, 192, 480): + n_sites = 7 + g = _gen(60 + rows) + hc0 = (torch.randn(rows, WIDTH, generator=g, device="cuda") * 2.0).to(torch.bfloat16) + inj = (torch.randn(rows, HC, generator=g, device="cuda") * 0.5 + 1.0).to(torch.bfloat16) + norms = [ + (torch.randn(WIDTH, generator=g, device="cuda") * 0.4).to(torch.bfloat16) + for _ in range(n_sites) + ] + shared = (torch.randn(rows, HIDDEN, generator=g, device="cuda") * 0.5).to(torch.bfloat16) + gate = torch.randn(rows, 1, generator=g, device="cuda").to(torch.bfloat16) + parts = [ + torch.empty(rows, HIDDEN, dtype=torch.bfloat16, device="cuda") for _ in range(n_sites) + ] + + def chain(kinds): + hc, outs = hc0, [] + for k in range(n_sites): + fn = _fused if kinds[k] else _plain + sh, gt = (shared, gate) if k % 2 else (None, None) + hc, normed = fn(hc, inj, norms[k], sh, gt, parts[k]) + outs.append(normed) + return hc, outs + + kinds = [k % 3 != 1 for k in range(n_sites)] + for p in parts: + p.copy_( + torch.randn(rows, HIDDEN, generator=_gen(1 + RANK), device="cuda").to( + torch.bfloat16 + ) + ) + chain(kinds) # warm the compiles + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + g_hc, g_outs = chain(kinds) + torch.cuda.synchronize() + MPI_RANK.barrier() + for rep in range(5): + pg = _gen(1000 * rep + 31 * rows + RANK) + for p in parts: + p.copy_( + (torch.randn(rows, HIDDEN, generator=pg, device="cuda") * 0.3).to( + torch.bfloat16 + ) + ) + graph.replay() + torch.cuda.synchronize() + got = (g_hc.clone(), [o.clone() for o in g_outs]) + MPI_RANK.barrier() + want = chain([False] * n_sites) + torch.cuda.synchronize() + assert torch.equal(got[0], want[0]), f"{rows} rows replay {rep}: hc differs" + for k in range(n_sites): + assert torch.equal(got[1][k], want[1][k]), ( + f"{rows} rows replay {rep}: site {k} normed differs" + ) + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] CUDA graph chains replay bitwise", flush=True) + + +CHECKS = ( + check_fused_is_bitwise_the_all_reduce_then_the_injection, + check_cuda_graph_chain_replays_bitwise, +) + + +def _run_one_rank() -> int: + global modeling, fused_mod, tp_sum, MPI_RANK, RANK, WORLD, WS, ROWS + os.environ.pop("TLLM_DISABLE_MPI", None) + os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + assert WORLD in (2, 4), "the fused TP sum + injection is certified at 2 and 4 ranks" + from tensorrt_llm._torch.modeling_v2.catalog.comm import allreduce as entry + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _modeling, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + hc_allreduce_inject_norm__tp_sum as _fused_mod, + ) + from tensorrt_llm.functional import AllReduceFusionOp + + modeling, fused_mod = _modeling, _fused_mod + ROWS = tuple( + c.spec["rows"] for c in fused_mod.hc_allreduce_inject_norm.CELLS if c.spec["world"] == WORLD + ) + assert ROWS, f"no cell is certified at world size {WORLD}" + group = entry.allreduce_group_args(MPI_RANK.mapping) + WS = group["workspace"] + + def _tp_sum(x): + return entry.allreduce( + x, + None, + None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=EPS, + trigger_completion_at_end=False, + **group, + ) + + tp_sum = _tp_sum + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + import torch.distributed as dist + + if dist.is_initialized(): + dist.barrier() + dist.destroy_process_group() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + args = _lockstep.parse_args(sys.argv[1:]) + if args.rank_worker: + sys.exit(_run_one_rank()) + import _launch # beside this file + + world = args.world_size or len(_launch.visible_devices()) + assert world in (2, 4), "the fused sum runs at 2 or 4 ranks" + worker = [_lockstep.WORKER_FLAG, "--world-size", str(world)] + worker += ["--fabric-handle", args.fabric_handle] + _launch.mpirun_ranks(__file__, world, worker, DEADLINE_S) + print("OK: fused TP sum + injection") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py new file mode 100644 index 000000000000..eac810b4f4db --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_lamport_allgather__index_split_op_matrix.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's gather of a decode step's QSA selection rows over the +TP group (`lamport_allgather__index_split`): every output bitwise the NCCL all-gather, over mixed +trunk and draft calls and with the split's row map, eagerly and replayed from a CUDA graph.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import torch + +assert torch.cuda.is_available(), "the selection gather requires CUDA devices" + +DEADLINE_S = 900 +WIDTH = 2051 +#: Requests a step (each rank scores `requests / world` of them): the entry's CELLS at this world +#: (tep2 / tep4 HT batches), set by `_run_one_rank`. +REQUESTS: tuple = () +MAX_ROWS = 0 +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +lam = gather = MPI_RANK = None +RANK = WORLD = 0 + + +def _share(rows: int, seed: int) -> torch.Tensor: + """This rank's `[rows / WORLD, WIDTH]` selection rows: token indices, -1 padding, and the + int32 extremes next to the not-arrived marker (int32 min itself never occurs).""" + g = torch.Generator(device="cuda").manual_seed(seed * 131 + RANK) + share = torch.randint(-1, 1 << 24, (rows // WORLD, WIDTH), generator=g, device="cuda") + share = share.to(torch.int32) + share[:, :3] = -1 + share[:, 3] = 2**31 - 1 + share[:, 4] = -(2**31) + 1 + return share + + +def _nccl(share: torch.Tensor) -> torch.Tensor: + import torch.distributed as dist + + out = torch.empty((WORLD * share.shape[0], WIDTH), dtype=torch.int32, device="cuda") + dist.all_gather_into_tensor(out, share) + return out + + +def _ensure_process_group() -> None: + """A process group for the NCCL reference, from the environment or a port rank 0 broadcasts.""" + import torch.distributed as dist + + if dist.is_initialized(): + return + if not all(os.environ.get(k) for k in ("MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE")): + addr = None + if RANK == 0: + import socket + + with socket.socket() as sock: + sock.bind(("", 0)) + addr = (socket.gethostbyname(socket.gethostname()), sock.getsockname()[1]) + host, port = MPI_RANK.comm.bcast(addr, root=0) + os.environ["MASTER_ADDR"] = host + os.environ["MASTER_PORT"] = str(port) + dist.init_process_group( + backend="nccl", + rank=RANK, + world_size=WORLD, + device_id=torch.device("cuda", torch.cuda.current_device()), + ) + + +def _same(got: torch.Tensor, want: torch.Tensor, what: str) -> None: + assert got.shape == want.shape and torch.equal(got, want), f"{what}: not bitwise NCCL's" + + +def _step_rows() -> list[int]: + """One served step per batch: 12 trunk layers at 3 rows a request, 2 draft steps at 1.""" + seq = [] + for requests in REQUESTS: + seq += [3 * requests] * 12 + [requests] * 2 + return seq + + +def check_eager_mixed_sequence_is_bitwise_nccl() -> None: + """Every call of a long mixed sequence equals NCCL's while the buffers' three slots rotate.""" + for i, rows in enumerate(_step_rows() * 2): + share = _share(rows, 100 + i) + got = gather(share, lam) + _same(got, _nccl(share), f"call {i} ({rows} rows)") + torch.cuda.synchronize() + assert MPI_RANK.same_on_ranks(lam.state[:2]) + + +def _row_map(rows: int, seed: int) -> torch.Tensor: + """A permutation of `rows` rows, the same on every rank (the split plan's `perm_rows`).""" + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randperm(rows, generator=g, device="cuda").to(torch.int32) + + +def check_row_map_stores_each_gathered_row_at_its_row() -> None: + """Gathered row `j` lands at row `dst_rows[j]`, bitwise NCCL's gather permuted the same way, + interleaved with plain calls, eagerly and in a CUDA graph replayed with new rows.""" + for i, rows in enumerate([3 * 64, 64, 3 * 192, 3 * 16, 192, 3 * 128]): + share = _share(rows, 7000 + i) + row_map = _row_map(rows, 7100 + i) + want = torch.empty_like(_nccl(share)) + want[row_map.long()] = _nccl(share) + _same(gather(share, lam, row_map), want, f"mapped call {i} ({rows} rows)") + plain = _share(rows, 7200 + i) + _same(gather(plain, lam), _nccl(plain), f"plain call after mapped call {i}") + torch.cuda.synchronize() + for requests in (32, 96): + rows_seq = [3 * requests] * 12 + [requests] * 2 + maps = [ + _row_map(r, 7300 + j + requests) if j < 12 else None for j, r in enumerate(rows_seq) + ] + shares = [ + torch.zeros((r // WORLD, WIDTH), dtype=torch.int32, device="cuda") for r in rows_seq + ] + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for s_, m_ in zip(shares, maps): + gather(s_, lam, m_) + torch.cuda.current_stream().wait_stream(stream) + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = [gather(s_, lam, m_) for s_, m_ in zip(shares, maps)] + torch.cuda.synchronize() + MPI_RANK.barrier() + for rep in range(6): + for j, s_ in enumerate(shares): + s_.copy_(_share(s_.shape[0] * WORLD, 7500 + 100 * rep + j + requests)) + torch.cuda.synchronize() + MPI_RANK.barrier() + graph.replay() + torch.cuda.synchronize() + for j, (s_, m_, o) in enumerate(zip(shares, maps, outs)): + want = _nccl(s_) + if m_ is not None: + w2 = torch.empty_like(want) + w2[m_.long()] = want + want = w2 + _same(o, want, f"{requests} requests, replay {rep}, call {j}") + MPI_RANK.barrier() + del graph + + +CHECKS = ( + check_eager_mixed_sequence_is_bitwise_nccl, + check_row_map_stores_each_gathered_row_at_its_row, +) + + +def _run_one_rank() -> int: + global lam, gather, MPI_RANK, RANK, WORLD, REQUESTS, MAX_ROWS + os.environ.pop("TLLM_DISABLE_MPI", None) + os.environ.setdefault("TRTLLM_ENABLE_PDL", "1") + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + lamport_allgather__index_split as entry, + ) + + gather = entry.lamport_allgather + cells = [c.spec for c in gather.CELLS if c.spec["world"] == WORLD] + assert cells and all(s["width"] == WIDTH for s in cells), f"no cell at world {WORLD}" + REQUESTS = tuple(s["requests"] for s in cells) + MAX_ROWS = 3 * max(REQUESTS) + # The buffers' handles travel over MPI (the engine's TP communicator); the NCCL reference + # needs a torch process group. + lam = entry.build(MPI_RANK.mapping, MAX_ROWS * WIDTH) + _ensure_process_group() + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + import torch.distributed as dist + + dist.barrier() + dist.destroy_process_group() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + + args = _lockstep.parse_args(sys.argv[1:]) + if args.rank_worker: + sys.exit(_run_one_rank()) + import _launch # beside this file + + world = args.world_size or len(_launch.visible_devices()) + worker = [_lockstep.WORKER_FLAG, "--world-size", str(world)] + _launch.mpirun_ranks( + __file__, world, worker + ["--fabric-handle", args.fabric_handle], DEADLINE_S + ) + print("OK: lamport selection gather") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py new file mode 100644 index 000000000000..7af7e3bf0c52 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_launch.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Rank launchers for the collective matrices beside it, which import it from launcher code only. +Module scope imports only the standard library, since `_rank_job` runs a matrix's module scope +from pytest. The deadlines are there because a broken collective hangs rather than raising.""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import Sequence + +#: Slurm step variables an mpirun inside a one-task step must not inherit: with them OpenMPI +#: sizes its slots from the step (one) and every rank reads the step's own rank. +SLURM_STEP_VARS = ( + "SLURM_PROCID", + "SLURM_NTASKS", + "SLURM_LOCALID", + "SLURM_NODEID", + "SLURM_STEP_ID", + "SLURM_TASK_PID", + "SLURM_STEP_NUM_TASKS", +) + + +def visible_devices() -> list[str]: + """The devices this run owns: CUDA_VISIBLE_DEVICES, which must name them.""" + visible = [d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()] + assert visible, "set CUDA_VISIBLE_DEVICES to the devices this run owns, e.g. 0,1,2,3" + return visible + + +def _claim(world: int) -> str: + devices = visible_devices() + assert 2 <= world <= len(devices), ( + f"world size {world} needs 2..{len(devices)} visible devices (one rank per device)" + ) + return ",".join(devices[:world]) + + +def mpirun_ranks(entry_file: str, world: int, args: Sequence[str], deadline_s: int) -> None: + """Re-execute `entry_file` as `world` MPI ranks on the first `world` visible devices, without + `TLLM_DISABLE_MPI` (the ranks need tensorrt_llm's MPI session); raise on a nonzero exit or + past `deadline_s`, killing the process group.""" + command = [ + "mpirun", + "--allow-run-as-root", + "--oversubscribe", + "-n", + str(world), + sys.executable, + str(Path(entry_file).resolve()), + *args, + ] + env = { + k: v for k, v in os.environ.items() if k not in SLURM_STEP_VARS and k != "TLLM_DISABLE_MPI" + } + env["CUDA_VISIBLE_DEVICES"] = _claim(world) + print(f"[launcher] {' '.join(command)}", flush=True) + process = subprocess.Popen(command, env=env, start_new_session=True) + try: + code = process.wait(timeout=deadline_s) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise AssertionError( + f"the {world}-rank MPI run did not finish in {deadline_s}s (wedged)" + ) from None + assert code == 0, f"the {world}-rank MPI run exited {code}" + + +def spawn_pg_ranks(entry_file: str, world: int, args: Sequence[str], deadline_s: int) -> None: + """Start `entry_file` as `world` torch.distributed ranks (`env://` on a free local port), each + in its own session; kill them all and raise on the first nonzero exit or past `deadline_s`.""" + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + env = dict( + os.environ, + CUDA_VISIBLE_DEVICES=_claim(world), + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + WORLD_SIZE=str(world), + TLLM_DISABLE_MPI="1", + ) + # Drop the srun step's PMIx/PMI/OMPI namespace: a child that inherits it joins the step, and + # its hard exit makes srun SIGKILL the whole step before any error is reported. + env = {k: v for k, v in env.items() if not k.startswith(("PMIX_", "PMI_", "OMPI_", "SLURM_"))} + print(f"[launcher] spawning {world} torch.distributed ranks, rendezvous on :{port}", flush=True) + children = [ + subprocess.Popen( + [sys.executable, str(Path(entry_file).resolve()), *args], + env=dict(env, RANK=str(r)), + start_new_session=True, + ) + for r in range(world) + ] + deadline = time.monotonic() + deadline_s + live = set(range(world)) + failure = None + while live and failure is None: + if time.monotonic() > deadline: + failure = f"did not finish in {deadline_s}s (wedged)" + break + for r in sorted(live): + code = children[r].poll() + if code is None: + continue + live.discard(r) + if code != 0: + failure = f"rank {r} exited {code}" + break + time.sleep(0.5) + if failure is not None: + for p in children: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for p in children: + p.wait() + raise AssertionError(f"the {world}-rank torch.distributed run failed: {failure}") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py new file mode 100644 index 000000000000..cc871b086ffe --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_mnnvl_allreduce__tp_sum_op_matrix.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's `mnnvl_allreduce__tp_sum` under mpirun: each sum is +bitwise the IPC op's and within `_compare`'s band of the fp32 sum, eagerly and graph-replayed. +The MPI-disabled launch (`TLLM_DISABLE_MPI=1`) is not certified here.""" + +import argparse +import functools +import os +import sys +from pathlib import Path +from typing import Any + +import torch + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: tests/unittest/_torch/modeling_v2: `_validating` and the shared `comm/` helpers. +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] + +# Bound inside the rank body: module scope imports only torch and the standard library, since +# `_rank_job` executes it to read `DEADLINE_S`. +entry: Any = None +ipc: Any = None +mnnvl: Any = None +modeling: Any = None +validating: Any = None +WORKSPACE: dict = {} +MPI_RANK: Any = None +RANK = 0 +WORLD = 1 + +#: Row counts a captured `tp_sum` meets inside the budget: decode batches 1..8 and the MTP-3 +#: verify steps of 1..8 requests (4..32 rows). +GRAPH_ROWS = (1, 2, 3, 4, 8, 12, 16, 24, 32) + + +def _compare(out, ref) -> None: + """`catalog/comm/allreduce`'s band: 5 ulp per element against the row scale, 1.25 ulp RMS.""" + from _validating import assert_within_ulp + + assert_within_ulp(out, ref, element_ulp=5.0, rms_ulp=1.25) + + +def _payload(rank: int, rows: int, seed: int) -> torch.Tensor: + """One rank's partial: every rank can regenerate every other rank's from the seed.""" + gen = torch.Generator(device="cuda").manual_seed(seed * 977 + rank + 1) + return torch.randn(rows, entry.HIDDEN, generator=gen, device="cuda").to(torch.bfloat16) + + +def _reference(rows: int, seed: int) -> torch.Tensor: + return entry.mnnvl_tp_sum.reference([_payload(r, rows, seed) for r in range(WORLD)]) + + +def _exact(out, ref) -> float: + return (out == ref).float().mean().item() + + +def check_certified_cells() -> None: + cells = [c for c in entry.mnnvl_tp_sum.CELLS if c.spec["world"] == WORLD] + assert cells, f"no cell is certified at world size {WORLD}" + for i, cell in enumerate(cells): + rows = cell.spec["rows"] + for s in range(3): + seed = 4000 + 31 * i + 1000 * s + x = _payload(RANK, rows, seed) + with validating(entry.mnnvl_tp_sum): + out = mnnvl(x) + twin = ipc(x) + ref = _reference(rows, seed) + torch.cuda.synchronize() + assert out.shape == x.shape and out.dtype == x.dtype, (out.shape, out.dtype) + assert torch.equal(out, twin), f"{cell.why}: not the IPC op's bits (seed {seed})" + _compare(out, ref) + if RANK == 0 and s == 0: + print( + f"[world {WORLD}] {cell.why}: bitwise the IPC op; " + f"{100 * _exact(out, ref):.4f}% bit-exact to the fp32 sum rounded once", + flush=True, + ) + MPI_RANK.barrier() + + +def check_cuda_graph_replay_matches_eager() -> None: + """Captured once per row count, refilled with fresh partials and replayed: bitwise the + eager call and the IPC op on those partials (the Lamport buffers rotate per call).""" + for rows in GRAPH_ROWS: + static = _payload(RANK, rows, 900 + rows) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + for _ in range(3): + mnnvl(static) + torch.cuda.current_stream().wait_stream(side) + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = mnnvl(static) + MPI_RANK.barrier() + for step in range(3): + fresh = _payload(RANK, rows, 1900 + 10 * rows + step) + static.copy_(fresh) + graph.replay() + eager = mnnvl(fresh) + twin = ipc(fresh) + torch.cuda.synchronize() + assert torch.equal(captured, eager), f"{rows} rows step {step}: replay != eager" + assert torch.equal(captured, twin), f"{rows} rows step {step}: replay != IPC op" + MPI_RANK.barrier() + if RANK == 0: + print(f"[world {WORLD}] captured {rows} rows: replay == eager == IPC op", flush=True) + del graph + + +CHECKS = (check_certified_cells, check_cuda_graph_replay_matches_eager) + + +def _run_one_mpi_rank() -> int: + global entry, ipc, mnnvl, modeling, validating, WORKSPACE, MPI_RANK, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm._torch.modeling_v2.catalog.comm import allreduce as allreduce_mod + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _modeling, + ) + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm import ( # noqa: E501 + mnnvl_allreduce__tp_sum as _entry, + ) + from tensorrt_llm.functional import AllReduceFusionOp + + entry, modeling, validating = _entry, _modeling, _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + group = allreduce_mod.allreduce_group_args(MPI_RANK.mapping) + ipc = functools.partial( + allreduce_mod.allreduce, + residual=None, + norm_weight=None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=1e-6, + trigger_completion_at_end=False, + **group, + ) + WORKSPACE = entry.mnnvl_workspace_args( + MPI_RANK.mapping, modeling._TP_SUM_MNNVL_MAX_ROWS, entry.HIDDEN, torch.bfloat16 + ) + mnnvl = functools.partial(entry.mnnvl_tp_sum, **WORKSPACE) + + code = _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + os._exit(code) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + parser = argparse.ArgumentParser() + parser.add_argument("--world-size", type=int, default=None) + world_size = parser.parse_args().world_size or len(_launch.visible_devices()) + _launch.mpirun_ranks( + __file__, world_size, [_MPI_WORKER_FLAG, "--world-size", str(world_size)], DEADLINE_S + ) + print("OK: MPI session (mnnvl_fusion_allreduce, fusion NONE)", flush=True) + print("OK") diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py new file mode 100644 index 000000000000..e84444c1cec2 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/_moe_a2a__dp_dispatch_combine_op_matrix.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Certification matrix for the FP8 target's MoE all-to-all round trip (`moe_a2a_dispatch` and +`moe_a2a_combine` on the stock NVLink one-sided all-to-all) at dep2 and dep4 under mpirun, eagerly +and graph-replayed. Fails where the platform has no MNNVL (the collected test skips there first).""" + +import os +import sys +from pathlib import Path +from typing import Any, List, Tuple + +import torch + +assert torch.cuda.is_available(), "the MoE all-to-all requires CUDA devices" + +_MPI_WORKER_FLAG = "--mpi-rank-worker" +DEADLINE_S = 900 + +#: The attention-DP group sizes this target serves (dep2, dep4). +CERTIFIED_WORLDS: Tuple[int, ...] = (2, 4) + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +_MODELING_V2_TESTS = Path(__file__).resolve().parents[4] +_TARGET = "tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified" + +# Bound inside the rank body: importing the entry pulls in tensorrt_llm, which calls MPI_Init at +# import, and the launcher must stay free of MPI and CUDA state. +entry: Any = None +validating: Any = None +MPI_RANK: Any = None +STRATEGY: Any = None +RANK = 0 +WORLD = 1 + + +def _gen(*key: int) -> torch.Generator: + """Seeded alike on every rank for a key, so each rank can rebuild what any other rank holds.""" + seed = 0 + for k in key: + seed = seed * 1000003 + int(k) + return torch.Generator(device="cuda").manual_seed(seed % (2**62)) + + +def _inputs(rows: int, src: int, salt: int) -> tuple: + """Rank `src`'s rows and routing: bf16 rows, ten distinct experts a row, softmax weights.""" + g = _gen(salt, src) + x = torch.randn(rows, HIDDEN, generator=g, device="cuda").to(torch.bfloat16) + ids = torch.rand(rows, NUM_EXPERTS, generator=g, device="cuda").topk(TOPK, dim=1).indices + w = torch.softmax(torch.randn(rows, TOPK, generator=g, device="cuda"), dim=1) + return x, ids.to(torch.int32), w.to(torch.bfloat16) + + +def _window(rows, ids, w, rank: int) -> torch.Tensor: + """Rank `rank`'s stand-in expert window: each row scaled by its in-window experts' weights times + their ids, in fp32, rounded once; a row with no expert there (an empty slot too) is zero.""" + epr = NUM_EXPERTS // WORLD + keep = (ids >= rank * epr) & (ids < (rank + 1) * epr) + scale = (w.float() * keep * (ids.float() + 1) / NUM_EXPERTS).sum(-1, keepdim=True) + out = scale * rows.float() + return torch.where(keep.any(-1, keepdim=True), out, torch.zeros_like(out)).to(torch.bfloat16) + + +def _cells() -> list: + return [c for c in entry.moe_a2a_dispatch.CELLS if c.spec["world"] == WORLD] + + +def _check_received(got, everyone, counts, what: str) -> None: + """Received token slots equal the reference rows as a set: the kernel orders a block itself.""" + want = entry.moe_a2a_dispatch.reference(everyone, counts, NUM_EXPERTS // WORLD, rank=RANK) + by_key = {} + for x, ids, w in want: + for i in range(ids.shape[0]): + by_key[tuple(ids[i].tolist())] = (x[i], w[i]) + valid = (got.topk_ids != entry.NO_EXPERT).any(1) + assert got.rows.shape[0] == WORLD * max(counts), (what, got.rows.shape, counts) + assert int(valid.sum()) == len(by_key), (what, int(valid.sum()), len(by_key)) + for i in torch.nonzero(valid).flatten().tolist(): + key = tuple(got.topk_ids[i].tolist()) + assert key in by_key, f"{what}: slot {i} holds a row no source routed here" + x, w = by_key[key] + assert torch.equal(got.rows[i], x) and torch.equal(got.topk_weights[i], w), (what, i) + + +def _check_combined(out, mine, what: str) -> None: + x, ids, w = mine + want = entry.moe_a2a_combine.reference([_window(x, ids, w, r) for r in range(WORLD)]) + assert out.shape == want.shape, (what, out.shape, want.shape) + torch.testing.assert_close(out.float(), want.float(), rtol=2 * 2**-8, atol=1e-3, msg=what) + + +def _round_trip(x, ids, w, counts): + got = entry.moe_a2a_dispatch(STRATEGY, x, ids, w, counts) + partials = _window(got.rows, got.topk_ids, got.topk_weights, RANK) + return got, entry.moe_a2a_combine(STRATEGY, partials) + + +def check_cells() -> None: + """Every cell of this world size: dispatch as a set against its reference, then the round + trip through the stand-in window against the combine reference.""" + cells = _cells() + assert cells, f"no cell at world {WORLD}" + for i, cell in enumerate(cells): + counts = list(cell.spec["counts"]) + everyone = [_inputs(n, r, 10 + i) for r, n in enumerate(counts)] + with validating(entry.moe_a2a_dispatch): + got, out = _round_trip(*everyone[RANK], counts) + torch.cuda.synchronize() + _check_received(got, everyone, counts, cell.why) + _check_combined(out, everyone[RANK], cell.why) + MPI_RANK.barrier() + + +def check_graph() -> None: + """The round trip captured once at a uniform cell and replayed with fresh rows and routing.""" + cell = max( + (c for c in _cells() if len(set(c.spec["counts"])) == 1), key=lambda c: c.spec["counts"][0] + ) + counts = list(cell.spec["counts"]) + static = [t.clone() for t in _inputs(counts[RANK], RANK, 50)] + _round_trip(*static, counts) # eager first: the graph replays what the warm call set up + torch.cuda.synchronize() + MPI_RANK.barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got, out = _round_trip(*static, counts) + for salt in (51, 52): + everyone = [_inputs(n, r, salt) for r, n in enumerate(counts)] + for dst, src in zip(static, everyone[RANK]): + dst.copy_(src) + MPI_RANK.barrier() + graph.replay() + torch.cuda.synchronize() + _check_received(got, everyone, counts, f"replay {salt}") + _check_combined(out, everyone[RANK], f"replay {salt}") + MPI_RANK.barrier() + del graph + torch.cuda.synchronize() + MPI_RANK.barrier() + + +CHECKS = (check_cells, check_graph) + + +def _run_one_mpi_rank() -> int: + """One MPI rank's checks; a failing check aborts the MPI job, which ends its siblings.""" + global entry, validating, MPI_RANK, STRATEGY, RANK, WORLD + os.environ.pop("TLLM_DISABLE_MPI", None) + sys.path.insert(0, str(_MODELING_V2_TESTS)) + sys.path.insert(0, str(_MODELING_V2_TESTS / "comm")) + import importlib + + import _lockstep + from _validating import validating as _validating_cm + + MPI_RANK = _lockstep.Rank(_lockstep.parse_args(sys.argv[1:])) + from tensorrt_llm.mapping import Mapping + + entry = importlib.import_module(f"{_TARGET}.ops.comm.moe_a2a__dp_dispatch_combine") + validating = _validating_cm + RANK, WORLD = MPI_RANK.rank, MPI_RANK.world + mapping = Mapping( + world_size=WORLD, + rank=RANK, + gpus_per_node=WORLD, + tp_size=WORLD, + moe_tp_size=1, + moe_ep_size=WORLD, + enable_attention_dp=True, + ) + max_rows = max(max(c.spec["counts"]) for c in entry.moe_a2a_dispatch.CELLS) + STRATEGY = entry.build_strategy(mapping, max_rows) + if STRATEGY is None: + print(f"[rank {RANK}] no MNNVL on this platform: nothing certified", flush=True) + MPI_RANK.MPI.Finalize() + os._exit(1) + _lockstep.run_checks(MPI_RANK, list(CHECKS)) + torch.cuda.synchronize() + print(f"[rank {RANK}] {len(CHECKS)} checks passed over the MPI session", flush=True) + sys.stdout.flush() + sys.stderr.flush() + MPI_RANK.MPI.Finalize() + # Skip static destructors: the symmetric workspace's teardown adds nothing to the verdict. + os._exit(0) + + +def _world_size(argv: List[str]) -> int: + if "--world-size" in argv: + return int(argv[argv.index("--world-size") + 1]) + return len([d for d in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") if d.strip()]) + + +if __name__ == "__main__": + if _MPI_WORKER_FLAG in sys.argv: + sys.exit(_run_one_mpi_rank()) + import _launch # beside this file + + WORLD_SIZE = _world_size(sys.argv) + assert WORLD_SIZE in CERTIFIED_WORLDS, (WORLD_SIZE, CERTIFIED_WORLDS) + _launch.mpirun_ranks( + __file__, WORLD_SIZE, [_MPI_WORKER_FLAG, "--world-size", str(WORLD_SIZE)], DEADLINE_S + ) + print(f"OK: {WORLD_SIZE} ranks over the MPI session", flush=True) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py new file mode 100644 index 000000000000..a8256b92fef8 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/comm/test_fp8_unified_op_matrices.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Collected entry point for the FP8 target's collective matrices: each `__op_matrix.py` +beside this file is its own launcher, which `_rank_job` runs in a fresh interpreter on `world` +sm_103 GPUs of one node. + +Pre-merge CI runs this directory in the GB300 multi-GPU stage (`l0_gb300.yml`), which it +schedules only when a file listed in `getMultiGpuFileChanged` (`jenkins/L0_MergeRequest.groovy`) +changes; neither the target's `csrc/` and `ops/comm/` nor this directory is listed. A change to +them is run with `/bot run --add-multi-gpu-test` +(`docs/source/developer-guide/ci-overview.md`, Multi-GPU Tests).""" + +from pathlib import Path + +import pytest +import torch + +__extra_import_path__ = ["../../../../comm"] +import _rank_job # noqa: E402 — needs the path declared above + +_HERE = Path(__file__).resolve().parent + +#: The group sizes the target runs each entry at (each launcher's `CERTIFIED_WORLDS`, restated +#: so that collecting this module on a machine without GPUs loads no launcher). +_WORLDS = { + "mnnvl_allreduce__tp_sum": (2, 4), + "ce_exchange__sp_pair": (2,), + "lamport_allgather__index_split": (2, 4), + "hc_ar_inject_norm": (2, 4), + "moe_a2a__dp_dispatch_combine": (2, 4), +} +_CASES = [(entry, world) for entry, worlds in _WORLDS.items() for world in worlds] + + +def _sm103_devices(world: int) -> int: + """How many of the devices a `world`-rank job would claim are sm_103.""" + if not torch.cuda.is_available(): + return 0 + claimed = range(min(torch.cuda.device_count(), world)) + return sum(torch.cuda.get_device_capability(i) == (10, 3) for i in claimed) + + +# Each case spawns its own ranks over the visible devices, so xdist workers would share GPUs. +@pytest.mark.no_xdist +@pytest.mark.parametrize("entry, world", _CASES, ids=[f"{e}-world{w}" for e, w in _CASES]) +def test_op_matrix(entry: str, world: int) -> None: + if _sm103_devices(world) < world: + pytest.skip(f"the {entry} matrix at world {world} runs on {world} sm_103 GPUs in one node") + if entry == "moe_a2a__dp_dispatch_combine": + from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided + + if not NVLinkOneSided.is_platform_supported(): + pytest.skip(f"the {entry} matrix needs MNNVL, which this platform lacks") + _rank_job.run(entry, world_size=world, directory=_HERE) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py new file mode 100644 index 000000000000..6c32927aadc2 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_attention_ops.py @@ -0,0 +1,2359 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's attention entries other than `thop_attention`: every +certified cell against the entry's `reference` over the engine's paged state, plus the TP index +split, captured stores, kernels without cells, the call sites and the FP8 KV range probe.""" + +import math +import types +import zlib +from types import SimpleNamespace +from unittest.mock import Mock + +import cuda.bindings.driver as cuda_drv +import pytest +import torch + +from tensorrt_llm._torch.attention.backends.sparse.qsa.kernels import ( + triton_qsa_generation_token_mapping, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_topk_expand_decode import ( # noqa: E501 + qsa_topk_expand_decode as family_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + pool as P, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_index_scores_cute__prefill as _isc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_index_scores_groups_cute__decode as _igc, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_sparse_gqa_rowtile__prefill as _module, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_sparse_gqa_splitk_cute__decode as mod, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention import ( # noqa: E501 + qsa_triton_kernels__stable_jit as _stable, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.draft_continue_rows__decode import ( # noqa: E501 + draft_continue_select, + draft_rows_gather, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_fp8_pages_to_bf16__prefill import ( # noqa: E501 + qsa_fp8_pages_to_bf16 as _f2b_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_index_split_plan__decode import ( # noqa: E501 + index_split_plan, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_index_store import ( # noqa: E501 + qsa_paged_index_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_kv_store import ( # noqa: E501 + qsa_paged_kv_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_sparse_gqa import ( # noqa: E501 + qsa_paged_sparse_gqa, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_grouped_cute__decode import ( # noqa: E501 + qsa_sparse_gqa_grouped_cute as _sgg_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_rowtile__prefill import ( # noqa: E501 + qsa_sparse_gqa_rowtile as _rtl_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_rowtile__prefill import ( # noqa: E501 + rows_per_group, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_sparse_gqa_splitk_cute__decode import ( # noqa: E501 + qsa_sparse_gqa_splitk_cute as _sgc_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_spec_step_metadata__decode import ( # noqa: E501 + qsa_spec_step_metadata, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_expand_cluster__decode import ( # noqa: E501 + qsa_topk_expand_cluster as _tkc_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_expand_cluster__decode import ( # noqa: E501 + supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_rows_cute__prefill import ( # noqa: E501 + qsa_topk_rows as _tkr_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_topk_rows_cute__prefill import ( # noqa: E501 + supports as _tkr_supports, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qwen4_exp_attn_decode_prologue__decode import ( # noqa: E501 + qwen4_exp_attn_decode_prologue as _pro_op, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — needs the path declared above; fixture by name + assert_within_ulp, + certifying, +) + +assert torch.cuda.is_available(), "the attention entries require a CUDA device" + +DEV = "cuda" +#: The element offset a straddling cell's pages lie on both sides of: a 32-bit offset breaks. +_OFFSET_EDGE = 2**31 + + +def _seed(text: str) -> int: + """A seed that is the same in every process; `hash()` of a str is not.""" + return zlib.crc32(text.encode()) + + +def _bits(tensor: torch.Tensor) -> torch.Tensor: + as_int = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[tensor.element_size()] + return tensor.contiguous().view(as_int) + + +def _hole(spec, table, allocated_cols): + """One row that resolves to no slot: its (request, position).""" + assert spec["hole"] == "unallocated_page", spec["hole"] + assert int(table[0, allocated_cols[0]]) == -1 + return 0, allocated_cols[0] * spec["tokens_per_block"] + + +def _index_store_compare(out, ref) -> None: + """Bitwise over both whole planes, as raw bits: a store is a move with one conversion that + `reference` performs too, and comparing every slot is the stray-write check.""" + for name, got, want in zip(("index_cache", "position_cache"), out, ref): + assert got.dtype == want.dtype, (name, got.dtype, want.dtype) + assert got.shape == want.shape, (name, got.shape, want.shape) + torch.testing.assert_close( + _bits(got), _bits(want), rtol=0, atol=0, msg=lambda m, n=name: f"{n}: {m}" + ) + + +KEY_FILL = -7.0 +POS_FILL = -7 +ENGINE_MAX_NUM_TOKENS = 16384 + + +def _key_fill_bits() -> int: + return int(_bits(torch.tensor([KEY_FILL], dtype=torch.bfloat16))[0]) + + +def _shared_page_planes(spec, extras: dict): + """The NVFP4-KV pool's side planes: a page holds the `layers` index-K pages then the position + page, so both caches are strided views of one allocation.""" + layers, tpb, hkv, head_dim, axes, pages = ( + spec[k] for k in ("layers", "tokens_per_block", "hkv", "head_dim", "axes", "pages") + ) + index_bytes = tpb * hkv * head_dim * 2 + page = torch.empty(pages, layers * index_bytes + tpb * axes * 4, dtype=torch.uint8, device=DEV) + page[:, : layers * index_bytes].view(torch.bfloat16).fill_(KEY_FILL) + page[:, layers * index_bytes :].view(torch.int32).fill_(POS_FILL) + index_cache = page[:, (layers - 1) * index_bytes : layers * index_bytes].view(torch.bfloat16) + index_cache = index_cache.view(pages, tpb, hkv, head_dim) + position_cache = page[:, layers * index_bytes :].view(torch.int32).view(pages, tpb, axes) + assert index_cache.stride() == (spec["page_stride"], hkv * head_dim, head_dim, 1) + assert position_cache.stride() == spec["position_strides"] + extras["roles"] = page[:, : (layers - 1) * index_bytes] + return index_cache, position_cache + + +def _coalesced_planes(spec, extras: dict): + """The default target's bf16 index pool: the sparse layers' INDEX_KEY buffers coalesced into + one allocation, the cell's plane a strided view; positions in their own pool.""" + layers, layer, tpb, hkv, head_dim, axes, pages = ( + spec[k] for k in ("layers", "layer", "tokens_per_block", "hkv", "head_dim", "axes", "pages") + ) + pool = torch.full( + (pages, layers, tpb, hkv, head_dim), KEY_FILL, dtype=torch.bfloat16, device=DEV + ) + index_cache = pool[:, layer] + assert index_cache.stride() == (spec["page_stride"], hkv * head_dim, head_dim, 1) + position_cache = torch.full((pages, tpb, axes), POS_FILL, dtype=torch.int32, device=DEV) + extras["coalesced"] = (pool, layer) + return index_cache, position_cache + + +def _other_layers_untouched(pool: torch.Tensor, layer: int) -> None: + fill, bits = _key_fill_bits(), pool.view(torch.int16) + for start in range(0, pool.shape[0], 16384): + band = bits[start : start + 16384] + for other in range(pool.shape[1]): + if other != layer: + assert bool((band[:, other] == fill).all()), ( + f"layer {other}'s index plane moved in pages {start}..{start + band.shape[0]}" + ) + + +def _index_store_build(spec, seed: int, planes=None): + """The op's kwargs and extras for one cell, storing into `planes` when given.""" + gen = torch.Generator(device="cpu").manual_seed(seed) + tpb, hkv, head_dim, axes = (spec[k] for k in ("tokens_per_block", "hkv", "head_dim", "axes")) + slots, width, pages = spec["slots"], spec["width"], spec["pages"] + segments = spec["segments"] + requests = [r for r, (_, n) in enumerate(segments) for _ in range(n)] + positions = [p for start, n in segments for p in range(start, start + n)] + rows = len(positions) + + table = torch.randint(0, pages, (slots, width), generator=gen, dtype=torch.int32) + order = torch.randperm(pages, generator=gen).to(torch.int32) + used, allocated_cols = 0, [] + for r, (start, n) in enumerate(segments): + cols = (start + n - 1) // tpb + 1 + table[r] = -1 + table[r, :cols] = order[used : used + cols] + used += cols + allocated_cols.append(cols) + assert used <= pages and len(segments) <= slots, "pool or table too small for the batch" + if spec.get("straddle"): + # The last row stores into the pool's last page: rows land on both sides of 2^31. + last_start, last_rows = segments[-1] + column = (last_start + last_rows - 1) // tpb + live = table[: len(segments)] + holder = (live == pages - 1).nonzero() + if len(holder): + live[holder[0, 0], holder[0, 1]] = live[len(segments) - 1, column] + live[len(segments) - 1, column] = pages - 1 + if spec["hole"] is not None: + request, position = _hole(spec, table, allocated_cols) + requests.append(request) + positions.append(position) + rows += 1 + table = table.to(DEV) + + projection = torch.randn( + rows, spec["index_heads"] + hkv, head_dim, generator=gen, dtype=torch.float32 + ).to(device=DEV, dtype=spec["key_dtype"]) + token_k = projection[:, spec["index_heads"] :] + if spec["key_layout"] == "fused_row": + # The fused projection row in heads of 128; the raw index key is the second to last. + heads = spec.get("row_width", 14080) // head_dim + row = torch.randn(rows, heads, head_dim, generator=gen, dtype=torch.float32) + token_k = row.to(device=DEV, dtype=spec["key_dtype"])[:, heads - 2 : heads - 2 + hkv] + assert token_k.stride(0) == heads * head_dim + + request_indices = torch.tensor(requests, dtype=spec["request_dtype"], device=DEV) + logical_positions = torch.tensor(positions, dtype=spec["position_dtype"], device=DEV) + # Three distinct axes (an image span) in the engine's `[3, max_num_tokens]` buffer. + coords = logical_positions.to(torch.int32)[:, None] + 7 * torch.arange( + 3, dtype=torch.int32, device=DEV + ) + buffer = torch.full((3, ENGINE_MAX_NUM_TOKENS), -1, dtype=torch.int32, device=DEV) + buffer[:, :rows] = coords.t() + coords = buffer[:, :rows].t() + + extras = {} + position_cache = None + if planes is not None: + index_cache, position_cache = planes + elif spec["cache_layout"] == "shared_page": + index_cache, position_cache = _shared_page_planes(spec, extras) + elif spec["cache_layout"] == "coalesced": + index_cache, position_cache = _coalesced_planes(spec, extras) + else: + index_cache = torch.full((pages, tpb, hkv, head_dim), KEY_FILL, device=DEV) + index_cache = index_cache.to(torch.bfloat16) + if position_cache is None: + position_cache = torch.full((pages, tpb, axes), POS_FILL, dtype=torch.int32, device=DEV) + args = dict( + token_k=token_k, position_coordinates=coords, index_cache=index_cache, + position_cache=position_cache, request_indices=request_indices, + logical_positions=logical_positions, block_table=table, tokens_per_block=tpb, + ) # fmt: skip + return args, extras + + +def _check_index_store(cell, monkeypatch) -> None: + args, extras = _index_store_build(cell.spec, seed=_seed(cell.why)) + if cell.spec.get("straddle"): + assert (cell.spec["pages"] - 1) * args["index_cache"].stride(0) >= _OFFSET_EDGE + ref = qsa_paged_index_store.reference(**args) + out = qsa_paged_index_store(**args) + assert out[0] is args["index_cache"] and out[1] is args["position_cache"] + _index_store_compare(out, ref) + if "roles" in extras: + assert bool((extras["roles"].view(torch.int16) == _key_fill_bits()).all()), ( + "another index-K role of the shared page moved" + ) + if "coalesced" in extras: + _other_layers_untouched(*extras["coalesced"]) + + +def _kv_store_compare(out, ref) -> None: + """Bitwise over both role planes, which together are the whole pool: the stray-write check.""" + for name, got, want in zip(("k_cache", "v_cache"), out, ref): + assert got.dtype == want.dtype, (name, got.dtype, want.dtype) + assert got.shape == want.shape, (name, got.shape, want.shape) + torch.testing.assert_close( + _bits(got), _bits(want), rtol=0, atol=0, msg=lambda m, n=name: f"{n}: {m}" + ) + + +FILL = -2.0 + + +def _check(result): + err, *value = result + assert err == cuda_drv.CUresult.CUDA_SUCCESS, err + return value[0] if len(value) == 1 else value + + +class _ArrayInterface: + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = dict( + shape=(nbytes,), strides=None, typestr="|u1", data=(ptr, False), version=2 + ) + + +class _KvEnginePool: + """Layer `layer_slot`'s K and V views of the stock manager's QSA pool at `pages` pages. The + range is reserved whole; memory is mapped only under the K and V planes of `touched` pages + and page 0 (the tensor's base pointer).""" + + def __init__(self, spec, touched): + dtype = spec["cache_dtype"] + elem = torch.empty((), dtype=dtype).element_size() + hkv, tpb, dim = spec["hkv"], spec["tokens_per_block"], spec["head_dim"] + self.pages, self.touched = spec["pages"], sorted(set(touched)) + self.role = hkv * tpb * dim + self.stride = spec["role_pages"] * self.role + self.own = spec["layer_slot"] * 2 * self.role + + def planes(page): + """This layer's K and V planes of `page`, in bytes of the range.""" + lo = (page * self.stride + self.own) * elem + return lo, lo + 2 * self.role * elem + + self.written = [planes(p) for p in self.touched] + dev = torch.cuda.current_device() + prop = cuda_drv.CUmemAllocationProp() + prop.type = cuda_drv.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = cuda_drv.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + prop.location.id = dev + gran = _check( + cuda_drv.cuMemGetAllocationGranularity( + prop, + cuda_drv.CUmemAllocationGranularity_flags.CU_MEM_ALLOC_GRANULARITY_RECOMMENDED, + ) + ) + self.size = -(-self.pages * self.stride * elem // gran) * gran + self.va = int(_check(cuda_drv.cuMemAddressReserve(self.size, gran, 0, 0))) + merged = [] + for lo, hi in sorted({planes(0), *self.written}): + lo, hi = lo // gran * gran, min(-(-hi // gran) * gran, self.size) + if merged and lo <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], hi) + else: + merged.append([lo, hi]) + access = cuda_drv.CUmemAccessDesc() + access.location.type = cuda_drv.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + access.location.id = dev + access.flags = cuda_drv.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + self.maps, self.spans = [], merged + for lo, hi in merged: + handle = _check(cuda_drv.cuMemCreate(hi - lo, prop, 0)) + ptr = cuda_drv.CUdeviceptr(self.va + lo) + _check(cuda_drv.cuMemMap(ptr, hi - lo, 0, handle, 0)) + self.maps.append((ptr, hi - lo, handle)) + _check(cuda_drv.cuMemSetAccess(ptr, hi - lo, [access], 1)) + + self.raw = torch.as_tensor(_ArrayInterface(self.va, self.size), device=DEV) + pool = self.raw.view(dtype).as_strided( + (self.pages, 2, hkv, tpb, dim), + (self.stride, self.role, tpb * dim, dim, 1), + self.own, + ) + self.k, self.v = pool[:, 0], pool[:, 1] + self.index = torch.tensor(self.touched, dtype=torch.long, device=DEV) + self.fill() + + def fill(self) -> None: + pattern = _bits(torch.full((2,), FILL, dtype=torch.bfloat16).to(self.k.dtype)) + for lo, hi in self.spans: + self.raw[lo:hi].view(pattern.dtype).fill_(int(pattern[0])) + torch.cuda.synchronize() + self.before = [self.raw[lo:hi].clone() for lo, hi in self.spans] + + def dense(self, table: torch.Tensor): + """`(k, v, table)`: copies of the touched pages and `table` renumbered into them.""" + remap = torch.full((self.pages,), -1, dtype=torch.int32, device=DEV) + remap[self.index] = torch.arange(len(self.touched), dtype=torch.int32, device=DEV) + renamed = torch.where(table >= 0, remap[table.clamp_min(0).long()], -1).to(torch.int32) + return self.k[self.index].clone(), self.v[self.index].clone(), renamed.contiguous() + + def gathered(self): + return self.k[self.index], self.v[self.index] + + def assert_rest_unchanged(self) -> None: + torch.cuda.synchronize() + for (lo, hi), before in zip(self.spans, self.before): + keep = torch.ones(hi - lo, dtype=torch.bool, device=DEV) + for p_lo, p_hi in self.written: + if p_hi > lo and p_lo < hi: + keep[max(p_lo, lo) - lo : min(p_hi, hi) - lo] = False + assert torch.equal(self.raw[lo:hi][keep], before[keep]), ( + f"bytes [{lo}, {hi}) of the pool changed outside the touched pages' planes" + ) + + def close(self) -> None: + torch.cuda.synchronize() + del self.k, self.v, self.raw + for ptr, size, handle in self.maps: + _check(cuda_drv.cuMemUnmap(ptr, size)) + _check(cuda_drv.cuMemRelease(handle)) + _check(cuda_drv.cuMemAddressFree(cuda_drv.CUdeviceptr(self.va), self.size)) + + +def _kv_store_build(spec, seed: int): + """`(op kwargs, pool)`: a contiguous pool, or a `_KvEnginePool` for a `pool="engine"` cell.""" + gen = torch.Generator(device="cpu").manual_seed(seed) + tpb, hkv, head_dim = spec["tokens_per_block"], spec["hkv"], spec["head_dim"] + slots, width, pages = spec["slots"], spec["width"], spec["pages"] + engine = spec.get("pool") == "engine" + segments = spec["segments"] + requests = [r for r, (_, n) in enumerate(segments) for _ in range(n)] + positions = [p for start, n in segments for p in range(start, start + n)] + + table = torch.randint(0, pages, (slots, width), generator=gen, dtype=torch.int32) + order = torch.randperm(pages, generator=gen).to(torch.int32) + if engine: + # The first row's page is the pool's last one, whose planes are its highest bytes. + first = segments[0][0] // tpb + last = int((order == pages - 1).nonzero()[0]) + order[[first, last]] = order[[last, first]] + used, allocated_cols = 0, [] + for r, (start, n) in enumerate(segments): + cols = (start + n - 1) // tpb + 1 + assert cols <= width, f"request {r} ends past the table's {width} columns" + table[r] = -1 + table[r, :cols] = order[used : used + cols] + used += cols + allocated_cols.append(cols) + assert used <= pages and len(segments) <= slots, "pool or table too small for the batch" + if spec["hole"] is not None: + request, position = _hole(spec, table, allocated_cols) + requests.append(request) + positions.append(position) + + rows = len(positions) + assert rows == spec.get("rows", rows), (rows, spec["rows"]) + heads = spec["heads_q"] + 2 * hkv + spec.get("row_pad_heads", 0) + qkv = torch.randn(rows, heads, head_dim, generator=gen, dtype=torch.float32) + qkv = qkv.to(device=DEV, dtype=spec["row_dtype"]) + assert qkv.stride(0) == spec.get("row_stride", qkv.stride(0)), qkv.stride(0) + k = qkv[:, spec["heads_q"] : spec["heads_q"] + hkv] + v = qkv[:, spec["heads_q"] + hkv : spec["heads_q"] + 2 * hkv] + touched = [int(table[r, p // tpb]) for r, p in zip(requests, positions)] + table = table.to(DEV) + + if engine: + pool = _KvEnginePool(spec, [t for t in touched if t >= 0]) + k_cache, v_cache = pool.k, pool.v + else: + pool = torch.full( + (pages, 2, hkv, tpb, head_dim), FILL, dtype=torch.bfloat16, device=DEV + ).to(spec["cache_dtype"]) + k_cache, v_cache = pool[:, 0], pool[:, 1] + args = dict( + k=k, v=v, k_cache=k_cache, v_cache=v_cache, + request_indices=torch.tensor(requests, dtype=spec["request_dtype"], device=DEV), + logical_positions=torch.tensor(positions, dtype=spec["position_dtype"], device=DEV), + block_table=table, tokens_per_block=tpb, + ) # fmt: skip + return args, pool + + +def _kv_engine_reference(args, pool: _KvEnginePool): + """The entry's reference over a dense copy of the touched pages.""" + dense_k, dense_v, renamed = pool.dense(args["block_table"]) + return qsa_paged_kv_store.reference( + **{**args, "k_cache": dense_k, "v_cache": dense_v, "block_table": renamed} + ) + + +def _check_kv_store(cell, monkeypatch) -> None: + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + if not isinstance(pool, _KvEnginePool): + before = pool.clone() + ref = qsa_paged_kv_store.reference(**args) + out = qsa_paged_kv_store(**args) + assert out[0] is args["k_cache"] and out[1] is args["v_cache"] + _kv_store_compare(out, ref) + wrote = not torch.equal(_bits(pool), _bits(before)) + assert wrote == bool(cell.spec["segments"]), "a cell with rows writes, an empty one not" + return + try: + top = max(pool.touched) + assert top == pool.pages - 1, (top, pool.pages) + assert top * pool.k.stride(0) >= 2**32, top * pool.k.stride(0) + before = tuple(t.clone() for t in pool.gathered()) + ref = _kv_engine_reference(args, pool) + out = qsa_paged_kv_store(**args) + assert out[0] is args["k_cache"] and out[1] is args["v_cache"] + got = pool.gathered() + _kv_store_compare(got, ref) + pool.assert_rest_unchanged() + assert any(not torch.equal(_bits(g), _bits(b)) for g, b in zip(got, before)), "no write" + finally: + pool.close() + + +def _gqa_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """4 ulp of each (row, head)'s magnitude and 2 ulp of relative RMS in q's dtype, the band of + every sparse GQA entry here. The kernels round softmax probabilities to the cache dtype + before P.V, so the error tracks the averaged values rather than the output, and a gated + output is rounded twice; a page-shifted table or a dropped scale or token falls outside it.""" + assert out.dtype == ref.dtype, (out.dtype, ref.dtype) + assert out.shape == ref.shape, (out.shape, ref.shape) + if out.numel() == 0: + return + assert_within_ulp(out, ref, element_ulp=4.0, rms_ulp=2.0) + + +#: NaN's bit pattern per pool dtype, filled through an integer view (not every fill takes e4m3). +_NAN_BITS = {torch.bfloat16: (torch.int16, 0x7FC0), torch.float8_e4m3fn: (torch.uint8, 0x7F)} + + +def _selected_tokens(positions, seed, compress_ratio, token_topk, shared=False, chunk=None): + """QSA's expansion layout: a random `min(available, block_topk)`-subset of the complete + visible groups expanded to their tokens, the incomplete trailing group, then `-1`, front-packed. + `shared`: every row ranks the groups by one score vector, as real selections overlap.""" + block_topk = token_topk // compress_ratio + final_topk = token_topk + compress_ratio - 1 + gen = torch.Generator(device=DEV).manual_seed(seed) + rows = positions.numel() + visible = positions.long() + 1 + available = visible // compress_ratio + taken = available.clamp(max=block_topk) + width = max(block_topk, int(available.max().item()) if rows else 0) + out = torch.empty(rows, final_topk, dtype=torch.int32, device=DEV) + column = torch.arange(final_topk, device=DEV)[None, :] + step = chunk or max(rows, 1) + for start in range(0, rows, step): + stop = min(start + step, rows) + avail, take, vis = available[start:stop], taken[start:stop], visible[start:stop] + if shared: + scores = torch.rand(1, width, generator=gen, device=DEV).expand(stop - start, -1) + gen.manual_seed(seed) + else: + scores = torch.rand(stop - start, width, generator=gen, device=DEV) + group_ids = torch.arange(width, device=DEV)[None, :] + scores = torch.where(group_ids < avail[:, None], scores, torch.inf) + chosen = scores.argsort(dim=1)[:, :block_topk] + group_of_column = (column // compress_ratio).clamp(max=block_topk - 1) + body = torch.gather(chosen, 1, group_of_column.expand(stop - start, -1)) + body = body * compress_ratio + column % compress_ratio + in_body = column < (take * compress_ratio)[:, None] + tail_column = column - (take * compress_ratio)[:, None] + tail = (avail * compress_ratio)[:, None] + tail_column + in_tail = ~in_body & (tail_column >= 0) & (tail < vis[:, None]) + out[start:stop] = torch.where(in_body, body, torch.where(in_tail, tail, -1)).to(torch.int32) + return out + + +def _straddling_pages(spec, needed: int, edge: int = _OFFSET_EDGE) -> int: + """Pool size whose last `needed` pages lie on both sides of `edge` elements of the K plane.""" + role_page = spec["kv_heads"] * spec["tokens_per_block"] * spec["head_dim"] + edge_page = math.ceil((edge - 2 * spec["layer"] * role_page) / (spec["slot_pages"] * role_page)) + return max(edge_page + needed // 2, needed) + + +def _paged_kv_state(spec, seed, dtype, pool_pages_of, slot_pages=2, layer=0, chunk=None, + scratch=False) -> SimpleNamespace: # fmt: skip + """The engine's paged K/V: a `[slots, pages]` table over seeded-permuted slots, `-1` past each + sequence; K and V layer `layer`'s role planes of a NaN `[pool_pages, slot_pages, kv, tpb, D]` + pool, or with `scratch` two contiguous pools (what `qsa_fp8_pages_to_bf16` writes).""" + gen = torch.Generator(device=DEV).manual_seed(seed) + tpb, kv, d = spec["tokens_per_block"], spec["kv_heads"], spec["head_dim"] + segments = list(spec["segments"]) + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in segments]) + req = torch.cat([torch.full((n,), i, device=DEV) for i, (_, n) in enumerate(segments)]) + slots = spec["slots"] + assert len(segments) <= slots + slot_of = torch.randperm(slots, generator=torch.Generator().manual_seed(seed))[: len(segments)] + mapped = [math.ceil((s + n) / tpb) for s, n in segments] + assert max(mapped) <= spec["pages"] + needed = sum(mapped) + 7 + pool_pages = pool_pages_of(needed) + assert pool_pages >= needed + first = pool_pages - needed + page_ids = first + torch.randperm(needed, generator=gen, device=DEV).to(torch.int32) + table = torch.full((slots, spec["pages"]), -1, dtype=torch.int32, device=DEV) + cursor = 0 + for slot, count in zip(slot_of.tolist(), mapped): + table[slot, :count] = page_ids[cursor : cursor + count] + cursor += count + if scratch: + k, v = (torch.randn((pool_pages, kv, tpb, d), generator=gen, device=DEV).to(dtype) + for _ in range(2)) # fmt: skip + else: + bits, nan = _NAN_BITS[dtype] + pool = torch.full((pool_pages, slot_pages, kv, tpb, d), nan, dtype=bits, device=DEV) + pool = pool.view(dtype) + for p0 in range(first, pool_pages, chunk or needed): + p1 = min(p0 + (chunk or needed), pool_pages) + fresh = torch.randn(p1 - p0, 2, kv, tpb, d, generator=gen, device=DEV) + pool[p0:p1, 2 * layer : 2 * layer + 2] = fresh.to(dtype) + k, v = pool[:, 2 * layer], pool[:, 2 * layer + 1] + rows = slot_of.to(DEV)[req] + return SimpleNamespace(gen=gen, positions=positions, rows=rows, table=table, k=k, v=v) + + +def _assert_straddles(args, edge: int = _OFFSET_EDGE) -> None: + """The mapped pages put both planes' element offsets on both sides of `edge`.""" + table = args["block_table"] + pages = table[(table >= 0) & (table < args["k_cache"].shape[0])].long() + for name in ("k_cache", "v_cache"): + plane = args[name] + ends = plane.storage_offset() + pages * plane.stride(0) + plane[0].numel() - 1 + assert bool((ends >= edge).any()), f"{name}: no mapped page lies past {edge}" + assert bool((ends < edge).any()), f"{name}: every mapped page lies past {edge}" + + +def _assert_prefill_selection(selected, positions, spec) -> None: + """The precondition the prefill kernels cannot check: front-packed, and exactly as many + valid entries as the per-row scan bound the positions give.""" + valid = selected >= 0 + columns = torch.arange(selected.shape[1], device=selected.device)[None, :] + assert bool((valid == (columns < valid.sum(1)[:, None])).all()) + cr, visible = spec["compress_ratio"], positions.long() + 1 + bound = (visible // cr).clamp(max=spec["token_topk"] // cr) * cr + visible % cr + assert torch.equal(valid.sum(1), bound) + + +def _refill(args, seed, selection) -> None: + """The engine's next step in the call's own buffers: new q, gate, selection and mapped K/V.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + for name in ("q", "output_gate"): + if args.get(name) is not None: + view = args[name] + view.copy_(torch.randn(view.shape, generator=gen, device=DEV).to(view.dtype)) + args["selected_tokens"].copy_(selection(args)) + pages = args["block_table"][args["block_table"] >= 0].long() + for name in ("k_cache", "v_cache"): + plane = args[name] + bits = _NAN_BITS[plane.dtype][0] + fresh = torch.randn((pages.numel(), *plane.shape[1:]), generator=gen, device=DEV) + plane.view(bits)[pages] = fresh.to(plane.dtype).view(bits) + + +def _captured(fn, refill): + """`fn()` captured in a CUDA graph, `refill()` run, the graph replayed: the captured output.""" + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fn() + refill() + graph.replay() + torch.cuda.synchronize() + return out + + +def _replay_in_a_graph(entry, args, refill, first) -> None: + """Captured once, inputs refilled in place, replayed: within the band of the new inputs' + reference and bitwise an eager call on them.""" + replayed = _captured(lambda: entry(**args), refill).clone() + assert not torch.equal(replayed, first), "the replay did not read the refilled inputs" + _gqa_compare(replayed, entry.reference(**args)) + assert torch.equal(replayed, entry(**args)), "the replay differs from an eager call" + + +class _LaunchRecorder: + """Records a kernel's grid and PDL flag, then delegates to the kernel.""" + + def __init__(self, inner, name, log): + self._inner, self._name, self._log = inner, name, log + + def __getitem__(self, grid): + entry = [self._name, tuple(grid), False] + self._log.append(entry) + launch = self._inner[grid] + + def run(*args, **kwargs): + entry[2] = bool(kwargs.get("launch_pdl", False)) + return launch(*args, **kwargs) + + return run + + def __call__(self, selected_tokens, *args, **kwargs): + # the row-tile token plan: one CTA a group (its 10th argument); the block plan: its 9th + grid = (args[8],) if self._name == "_group_plan" else (args[7],) + self._log.append([self._name, grid, False]) + return self._inner(selected_tokens, *args, **kwargs) + + def warmup(self, *args, **kwargs): + """A compile-only call (`jit_variants.compile_only`): no launch to record.""" + return self._inner.warmup(*args, **kwargs) + + +def _record_launches(monkeypatch, module, names) -> list: + """Which of `module`'s kernels the next calls launch, in order: [name, grid, PDL].""" + log: list = [] + for name in names: + monkeypatch.setattr(module, name, _LaunchRecorder(getattr(module, name), name, log)) + return log + + +_KERNELS = { + "_qsa_prefill_pair_stable_kernel": "prefill", + "_qsa_splitk_stable_kernel": "splitk", + "_qsa_merge_stable_kernel": "merge", +} + + +def _row_gate(q, spec): + """The output gate's columns of the fused row `q` is read off: after q, k and v.""" + rows, d = q.shape[0], spec["head_dim"] + row = q.as_strided((rows, q.stride(0)), (q.stride(0), 1)) + start = (spec["q_heads"] + 2 * spec["kv_heads"]) * d + return row[:, start : start + spec["q_heads"] * d] + + +def _sparse_gqa_selection(spec, seed): + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in spec["segments"]]) + return _selected_tokens(positions, seed, spec["compress_ratio"], spec["token_topk"]) + + +def _sparse_gqa_build(spec, seed): + """One cell's args: q off the packed or the fused projection row (`row_heads` heads).""" + st = _paged_kv_state( + spec, seed, spec["cache_dtype"], lambda needed: spec.get("pool_pages", needed), + slot_pages=spec.get("slot_pages", 2), layer=spec.get("layer", 0), + ) # fmt: skip + rows, d, q_heads = st.positions.numel(), spec["head_dim"], spec["q_heads"] + fused = spec["q_layout"] == "fused_row" + row_heads = spec.get("row_heads", 55 if fused else q_heads + 2 * spec["kv_heads"]) + row = torch.randn(rows, row_heads * d, generator=st.gen, device=DEV).to(spec["q_dtype"]) + q = row.view(rows, row_heads, d)[:, :q_heads] + args = dict( + q=q, k_cache=st.k, v_cache=st.v, block_table=st.table, + selected_tokens=_sparse_gqa_selection(spec, seed + 3), + request_indices=st.rows.to(spec["request_dtype"]), + tokens_per_block=spec["tokens_per_block"], softmax_scale=d**-0.5, + ) # fmt: skip + if spec["prefill_pair"]: + args.update(query_positions=st.positions, compress_ratio=spec["compress_ratio"]) + if spec.get("gate") == "fused_row": + assert fused, "the gate is read off the fused row" + if spec.get("gate_fused", True): + args.update(output_gate=_row_gate(q, spec)) + return args + + +def _check_sparse_gqa(cell, monkeypatch) -> None: + """The cell's branch (its kernels, grid and PDL, recorded), then the band; a launch with no + merge refuses the gate (the call site applies it after); a `capture` cell is replayed.""" + launches = _record_launches(monkeypatch, _stable, _KERNELS) + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sparse_gqa_build(spec, seed) + q, rows = args["q"], args["q"].shape[0] + assert rows == spec.get("rows", rows) + if "row_heads" in spec: + assert q.stride(0) == spec["row_heads"] * spec["head_dim"] + if "output_gate" in args: + gate = args["output_gate"] + assert gate.stride(0) == q.stride(0) + assert gate.untyped_storage().data_ptr() == q.untyped_storage().data_ptr() + if "rows_per_request" in spec: + assert {n for _, n in spec["segments"]} == {spec["rows_per_request"]} + if "offset_edge" in spec: + _assert_straddles(args, spec["offset_edge"]) + if spec["prefill_pair"]: + _assert_prefill_selection(args["selected_tokens"], args["query_positions"], spec) + out = qsa_paged_sparse_gqa(**args) + assert [_KERNELS[name] for name, _, _ in launches] == list(spec["launches"]) + assert launches[0][1][:2] == (rows, spec["kv_heads"]) + if "splits" in spec: + assert launches[0][1][2] == spec["splits"], launches + assert all(pdl == spec.get("pdl", False) for _, _, pdl in launches), launches + if "output_gate" in args: + gated = (q, args["k_cache"], args["selected_tokens"], args["output_gate"]) + assert qsa_paged_sparse_gqa.fuses_output_gate(*gated) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == q.dtype and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, qsa_paged_sparse_gqa.reference(**args)) + if spec.get("gate") == "fused_row" and not spec.get("gate_fused", True): + gate = _row_gate(q, spec) + assert not qsa_paged_sparse_gqa.fuses_output_gate( + q, args["k_cache"], args["selected_tokens"], gate + ) + with pytest.raises(ValueError, match="output-gate fusion requires"): + qsa_paged_sparse_gqa(**args, output_gate=gate) + if spec.get("capture"): + eager = [tuple(entry) for entry in launches] + del launches[:] + + def refill(): + assert [tuple(entry) for entry in launches] == eager, "the capture's launches" + _refill(args, seed + 1, lambda a: _sparse_gqa_selection(spec, seed + 4)) + + _replay_in_a_graph(qsa_paged_sparse_gqa, args, refill, out) + + +PRO_D, PRO_IH, PRO_ID, PRO_TPB, PRO_CR, PRO_ROT, PRO_WIDTH, PRO_SLOTS = ( + 256, 4, 128, 32, 4, 64, 8196, 8 +) # fmt: skip +#: (q heads, KV heads) per rank; fused row `[q | k | v | gate | index q 4 x 128, k | v slot]`. +PRO_GEOMS = {"tp4": (6, 1), "tp2": (12, 1), "dep4": (24, 2)} +PRO_ROLE_PAGES, PRO_LAYER_SLOT, PRO_INDEX_LAYERS = 24, 11, 12 +#: One bf16 step in 2^-8 units of the row scale; the relative RMS bound is empirical. +PRO_ELEMENT_ULP, PRO_RMS_ULP = 2.0, 0.05 + + +def _pro_geometry(geom): + hq, hk = PRO_GEOMS[geom] + qkv = (hq + 2 * hk) * PRO_D + iq_col = qkv + hq * PRO_D + return hq, hk, qkv, iq_col, iq_col + (PRO_IH + 2) * PRO_ID + + +def _pro_edge_rows(proj, hq, hk) -> None: + """Zero, -0.0 and outlier blocks in the k head past the rotary prefix and in the v head.""" + k0, v0 = hq * PRO_D, (hq + hk) * PRO_D + edges = ( + (0, slice(k0 + 128, k0 + 144), 0.0), + (1, slice(k0 + 160, k0 + 176), -0.0), + (2, slice(k0 + 200, k0 + 201), 900.0), + (0, slice(v0 + 16, v0 + 32), 0.0), + (1, slice(v0 + 48, v0 + 64), torch.tensor([-0.0] * 8 + [0.0] * 8)), + (3, slice(v0 + 40, v0 + 41), -900.0), + (2, slice(v0 + 64, v0 + 80), 1e-30), + ) + for r, cols, value in edges: + if r < proj.shape[0]: + proj[r, cols] = value + + +def _pro_build(spec, seed): + g = torch.Generator(device=DEV).manual_seed(seed) + hq, hk, qkv_width, iq_col, row_width = _pro_geometry(spec["geom"]) + per = spec.get("rows_per_request", 1) + layout = [(r, p + i) for r, p in enumerate(spec["pos"]) for i in range(per)] + pos = [p for _, p in layout] + rows = len(pos) + last = {} + for req, p in layout: + last[req] = max(last.get(req, 0), p) + slots = max(PRO_SLOTS, len(last) + 3) + engine = spec.get("engine", False) + if engine: + # Enough pages that the last ones start past 2^31 elements; rows go on those. + page_elems = PRO_ROLE_PAGES * hk * PRO_TPB * PRO_D + pages = _OFFSET_EDGE // page_elems + 2 + sum(p // PRO_TPB + 1 for p in last.values()) + perm = torch.arange(pages - 1, -1, -1, device=DEV, dtype=torch.int32) + else: + pages = sum(p // PRO_TPB + 1 for p in last.values()) + 64 + perm = torch.randperm(pages, generator=g, device=DEV).to(torch.int32) + block_table = torch.full((slots, PRO_WIDTH), -1, dtype=torch.int32, device=DEV) + used = 0 + for req, p in last.items(): + n = p // PRO_TPB + 1 + block_table[req, :n] = perm[used : used + n] + used += n + request = torch.tensor([req for req, _ in layout], dtype=torch.int32, device=DEV) + if spec.get("unmapped") is not None: + request[spec["unmapped"]] = slots + 5 + logical = torch.tensor(pos, dtype=torch.int64, device=DEV) + ids = torch.stack([logical, logical + 3, logical + 7]).to(torch.int32).contiguous() + buf = torch.empty(3, 1, rows + 5, dtype=torch.int32, device=DEV) + buf[:, 0, :rows] = ids + coords = torch.transpose(buf[:, 0, :rows], 0, 1) + table_rows = max(pos) + 8 + angles = torch.rand(table_rows, PRO_ROT // 2, generator=g, device=DEV) * 6.28 + cos_sin = torch.cat([torch.cos(angles), torch.sin(angles)], dim=1).contiguous() + proj = (torch.randn(rows, row_width, generator=g, device=DEV) * 2.0).to(torch.bfloat16) + if spec.get("edge"): + _pro_edge_rows(proj, hq, hk) + out = torch.randn(rows, qkv_width + 64, generator=g, device=DEV).to(torch.bfloat16) + index_scratch = torch.randn(rows, PRO_IH + 2, PRO_ID, generator=g, device=DEV) + index_scratch = index_scratch.to(torch.bfloat16) + weights = [ + (0.3 * torch.randn(n, generator=g, device=DEV)).to(torch.bfloat16) + for n in (PRO_D, PRO_D, PRO_ID, PRO_ID) + ] + kv_dtype = torch.float8_e4m3fn if spec.get("fp8") else torch.bfloat16 + if engine: + roles = torch.zeros(pages, PRO_ROLE_PAGES, hk, PRO_TPB, PRO_D, dtype=kv_dtype, device=DEV) + pool = roles[:, 2 * PRO_LAYER_SLOT : 2 * PRO_LAYER_SLOT + 2] + planes = torch.randn(pages, PRO_INDEX_LAYERS, PRO_TPB, 1, PRO_ID, generator=g, device=DEV) + index_cache = planes.to(torch.bfloat16)[:, PRO_LAYER_SLOT] + else: + pool = torch.randn(pages, 2, hk, PRO_TPB, PRO_D, generator=g, device=DEV).to(kv_dtype) + index_cache = torch.randn(pages, PRO_TPB, 1, PRO_ID, generator=g, device=DEV) + index_cache = index_cache.to(torch.bfloat16) + position_cache = torch.randint( + 0, table_rows, (pages, PRO_TPB, 3), generator=g, device=DEV, dtype=torch.int32 + ) + return dict( + proj=proj, out=out[:, :qkv_width], index_q=index_scratch[:, :PRO_IH], position_ids=ids, + coords=coords, cos_sin=cos_sin, q_weight=weights[0], k_weight=weights[1], + index_q_weight=weights[2], index_k_weight=weights[3], k_cache=pool[:, 0], + v_cache=pool[:, 1], index_cache=index_cache, position_cache=position_cache, + request_indices=request, logical_positions=logical, block_table=block_table, + tokens_per_block=PRO_TPB, compress_ratio=PRO_CR, num_heads_q=hq, num_heads_k=hk, head_dim=PRO_D, + index_heads=PRO_IH, index_head_dim=PRO_ID, index_q_col=iq_col, rotary_dim=PRO_ROT, eps=1e-6, + mrope_section1=11, mrope_section2=10, rows_per_request=per, + ) # fmt: skip + + +def _pro_clone(args): + """The arguments over fresh copies of every allocation they view, and the copies keyed by + the first argument viewing each.""" + out, copies, keys = dict(args), {}, {} + for name, t in args.items(): + if not isinstance(t, torch.Tensor): + continue + key = keys.setdefault(t.untyped_storage().data_ptr(), name) + if key not in copies: + storage = torch.empty(0, dtype=torch.uint8, device=t.device).set_(t.untyped_storage()) + copies[key] = storage.clone() + view = torch.empty(0, dtype=t.dtype, device=t.device) + view.set_(copies[key].untyped_storage(), t.storage_offset(), t.shape, t.stride()) + out[name] = view + return out, copies + + +def _pro_written_slots(args): + """(row, page, slot) of every row whose page lookup holds, from the host's view.""" + table = args["block_table"].cpu() + pages = args["k_cache"].shape[0] + found = [] + for t, (req, p) in enumerate( + zip(args["request_indices"].tolist(), args["logical_positions"].tolist()) + ): + column = p // PRO_TPB + if 0 <= req < table.shape[0] and column < table.shape[1]: + page = int(table[req, column]) + if 0 <= page < pages: + found.append((t, page, p % PRO_TPB)) + return found + + +def _pro_check(got, want) -> None: + """Band the rotated values (only the rms's reduction order differs), overwrite them in `got`, + then require every byte bitwise.""" + g, w = got[0], want[0] + hq, hk = g["num_heads_q"], g["num_heads_k"] + width = (hq + hk) * PRO_D + + def banded(a, b): + a, b = a.float().reshape(-1, a.shape[-1]), b.float().reshape(-1, b.shape[-1]) + assert_within_ulp(a.to(torch.bfloat16), b.to(torch.bfloat16), PRO_ELEMENT_ULP, PRO_RMS_ULP) + + banded(g["out"][:, :width], w["out"][:, :width]) + banded(g["index_q"], w["index_q"]) + slots = _pro_written_slots(g) + k_got = g["out"][:, hq * PRO_D : width].reshape(-1, hk, PRO_D) + for t, page, slot in slots: + own = k_got[t].to(g["k_cache"].dtype) + assert torch.equal(g["k_cache"][page, :, slot].view(torch.uint8), own.view(torch.uint8)), ( + f"row {t}: the K pool's row is not the packed k converted" + ) + g["k_cache"][page, :, slot] = w["k_cache"][page, :, slot] + ends = [(t, page, slot) for t, page, slot in slots if (slot + 1) % PRO_CR == 0] + if ends: + a = torch.stack([g["index_cache"][page, slot, 0] for _, page, slot in ends]) + b = torch.stack([w["index_cache"][page, slot, 0] for _, page, slot in ends]) + banded(a, b) + for _, page, slot in ends: + g["index_cache"][page, slot, 0] = w["index_cache"][page, slot, 0] + g["out"][:, :width] = w["out"][:, :width] + g["index_q"].copy_(w["index_q"]) + assert got[1].keys() == want[1].keys() + for key, copy in got[1].items(): + differ = (copy != want[1][key]).nonzero() + assert differ.numel() == 0, f"{key}: {differ.shape[0]} bytes differ" + + +def _check_decode_prologue(cell, monkeypatch) -> None: + args = _pro_build(cell.spec, seed=_seed(cell.why) % (2**31)) + want = _pro_clone(args) + _pro_op.reference(**want[0]) + got = _pro_clone(args) + _pro_op(**got[0]) + torch.cuda.synchronize() + _pro_check(got, want) + + +def _plan_operands(spec, g): + n, p = spec["requests"], spec["rows_per_request"] + kind = spec["kind"] + if kind == "equal": + windows = torch.full((n,), 25000, dtype=torch.int64) + else: + windows = torch.randint(500, 62000, (n,), generator=g) + if kind == "padded": + windows[-3:] = 1 + rows = n * p + visible = windows.repeat_interleave(p) + torch.arange(p).repeat(n) // 2 + return ( + visible.to(torch.int32).to(DEV), + torch.arange(n, dtype=torch.int32).repeat_interleave(p).to(DEV), + (visible * 4 + torch.randint(0, 4, (rows,), generator=g)).to(DEV), + (visible * 4 + 7).to(torch.int32).to(DEV), + torch.zeros(rows, dtype=torch.int32, device=DEV), + ) + + +def _check_index_split_plan(cell, monkeypatch) -> None: + """Bitwise the reference, eager and replayed on new windows; the ranks' summed windows + balanced and the padding requests spread over them.""" + sp = cell.spec + n, p, world, rank = sp["requests"], sp["rows_per_request"], sp["world"], sp["rank"] + g = torch.Generator().manual_seed(_seed(cell.why)) + + def check(got, vectors): + want = index_split_plan.reference(*vectors, n, p, world, rank) + for a, b in zip(got, want): + assert a.dtype == b.dtype and torch.equal(a, b) + perm = got.perm_rows.long() + share = n // world * p + win = vectors[0].long() + sums = [int(win[perm[r * share : (r + 1) * share]].sum()) for r in range(world)] + assert max(sums) - min(sums) <= int(win.max()) * p, sums + if sp["kind"] == "padded": + pads = set(range((n - 3) * p, n * p)) + owners = {int(torch.nonzero(perm == r_).item()) // share for r_ in pads} + assert len(owners) > 1, "the padding requests land on more than one rank" + + vectors = _plan_operands(sp, g) + check(index_split_plan(*vectors, n, p, world, rank), vectors) + s_vec, fresh = tuple(v.clone() for v in vectors), _plan_operands(sp, g) + out = _captured( + lambda: index_split_plan(*s_vec, n, p, world, rank), + lambda: [a.copy_(b) for a, b in zip(s_vec, fresh)], + ) + check(out, fresh) + + +def _check_draft_continue_select(cell, monkeypatch) -> None: + """Bitwise `_draft_continue_rows` and the row metadata's `index_select`s, eager and replayed.""" + seqs, p = cell.spec["seqs"], cell.spec["rows_per_request"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + rows = seqs * p + 5 + + def operands(): + accepted = torch.randint(1, p + 1, (seqs + 3,), device=DEV, generator=g).to(torch.int32) + vectors = ( + torch.randint(0, 1 << 20, (rows,), device=DEV, generator=g).to(torch.int32), + torch.randint(0, 1 << 20, (rows,), device=DEV, generator=g).to(torch.int32), + torch.randint(0, 1 << 40, (rows,), device=DEV, generator=g), + torch.randint(0, seqs, (rows,), device=DEV, generator=g).to(torch.int32), + ) + return accepted, vectors + + def check(got, accepted, vectors): + want = draft_continue_select.reference(accepted, seqs, p, vectors) + assert torch.equal(got[0], want[0]) + for a, b in zip(got[1], want[1]): + assert a.dtype == b.dtype and torch.equal(a, b) + core = types.SimpleNamespace(dp_size=1) + md = types.SimpleNamespace(qsa_mtp_num_accepted=accepted) + assert torch.equal(got[0], M._draft_continue_rows(core, md, seqs, p, False)) + + accepted, vectors = operands() + check(draft_continue_select(accepted, seqs, p, vectors), accepted, vectors) + s_acc, s_vec = accepted.clone(), tuple(v.clone() for v in vectors) + accepted, vectors = operands() + out = _captured( + lambda: draft_continue_select(s_acc, seqs, p, s_vec), + lambda: [a.copy_(b) for a, b in zip((s_acc, *s_vec), (accepted, *vectors))], + ) + check(out, accepted, vectors) + + +def _check_draft_rows_gather(cell, monkeypatch) -> None: + """Bitwise `index_select` of strided and one-column row operands, eager and replayed.""" + rows, widths = cell.spec["rows"], cell.spec["widths"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + src_rows = 3 * rows + 2 + index = torch.randperm(src_rows, device=DEV, generator=g)[:rows].to(torch.int64) + operands = [] + for k, w in enumerate(widths): + pad = 64 * (k % 2) # every other operand a column slice of a wider row + buf = torch.randn(src_rows, w + pad, device=DEV, generator=g).to(torch.bfloat16) + operands.append(buf[:, pad : pad + w]) + got = draft_rows_gather(index, tuple(operands)) + want = draft_rows_gather.reference(index, tuple(operands)) + for a, b, t in zip(got, want, operands): + assert a.is_contiguous() and torch.equal(a, b) + assert torch.equal(a, torch.index_select(t, 0, index)) + s_idx = index.clone() + index2 = torch.randperm(src_rows, device=DEV, generator=g)[:rows].to(torch.int64) + out = _captured(lambda: draft_rows_gather(s_idx, tuple(operands)), lambda: s_idx.copy_(index2)) + for a, t in zip(out, operands): + assert torch.equal(a, torch.index_select(t, 0, index2)) + + +def _sgc_build(spec, seed): + st = _paged_kv_state( + spec, seed, spec["kv"], lambda needed: _straddling_pages(spec, needed), + slot_pages=spec["slot_pages"], layer=spec["layer"], chunk=4096, + ) # fmt: skip + rows, d, hq, kv = st.positions.numel(), spec["head_dim"], spec["q_heads"], spec["kv_heads"] + qkv = torch.randn(rows, hq + 2 * kv, d, generator=st.gen, device=DEV).to(torch.bfloat16) + q = qkv[:, :hq] + cr, topk = spec["compress_ratio"], spec["token_topk"] + selected = _selected_tokens(st.positions, seed + 3, cr, topk) + if spec.get("holes"): + selected[:, 1::3] = -1 + offset = spec.get("row_offset", 0) + if offset: + # The generation rows of a mixed step: a row slice of a wider selection buffer. + wide = torch.full((offset + rows, selected.shape[1]), -1, dtype=torch.int32, device=DEV) + wide[offset:] = selected + selected = wide[offset:] + request_indices = st.rows.to(torch.int32) + if spec.get("foreign"): + request_indices[0] = -1 + args = dict( + q=q, k_cache=st.k, v_cache=st.v, block_table=st.table, selected_tokens=selected, + request_indices=request_indices, tokens_per_block=spec["tokens_per_block"], + softmax_scale=d**-0.5, + ) # fmt: skip + gate = spec.get("gate") + if gate == "row": + # The trunk's fused input projection row `[q | k | v | gate | index q, k]`. + row = torch.randn(rows, spec["row_heads"] * d, generator=st.gen, device=DEV).to(q.dtype) + start = (hq + 2 * kv) * d + args["output_gate"] = row[:, start : start + hq * d] + elif gate == "interleaved": + # The draft layer's q projection `[T, Hq, 2 D]`: q then the gate, per head. + qg = torch.randn(rows, hq, 2 * d, generator=st.gen, device=DEV).to(q.dtype) + args["output_gate"] = qg[:, :, d:] + elif gate == "contiguous": + # A gate gathered out of the projection row (a draft-extend step's continuing rows). + args["output_gate"] = torch.randn(rows, hq * d, generator=st.gen, device=DEV).to(q.dtype) + return args + + +def _rolled_selection(args): + """The same selection rolled by one column: other tokens, the layout kept.""" + sel = args["selected_tokens"] + return torch.where(sel >= 0, torch.roll(sel, 1, dims=1).clamp_min(0), sel) + + +def _check_sparse_gqa_splitk(cell, monkeypatch) -> None: + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sgc_build(spec, seed) + _assert_straddles(args) + q, k = args["q"], args["k_cache"] + rows = q.shape[0] + assert _sgc_op.serves(q, k, args["selected_tokens"]), "the target sends this call here" + if "splits" in spec: + width = args["selected_tokens"].shape[1] + elem = 1 if spec["kv"] == torch.float8_e4m3fn else 2 + assert mod.plan_splits(rows, spec["kv_heads"], width, elem)[0] == spec["splits"] + out = _sgc_op(**args) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == torch.bfloat16 and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _sgc_op.reference(**args)) + if spec.get("foreign"): + assert bool((out[0] == 0).all()), "a row of no request attends nothing" + if spec.get("capture"): + _replay_in_a_graph(_sgc_op, args, lambda: _refill(args, seed + 1, _rolled_selection), out) + + +@pytest.mark.parametrize( + "spec", + [ + pytest.param( + mod._verify(dict(mod._TEP2, slots=128), 128, last=30000, step=150, **mod._TEP2_ROW), + id="tep2", + ), + pytest.param(mod._verify(mod._DEP4, 32, last=60000, step=1700, **mod._DEP4_ROW), id="dep4"), + pytest.param(mod._verify(mod._TEP4, 4, **mod._TEP4_ROW), id="tep4"), + ], +) +def test_sparse_gqa_splitk_first_decode_call_compiles_every_plan(spec, monkeypatch) -> None: + """A geometry's first gated decode call compiles the split pass and both merge flavours of + every plan its row counts reach (`MAX_ROWS`). A mixed step's ungated generation rows then + compile nothing at any plan (at tp2/tep2, 164..256 and 324..512 rows take three splits), + nor do gated calls, and the results stay within the reference band.""" + monkeypatch.setattr(mod, "_COMPILED", {}) + monkeypatch.setattr(mod, "_PLANNED", set()) + args = _sgc_build(spec, seed=4242) + q, k = args["q"], args["k_cache"] + cap = mod.MAX_ROWS[(q.shape[1], k.shape[1], k.dtype)] + assert q.shape[0] == cap, "the build covers every routed row count" + width = args["selected_tokens"].shape[1] + elem = 1 if k.dtype == torch.float8_e4m3fn else 2 + sliced = ("q", "selected_tokens", "request_indices", "output_gate") + + def call(rows, gated): + a = {n: (v[:rows] if n in sliced else v) for n, v in args.items()} + if not gated: + del a["output_gate"] + return a, _sgc_op(**a) + + call(4, True) # the decode warmup's first call of the geometry + compiled = set(mod._COMPILED) + by_plan = {} + for r in range(1, cap + 1): + by_plan.setdefault(mod.plan_splits(r, k.shape[1], width, elem), []).append(r) + assert sorted(by_plan) == mod.reachable_plans(cap, k.shape[1], width, elem) + pdl = bool(mod.get_env_enable_pdl()) + for splits, _ in by_plan: + if splits > 1: + assert ("merge", q.shape[1], splits, False, 3, pdl) in compiled, splits + assert ("merge", q.shape[1], splits, True, 2, pdl) in compiled, splits + # Each plan's first and last row count; at tp2/tep2 also the mixed steps' three-split ranges. + picks = {r for rows in by_plan.values() for r in (rows[0], rows[-1])} + if cap == 512: + extra = (164, 256, 324, 512) + assert {mod.plan_splits(r, 1, width, elem)[0] for r in extra} == {3} + picks |= set(extra) + for r in sorted(picks): + for gated in (False, True): + a, out = call(r, gated) + assert set(mod._COMPILED) == compiled, f"{r} rows, gated={gated} compiled" + if r == by_plan[mod.plan_splits(r, k.shape[1], width, elem)][-1]: + _gqa_compare(out, _sgc_op.reference(**a)) # each plan's widest call + + +def _sgg_selection(spec, seed): + """Rows of a request select from its block scores plus `overlap` x noise (None: independent).""" + cr, token_topk, spread = spec["compress_ratio"], spec["token_topk"], spec.get("overlap") + block_topk, final_topk = token_topk // cr, token_topk + cr - 1 + segments = list(spec["segments"]) + positions = torch.cat([torch.arange(s, s + n, device=DEV) for s, n in segments]) + requests = torch.cat([torch.full((n,), i, device=DEV) for i, (_, n) in enumerate(segments)]) + gen = torch.Generator(device=DEV).manual_seed(seed) + rows = positions.numel() + visible = positions.long() + 1 + available = visible // cr + taken = available.clamp(max=block_topk) + width = max(block_topk, int(available.max().item())) + if spread is None: + scores = torch.rand(rows, width, generator=gen, device=DEV) + else: + base = torch.rand(int(requests.max()) + 1, width, generator=gen, device=DEV) + scores = base[requests] + spread * torch.rand(rows, width, generator=gen, device=DEV) + group_ids = torch.arange(width, device=DEV)[None, :] + scores = torch.where(group_ids < available[:, None], scores, -torch.inf) + chosen = scores.topk(block_topk, dim=1).indices.sort(dim=1).values + column = torch.arange(final_topk, device=DEV)[None, :] + group_of_column = (column // cr).clamp(max=block_topk - 1).expand(rows, -1) + body = torch.gather(chosen, 1, group_of_column) * cr + column % cr + in_body = column < (taken * cr)[:, None] + tail_column = column - (taken * cr)[:, None] + tail = (available * cr)[:, None] + tail_column + in_tail = ~in_body & (tail_column >= 0) & (tail < visible[:, None]) + selected = torch.where(in_body, body, torch.where(in_tail, tail, torch.full_like(body, -1))) + return selected.to(torch.int32) + + +def _check_sparse_gqa_grouped(cell, monkeypatch) -> None: + """Within the band of the reference and of the per-row entry, and deterministic.""" + spec, seed = cell.spec, sum(map(ord, cell.why)) + args = _sgc_build({k: v for k, v in spec.items() if k != "holes"}, seed) + args["selected_tokens"].copy_(_sgg_selection(spec, seed + 5)) + args["rows_per_request"] = spec["rows_per_request"] + _assert_straddles(args) + q, k = args["q"], args["k_cache"] + rows = q.shape[0] + assert _sgg_op.serves(q, k, args["selected_tokens"], args["rows_per_request"]) + out = _sgg_op(**args) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == torch.bfloat16 and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _sgg_op.reference(**args)) + _gqa_compare(out, _sgc_op(**{k_: v for k_, v in args.items() if k_ != "rows_per_request"})) + assert torch.equal(out, _sgg_op(**args)), "two calls on the same inputs differ" + if spec.get("foreign"): + assert bool((out[0] == 0).all()), "a row of no request attends nothing" + if spec.get("capture"): + refill = lambda: _refill(args, seed + 1, lambda a: _sgg_selection(spec, seed + 8)) # noqa: E731 + _replay_in_a_graph(_sgg_op, args, refill, out) + + +TKC_K, TKC_CR, TKC_TOKEN_TOPK = 512, 4, 2048 + + +def _tkc_build(spec, seed): + """One cell's arguments, with garbage past each row's visible blocks that must not be read.""" + g = torch.Generator(device=DEV).manual_seed(seed) + W = spec["width"] + if "requests" in spec: + # A verify step's 1 + draft rows: every row's KV length is the request's last row + 1. + pos = torch.cat( + [torch.arange(s, s + c, dtype=torch.int64, device=DEV) for s, c in spec["requests"]] + ) + seqlen = torch.cat( + [torch.full((c,), s + c, dtype=torch.int32, device=DEV) for s, c in spec["requests"]] + ) + else: + pos = torch.tensor(spec["pos"], dtype=torch.int64, device=DEV) + seqlen = (pos + 1).to(torch.int32) + rows = pos.numel() + assert rows == spec["rows"], (rows, spec["rows"]) + scores = torch.randn(rows, W, generator=g, device=DEV).abs() * 3.0 + if spec.get("relu_sparse"): + # Fewer positive scores than the budget, so the zero class straddles it. + scores = torch.relu(torch.randn(rows, W, generator=g, device=DEV) - 2.2) + scores[:, ::7] = -0.0 # negative zero ties with zero + if spec.get("relu_sum"): + # The indexer's form: four heads' ReLUs summed, times a scale (zero in 1 of 16). + scores = torch.zeros(rows, W, device=DEV) + for _ in range(4): + scores += torch.relu(torch.randn(rows, W, generator=g, device=DEV)) + scores *= 0.0884 + if spec.get("equal"): + # Every score tied: the budget is the highest-index blocks. + scores = torch.full((rows, W), 0.5, device=DEV) + if spec.get("lognormal"): + # bf16-like resolution, so equal scores tie; a zero in every other row. + scores = torch.exp(1.74 + 0.2 * torch.randn(rows, W, generator=g, device=DEV)) + scores = scores.to(torch.bfloat16).float() + scores[::2, 5] = 0.0 + ends = ((pos + 1) // TKC_CR).to(torch.int32) + assert int(ends.max()) <= W, "a window wider than the score table" + cols = torch.arange(W, device=DEV) + scores = torch.where(cols[None, :] >= ends[:, None], 1.0e30, scores) + scores = torch.where(cols[None, :] == ends[:, None] + 3, float("nan"), scores) + topk = torch.full((rows, TKC_K), -7, dtype=torch.int32, device=DEV) + return [scores, ends, pos, seqlen, topk, TKC_CR, TKC_TOKEN_TOPK] + + +def _check_topk_expand_cluster(cell, monkeypatch) -> None: + args = _tkc_build(cell.spec, seed=_seed(cell.why) % 2**31) + assert supports(args[0]), "a cell outside the target's routing to this entry" + want_topk, want_sel = _tkc_op.reference(*args) + sel = _tkc_op(*args) + assert sel.dtype == torch.int32 and sel.shape == want_sel.shape + assert torch.equal(args[4], want_topk), "block list" + assert torch.equal(sel, want_sel), "token selection" + + +# Scratch and table are pre-filled with sentinels so a missing or a stray write shows. + +F2B_SCRATCH_FILL = 0x7FC1 # a bf16 NaN +F2B_TABLE_FILL = -7 + + +def _f2b_build(spec, seed): + gen = torch.Generator(device=DEV).manual_seed(seed) + tpb, kv, d = spec["tokens_per_block"], spec["kv_heads"], spec["head_dim"] + counts = [math.ceil(n / tpb) for n in spec["kv_lens"]] + needed = sum(counts) + 7 + pool_pages = (_straddling_pages(spec, needed) if spec["straddle"] else needed) + 5 + role_pages, layer = spec["slot_pages"], spec["layer"] + # Random e4m3 codes (NaN codes 0x7F / 0xFF excluded), in the layer's two planes only. + data = torch.zeros((pool_pages, role_pages, kv, tpb, d), dtype=torch.uint8, device=DEV) + first = pool_pages - needed + for p0 in range(first, pool_pages, 1024): + p1 = min(p0 + 1024, pool_pages) + codes = torch.randint(0, 254, (p1 - p0, 2, kv, tpb, d), generator=gen, device=DEV) + codes = torch.where(codes >= 0x7F, codes + 1, codes) + data[p0:p1, 2 * layer : 2 * layer + 2] = codes.to(torch.uint8) + pool = data.view(torch.float8_e4m3fn) + width, slots_total = spec["width"], spec["slots"] + table = torch.full((slots_total, width), -1, dtype=torch.int32, device=DEV) + slots = torch.randperm(slots_total, generator=torch.Generator().manual_seed(seed)) + slots = slots[: len(counts)] + page_ids = first + torch.randperm(needed, generator=gen, device=DEV).to(torch.int32) + cursor = 0 + for slot, count in zip(slots.tolist(), counts): + table[slot, :count] = page_ids[cursor : cursor + count] + cursor += count + if spec["holes"]: + head = slots[0].item() + table[head, 1] = -1 # unmapped inside the sequence + table[head, 2] = pool_pages + 3 # past the pool + counts_t = torch.tensor(counts, dtype=torch.int64, device=DEV) + out_shape = (int(counts_t.sum()), kv, tpb, d) + + def scratch(): + fill = torch.full(out_shape, F2B_SCRATCH_FILL, dtype=torch.int16, device=DEV) + return fill.view(torch.bfloat16) + + return dict( + k_cache=pool[:, 2 * layer], v_cache=pool[:, 2 * layer + 1], block_table=table, + slots=slots.to(DEV, torch.int32), counts=counts_t, + offsets=torch.cumsum(counts_t, 0) - counts_t, table=torch.full_like(table, F2B_TABLE_FILL), + k_out=scratch(), v_out=scratch(), + ) # fmt: skip + + +def _check_fp8_pages_to_bf16(cell, monkeypatch) -> None: + args = _f2b_build(cell.spec, _seed(cell.why)) + if cell.spec["straddle"]: + _assert_straddles(args) + ref = _f2b_op.reference(**{k: v.clone() if torch.is_tensor(v) else v for k, v in args.items()}) + out = _f2b_op(**args) + for name, got, want in zip(("k_out", "v_out", "table"), out, ref): + assert got.shape == want.shape and torch.equal(_bits(got), _bits(want)), name + # Every listed page was written, and only the listed requests' table entries. + assert not bool(out[0].isnan().any()) and not bool(out[1].isnan().any()) + listed_rows = torch.zeros_like(args["block_table"], dtype=torch.bool) + for slot, count in zip(args["slots"].tolist(), args["counts"].tolist()): + listed_rows[slot, :count] = True + assert bool((out[2][~listed_rows] == F2B_TABLE_FILL).all()) + + +#: The entry's launches with the token plan and with the block plan (where it serves). +_RTL_FUSED = ("_group_plan", "_rowtile_lone_kernel", "_rowtile_attention_cute") +_RTL_BLOCK = ("_block_plan", "_rowtile_lone_kernel", "_rowtile_attention_cute") + + +def _rtl_build(spec, seed): + trunk = spec["cache_layout"] == "trunk" + assert trunk or spec["cache_layout"] == "scratch" + assert spec["cache_dtype"] == torch.bfloat16 and spec["q_layout"] == "qkv_slice" + st = _paged_kv_state( + spec, seed, spec["cache_dtype"], + lambda needed: _straddling_pages(spec, needed, spec["offset_edge"]) if trunk else needed, + slot_pages=spec.get("slot_pages", 2), layer=spec.get("layer", 0), chunk=1024, + scratch=not trunk, + ) # fmt: skip + rows, kv, d, q_heads = st.positions.numel(), spec["kv_heads"], spec["head_dim"], spec["q_heads"] + qkv = torch.randn(rows, q_heads + 2 * kv, d, generator=st.gen, device=DEV).to(spec["q_dtype"]) + selected = _selected_tokens( + st.positions, seed + 3, spec["compress_ratio"], spec["token_topk"], + shared=spec.get("shared_scores", False), chunk=1024, + ) # fmt: skip + return dict( + q=qkv[:, :q_heads], k_cache=st.k, v_cache=st.v, block_table=st.table, + selected_tokens=selected, request_indices=st.rows.to(spec["request_dtype"]), + tokens_per_block=spec["tokens_per_block"], softmax_scale=d**-0.5, + query_positions=st.positions.to(spec.get("position_dtype", spec["request_dtype"])), + compress_ratio=spec["compress_ratio"], + ) # fmt: skip + + +def _check_sparse_gqa_rowtile(cell, monkeypatch) -> None: + """The plan, lone-row and row-tile passes with their grids, then the band of the reference.""" + launches = _record_launches(monkeypatch, _module, dict.fromkeys(_RTL_FUSED + _RTL_BLOCK)) + spec = cell.spec + args = _rtl_build(spec, seed=sum(map(ord, cell.why))) + if spec["cache_layout"] == "trunk": + _assert_straddles(args, spec["offset_edge"]) + rows = args["q"].shape[0] + _assert_prefill_selection(args["selected_tokens"], args["query_positions"], spec) + out = _rtl_op(**args) + group_rows = rows_per_group(spec["q_heads"] // spec["kv_heads"]) + assert group_rows == {12: 10, 6: 20}[spec["q_heads"] // spec["kv_heads"]] + groups = math.ceil(rows / group_rows) + blocks = _module.block_plan_serves(spec["compress_ratio"], spec["tokens_per_block"], group_rows) + assert [name for name, _, _ in launches] == list(_RTL_BLOCK if blocks else _RTL_FUSED) + assert launches[0][1] == (groups,) + assert launches[1][1] == (min(rows, 256), spec["kv_heads"]) + # The row-tile pass is persistent: one CTA per SM, never more than (group, KV head) items. + sms = torch.cuda.get_device_properties(0).multi_processor_count + assert launches[2][1] == (min(groups * spec["kv_heads"], sms),) + assert out.shape == (rows, spec["q_heads"], spec["head_dim"]) + assert out.dtype == args["q"].dtype and out.is_contiguous() + assert not bool(out.isnan().any()) + _gqa_compare(out, _rtl_op.reference(**args)) + + +def _check_spec_step_metadata(cell, monkeypatch) -> None: + """Bitwise the engine's composed launches over a column slice of a wider position buffer, + eager and replayed on new offsets.""" + n, p = cell.spec["n"], cell.spec["p"] + rows = n * p + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why) % 2**31) + + def operands(): + buf = torch.randint(0, 200000, (3, 1, rows + 9), generator=g, device=DEV, dtype=torch.int32) + pos_off = torch.randint(0, 4, (rows + 5,), generator=g, device=DEV, dtype=torch.int32) + kv = torch.randint(90000, 100000, (n,), generator=g, device=DEV, dtype=torch.int32) + kv_off = torch.randint(-3, 1, (n + 2,), generator=g, device=DEV, dtype=torch.int32) + seq = torch.full((n + 1,), p, device=DEV, dtype=torch.int32) + mapping = tuple( + torch.full((rows + 4,), -9, device=DEV, dtype=dt) + for dt in (torch.int32, torch.int64, torch.int32, torch.int32) + ) + return buf, buf[:, :, :rows], pos_off, kv, kv_off, seq, mapping + + def copy(ops): + c = tuple(x.clone() if torch.is_tensor(x) else tuple(y.clone() for y in x) for x in ops) + return (c[0], c[0][:, :, :rows]) + c[2:] + + def composed(buf, pos, pos_off, kv, kv_off, seq, mapping): + pos += pos_off[:rows] + kv += kv_off[:n] + triton_qsa_generation_token_mapping( + kv_lens=kv, seq_lens=seq[:n], request_indices=mapping[0], logical_positions=mapping[1], + sequence_lengths=mapping[2], visible_blocks=mapping[3], num_rows=rows, compress_ratio=4, + ) # fmt: skip + + a = operands() + b, c = copy(a), copy(a) + pos_out = torch.full((3, rows), -9, device=DEV, dtype=torch.int32) + qsa_spec_step_metadata(a[1], a[2], a[3], a[4], a[5][:n], a[6], 4, pos_out=pos_out) + composed(*b) + assert torch.equal(pos_out, b[1].reshape(3, rows)), "the contiguous positions" + qsa_spec_step_metadata.reference(c[1], c[2], c[3], c[4], c[5][:n], c[6], 4) + for got, want, ref in zip((a[0], a[3]) + a[6], (b[0], b[3]) + b[6], (c[0], c[3]) + c[6]): + assert torch.equal(got, want) and torch.equal(got, ref) + if not cell.spec["capture"]: + return + s, fresh = operands(), operands() + want = copy(fresh) + _captured( + lambda: qsa_spec_step_metadata(s[1], s[2], s[3], s[4], s[5][:n], s[6], 4), + lambda: [dst.copy_(fresh[i]) for i, dst in ((0, s[0]), (2, s[2]), (3, s[3]), (4, s[4]))], + ) + composed(*want) + assert torch.equal(s[0], want[0]) and torch.equal(s[3], want[3]) + for got, exp in zip(s[6], want[6]): + assert torch.equal(got[:rows], exp[:rows]) + + +_ENTRIES = ( + (qsa_paged_index_store, qsa_paged_index_store.CELLS, _check_index_store), + (qsa_paged_kv_store, qsa_paged_kv_store.CELLS, _check_kv_store), + (qsa_paged_sparse_gqa, qsa_paged_sparse_gqa.CELLS, _check_sparse_gqa), + (_pro_op, _pro_op.CELLS, _check_decode_prologue), + (index_split_plan, index_split_plan.CELLS, _check_index_split_plan), + (draft_continue_select, draft_continue_select.CELLS, _check_draft_continue_select), + (draft_rows_gather, draft_rows_gather.CELLS, _check_draft_rows_gather), + (_sgc_op, _sgc_op.CELLS, _check_sparse_gqa_splitk), + (_sgg_op, _sgg_op.CELLS, _check_sparse_gqa_grouped), + (_tkc_op, _tkc_op.CELLS, _check_topk_expand_cluster), + (_f2b_op, _f2b_op.CELLS, _check_fp8_pages_to_bf16), + (_rtl_op, _rtl_op.CELLS, _check_sparse_gqa_rowtile), + (qsa_spec_step_metadata, qsa_spec_step_metadata.CELLS, _check_spec_step_metadata), +) +CASES = [ + pytest.param( + entry, cell, check, + id=f"{type(entry).__name__}: {cell.why[:44]}", + marks=[pytest.mark.high_cuda_memory] if cell.spec.get("large") else [], + ) + for entry, cells, check in _ENTRIES + for cell in cells +] # fmt: skip + + +@pytest.mark.parametrize("entry, cell, check", CASES) +def test_certified_cells(entry, cell, check, certifying, monkeypatch) -> None: # noqa: F811 + """Every (entry, cell) pair against the entry's own `reference`, with `certifying` armed.""" + certifying(entry) + check(cell, monkeypatch) + + +def _stable_kv_store_is_shared(cell) -> None: + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + if not isinstance(pool, _KvEnginePool): + shared = _stable.qsa_paged_kv_store.reference(**args) + _stable.qsa_paged_kv_store(**args) + _kv_store_compare((args["k_cache"], args["v_cache"]), shared) + return + try: + # The shared launcher stores into dense copies of the touched pages; the copy into the + # engine's pool, whose pages straddle 2^32 elements. + dense_k, dense_v, renamed = pool.dense(args["block_table"]) + shared = _stable.qsa_paged_kv_store.reference( + **{**args, "k_cache": dense_k, "v_cache": dense_v, "block_table": renamed} + ) + _stable.qsa_paged_kv_store(**args) + _kv_store_compare(pool.gathered(), shared) + finally: + pool.close() + + +def _stable_sparse_gqa_is_shared(cell) -> None: + args = _sparse_gqa_build(cell.spec, _seed(cell.why)) + got = _stable.qsa_paged_sparse_gqa(**args) + want = _stable.qsa_paged_sparse_gqa.reference(**args) + assert got.dtype == want.dtype and got.shape == want.shape, (got.shape, want.shape) + assert torch.equal(_bits(got), _bits(want)), "the stable-JIT kernels differ from the shared" + + +_STABLE_CASES = [ + pytest.param( + check, cell, + id=f"{name}: {cell.why[:44]}", + marks=[pytest.mark.high_cuda_memory] if cell.spec.get("large") else [], + ) + for name, cells, check in ( + ("kv_store", qsa_paged_kv_store.CELLS, _stable_kv_store_is_shared), + ("sparse_gqa", qsa_paged_sparse_gqa.CELLS, _stable_sparse_gqa_is_shared), + ) + for cell in cells +] # fmt: skip + + +@pytest.mark.parametrize("check, cell", _STABLE_CASES) +def test_stable_kernels_are_the_shared_launchers(check, cell) -> None: + """`qsa_triton_kernels__stable_jit`'s kernel copies write bitwise what the shared QSA + launchers they copy write, at every cell of the entries that launch them.""" + check(cell) + + +# `DecodeTarget._attention_block` at the checkpoint's shapes for one rank of a layout. + +PRL_DEV, PRL_BF, PRL_F8 = "cuda", torch.bfloat16, torch.float8_e4m3fn +PRL_D, PRL_ROT, PRL_IH, PRL_ID, PRL_TPB, PRL_CR, PRL_HIDDEN, PRL_LAYERS, PRL_LAYER = ( + 256, 64, 4, 128, 32, 4, 2560, 12, 5 +) # fmt: skip +PRL_CFG = dict(theta=1.0e7, eps=1e-6, mrope_section1=11, mrope_section2=10, max_positions=262144) +#: (q heads, KV heads, context lengths after the step, rows a request, pool dtype). +PRL_LAYOUTS = { + "tep4_verify_bf16": (6, 1, (7003,), 4, PRL_BF), + "tep2_decode_bf16": (12, 1, (4096, 5003, 6150), 1, PRL_BF), + "dep4_verify_e4m3": (24, 2, (2500, 4100, 6000, 8000), 4, PRL_F8), +} +#: A tep2 rank, kv_fp8, MTP2 verify (3 rows a request) at 90 requests (270 rows). +IDX_LAYOUT = (12, 1, tuple(2400 + 61 * r for r in range(90)), 3, PRL_F8) + + +def _prl_core(hq, hk): + c = types.SimpleNamespace( + **PRL_CFG, head_dim=PRL_D, rotary_dim=PRL_ROT, heads_q=hq, heads_kv=hk + ) + c.index_heads, c.index_kv_heads, c.index_head_dim = PRL_IH, 1, PRL_ID + c.token_topk, c.compress_ratio = 2048, PRL_CR + c.block_topk = c.token_topk // PRL_CR + c.index_width = (PRL_IH + 1) * PRL_ID + c.index_scratch_width = c.index_width + PRL_ID + c.softmax_scale = PRL_D**-0.5 + c.q_width, c.kv_width = hq * PRL_D, hk * PRL_D + c.qkv_width = c.q_width + 2 * c.kv_width + c.attn_in_width = c.qkv_width + c.q_width + c.index_scratch_width + c.attn_in_tail_heads = c.attn_in_width // PRL_D - hq - hk + c.index_col = c.qkv_width + c.q_width + c._cos_sin = M.Qwen4ExpUnifiedModelingV2._index_rope_table(c, c.theta, PRL_DEV) + c.tp_sum = lambda x: x + c._decode_graph_step = True # the trunk sites run as in a captured decode step + return c + + +def _prl_build(hq, hk, ctx, p, kv_dtype, seed): + g = torch.Generator(device=PRL_DEV).manual_seed(seed) + core = _prl_core(hq, hk) + need = [-(-n // PRL_TPB) for n in ctx] + pages = sum(need) + 16 + perm = torch.randperm(pages, generator=g, device=PRL_DEV).to(torch.int32) + table = torch.full((max(8, len(ctx)), 8196), -1, dtype=torch.int32, device=PRL_DEV) + used = 0 + for r, n in enumerate(need): + table[r, :n] = perm[used : used + n] + used += n + roles = torch.randn(pages, 2 * PRL_LAYERS, hk, PRL_TPB, PRL_D, generator=g, device=PRL_DEV) + roles = roles.to(kv_dtype) + # Index keys 2..3x larger at 512 groups a request win the block top-k by a 2x margin, so + # paths a sub-ulp apart in the query select the same tokens. + scale = 0.25 + 0.75 * torch.rand(pages, PRL_TPB, generator=g, device=PRL_DEV) + for r, n in enumerate(ctx): + hot = torch.randperm((n - p) // PRL_CR - 1, generator=g, device=PRL_DEV)[:512] + token = hot * PRL_CR + PRL_CR - 1 + scale[table[r, token // PRL_TPB].long(), token % PRL_TPB] = 2.0 + torch.rand( + hot.shape[0], generator=g, device=PRL_DEV + ) + planes = ( + scale[:, None, :, None, None] + * torch.randn(PRL_LAYERS, 1, 1, PRL_ID, generator=g, device=PRL_DEV)[None] + ).to(PRL_BF) + position = torch.randint( + 0, 8192, (pages, PRL_TPB, 3), generator=g, device=PRL_DEV, dtype=torch.int32 + ) + core._bound = types.SimpleNamespace( + pools=types.SimpleNamespace( + kv={PRL_LAYER: P.KvPlanes(roles[:, 2 * PRL_LAYER], roles[:, 2 * PRL_LAYER + 1])}, + index_cache={PRL_LAYER: planes[:, PRL_LAYER]}, + position_cache=position, + ) + ) + n = len(ctx) + req = torch.arange(n, device=PRL_DEV, dtype=torch.int32).repeat_interleave(p) + lens = torch.tensor(ctx, device=PRL_DEV, dtype=torch.int64).repeat_interleave(p) + logical = lens - p + torch.arange(p, device=PRL_DEV, dtype=torch.int64).repeat(n) + ids = logical.to(torch.int32).expand(3, n * p).contiguous() + qsa = ( + req, logical, lens.to(torch.int32), ((logical + 1) // PRL_CR).to(torch.int32), + torch.empty(n * p, core.block_topk, dtype=torch.int32, device=PRL_DEV), + torch.zeros(n * p, dtype=torch.int32, device=PRL_DEV), table, PRL_TPB, ids.clone(), ids[0], + torch.transpose(ids, 0, 1), True, + ) # fmt: skip + w = lambda *shape: torch.randn(*shape, generator=g, device=PRL_DEV) # noqa: E731 + L = types.SimpleNamespace( + attn_in=(w(core.attn_in_width, PRL_HIDDEN) * 0.02).to(PRL_BF).t(), + o=(w(PRL_HIDDEN, core.q_width) * 0.02).to(PRL_BF).t(), + q_norm=(w(PRL_D) * 0.3).to(PRL_BF), k_norm=(w(PRL_D) * 0.3).to(PRL_BF), + idx_qn=(w(PRL_ID) * 0.3).to(PRL_BF), idx_kn=(w(PRL_ID) * 0.3).to(PRL_BF), + ) # fmt: skip + x = w(n * p, PRL_HIDDEN).to(PRL_BF) + return core, L, x, qsa, (roles, planes, position), p + + +def test_decode_index_selection_split_over_tp_is_the_replicated_selection(monkeypatch) -> None: + """The decode index selection split over two attention-TP ranks, gathers emulated in one + process: split by position or by `_index_split_plan`, the gathered selection and the layer + over it are bitwise the replicated ones.""" + core, L, x, qsa, pools, p = _prl_build(*IDX_LAYOUT, seed=37) + rows = x.shape[0] + monkeypatch.setattr(M, "_INDEX_SPLIT_MIN_REQUESTS", 32) + # bitwise halves need one scoring kernel for 45 and 90 groups alike (the family's) + monkeypatch.setattr(M, "_INDEX_SCORES_CUTE_MIN_GROUPS", 1 << 30) + q8 = qsa[:6] + (qsa[6], qsa[7]) + cache = core._bound.pools.index_cache[PRL_LAYER] + saved = [t.clone() for t in pools] + assert M._index_split_rows(None, rows, p) is None + assert M._index_split_rows(([0, 1], None, 1), rows, p) == (135, 270) + assert M._index_split_rows(([0, 1], None, 1), 31 * p, p) is None # below the threshold + + def restore(): + for t, sv in zip(pools, saved): + t.copy_(sv) + + def layer(target, gather): + qkv = torch.empty(rows, core.qkv_width, dtype=PRL_BF, device=PRL_DEV) + scratch = torch.empty(rows, core.index_scratch_width, dtype=PRL_BF, device=PRL_DEV) + with monkeypatch.context() as m: + m.setattr(M, "allgather", gather) + out = M.DecodeTarget._attention_block( + target, x, PRL_LAYER, L, qkv, scratch, None, qsa, + {"predicted_tokens_per_seq": p}, False, + ) # fmt: skip + torch.cuda.synchronize() + return out.clone(), scratch + + def select(rank, q, vectors, n, rpr, gather=lambda part, sizes, g_, pg: part, **kw): + with monkeypatch.context() as m: + m.setattr(M, "allgather", gather) + split = None if rank is None else ([0, 1], None, rank) + return M._decode_index_selection(core, split, q, vectors, n, rpr, PRL_DEV, cache, **kw) + + # The selections are taken over the planes the layer left (its rows' index keys stored). + full_out, scratch = layer( + types.SimpleNamespace(core=core, _index_split=None, _kv_probe=None), None + ) + q = M._index_query(core, scratch) + full = select(None, q, q8, rows, p) + halves = [select(rank, q, q8, rows, p) for rank in (0, 1)] + assert [h.shape[0] for h in halves] == [135, 135] + assert torch.equal(torch.cat(halves), full), "the halves are bitwise the replicated selection" + # A draft step's shape: one row a request (each request's last verify row here). + last = torch.arange(p - 1, rows, p, device=PRL_DEV) + q1, d8 = q.index_select(0, last), tuple(t.index_select(0, last) for t in q8[:6]) + q8[6:] + halves1 = [select(rank, q1, d8, last.numel(), 1) for rank in (0, 1)] + assert [h.shape[0] for h in halves1] == [45, 45] + assert torch.equal(torch.cat(halves1), select(None, q1, d8, last.numel(), 1)) + + lam = object() # `_index_split_plan` needs a Lamport gather; the emulation below is NCCL's + plans = [M._index_split_plan(([0, 1], None, r, lam), q8, rows, p) for r in (0, 1)] + assert all(pl is not None for pl in plans) + assert torch.equal(plans[0].perm_rows, plans[1].perm_rows) + perm = plans[0].perm_rows.long() + assert torch.equal(torch.sort(perm).values, torch.arange(rows, device=PRL_DEV)) + assert torch.equal(plans[0].iq_rows.long()[perm], torch.arange(rows, device=PRL_DEV)) + windows = qsa[3].long() + by_plan = [int(windows[perm[r * 135 : (r + 1) * 135]].sum()) for r in (0, 1)] + by_position = [int(windows[r * 135 : (r + 1) * 135].sum()) for r in (0, 1)] + assert max(by_position) / min(by_position) > 1.1 + assert max(by_plan) / min(by_plan) < 1.01, by_plan + parts = {} + for permuted in (False, True): + qs = q.index_select(0, perm) if permuted else q + for rank in (0, 1): + kw = dict(plan=plans[rank], q_permuted=permuted) + keep = lambda part, sizes, g_, pg, key=(permuted, rank): parts.setdefault( # noqa: E731 + key, part.clone() + ).repeat(2, 1) + select(rank, qs, q8, rows, p, gather=keep, **kw) + gathered = torch.cat([parts[(permuted, 0)], parts[(permuted, 1)]]) + assert torch.equal(gathered, full.index_select(0, perm)) + for rank in (0, 1): + kw = dict(plan=plans[rank], q_permuted=permuted) + got = select(rank, qs, q8, rows, p, gather=lambda *a, g=gathered: g, **kw) + assert torch.equal(got, full), (permuted, rank) + restore() + + for rank in (0, 1): + # By position: this rank's half handed over, the gathered selection back. + handed = [] + + def nccl(part, sizes, group, process_group): + assert sizes is None and group == [0, 1] + handed.append(part.clone()) + return torch.cat(halves) + + out, _ = layer( + types.SimpleNamespace(core=core, _index_split=([0, 1], None, rank), _kv_probe=None), + nccl, + ) + restore() + assert len(handed) == 1 and torch.equal(handed[0], halves[rank]) + assert torch.equal(out, full_out), "the layer over the gathered selection" + # The plan: the prologue stores the index queries in the permuted order. + target = types.SimpleNamespace( + core=core, _index_split=([0, 1], None, rank), _index_plan=plans[rank], _kv_probe=None + ) + out, scr = layer(target, lambda part, sizes, g_, pg: gathered) + restore() + assert torch.equal(M._index_query(core, scr), q.index_select(0, perm)) + assert torch.equal(out, full_out), rank + + +def _replay_index_store(prefix, monkeypatch) -> None: + """A store captured at the cell's batch, replayed after the next step's metadata is written + into the captured buffers, stores that step bitwise (the pool is sized to the batch).""" + cell = next(c.spec for c in qsa_paged_index_store.CELLS if c.why.startswith(prefix)) + spec = dict(cell, pages=24576, straddle=False) + args, extras = _index_store_build(spec, seed=101) + index_cache, position_cache = args["index_cache"], args["position_cache"] + qsa_paged_index_store(**args) # compiles the launch before the capture + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_paged_index_store(**args) + shifted = dict(spec, segments=tuple((max(0, s - 3), n) for s, n in spec["segments"])) + following, _ = _index_store_build(shifted, seed=202, planes=(index_cache, position_cache)) + before = qsa_paged_index_store.reference(**args) + names = ("token_k", "position_coordinates", "request_indices", "logical_positions") + for name in (*names, "block_table"): + args[name].copy_(following[name]) + index_cache.fill_(KEY_FILL) + position_cache.fill_(POS_FILL) + ref = qsa_paged_index_store.reference(**args) + assert not torch.equal(_bits(ref[0]), _bits(before[0])), "the next step stores nothing new" + graph.replay() + torch.cuda.synchronize() + _index_store_compare((index_cache, position_cache), ref) + _other_layers_untouched(*extras["coalesced"]) + + +def _replay_kv_store(prefix, monkeypatch) -> None: + """A step over the engine's pool captured with every position at -1 (nothing read or + written), replayed after the real positions are written into the same buffer.""" + cell = next(c for c in qsa_paged_kv_store.CELLS if c.why.startswith(prefix)) + args, pool = _kv_store_build(cell.spec, seed=_seed(cell.why)) + try: + positions = args["logical_positions"] + saved = positions.clone() + positions.fill_(-1) + before = tuple(t.clone() for t in pool.gathered()) + for _ in range(3): # compile the Triton kernel before capture + qsa_paged_kv_store(**args) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_paged_kv_store(**args) + torch.cuda.synchronize() + _kv_store_compare(pool.gathered(), before) # a row at position -1 writes nothing + positions.copy_(saved) + ref = _kv_engine_reference(args, pool) + graph.replay() + torch.cuda.synchronize() + _kv_store_compare(pool.gathered(), ref) + pool.assert_rest_unchanged() + finally: + pool.close() + + +@pytest.mark.parametrize( + "replay, prefix", + [ + pytest.param(_replay_index_store, "tp2 decode: 128 requests", id="index tp2-decode"), + pytest.param(_replay_index_store, "dep4 MTP3 verify", id="index dep4-mtp3"), + pytest.param(_replay_kv_store, "tp2 decode, 128 rows (max_batch_size)", id="kv tp2-decode"), + pytest.param(_replay_kv_store, "dep4 MTP3 verify, 128 requests x 4", id="kv dep4-mtp3"), + ], +) +def test_captured_stores_replay_rewritten_metadata(replay, prefix, monkeypatch) -> None: + replay(prefix, monkeypatch) + + +def _index_pool(n_requests, need, seed, pool_pages, slots): + """A trunk-like index pool's layer-11 plane and a `[slots, 8196]` table of random pages.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + pages = max(pool_pages, sum(need) + 8) + pool = torch.randn(pages, 12, 32, 1, 128, generator=gen, device=DEV).to(torch.bfloat16) + perm = torch.randperm(pages, generator=gen, device=DEV).to(torch.int32) + table = torch.full((slots, 8196), -1, dtype=torch.int32, device=DEV) + used = 0 + slot_of = torch.randperm(slots, generator=torch.Generator().manual_seed(seed))[:n_requests] + for i, k in enumerate(need): + table[int(slot_of[i]), :k] = perm[used : used + k] + used += k + return gen, pool[:, 11], table, slot_of + + +def _family_index_scores_groups(contexts, monkeypatch) -> None: + """The CuTe request-group index scores: within fp32 summation-order noise of the family's + kernel, the same top-512 except at near-ties, deterministic, and replayable.""" + rpr = 3 + gen, cache, table, slot_of = _index_pool( + len(contexts), [-(-c // 32) for c in contexts], len(contexts) * 7 + 1, 6000, 128 + ) + req = slot_of.to(DEV).to(torch.int32).repeat_interleave(rpr) + lens = torch.tensor(contexts, device=DEV, dtype=torch.int64).repeat_interleave(rpr) + pos = lens - rpr + torch.arange(rpr, device=DEV, dtype=torch.int64).repeat(len(contexts)) + q = torch.randn(len(contexts) * rpr, 6, 128, generator=gen, device=DEV) + q = q.to(torch.bfloat16)[:, :4] + assert _igc.serves(q, cache, rpr) + fam = M.qsa_paged_index_scores( + q, cache, table, pos, req, 32, 4, only_visible_blocks=True, context_rows=False, + rows_per_request=rpr, + ) # fmt: skip + out = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + vis = (pos + 1) // 4 + inside = torch.arange(out.shape[1], device=DEV)[None, :] < vis[:, None] + torch.testing.assert_close(out[inside], fam[inside], rtol=1e-5, atol=1e-6) + neg = torch.full_like(out, -1.0) + k = min(512, int(vis.min().item())) if int(vis.min().item()) > 0 else 0 + if k: + a = torch.where(inside, out, neg).topk(k, dim=1) + b = torch.where(inside, fam, neg).topk(k, dim=1) + differ = (a.indices.sort(dim=1).values != b.indices.sort(dim=1).values).any(dim=1) + # a row may differ only where its k-th and (k+1)-th scores are within the noise + next_value = torch.where(inside, fam, neg).topk(k + 1, dim=1).values[:, -1] + gap = (b.values[:, -1] - next_value).abs() + assert bool((~differ | (gap <= 1e-5 * b.values[:, -1].abs())).all()) + out2 = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + assert torch.equal(out2[inside], out[inside]), "two calls on the same inputs differ" + refill = lambda: q.copy_(torch.randn(q.shape, device=DEV).to(q.dtype)) # noqa: E731 + captured = _captured( + lambda: _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr), refill + ) + eager = _igc.index_scores_groups(q, cache, table, pos, req, 32, 4, False, rpr) + assert torch.equal(captured[inside], eager[inside]), "the replay differs from an eager call" + + +def _family_index_scores_prefill(case, monkeypatch) -> None: + """The tcgen05 context-row index scores, `case` = (((prefix, rows), ...), position dtype): + bitwise the family's kernel over every row's visible columns, and replayable.""" + requests, pos_dtype = case + need = [-(-(p + r) // 32) for p, r in requests] + gen, cache, table, slot_of = _index_pool( + len(requests), need, sum(r for _, r in requests), 12000, 64 + ) + req = torch.cat( + [torch.full((r,), int(slot_of[i]), dtype=torch.int32) for i, (_, r) in enumerate(requests)] + ).to(DEV) + pos = torch.cat([torch.arange(p, p + r, dtype=pos_dtype) for p, r in requests]).to(DEV) + q = torch.randn(pos.numel(), 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + assert _isc.supports(q, cache, 32, 4) + assert not _isc.supports(q, cache, 64, 4) + assert _isc.supports(q, cache, 16, 2) + fam = M.qsa_paged_index_scores( + q, cache, table, pos, req, 32, 4, only_visible_blocks=True, context_rows=True + ) + out = _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4) + vis = ((pos + 1) // 4).long() + inside = torch.arange(out.shape[1], device=DEV)[None, :] < vis[:, None] + assert torch.equal(out[inside], fam[inside]), "visible scores differ from the family's" + refill = lambda: q.copy_(torch.randn(q.shape, device=DEV).to(q.dtype)) # noqa: E731 + captured = _captured( + lambda: _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4), refill + ) + eager = _isc.qsa_index_scores_prefill(q, cache, table, pos, req, 32, 4) + assert torch.equal(captured[inside], eager[inside]), "the replay differs from an eager call" + + +def _family_topk_rows(case, monkeypatch) -> None: + """The prefill row top-k: bitwise the family's `qsa_topk_expand_decode` on a wide table.""" + name, spec = case + args = _tkc_build(dict(spec, width=65568), seed=_seed(name) % 2**31) + assert _tkr_supports(args[0], args[4]), "a table outside the row kernel's range" + family_args = [a.clone() if torch.is_tensor(a) else a for a in args] + sel = _tkr_op(*args) + family_sel = family_op(*family_args) + assert torch.equal(args[4], family_args[4]), "block list" + assert torch.equal(sel, family_sel), "token selection" + + +_FAMILY_CASES = [ + *( + pytest.param(_family_index_scores_groups, contexts, id=f"index scores groups {name}") + for name, contexts in ( + ("32x50k", tuple([50000] * 32)), + ("64x20k", tuple([20000] * 64)), + ("32_spread", tuple(4400 + 6007 * i for i in range(32))), + ("edges", (3, 4, 5, 515, 516, 517, 2051, 66000, 131071, 262143, 9000, 31)), + ("8x100k", tuple([100000] * 8)), + ("20_spread", tuple(70000 + 1500 * i for i in range(20))), + ) + ), + *( + pytest.param(_family_index_scores_prefill, (requests, pos_dtype), + id=f"index scores prefill {name}") + for name, requests, pos_dtype in ( + ("88k_one", ((88064, 1024),), torch.int32), + ("three_requests", ((88064, 700), (40000, 333), (3, 300)), torch.int32), + ("windows_from_0", ((0, 1500),), torch.int32), + ("131k_250k", ((131072, 320), (250000, 64)), torch.int32), + ("three_requests_int64", ((88064, 700), (40000, 333), (3, 300)), torch.int64), + ) + ), + *( + pytest.param(_family_topk_rows, (name, spec), id=f"topk rows {name}") + for name, spec in ( + ("chunk_88k_relu_sum", dict(rows=1024, requests=((88064, 1024),), relu_sum=True)), + ("chunk_131k_relu_sum", dict(rows=512, requests=((131072, 512),), relu_sum=True)), + ("requests_12_relu_sum", dict(rows=768, requests=tuple((88064 + 97 * j, 64) + for j in range(12)), relu_sum=True)), + ("chunk_250k_relu_sum", dict(rows=256, requests=((250000, 256),), relu_sum=True)), + ("zero_class_straddles", dict(rows=256, requests=((70000, 256),), relu_sparse=True)), + ("every_score_tied", dict(rows=256, requests=((88064, 256),), equal=True)), + ("lognormal_bf16_ties", dict(rows=512, requests=((60000, 512),), lognormal=True)), + ("windows_0_to_past_k", dict(rows=2600, requests=((0, 2600),), relu_sum=True)), + ) + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, case", _FAMILY_CASES) +def test_kernels_without_cells_match_the_family(check, case, monkeypatch) -> None: + check(case, monkeypatch) + + +def _prl_site(monkeypatch, built, prologue, site): + """One `trunk` (`DecodeTarget._attention_block`) or `draft` (`_mtp_attention_planes`) run + over the built pools, restored after, with the fused prologue on or off.""" + core, L, x, qsa, pools, p = built + saved = [t.clone() for t in pools] + calls, entry = [], M.qwen4_exp_attn_decode_prologue + + def counted(*args, **kwargs): + calls.append(kwargs["rows_per_request"]) + return entry(*args, **kwargs) + + rows = x.shape[0] + with monkeypatch.context() as m: + m.setattr(M, "qwen4_exp_attn_decode_prologue", counted) + if not prologue: + m.setattr(M, "_decode_prologue", lambda kv, sparse: False) + if site == "trunk": + qkv = torch.empty(rows, core.qkv_width, dtype=PRL_BF, device=PRL_DEV) + scratch = torch.empty(rows, core.index_scratch_width, dtype=PRL_BF, device=PRL_DEV) + # The FP8 KV range probe reads the packed K/V on both branches; this is its last layer. + target = types.SimpleNamespace(core=core, _kv_probe=M._KvRangeProbe(PRL_LAYER)) + out = M.DecodeTarget._attention_block( + target, x, PRL_LAYER, L, qkv, scratch, None, qsa, + {"predicted_tokens_per_seq": p}, False, + ) # fmt: skip + assert target._kv_probe is None, "the probe reported on its last layer and was dropped" + else: + roles, planes, position = pools + draft = types.SimpleNamespace( + kv=P.KvPlanes(roles[:, 2 * PRL_LAYER], roles[:, 2 * PRL_LAYER + 1]), + index_cache=planes[:, PRL_LAYER], + position_cache=position, + ) + vectors = (qsa[0], qsa[1], qsa[6], qsa[7], qsa[8], qsa[9], qsa[10]) + qkv, gate, iq = M._mtp_attention_planes( + core, L, x, draft, vectors, p, True, True, M._decode_projection + ) + torch.cuda.synchronize() + if site == "trunk": + got = dict(out=out, qkv=qkv, iq=M._index_query(core, scratch).clone()) + else: + got = dict(qkv=qkv.clone(), gate=gate.clone(), iq=iq.clone()) + got.update(calls=calls, planes=[t.clone() for t in pools]) + for t, s in zip(pools, saved): + t.copy_(s) + return got + + +def _site_prologue(case, monkeypatch) -> None: + """The one-launch prologue against the composition it replaces: v, index keys, coordinates + and the draft's gate bitwise; rotated values within rounding, since the entry reads the fp32 + table and `fused_qk_norm_rope` takes `__sincosf`.""" + site, layout = case + hq, hk, ctx, p, kv_dtype = PRL_LAYOUTS[layout] + built = _prl_build(hq, hk, ctx, p, kv_dtype, seed=len(layout) + (7 if site == "draft" else 0)) + if site == "draft": + core = built[0] + core.mtp_theta = 5.0e6 + core._mtp_cos_sin = M.Qwen4ExpUnifiedModelingV2._index_rope_table(core, 5.0e6, PRL_DEV) + fused = _prl_site(monkeypatch, built, True, site) + composed = _prl_site(monkeypatch, built, False, site) + assert fused["calls"] == [p] and composed["calls"] == [] + width = (hq + hk) * PRL_D + assert_within_ulp(fused["qkv"][:, :width], composed["qkv"][:, :width], 2.0, 0.25) + assert torch.equal(fused["qkv"][:, width:], composed["qkv"][:, width:]), "v" + if site == "draft": + assert torch.equal(fused["gate"], composed["gate"]), "the gate columns" + assert_within_ulp(fused["iq"], composed["iq"], 2.0, 0.25) + (roles_f, index_f, pos_f), (roles_c, index_c, pos_c) = fused["planes"], composed["planes"] + assert torch.equal(index_f, index_c), "raw and compressed index keys" + assert torch.equal(pos_f, pos_c), "coordinates" + v_f, v_c = roles_f[:, 2 * PRL_LAYER + 1], roles_c[:, 2 * PRL_LAYER + 1] + assert torch.equal(v_f.view(torch.uint8), v_c.view(torch.uint8)), "the V pool" + others = [s for s in range(2 * PRL_LAYERS) if s != 2 * PRL_LAYER] + assert torch.equal(roles_f[:, others].view(torch.uint8), roles_c[:, others].view(torch.uint8)) + k_f, k_c = roles_f[:, 2 * PRL_LAYER], roles_c[:, 2 * PRL_LAYER] + if kv_dtype == PRL_BF: + assert_within_ulp(k_f, k_c, 2.0, 0.25) + else: + a, b = k_f.view(torch.uint8).int(), k_c.view(torch.uint8).int() + moved = a != b + assert moved.float().mean().item() < 1e-3 + if site == "trunk": + # One bf16 step of k moves an e4m3 rounding by at most one code: the codes' + # magnitudes are monotonic, a sign may differ only next to zero. + mag = ((a & 0x7F) - (b & 0x7F)).abs() + near_zero = ((a & 0x7F) <= 1) & ((b & 0x7F) <= 1) + assert ((mag <= 1) & (((a ^ b) & 0x80 == 0) | near_zero))[moved].all() + if site == "trunk": + # The layer output follows within the selection both paths make from those keys. + o_f, o_c = fused["out"].float(), composed["out"].float() + assert ((o_f - o_c).norm() / o_c.norm()).item() < 1e-2 + + +def _site_rowtile_q_in_place(segments, monkeypatch) -> None: + """q in place on the attention projection row gives the compact row's bits, and the + lone-row kernel (q's row stride a runtime argument) compiles once for both strides.""" + spec = {**_rtl_op.CELLS[0].spec, "segments": tuple(segments), "slots": 16} + assert (spec["q_heads"], spec["kv_heads"], spec["head_dim"]) == (6, 1, 256) + args = _rtl_build(spec, seed=7900 + len(segments)) + q = args["q"] + rows, heads, d = q.shape + # A projection row: q | k | v | gate | index planes, wider than the compact row. + width = (2 * heads + 5) * d + proj = torch.randn(rows, width, device=DEV).to(q.dtype) + proj[:, : heads * d].copy_(q.reshape(rows, heads * d)) + in_place = torch.reshape(proj, [rows, width // d, d])[:, :heads] + assert in_place.stride(0) != q.stride(0) and torch.equal(in_place, q) + dev = torch.cuda.current_device() + compact = _rtl_op(**args) + variants = len(_module._rowtile_lone_kernel.device_caches[dev][0]) + got = _rtl_op(**{**args, "q": in_place}) + assert len(_module._rowtile_lone_kernel.device_caches[dev][0]) == variants, "recompiled" + assert torch.equal(_bits(got), _bits(compact)), "q in place vs compact" + _gqa_compare(got, _rtl_op.reference(**args)) + + +def _site_rowtile_scratch_groups(_, monkeypatch) -> None: + """`_rowtile_scratch_groups`: one launch (None) while the pages fit the scratch, else the + fewest launches of consecutive requests that fit, each request in exactly one.""" + tpb = 32 + page_bytes = 32 * 256 * 4 # one fp8 K/V head of 256 at 32 tokens a page, widened to bf16 + cap_pages = M._FP8_ROWTILE_MAX_SCRATCH // page_bytes + + def md(q_lens, kv_lens): + return types.SimpleNamespace( + num_contexts=len(q_lens), + tokens_per_block=tpb, + seq_lens=torch.tensor(q_lens + [1, 1], dtype=torch.int32), + kv_lens_runtime=torch.tensor(kv_lens + [9, 9], dtype=torch.int32), + ) + + assert M._rowtile_scratch_groups(md([1024] * 4, [68608] * 4), page_bytes) is None + # Sixteen 1024-token chunks at a 67k prefix: 2144 pages each, 34304 in all, past the cap. + q_lens, kv_lens = [1024] * 16, [67584 + 1024] * 16 + groups = M._rowtile_scratch_groups(md(q_lens, kv_lens), page_bytes) + pages = [-(-k // tpb) for k in kv_lens] + assert sum(pages) > cap_pages and groups is not None and len(groups) > 1 + starts = [0] + for q in q_lens: + starts.append(starts[-1] + q) + first = 0 + for a, b, n in groups: + assert n <= cap_pages + last = starts.index(b) + assert a == starts[first] and n == sum(pages[first:last]) + first = last + assert first == len(q_lens) and groups[-1][1] == sum(q_lens) + assert len(groups) == -(-sum(pages) // cap_pages) == 2 + assert [b - a for a, b, _ in groups] == [8192, 8192] + # Uneven requests: still launches that fit, every page in one. + q_lens = [512, 4096, 256, 2048, 1024, 6000, 300, 2000] + kv_lens = [200000, 150000, 180000, 90000, 210000, 120000, 160000, 70000] + groups = M._rowtile_scratch_groups(md(q_lens, kv_lens), page_bytes) + assert groups[0][0] == 0 and groups[-1][1] == sum(q_lens) + assert all(n <= cap_pages for _, _, n in groups) + assert sum(n for _, _, n in groups) == sum(-(-k // tpb) for k in kv_lens) + assert all(prev[1] == nxt[0] for prev, nxt in zip(groups, groups[1:])) + + +def _site_draft_share_chunks(case, monkeypatch) -> None: + """The IndexShare draft-extend step's context selection (`_score_and_select`, as + `mtp_forward` calls it) in equal chunks of a lowered score-table bound, `case` = + (((prefix, rows), ...), bound, chunk rows): bitwise the one-call scores and `_selection` + the step ran unchunked, block lists and token selection alike.""" + requests, bound, want = case + tpb, cr, token_topk = 32, 4, TKC_TOKEN_TOPK + need = [-(-(p + r) // tpb) for p, r in requests] + rows = sum(r for _, r in requests) + gen, cache, table, slot_of = _index_pool(len(requests), need, rows, 12000, 64) + req = torch.cat( + [torch.full((r,), int(slot_of[i]), dtype=torch.int32) for i, (_, r) in enumerate(requests)] + ).to(DEV) + logical = torch.cat([torch.arange(p, p + r, dtype=torch.int64) for p, r in requests]).to(DEV) + seq_lens = torch.cat([torch.full((r,), p + r, dtype=torch.int32) for p, r in requests]).to(DEV) + visible = ((logical + 1) // cr).to(torch.int32) + row_starts = torch.zeros(rows, dtype=torch.int32, device=DEV) + q = torch.randn(rows, 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + core = SimpleNamespace(compress_ratio=cr, token_topk=token_topk) + columns = table.shape[1] * tpb // cr + # At the engine's bound, `max_seq_len` 262144 (8196 pages of 32) scores 32704 rows a call. + assert M._score_chunk_rows(core, table, tpb) == 32704 + + def topk(): + return torch.full((rows, token_topk // cr), -7, dtype=torch.int32, device=DEV) + + def selected(): + return torch.full((rows, token_topk + cr - 1), -7, dtype=torch.int32, device=DEV) + + one_topk, one = topk(), selected() + scores = M._context_index_scores(q, cache, table, logical, req, tpb, cr, True) + M._selection(core, scores, visible, logical, seq_lens, one_topk, row_starts, one) + del scores + + monkeypatch.setattr(M, "_SCORE_TABLE_ELEMENTS", bound * columns + 1) + assert M._score_chunk_rows(core, table, tpb) == bound + calls = [] + scorer = M._context_index_scores + + def counted(q_rows, *args): + calls.append(q_rows.shape[0]) + return scorer(q_rows, *args) + + monkeypatch.setattr(M, "_context_index_scores", counted) + chunked_topk, chunked = topk(), selected() + M._score_and_select( + core, cache, q, table, logical, req, tpb, visible, seq_lens, chunked_topk, row_starts, + 0, rows, True, chunked, + ) # fmt: skip + assert calls == list(want) + assert torch.equal(chunked_topk, one_topk), "block lists" + assert torch.equal(chunked, one), "token selection" + + +def _site_gen_rows_order(rows, monkeypatch) -> None: + """A mixed step's generation rows through `_score_and_select` with the step's `_RowOrder`: + past one wave of rows on the wide (65568-column) table the selection launches in that + order, bitwise the unordered launch, block lists and token selection alike.""" + tpb, cr, token_topk = 32, 4, TKC_TOKEN_TOPK + gen, cache, table, slot_of = _index_pool(1, [2900], rows, 3000, rows) + table[:] = table[int(slot_of[0])] # every request reads the same pages + req = torch.arange(rows, dtype=torch.int32, device=DEV) + logical = torch.randint(1, 2900 * tpb - 1, (rows,), generator=gen, device=DEV) + seq_lens = (logical + 1).to(torch.int32) + visible = ((logical + 1) // cr).to(torch.int32) + row_starts = torch.zeros(rows, dtype=torch.int32, device=DEV) + q = torch.randn(rows, 6, 128, generator=gen, device=DEV).to(torch.bfloat16)[:, :4] + core = SimpleNamespace(compress_ratio=cr, token_topk=token_topk) + seen = [] + op = M.qsa_topk_expand_decode + + def spy(*args, row_order=None, **kwargs): + seen.append(row_order) + return op(*args, row_order=row_order, **kwargs) + + monkeypatch.setattr(M, "qsa_topk_expand_decode", spy) + outs = [] + for order in (None, M._RowOrder()): + topk = torch.full((rows, token_topk // cr), -7, dtype=torch.int32, device=DEV) + sel = torch.full((rows, token_topk + cr - 1), -7, dtype=torch.int32, device=DEV) + M._score_and_select( + core, cache, q, table, logical, req, tpb, visible, seq_lens, topk, row_starts, + 0, rows, False, sel, order, + ) # fmt: skip + outs.append((topk, sel)) + assert seen[0] is None and seen[1] is not None, "only the ordered call passes a row order" + assert torch.equal(outs[0][0], outs[1][0]), "block lists" + assert torch.equal(outs[0][1], outs[1][1]), "token selection" + + +_SITE_CASES = [ + *( + pytest.param(_site_prologue, (site, layout), id=f"{site} prologue {layout}") + for site in ("trunk", "draft") + for layout in PRL_LAYOUTS + ), + pytest.param(_site_rowtile_q_in_place, [(5000, 2048)], id="rowtile q in place, one request"), + pytest.param( + _site_rowtile_q_in_place, [(3000, 1040), (7000, 1043)], id="rowtile q in place, lone group" + ), + pytest.param(_site_rowtile_scratch_groups, None, id="rowtile fp8 scratch groups"), + pytest.param( + _site_draft_share_chunks, + (((88064, 900), (40000, 600), (3, 300)), 512, (512, 512, 512, 264)), + id="draft index share chunks, three requests", + ), + pytest.param( + _site_draft_share_chunks, + (((250000, 1100),), 512, (384, 384, 332)), + id="draft index share chunks, 250k", + ), + pytest.param(_site_gen_rows_order, 300, id="generation rows in the step's row order"), +] + + +@pytest.mark.parametrize("check, case", _SITE_CASES) +def test_call_sites(check, case, monkeypatch) -> None: + """What the target's call sites hand the entries beyond a cell.""" + check(case, monkeypatch) + + +# ---- the FP8 KV range probe (`TRTLLM_MODELING_V2_VALIDATE` over an fp8 pool) ---------------- +# The pool stores K/V at the unit scale, so a magnitude past the e4m3 maximum saturates. The probe +# reports the largest magnitude of the first eager forward's attention layers, once, and never +# reads a tensor under CUDA graph capture. + +PROBE_LAST = 11 + + +def _probe_kv(peak: float = 1.0): + """K and V of one layer, 5 rows x 2 heads x 8 dims, with `peak` as V's largest value.""" + k = torch.linspace(-1.0, 1.0, 80, device=DEV).reshape(5, 2, 8).to(torch.bfloat16) + v = torch.linspace(-0.5, 0.5, 80, device=DEV).reshape(5, 2, 8).to(torch.bfloat16) + v[3, 1, 4] = peak + return k, v + + +def _probe_forward(probe, peaks): + """One forward's attention layers 3, 7 and PROBE_LAST; returns each `observe` result.""" + return [ + probe.observe(layer, *_probe_kv(peak)) for layer, peak in zip((3, 7, PROBE_LAST), peaks) + ] + + +def test_a_forward_within_the_e4m3_range_is_silent(monkeypatch) -> None: + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + assert _probe_forward(M._KvRangeProbe(PROBE_LAST), (1.0, 300.0, 448.0)) == [True, True, False] + logger.warning.assert_not_called() + + +@pytest.mark.parametrize("peak", [-600.0, float("nan")]) +def test_a_magnitude_past_the_e4m3_maximum_is_reported_once(monkeypatch, peak) -> None: + """A negative value counts by its magnitude; a NaN counts as out of range. The oversized value + is in a middle layer: the report waits for the forward's last one.""" + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + assert _probe_forward(M._KvRangeProbe(PROBE_LAST), (1.0, peak, 1.0)) == [True, True, False] + logger.warning.assert_called_once() + assert "e4m3 maximum 448" in logger.warning.call_args.args[0] + + +def test_the_probe_runs_only_when_asked_over_an_fp8_pool() -> None: + core = SimpleNamespace(quant_mode=M.QM_FP8_KV_CACHE, attn_layers=[3, 7, PROBE_LAST]) + assert M._KvRangeProbe.for_target(core, True)._last == PROBE_LAST + assert M._KvRangeProbe.for_target(core, False) is None + for quant_mode in (0, M.QM_NVFP4_KV_CACHE): + other = SimpleNamespace(quant_mode=quant_mode, attn_layers=core.attn_layers) + assert M._KvRangeProbe.for_target(other, True) is None + + +def test_observing_under_capture_reads_nothing(monkeypatch) -> None: + logger = Mock() + monkeypatch.setattr(M, "logger", logger) + probe = M._KvRangeProbe(PROBE_LAST) + k, v = _probe_kv(1000.0) + scratch = torch.zeros(1, device=DEV) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + scratch.add_(1) # warm the allocator off the capture stream + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + kept = [] + with torch.cuda.graph(graph): + scratch.add_(1) + # The last layer, with an out-of-range value: a read here would fail the capture (a host + # sync) and a report would fire. + kept.append(probe.observe(PROBE_LAST, k, v)) + assert kept == [True], "a captured forward does not end the probe" + logger.warning.assert_not_called() + # The first eager forward after capture is what reports. + assert _probe_forward(probe, (1.0, 1.0, 1000.0)) == [True, True, False] + logger.warning.assert_called_once() diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py new file mode 100644 index 000000000000..e34b53674d5b --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_hc_ops.py @@ -0,0 +1,340 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's hyper-connection entries and their sites' paths.""" + +import zlib +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_norm import ( # noqa: E501 + qwen4_exp_hc_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_hc_inject_rstd import ( # noqa: E501 + qwen4_exp_hc_inject_rstd as rstd_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( # noqa: E501 + qwen4_exp_hc_decode_rows_first as first_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_rows_tc__decode import ( # noqa: E501 + qwen4_exp_hc_decode_rows_mix as mix_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.hc.qwen4_exp_hc_decode_tc_first__decode import ( # noqa: E501 + qwen4_exp_hc_decode_tc_first as op, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + assert_within_ulp, + certifying, + validating, +) + +assert torch.cuda.is_available(), "the hyper-connection entries require a CUDA device" + +DEV = "cuda" +HC, HIDDEN, RANK, FIRST_ROWS = 4, 2560, 320, 384 +WIDTH = HC * HIDDEN +EPS = 1e-6 +CORE = SimpleNamespace(hc_count=HC, hc_lowrank=RANK, hc_first_cols=352, eps=EPS) + + +def _cell_seed(cell) -> int: + return zlib.crc32(cell.why.encode()) % (2**31) + + +def _ratio(out, ref) -> float: + """Worst error in bands of one ulp plus `eps / 16` of the max (SiLU cancels near zero).""" + eps = torch.finfo(ref.dtype).eps + r = ref.double() + return ((out.double() - r).abs() / (eps * r.abs() + eps / 16 * r.abs().max())).max().item() + + +def _site(rows, inject, seed): + """One site's normed rows and weights at the checkpoint's statistics.""" + g = torch.Generator(device=DEV).manual_seed(seed) + parts = [torch.randn(RANK, WIDTH, generator=g, device=DEV) * 0.0296] + if inject: + parts.append(torch.randn(HC, WIDTH, generator=g, device=DEV) * 0.0107) + w = torch.cat(parts) + first = torch.cat([w, torch.zeros(FIRST_ROWS - w.shape[0], WIDTH, device=DEV)]) + up = torch.randn(WIDTH, RANK, generator=g, device=DEV) * 0.05 + gamma = torch.randn(WIDTH, generator=g, device=DEV) * 0.477 + x = torch.randn(rows, HC, HIDDEN, generator=g, device=DEV) + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) + normed = (x.flatten(-2) * (1.0 + gamma)).to(torch.bfloat16) + return normed, first.to(torch.bfloat16).t(), up.to(torch.bfloat16).t() + + +def _check_tc_first(cell): + rows, inject = cell.spec["rows"], cell.spec.get("inject", True) + normed, first, _ = _site(rows, inject, _cell_seed(cell)) + args = (normed, first, RANK, HC, inject) + low, inj = op(*args) + assert low.shape == (rows, RANK) + for got, want in zip((low, inj), op.reference(*args)): + assert (got is None) == (want is None) + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape and got.is_contiguous() + assert _ratio(got, want) <= 1.0, f"{_ratio(got, want):.4g}x the band" + + +def _mixer(monkeypatch, cap, normed, first, up, inject): + calls = [] + entry = M.qwen4_exp_hc_decode_tc_first + + def counted(*args, **kwargs): + calls.append(args[0].shape[0]) + return entry(*args, **kwargs) + + with monkeypatch.context() as m: + m.setattr(M, "_HC_TC_FIRST_MAX_ROWS", cap) + m.setattr(M, "qwen4_exp_hc_decode_tc_first", counted) + mixed, inj = M._hc_mixer(CORE, normed, first, up, inject) + return mixed, inj, calls + + +def _mixer_first_halves(rows, inject, monkeypatch): + """`_hc_mixer`'s entry against its cuBLAS path: bf16 roundings of fp32 sums in other orders.""" + normed, first, up = _site(rows, inject, seed=100 + rows) + tc = _mixer(monkeypatch, 640, normed, first, up, inject) + cb = _mixer(monkeypatch, 0, normed, first, up, inject) + assert tc[2] == [rows] and cb[2] == [] + assert_within_ulp(tc[0], cb[0], element_ulp=2.0, rms_ulp=0.25) + if inject: + assert_within_ulp(tc[1], cb[1], element_ulp=2.0, rms_ulp=1.0) + else: + assert tc[1] is None and cb[1] is None + + +def _operands(rows, seed): + """One site in the forward's layouts; `gate` is column 1280 of a [rows, 1288] gate|up row.""" + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + + def rnd(*shape, scale=1.0): + return (torch.randn(*shape, generator=g, device=DEV) * scale).to(bf) + + w = torch.cat( + [ + torch.randn(RANK, WIDTH, generator=g, device=DEV) * 0.0296, + torch.randn(HC, WIDTH, generator=g, device=DEV) * 0.0107, + torch.zeros(FIRST_ROWS - RANK - HC, WIDTH, device=DEV), + ] + ) + return dict( + first=w.to(bf).t(), + up=rnd(WIDTH, RANK, scale=0.05).t(), + norm=rnd(WIDTH, scale=0.477), + hc=rnd(rows, WIDTH, scale=2.0), + block=rnd(rows, HIDDEN, scale=0.5), + shared=rnd(rows, HIDDEN, scale=0.5), + gate=rnd(rows, 1288)[:, 1280:1281], + inject=(2.0 * torch.sigmoid(torch.randn(rows, HC, generator=g, device=DEV))).to(bf), + ) + + +def _site_inputs(o, kind): + blk, inj = (None, None) if kind == "norm" else (o["block"], o["inject"]) + sh, gate = (o["shared"], o["gate"]) if kind == "combine" else (None, None) + return blk, inj, sh, gate + + +def _first_args(o, site, cols): + blk, inj, sh, gate = _site_inputs(o, site) + return (o["hc"], blk, inj, o["norm"], o["first"], HC, cols, EPS, sh, gate) + + +def _check_rows_first(cell): + rows, site, cols = cell.spec["rows"], cell.spec["site"], cell.spec["cols"] + args = _first_args(_operands(rows, _cell_seed(cell)), site, cols) + hc_out, low, inj, rstd = first_op(*args) + ref = first_op.reference(*args) + assert low.shape == (rows, RANK) and rstd.shape == (rows, HC) and rstd.dtype == torch.float32 + assert (inj is None) == (cols == RANK) + if site == "norm": + assert hc_out is args[0], "without a pending block the stream is the input" + assert _ratio(hc_out, ref[0]) <= 1.0, "hc_out" + assert _ratio(low, ref[1]) <= 1.0, f"low {_ratio(low, ref[1]):.3g}x the band" + if inj is not None: + assert _ratio(inj, ref[2]) <= 1.0, f"inject {_ratio(inj, ref[2]):.3g}x the band" + assert ((rstd - ref[3]).abs() / ref[3]).max() <= 1e-6, "rstd in fp32" + # The residual every later site reads: bitwise what `qwen4_exp_hc_inject_norm` writes. + want, _ = qwen4_exp_hc_inject_norm(*args[:4], EPS, HC, *args[8:]) + assert torch.equal(hc_out, want) + + +def _check_rows_mix(cell): + rows = cell.spec["rows"] + o = _operands(rows, _cell_seed(cell)) + with validating(first_op): + hc_out, low, _, rstd = first_op(*_first_args(o, "combine", RANK + HC)) + args = (low, rstd, o["up"], hc_out, o["norm"], HC) + mixed = mix_op(*args) + assert mixed.shape == (rows, HIDDEN) and mixed.is_contiguous() + assert _ratio(mixed, mix_op.reference(*args)) <= 1.0 + + +def _site_run(monkeypatch, lo, o, kind): + """`_hc_site` with the two-launch range starting at `lo`; counts its first launches.""" + calls = [] + entry = M.qwen4_exp_hc_decode_rows_first + + def counted(*args, **kwargs): + calls.append(args[0].shape[0]) + return entry(*args, **kwargs) + + blk, inj, sh, gate = _site_inputs(o, kind) + extra = (sh, gate) if kind == "combine" else () + with monkeypatch.context() as m: + m.setattr(M, "_HC_ROWS_TC_MIN_ROWS", lo) + m.setattr(M, "qwen4_exp_hc_decode_rows_first", counted) + hc, mixed, inject = M._hc_site( + CORE, o["hc"], blk, inj, o["norm"], o["first"], o["up"], kind != "norm", *extra + ) + return hc, mixed, inject, calls + + +def _rows_tc_sites(rows, kind, monkeypatch): + """`_hc_site`'s two launches against `qwen4_exp_hc_inject_norm` + `_hc_mixer`: the residual + bitwise, `mixed` within rounding since the grouped norm reads other bf16 operands.""" + o = _operands(rows, 100 + rows) + two = _site_run(monkeypatch, 1, o, kind) + old = _site_run(monkeypatch, 99, o, kind) + assert two[3] == [rows] and old[3] == [] + assert torch.equal(two[0], old[0]) + assert_within_ulp(two[1], old[1], element_ulp=3.0, rms_ulp=1.0) + if kind == "norm": + assert two[2] is None and old[2] is None + else: + a, b = two[2].double(), old[2].double() + step = torch.finfo(torch.bfloat16).eps * torch.maximum(a.abs(), b.abs()) + assert ((a - b).abs() <= step).all(), (a - b).abs().max().item() + + +def _mixer_rstd(rows, inject): + """The prefill rstd site (normed stream formed on chip in fp32) against `_hc_mixer` on the + stored bf16 stream; the planted |1 + gamma| = 7.6 column sets the 4-ulp element bound.""" + o = _operands(rows, 900 + rows) + o["norm"][0], o["norm"][WIDTH // 2] = 6.625, -6.625 # this checkpoint's largest |gamma| + hc, rstd = rstd_op(o["hc"], o["block"], o["inject"], EPS, HC) + hc2, normed = qwen4_exp_hc_inject_norm(o["hc"], o["block"], o["inject"], o["norm"], EPS, HC) + assert torch.equal(hc, hc2) + w1 = {o["norm"].data_ptr(): (1.0 + o["norm"].float()).contiguous()} + core = SimpleNamespace(**vars(CORE), _hc_norm_w1=w1) + a = M._hc_mixer_rstd(core, hc, rstd, o["norm"], o["first"], o["up"], inject) + b = M._hc_mixer(core, normed, o["first"], o["up"], inject) + assert_within_ulp(a[0], b[0], element_ulp=4.0, rms_ulp=1.0) + if inject: + assert_within_ulp(a[1], b[1], element_ulp=2.0, rms_ulp=1.0) + else: + assert a[1] is None and b[1] is None + + +def _tp_partial(): + """`_tp_partial` defers the sum only where the next site's fused all-reduce can take it; + `_hc_site` sums a partial it cannot fuse with `tp_sum` first, bitwise the summed call.""" + calls = [] + + def tp_sum(x): + calls.append(x.shape[0]) + return (x.float() * 0.75).to(x.dtype) + + lo = M._HC_AR_INJECT_MIN_ROWS + x_lo = torch.zeros(lo, HIDDEN, dtype=torch.bfloat16, device=DEV) + x16 = torch.zeros(16, HIDDEN, dtype=torch.bfloat16, device=DEV) + off = SimpleNamespace(tp_sum=tp_sum) + on = SimpleNamespace(hc_ar_inject=SimpleNamespace(max_rows=1 << 30), tp_sum=tp_sum) + assert not isinstance(M._tp_partial(off, x_lo), M._TpPartial) + assert isinstance(M._tp_partial(on, x_lo), M._TpPartial) + assert not isinstance(M._tp_partial(on, x16), M._TpPartial) + assert calls == [lo, 16] + o = _operands(lo, 77) + core = SimpleNamespace(**vars(CORE), tp_sum=tp_sum) + for sh in ((o["shared"], o["gate"]), ()): + rest = (o["norm"], o["first"], o["up"], True, *sh) + a = M._hc_site(core, o["hc"], M._TpPartial(o["block"]), o["inject"], *rest) + b = M._hc_site(core, o["hc"], tp_sum(o["block"]), o["inject"], *rest) + for x, y in zip(a, b): + assert torch.equal(x, y) + + +def _inject_norm_launch(rows, kind): + """`_hc_inject_norm_launch` against the family op: residual bitwise, normed stream within one + rounding at the site's warps; all bitwise at eight, eager and replayed behind a producer.""" + + def inputs(seed): + o = _operands(rows, seed) + blk, inj, sh, gate = _site_inputs(o, kind) + return o, (o["hc"], blk, inj, o["norm"], EPS, HC, sh, gate) + + _, args = inputs(500 + rows) + hc8, normed8 = qwen4_exp_hc_inject_norm(*args) + for warps in (M._HC_INJECT_WARPS, 2): + hc_w, normed_w = M._hc_inject_norm_launch(*args, warps) + assert torch.equal(hc_w, hc8) + # One bf16 rounding of an element is at most 2^-7 of its magnitude, so up to 2 ulp of the + # row's largest one (B300: one element of 288 x 10240 one rounding apart, 1.14 ulp). + assert_within_ulp(normed_w, normed8, element_ulp=2.0, rms_ulp=0.25) + o, args = inputs(700 + rows) + hc8, normed8 = qwen4_exp_hc_inject_norm(*args) + hc_l, normed_l = M._hc_inject_norm_launch(*args, 8) + assert torch.equal(hc_l, hc8) and torch.equal(normed_l, normed8) + src = o["hc"].clone() + hc_in = src.clone() + M._hc_inject_norm_launch(hc_in, *args[1:], 8) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + hc_in.copy_(src * 1.5) + hc_g, normed_g = M._hc_inject_norm_launch(hc_in, *args[1:], 8) + graph.replay() + torch.cuda.synchronize() + hc_e, normed_e = M._hc_inject_norm_launch(src * 1.5, *args[1:], 8) + torch.cuda.synchronize() + assert torch.equal(hc_g, hc_e) and torch.equal(normed_g, normed_e) + + +_CELL_CHECKS = ((op, _check_tc_first), (first_op, _check_rows_first), (mix_op, _check_rows_mix)) +CASES = [(entry, check, cell) for entry, check in _CELL_CHECKS for cell in entry.CELLS] + + +@pytest.mark.parametrize( + "entry, check, cell", CASES, ids=[f"{type(e).__name__}-{c.why[:40]}" for e, _, c in CASES] +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + certifying(entry) + check(cell) + + +_KINDS = ("combine", "inject", "norm") +#: `(path, rows, variant)`: `inject` for the mixers, the site kind for the decode sites. +# fmt: off +_SITE_CASES = ( + [("mixer_first_halves", r, i) for r in (4, 64, 288, 576) for i in (True, False)] + + [("rows_tc_sites", r, k) for r in (1, 8, 16) for k in _KINDS] + + [("mixer_rstd", r, i) for r in (3072, 8192) for i in (True, False)] + + [("tp_partial", 0, None)] + + [("inject_norm_launch", r, k) for r in (17, 192, 288) for k in _KINDS] +) +# fmt: on + + +@pytest.mark.parametrize("path, rows, variant", _SITE_CASES, ids=lambda v: str(v)) +def test_hc_site_paths(path, rows, variant, monkeypatch) -> None: + """Site paths no certified cell reaches, each against its alternative.""" + if path == "mixer_first_halves": + _mixer_first_halves(rows, variant, monkeypatch) + elif path == "rows_tc_sites": + _rows_tc_sites(rows, variant, monkeypatch) + elif path == "mixer_rstd": + _mixer_rstd(rows, variant) + elif path == "tp_partial": + _tp_partial() + else: + _inject_norm_launch(rows, variant) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py new file mode 100644 index 000000000000..ec202e0bcd40 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_inforward_snapshot.py @@ -0,0 +1,386 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""In-forward Mamba snapshots in the FP8 unified target's prefill: one step over [.., L) leaves +the state at P = L - tail in the live slot and stages the state at L, checked against the main +chunk [.., P) then the tail [P, L) alone in the next step, at a tep2 rank's GDN geometry.""" + +from __future__ import annotations + +import types + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.embedding.ngram_hash import ( # noqa: E501 + CKPT_MULTIPLIERS, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( # noqa: E501 + ngram_token_history, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ple_gate_conv_inject_norm import ( # noqa: E501 + CKPT_CLAMP_MIN, + CKPT_DILATION, + CKPT_HC_COUNT, + CKPT_HIDDEN, + ple_gate_conv_inject_norm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _m, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm import ( # noqa: E501 + flashinfer_chunk_gated_delta_rule as fi, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_inforward_tail import ( # noqa: E501 + gdn_inforward_tail_reference, +) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata +from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import InforwardSnapshotEntry + +assert torch.cuda.is_available(), "the in-forward snapshot test requires a CUDA device" + +DEV = "cuda" +H, HV, DK, DV = fi.RANK2_GEOMETRY +CONV_DIM, D_CONV, SLOTS, ROWS, EOS = 2 * H * DK + HV * DV, 4, 24, 8, 2 # ROWS: staging rows +NGRAM, PLE_STATE, STREAM = len(CKPT_MULTIPLIERS), 3 * CKPT_DILATION, CKPT_HC_COUNT * CKPT_HIDDEN + +# (request id, live slot, chunk length, carries a state, tail or None). One request: CP pipeline. +BATCHES = { + "1req": [(31, 4, 3000, True, 2)], + "1fresh": [(32, 9, 16384, False, 2)], + "3req": [(11, 5, 300, True, None), (12, 2, 1000, True, 2), (13, 7, 150, False, 2)], + "8req": [(21, 3, 96, True, 2), (22, 9, 700, True, None), (23, 1, 67, True, 2), + (24, 17, 2048, False, None), (25, 12, 130, True, 2), (26, 0, 515, True, 2), + (27, 20, 64, True, None), (28, 6, 1500, False, 2)], +} # fmt: skip + + +def _batch_split(batch): + plan, row = {}, 0 + for rid, _, length, _, tail in batch: + if tail is not None: + plan[rid] = InforwardSnapshotEntry(row, length - tail, length - tail, tail) + row += 1 + slots = {rid: slot for rid, slot, *_ in batch} + manager = types.SimpleNamespace( + conv_state_shape=[CONV_DIM, D_CONV - 1], + inforward_snapshot_rows=ROWS, + inforward_snapshot_max_tail=2, + inforward_snapshot_plan=plan, + state_index=lambda rid: slots[rid], + ) + attn = types.SimpleNamespace( + request_ids=[rid for rid, *_ in batch], + seq_lens=torch.tensor([length for _, _, length, _, _ in batch], dtype=torch.int32), + ) + metadata = Mamba2Metadata(max_batch_size=32, chunk_size=128) + metadata._prepare_inforward_split(attn, manager, plan, len(batch), len(batch)) + return metadata + + +class _Rows: + def __init__(self, batch): + self.main, self.tail, self.main_cu, self.tail_cu = [], [], [0], [0] + self.tail_slots, self.starts = [], [] + start = 0 + for _, slot, length, _, tail in batch: + before = length - (tail or 0) + self.starts.append(start) + self.main += range(start, start + before) + self.main_cu.append(self.main_cu[-1] + before) + if tail: + self.tail += range(start + before, start + length) + self.tail_cu.append(self.tail_cu[-1] + tail) + self.tail_slots.append(slot) + start += length + self.total = start + self.slots = [slot for _, slot, *_ in batch] + self.carry = [c for *_, c, _ in batch] + self.split = [(slot, start, length - tail, start + length) for (_, slot, length, _, tail), start + in zip(batch, self.starts) if tail] # fmt: skip + + def t(self, values, dtype=torch.long): + return torch.tensor(values, dtype=dtype, device=DEV) + + +def _fake_target(metadata, batch, pools, stage, input_ids=None): + core = types.SimpleNamespace(_bound=types.SimpleNamespace(pools=pools, stage=stage)) + target = types.SimpleNamespace(core=core, _step=None) + hi = torch.tensor([c for *_, c, _ in batch], dtype=torch.bool, device=DEV) + total = sum(length for _, _, length, _, _ in batch) + split = _m.PrefillTarget._split_of(target, metadata, len(batch), 0, hi, total) + target._step = types.SimpleNamespace(split=split, input_ids=input_ids) + return target + + +def _strided_pool(gen, *shape, dtype=torch.bfloat16): + """`[SLOTS, *shape]` whose slot stride is two states: the coalesced page's layout.""" + page = (torch.randn(SLOTS, 2, *shape, generator=gen, device=DEV) * 0.1).to(dtype) + return page[:, 1] + + +def _stage(gdn): + bf = torch.bfloat16 + ssm = {0: torch.full((ROWS, HV, DV, DK), 7.0, dtype=bf, device=DEV)} if gdn else {} + conv = {0: torch.full((ROWS, CONV_DIM, D_CONV - 1), 7.0, dtype=bf, device=DEV)} if gdn else {} + return _m._InforwardStage( + ssm=ssm, + conv=conv, + ple_conv=torch.full((ROWS, STREAM, PLE_STATE), 7.0, dtype=bf, device=DEV), + ple_ngram=torch.full((ROWS, NGRAM - 1), -1, dtype=torch.long, device=DEV), + ) + + +def _gdn_inputs(total, gen): + q = torch.randn(1, total, H, DK, generator=gen, device=DEV) + k = torch.randn(1, total, H, DK, generator=gen, device=DEV) + q = (q / q.norm(dim=-1, keepdim=True)).bfloat16() + k = (k / k.norm(dim=-1, keepdim=True)).bfloat16() + v = torch.randn(1, total, HV, DV, generator=gen, device=DEV).bfloat16() + g = torch.exp(-torch.rand(1, total, HV, generator=gen, device=DEV) * 0.2) + beta = torch.rand(1, total, HV, generator=gen, device=DEV) + return q, k, v, g, beta + + +def _fi_step(qkvgb, sel, pool, slots, cu, carry): + out, _ = fi.flashinfer_chunk_gated_delta_rule( + *(x.index_select(1, sel) for x in qkvgb), scale=None, initial_state=pool, + initial_state_indices=slots, inplace_indexed_state_update=True, output_final_state=False, + cu_seqlens=cu, head_first=False, use_qk_l2norm_in_kernel=False, output=None, + state_workspace=None, has_initial_state=carry, linear_gate=True, + ) # fmt: skip + return out + + +def _ple_inputs(gen, n): + def rand(*shape, scale=1.0): + return (torch.randn(*shape, generator=gen, device=DEV) * scale).bfloat16() + + hc, key, value = rand(n, STREAM, scale=2.0), rand(n, STREAM), rand(n, CKPT_HIDDEN, scale=0.5) + norms = [rand(STREAM, scale=0.5) for _ in range(4)] + return hc, key, value, norms, rand(STREAM, 1, 4, scale=0.3) + + +def _ple(inputs, sel, pool, slots, cu, carry, max_len, **kw): + """`ple_gate_conv_inject_norm` over rows `sel` (None: every row) at the checkpoint's shapes.""" + hc, key, value, norms, conv_w = inputs + if sel is not None: + hc, key, value = (t.index_select(0, sel) for t in (hc, key, value)) + return ple_gate_conv_inject_norm( + hc, key, value, norms[0], norms[1], norms[2], conv_w, pool, slots, cu, norms[3], 1e-6, + CKPT_HC_COUNT, CKPT_CLAMP_MIN, CKPT_DILATION, -1, max_len, carry, **kw, + ) # fmt: skip + + +def _gdn_split(batch, metadata, rows): + """`_gdn_split`: the state at P, main rows and conv windows bitwise the main step's; the tails' + outputs and staged states within rounding of the tail step and the float64 reference.""" + gen = torch.Generator(device=DEV).manual_seed(7 + len(batch)) + qkvgb = _gdn_inputs(rows.total, gen) + ssm = _strided_pool(gen, HV, DV, DK) + conv = _strided_pool(gen, CONV_DIM, D_CONV - 1) + proj = torch.randn(rows.total, CONV_DIM + 96, generator=gen, device=DEV).bfloat16() + mixed_qkv = proj[:, :CONV_DIM] + main, tail = rows.t(rows.main), rows.t(rows.tail) + tail_slots = rows.t(rows.tail_slots) + tail_carry = torch.ones(len(rows.tail_slots), dtype=torch.bool, device=DEV) + + ref_ssm = ssm.clone() + slots32, carry = rows.t(rows.slots, torch.int32), rows.t(rows.carry, torch.bool) + out_main = _fi_step(qkvgb, main, ref_ssm, slots32, rows.t(rows.main_cu), carry) + state_p = ref_ssm.clone() + tail32 = rows.t(rows.tail_slots, torch.int32) + out_tail = _fi_step(qkvgb, tail, ref_ssm, tail32, rows.t(rows.tail_cu), tail_carry) + state_l = ref_ssm.index_select(0, tail_slots) + + for slot, start, _, end in rows.split: + conv[slot] = mixed_qkv[end - (D_CONV - 1) : end].t() + conv_before = conv.clone() + stage = _stage(gdn=True) + pools = types.SimpleNamespace(ssm={0: ssm}, conv={0: conv}) + target = _fake_target(metadata, batch, pools, stage) + out = torch.full((1, rows.total, HV, DV), -7.0, dtype=torch.bfloat16, device=DEV) + _m.PrefillTarget._gdn_split(target, *qkvgb, mixed_qkv, 0, out) + torch.cuda.synchronize() + + assert torch.equal(ssm, state_p), "live slots differ from the main step's states" + assert torch.equal(out[0].index_select(0, main), out_main[0]), "main rows' outputs differ" + nsplit = len(rows.split) + staged = stage.ssm[0][:nsplit] + got_tail = out[0].index_select(0, tail) + torch.testing.assert_close(staged.float(), state_l.float(), rtol=2e-2, atol=2e-2) + torch.testing.assert_close(got_tail.float(), out_tail[0].float(), rtol=2e-2, atol=2e-2) + ref_stage, ref_out = torch.zeros_like(stage.ssm[0]), torch.zeros_like(out) + sp = target._step.split + gdn_inforward_tail_reference( + *qkvgb, sp.tail_tok, sp.cu_tail, state_p, sp.tail_slots, ref_stage, sp.stage_rows, ref_out + ) + torch.testing.assert_close(staged.float(), ref_stage[:nsplit].float(), rtol=1e-2, atol=1e-2) + torch.testing.assert_close( + got_tail.float(), ref_out[0].index_select(0, tail).float(), rtol=1e-2, atol=1e-2 + ) + for row, (slot, start, before, end) in enumerate(rows.split): + p = start + before + assert torch.equal(conv[slot], mixed_qkv[p - (D_CONV - 1) : p].t()) + assert torch.equal(stage.conv[0][row], conv_before[slot]) + others = [s for s in range(SLOTS) if s not in rows.tail_slots] + assert torch.equal(conv[others], conv_before[others]) + + +def _ple_split(batch, metadata, rows): + """`_split_ple_states`: PLE conv slots and n-gram histories bitwise the two-step schedule's.""" + gen = torch.Generator(device=DEV).manual_seed(100 + len(batch)) + n = rows.total + inputs = _ple_inputs(gen, n) + conv_pool = torch.randn(SLOTS, STREAM, PLE_STATE, generator=gen, device=DEV).bfloat16() + ids = torch.randint(3, 50000, (n,), generator=gen, device=DEV, dtype=torch.int32) + ngram_pool = torch.randint(3, 50000, (SLOTS, NGRAM - 1), generator=gen, device=DEV) + carry = rows.t(rows.carry, torch.bool) + slots32 = rows.t(rows.slots, torch.int32) + + def conv_step(pool, sel, cu, slots, carries, conv_input=None): + max_len = int((cu[1:] - cu[:-1]).max()) + _ple(inputs, sel, pool, slots, cu, carries, max_len, conv_input=conv_input) + + main, tail = rows.t(rows.main), rows.t(rows.tail) + tail_slots32 = rows.t(rows.tail_slots, torch.int32) + tail_carry = torch.ones(len(rows.tail_slots), dtype=torch.bool, device=DEV) + ref_conv = conv_pool.clone() + conv_step(ref_conv, main, rows.t(rows.main_cu), slots32, carry) + conv_p = ref_conv.clone() + conv_step(ref_conv, tail, rows.t(rows.tail_cu), tail_slots32, tail_carry) + _, ngram_p = ngram_token_history.reference( + ids.index_select(0, main), ngram_pool, slots32, carry, rows.t(rows.main_cu), EOS, NGRAM + ) + _, ngram_l = ngram_token_history.reference( + ids.index_select(0, tail), ngram_p, tail_slots32, tail_carry, rows.t(rows.tail_cu), EOS, + NGRAM, + ) # fmt: skip + + whole_cu = rows.t([0] + [s + length for s, (_, _, length, _, _) in zip(rows.starts, batch)]) + conv_input = torch.empty(n, STREAM, dtype=torch.bfloat16, device=DEV) + conv_step(conv_pool, None, whole_cu, slots32, carry, conv_input=conv_input) + _, ngram_whole = ngram_token_history.reference( + ids, ngram_pool, slots32, carry, whole_cu, EOS, NGRAM + ) + ngram_pool = ngram_whole.clone() + stage = _stage(gdn=False) + target = _fake_target(metadata, batch, types.SimpleNamespace(), stage, input_ids=ids) + _m.PrefillTarget._split_ple_states(target, conv_pool, ngram_pool, conv_input) + torch.cuda.synchronize() + + k = len(rows.split) + assert torch.equal(conv_pool, conv_p), "PLE conv slots differ from the main step's" + assert torch.equal(stage.ple_conv[:k], ref_conv.index_select(0, rows.t(rows.tail_slots))) + assert torch.equal(ngram_pool, ngram_p), "n-gram histories differ from the main step's" + assert torch.equal(stage.ple_ngram[:k], ngram_l.index_select(0, rows.t(rows.tail_slots))) + + +@pytest.mark.parametrize("state", ["gdn", "ple"]) +@pytest.mark.parametrize("name", list(BATCHES)) +def test_split_states_are_the_two_iteration_schedule(name, state) -> None: + batch = BATCHES[name] + (_gdn_split if state == "gdn" else _ple_split)(batch, _batch_split(batch), _Rows(batch)) + + +def _padded_gdn_call(rows, padded, seqs, zero_pad) -> None: + """`_gdn_core_graph`'s CP and carried calls over a bucket's `padded` rows are bitwise the + unpadded call; rows past the batch are zero or, without `zero_pad`, values never read.""" + gen = torch.Generator(device=DEV).manual_seed(rows + padded) + q, k, v, g, beta = _gdn_inputs(padded, gen) + lens = [rows // seqs] * seqs + lens[-1] += rows - sum(lens) + cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.long, device=DEV) + slots = torch.tensor([3 + 5 * i for i in range(seqs)], dtype=torch.int32, device=DEV) + carry = torch.tensor([i % 2 == 0 for i in range(seqs)], dtype=torch.bool, device=DEV) + dev = torch.device(DEV) + cp = fi._reaches_cp(seqs, HV, dev) + keep = (torch.arange(padded, device=DEV) < rows).view(1, padded) + padded_in = [ + torch.where(keep.view(1, padded, *([1] * (t.dim() - 2))), t, 0) if zero_pad else t + for t in (q, k, v, g, beta) + ] + chunk = fi.cp_chunk_len_for(rows, max(H, HV), dev) if cp else None + fi_kwargs = {"_cp_chunk_len": chunk, "max_seqlen": padded} if cp else None + for path in [fi._chunk_carried] + ([fi._chunk_cp_inplace] if cp else []): + results = [] + for unpadded in (True, False): + pool = _strided_pool(torch.Generator(device=DEV).manual_seed(5), HV, DV, DK) + out = torch.full((1, padded, HV, DV), -7.0, dtype=torch.bfloat16, device=DEV) + if unpadded: + path( + *(t[:, :rows] for t in (q, k, v, g, beta)), None, pool, slots, carry, cu, + out[:, :rows], True, + ) # fmt: skip + else: + path(*padded_in, None, pool, slots, carry, cu, out, True, fi_kwargs=fi_kwargs) + results.append((out[:, :rows].clone(), pool.clone())) + torch.cuda.synchronize() + assert torch.equal(results[0][0], results[1][0]), f"{path.__name__}: outputs" + assert torch.equal(results[0][1], results[1][1]), f"{path.__name__}: states" + + +def _ple_publish_end() -> None: + """`publish_end=k` (`DecodeTarget._ple_from_pool`): outputs bitwise the full launch's; conv + pool rows of sequences below `k` published, the later ones left exactly as they were.""" + gen = torch.Generator(device=DEV).manual_seed(7) + # A 5-row context-shaped prefix, then three 3-row verify requests. + inputs = _ple_inputs(gen, 14) + pool0 = torch.randn(SLOTS, STREAM, PLE_STATE, generator=gen, device=DEV).bfloat16() + slots = torch.tensor([2, 5, 7, 9], dtype=torch.int32, device=DEV) + cu = torch.tensor([0, 5, 8, 11, 14], dtype=torch.int32, device=DEV) + carry = torch.ones(4, dtype=torch.bool, device=DEV) + outs, pools = [], [pool0.clone(), pool0.clone()] + for publish_end, pool in zip((None, 1), pools): + outs.append(_ple(inputs, None, pool, slots, cu, carry, 5, publish_end=publish_end)) + torch.cuda.synchronize() + assert torch.equal(outs[0][0], outs[1][0]) and torch.equal(outs[0][1], outs[1][1]), "outputs" + assert torch.equal(pools[1][2], pools[0][2]), "the sequence below publish_end publishes" + for slot in (5, 7, 9): + assert torch.equal(pools[1][slot], pool0[slot]), f"slot {slot} must stay unpublished" + assert not torch.equal(pools[0][slot], pool0[slot]), "the full launch publishes it" + others = [s for s in range(SLOTS) if s not in (2, 5, 7, 9)] + assert torch.equal(pools[1][others], pool0[others]) + + +# (eager rows, padded bucket rows, sequences): every FlashInfer CP chunk-length class below 4096. +PADDED_CALLS = [ + (126, 128, 1), (300, 512, 1), (1000, 1024, 1), (1300, 2048, 1), (2046, 2048, 1), + (3070, 4096, 1), (900, 1024, 2), (3000, 4096, 3), +] # fmt: skip +_VARIANTS = [ + (f"padded-{r}-{p}-{s}-{'zero' if z else 'stale'}", _padded_gdn_call, (r, p, s, z)) + for r, p, s in PADDED_CALLS + for z in (True, False) +] + [("ple_publish_end", _ple_publish_end, ())] + + +@pytest.mark.parametrize("helper,args", [v[1:] for v in _VARIANTS], ids=[v[0] for v in _VARIANTS]) +def test_launch_variants_are_bitwise_the_plain_launch(helper, args) -> None: + """A padded CUDA-graph GDN call and a partly publishing PLE launch against the plain ones.""" + helper(*args) + + +def _manager(rows=ROWS, min_main=_m._INFORWARD_MIN_MAIN, ple=True): + """A manager stub that plans `rows` staging rows and splits from `min_main` tokens.""" + views = { + "inforward_snapshot_stage_views": lambda i: (f"ssm{i}", f"conv{i}"), + "inforward_ple_stage_views": lambda layer: ("ple_conv", "ple_ngram"), + } + if not ple: + del views["inforward_ple_stage_views"] + return types.SimpleNamespace( + inforward_snapshot_rows=rows, inforward_snapshot_min_main=min_main, **views + ) + + +def test_the_bind_takes_only_snapshots_the_split_serves() -> None: + """No staging rows: no stage. A manager that stages no PLE state, or plans splits of fewer + tokens before P than the main chunk's FlashInfer path takes (the stock manager's conv-width + minimum, 3), is declined at bind rather than asserted on mid-step.""" + assert _m._inforward_stage(_manager(rows=0), [0, 2], 1) is None + stage = _m._inforward_stage(_manager(), [0, 2], 1) + assert stage.ssm == {0: "ssm0", 2: "ssm2"} and stage.ple_conv == "ple_conv" + with pytest.raises(NotImplementedError, match=r"lacks \[\] and splits from 3 tokens"): + _m._inforward_stage(_manager(min_main=D_CONV - 1), [0, 2], 1) + with pytest.raises(NotImplementedError, match="lacks \\['inforward_ple_stage_views'\\]"): + _m._inforward_stage(_manager(ple=False), [0, 2], 1) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py new file mode 100644 index 000000000000..b592eadfa885 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_moe_and_dense_ops.py @@ -0,0 +1,1048 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's MoE and dense entries and the paths no cell reaches.""" + +import zlib +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.hc.qwen4_exp_post_inject import ( # noqa: E501 + qwen4_exp_post_inject, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.moe.renorm_moe_routing_op import ( # noqa: E501 + renorm_moe_routing_op, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.quantization.fp8_quantize_1x128 import ( # noqa: E501 + fp8_quantize_1x128, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.embedding.embed_rows__decode import ( # noqa: E501 + embed_rows, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + low_m_bf16_mm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + plan as low_m_plan, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + fp8_block_scale_moe_runner as _runner_mod, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + moe_prefill_grouped__prefill as _grouped, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe import ( # noqa: E501 + moe_tpb_decode__decode as _tpb, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.fp8_block_scale_moe_runner import ( # noqa: E501 + BLOCK, + WIDE_DECODE_MIN_ROWS, + WIDE_DECODE_SMALL_TACTICS, + WIDE_DECODE_TACTIC, + fp8_block_scale_moe_front, + fp8_block_scale_moe_runner, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.moe_front_router_quant__decode import ( # noqa: E501 + moe_front_router_quant, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.moe.mtp_moe_combine_inject__decode import ( # noqa: E501 + mtp_moe_combine_inject, +) +from tensorrt_llm._torch.moe.fused_shared_expert import fused_sigmoid_gate_mul_add + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + ULP, + assert_within_ulp, + certifying, +) + +assert torch.cuda.is_available(), "the MoE and dense entries require a CUDA device" + +DEV = "cuda" +_BF16_ULP = 2.0**-8 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) + + +# ------------------------------------------------------- fp8_block_scale_moe_runner --- + + +def _moe_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """Gate in bf16 ulp of each row's max: 16 element-wise, 0.75 RMS. Kernel and reference differ + only by accumulation order and FC1 sums at e4m3 rounding boundaries (measured 6.65 / 0.306).""" + assert out.dtype == ref.dtype == torch.bfloat16, (out.dtype, ref.dtype) + assert out.shape == ref.shape, (out.shape, ref.shape) + o, r = out.float(), ref.float() + rowmax = r.abs().amax(dim=-1, keepdim=True).clamp(min=1e-6) + d = (o - r) / (rowmax * 2.0**-8) + elt, rms = d.abs().max().item(), d.pow(2).mean().sqrt().item() + assert elt <= 16.0, f"{elt:.2f} ulp element-wise > 16" + assert rms <= 0.75, f"{rms:.3f} ulp RMS > 0.75" + + +CKPT = dict(hidden=2560, inter=640, num_experts=512, top_k=10) + +#: The arguments a step writes: what a captured graph reads afresh on each replay. +STEP_INPUTS = ("hidden_states", "hidden_states_scale", "topk_weights", "topk_ids") + +_WEIGHTS: dict = {} + + +def _weights(hidden: int, inter: int, experts: int, seed: int): + """`(w1, w1s, w2, w2s)` drawn directly in e4m3 / fp32, so every value is exact. Cached: the + checkpoint stack is 2.5 GB. Drawn 32 experts at a time to cap the fp32 transient.""" + key = (hidden, inter, experts, seed) + if key not in _WEIGHTS: + g = torch.Generator(device=DEV).manual_seed(seed) + w1 = torch.empty(experts, 2 * inter, hidden, dtype=torch.float8_e4m3fn, device=DEV) + w2 = torch.empty(experts, hidden, inter, dtype=torch.float8_e4m3fn, device=DEV) + for lo in range(0, experts, 32): + hi = min(lo + 32, experts) + w1[lo:hi] = torch.randn(hi - lo, 2 * inter, hidden, device=DEV, generator=g) + w2[lo:hi] = torch.randn(hi - lo, hidden, inter, device=DEV, generator=g) + w1s = 0.01 + 0.02 * torch.rand( + experts, 2 * inter // BLOCK, hidden // BLOCK, device=DEV, generator=g + ) + w2s = 0.01 + 0.02 * torch.rand( + experts, hidden // BLOCK, inter // BLOCK, device=DEV, generator=g + ) + _WEIGHTS[key] = (w1, w1s, w2, w2s) + return _WEIGHTS[key] + + +def _stack_window(offset: int, width: int): + """`(w1, w1s, w2, w2s)` of experts `[offset, offset + width)` of the checkpoint-shape stack.""" + win = slice(offset, offset + width) + return tuple(t[win] for t in _weights(2560, 640, 512, seed=2560 + 640 + 512)) + + +def _kwargs(hs, hs_scale, weights, offset, local, num_experts=512, top_k=10, inter=640, **routing): + w1, w1s, w2, w2s = weights + # fmt: off + base = dict( + routing_logits=None, routing_bias=None, hidden_states=hs, hidden_states_scale=hs_scale, + gemm1_weights=w1, gemm1_weights_scale=w1s, gemm2_weights=w2, gemm2_weights_scale=w2s, + num_experts=num_experts, top_k=top_k, num_fused_shared_experts=None, n_group=None, + topk_group=None, intermediate_size=inter, local_expert_offset=offset, + local_num_experts=local, routed_scaling_factor=None, routing_method_type=1, + topk_weights=None, topk_ids=None, gemm1_clamp_limit=None, + ) + # fmt: on + return dict(base, **routing) + + +def _runner_kwargs(xq, xsf, offset, width, **routing) -> dict: + return _kwargs(xq, xsf, _stack_window(offset, width), offset, width, **routing) + + +def _args( + tokens, hidden, inter, num_experts, top_k, local_expert_offset=0, local_num_experts=None, seed=0 +): + """Pre-routed op arguments over a window of one stack; one seed's windows split one layer.""" + local = num_experts if local_num_experts is None else local_num_experts + window = slice(local_expert_offset, local_expert_offset + local) + stack = _weights(hidden, inter, num_experts, seed=hidden + inter + num_experts) + g = torch.Generator(device=DEV).manual_seed(seed) + hs = (2.0 * torch.randn(tokens, hidden, device=DEV, generator=g)).to(torch.float8_e4m3fn) + # [H/128, T]: the transposed layout fp8_quantize_1x128 returns here. + hs_scale = 0.5 + torch.rand(hidden // BLOCK, tokens, device=DEV, generator=g) + logits = torch.randn(tokens, num_experts, device=DEV, generator=g) + weights, ids = torch.topk(torch.softmax(logits, dim=-1), top_k, dim=-1) + return _kwargs( + hs, + hs_scale, + tuple(t[window] for t in stack), + local_expert_offset, + local, + num_experts, + top_k, + inter, + topk_weights=weights.to(torch.bfloat16).contiguous(), + topk_ids=ids.to(torch.int32).contiguous(), + ) + + +def _window_args(cell, salt: int = 0) -> tuple[dict, torch.Tensor]: + """The cell's arguments and in-window slots, redrawn while the routing misses the window.""" + geometry = {k: v for k, v in cell.spec.items() if k != "capture"} + for attempt in range(8): + args = _args(**geometry, seed=_seed(cell.why) + salt + attempt) + hit = _in_window(args) + if hit.any(): + return args, hit + raise AssertionError(f"no slot routes into the window: {cell.why!r} would certify nothing") + + +def _in_window(args) -> torch.Tensor: + """`[T, top_k]`: which slots the call's window answers for.""" + local = args["topk_ids"].long() - args["local_expert_offset"] + return (local >= 0) & (local < args["local_num_experts"]) + + +def _moe_check(out: torch.Tensor, args: dict, hit: torch.Tensor) -> None: + assert out.shape == args["hidden_states"].shape and out.dtype == torch.bfloat16 + miss = ~hit.any(-1) + assert torch.equal(out[miss], torch.zeros_like(out[miss])) + _moe_compare(out, fp8_block_scale_moe_runner.reference(**args)) + + +def _warm_on_side_stream(fn) -> None: + """One eager call on a side stream before a capture, as `torch.cuda.graph` asks.""" + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + fn() + torch.cuda.current_stream().wait_stream(side) + + +def _check_moe_runner(cell): + args, hit = _window_args(cell) + out = fp8_block_scale_moe_runner(**args) + _moe_check(out, args, hit) + if not cell.spec.get("capture"): + return + static = {k: (v.clone() if k in STEP_INPUTS else v) for k, v in args.items()} + _warm_on_side_stream(lambda: fp8_block_scale_moe_runner(**static)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + replayed = fp8_block_scale_moe_runner(**static) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(replayed, out), "the replay differs from the eager call it captured" + second, hit2 = _window_args(cell, salt=1 << 20) + assert not torch.equal(second["topk_ids"], args["topk_ids"]), "the second step reroutes" + for name in STEP_INPUTS: + static[name].copy_(second[name]) + graph.replay() + torch.cuda.synchronize() + step2 = replayed.clone() + assert torch.equal(step2, fp8_block_scale_moe_runner(**second)), "second replay vs eager" + _moe_check(step2, second, hit2) + + +def _windows_sum(width, tokens): + """A member's windows summed in fp32 pass the 512-expert gate; without any one they fail.""" + seed = 900 + tokens + width + offsets = range(0, CKPT["num_experts"], width) + whole = _args(tokens, **CKPT, seed=seed) + parts = [ + fp8_block_scale_moe_runner( + **_args(tokens, **CKPT, local_expert_offset=o, local_num_experts=width, seed=seed) + ) + for o in offsets + ] + ref = fp8_block_scale_moe_runner.reference(**whole) + _moe_compare(torch.stack([p.float() for p in parts]).sum(0).to(torch.bfloat16), ref) + ids = whole["topk_ids"].long() + for i, o in enumerate(offsets): + if not ((ids >= o) & (ids < o + width)).any(): + continue + short = torch.stack([p.float() for j, p in enumerate(parts) if j != i]).sum(0) + with pytest.raises(AssertionError): + _moe_compare(short.to(torch.bfloat16), ref) + + +def _pad_window(offset: int, width: int): + return M._weights.ExpertWindow(*_stack_window(offset, width), offset=offset, count=width) + + +def _padded_rows(bucket: int, live: int, seed: int) -> torch.Tensor: + """`[bucket, 2560]` bf16: `live` random rows, then the zero rows of a bucket's padding.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.zeros(bucket, 2560, dtype=torch.bfloat16, device=DEV) + x[:live] = torch.randn(live, 2560, device=DEV, generator=g).to(torch.bfloat16) + return x + + +def _padding_rows(offset, width): + """A prefill graph's padding rows (`PrefillTarget._pad_mask`) route to expert -1 and come out + zero, real rows bitwise unchanged, eager and replayed; unmasked they tie onto the lowest ids.""" + core = SimpleNamespace(topk=10, num_experts=512, moe_inter=640) + g = torch.Generator(device=DEV).manual_seed(77) + router = (0.02 * torch.randn(512, 2560, device=DEV, generator=g)).to(torch.bfloat16).t() + win = _pad_window(offset, width) + bucket, live = 2048, 1500 + x = _padded_rows(bucket, live, seed=5) + pad_ids, _ = renorm_moe_routing_op( + cublas_mm(x[live:], router), topk=10, output_dtype=torch.bfloat16 + ) + assert int(pad_ids.max()) < 10, "padding rows no longer tie onto the lowest expert ids" + + plain = M._routed_experts(core, x, router, win) + mask = torch.unsqueeze(torch.arange(bucket, device=DEV) >= live, 1) + masked = M._routed_experts(core, x, router, win, pad_mask=mask) + assert torch.equal(masked[:live], plain[:live]), "the mask changed a real row" + assert not masked[live:].any() and not plain[live:].any(), "a padding row is not zero" + + # As the prefill graph captures it: the mask from a device row count rewritten each step. + live_dev = torch.full([1], live, dtype=torch.int32, device=DEV) + xs = x.clone() + + def body(): + rows = torch.arange(bucket, dtype=torch.int32, device=DEV) + return M._routed_experts( + core, xs, router, win, pad_mask=torch.unsqueeze(rows >= live_dev, 1) + ) + + _warm_on_side_stream(body) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = body() + for n, seed in ((live, 5), (700, 6)): + xs.copy_(_padded_rows(bucket, n, seed)) + live_dev.fill_(n) + graph.replay() + torch.cuda.synchronize() + eager = M._routed_experts(core, xs, router, win) + assert torch.equal(out[:n], eager[:n]), f"live {n}: the replay changed a real row" + assert not out[n:].any(), f"live {n}: a padding row is not zero" + + +@pytest.mark.parametrize( + "kind, width, offset", + [("sum", 128, 0), ("sum", 256, 0), ("pad", 512, 0), ("pad", 256, 0), ("pad", 256, 256)], + ids=lambda v: str(v), +) +def test_moe_windows(kind, width, offset): + """A member's MoE windows sum to the stack; a prefill graph's padding rows route to none.""" + if kind == "sum": + _windows_sum(width, tokens=width) + else: + _padding_rows(offset, width) + + +# ---------------------------------------------------------------- low_m_bf16_mm__decode --- + + +def _low_m_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """Accumulation order only, as `cublas_mm`: 4 bf16 ulp element-wise, 2 ulp relative RMS.""" + assert_within_ulp(out, ref, element_ulp=4.0, rms_ulp=2.0, ulp=ULP[torch.bfloat16]) + + +def _low_m_build(m, n, k, seed): + """`mat_b` is a `[N, K]` weight's `.t()`, as a target stores its projections.""" + g = torch.Generator(device="cuda").manual_seed(seed) + mat_a = torch.randn(m, k, generator=g, device="cuda").to(torch.bfloat16) + weight = (torch.randn(n, k, generator=g, device="cuda") * k**-0.5).to(torch.bfloat16) + return mat_a, weight.t() + + +def _check_low_m(cell): + s = cell.spec + assert low_m_plan(s["m"], s["n"], s["k"]).kind == s["kind"], low_m_plan(s["m"], s["n"], s["k"]) + mat_a, mat_b = _low_m_build(s["m"], s["n"], s["k"], seed=_seed(cell.why) % 2**31) + out = low_m_bf16_mm(mat_a, mat_b, s["pdl"]) + assert out.shape == (s["m"], s["n"]) and out.dtype == torch.bfloat16 and out.is_contiguous() + _low_m_compare(out, low_m_bf16_mm.reference(mat_a, mat_b, s["pdl"])) + + +def test_the_low_m_dependent_chain_under_graph_replay() -> None: + """Decode replays these in CUDA graphs with PDL, each link reading the one before it, so a + kernel that read its activation before `griddepcontrol.wait` would see the previous replay's + values. One chain per branch at the target's rows: warp at M=4, direct at M=2.""" + chains = [] + for m, branches in ((4, ("warp", "warp", "warp")), (2, ("direct", "direct", "warp"))): + o = _low_m_build(m, 2560, 1536, seed=m)[1] + gdn_in = _low_m_build(m, 4128, 2560, seed=10 + m)[1] + attn_in = _low_m_build(m, 4352, 2560, seed=20 + m)[1] + got = tuple(low_m_plan(m, w.shape[1], w.shape[0]).kind for w in (o, gdn_in, attn_in)) + assert got == branches, got + x = torch.zeros(m, 1536, dtype=torch.bfloat16, device="cuda") + chains.append((x, o, gdn_in, attn_in)) + + def run(): + outs = [] + for x, o, gdn_in, attn_in in chains: + h = low_m_bf16_mm(x, o, True) + outs.append((h, low_m_bf16_mm(h, gdn_in, True), low_m_bf16_mm(h, attn_in, True))) + return outs + + _warm_on_side_stream(lambda: (run(), run())) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = run() + for trial in range(3): + g = torch.Generator(device="cuda").manual_seed(100 + trial) + for x, *_ in chains: + x.copy_(torch.randn(x.shape, generator=g, device="cuda").to(x.dtype)) + graph.replay() + torch.cuda.synchronize() + for (x, o, gdn_in, attn_in), (h, g_out, a_out) in zip(chains, outs): + _low_m_compare(h, low_m_bf16_mm.reference(x, o)) + _low_m_compare(g_out, low_m_bf16_mm.reference(h, gdn_in)) + _low_m_compare(a_out, low_m_bf16_mm.reference(h, attn_in)) + + +# ------------------------------------------------------- moe_front_router_quant__decode --- + + +def _front_operands(rows: int, seed: int) -> tuple[torch.Tensor, torch.Tensor]: + """bf16 MoE rows whose 1x128 blocks span many octaves (one zero block), and a router.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(rows, 2560, device=DEV, generator=g) + octaves = torch.randint(-24, 12, (rows, 20, 1), device=DEV, generator=g).float() + x = (x.reshape(rows, 20, 128) * torch.exp2(octaves)).reshape(rows, 2560) + x[0, :128] = 0.0 + router = 0.02 * torch.randn(512, 2560, device=DEV, generator=g) + return x.to(torch.bfloat16).contiguous(), router.to(torch.bfloat16).contiguous() + + +def _front_check(x, router, logits, xq, xsf, label: str) -> None: + """Logits within one bf16 ulp of each row's largest |logit| of the exact product; codes and + scales bitwise `fp8_quantize_1x128`'s, the scales within 4 fp32 ulp of the exact quotient.""" + _, _, ref_scales = moe_front_router_quant.reference(x, router) + exact = x.double() @ router.double().t() + rowmax = exact.abs().amax(dim=-1, keepdim=True) + err = ((logits.double() - exact) / rowmax).abs().max().item() + assert err <= _BF16_ULP, f"{label}: logits {err / _BF16_ULP:.3f} bf16 ulp off" + want_q, want_s = fp8_quantize_1x128(x, use_ue8m0=False) + assert torch.equal(xq.view(torch.uint8), want_q.view(torch.uint8)), f"{label}: e4m3 codes" + assert torch.equal(xsf, want_s), f"{label}: activation scales" + ratio = (xsf.double() / ref_scales.double() - 1.0).abs().max().item() + assert ratio <= 4 * 2.0**-23, f"{label}: scales {ratio:.3g} from the exact quotient" + + +def _check_front(cell): + rows = cell.spec["rows"] + x, router = _front_operands(rows, _seed(cell.why)) + moe_front_router_quant.is_valid(x, router) + logits, xq, xsf = moe_front_router_quant(x, router) + _front_check(x, router, logits, xq, xsf, cell.why[:44]) + if not cell.spec["capture"]: + return + static = x.clone() + _warm_on_side_stream(lambda: moe_front_router_quant(static, router)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = moe_front_router_quant(static, router) + graph.replay() + torch.cuda.synchronize() + for got, want in zip(outs, (logits, xq, xsf)): + assert torch.equal(got.view(torch.uint8), want.view(torch.uint8)) + x2, _ = _front_operands(rows, _seed(cell.why) + 1) + static.copy_(x2) + graph.replay() + torch.cuda.synchronize() + for got, want in zip(outs, moe_front_router_quant(x2, router)): + assert torch.equal(got.view(torch.uint8), want.view(torch.uint8)), "second replay" + _front_check(x2, router, *[o.clone() for o in outs], f"{cell.why[:44]} (second replay)") + + +def _coarse(logits: torch.Tensor) -> torch.Tensor: + """Three values a row: exact ties decide most of the ranking.""" + return (logits.float().sign() * (logits.float().abs() > 0.5)).to(torch.bfloat16) + + +def _runner_routes_as_renorm(rows, width, offset, ties): + """`routing_logits=` is bitwise the runner pre-routed by `renorm_moe_routing_op`.""" + x, router = _front_operands(rows, 7 * rows + width + offset) + logits, xq, xsf = moe_front_router_quant(x, router) + if ties: + logits = _coarse(logits) + ids, weights = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + pre_routed = fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, topk_weights=weights, topk_ids=ids) + ) + routed = fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, routing_logits=logits) + ) + torch.cuda.synchronize() + assert torch.equal(routed, pre_routed), ( + f"{(routed != pre_routed).sum().item()} of {routed.numel()} elements differ" + ) + + +def _front_is_the_logits_routed_runner(rows, width, offset, ties): + """`fp8_block_scale_moe_front` and its folded quantization are bitwise the logits-routed + call and `fp8_quantize_1x128`; a capture replays on new rows (counters reset each launch).""" + + def operands(seed): + x, router = _front_operands(rows, seed) + return x, cublas_mm(x, router.t()) + + def want(x, logits): + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + return fp8_block_scale_moe_runner( + **_runner_kwargs(xq, xsf, offset, width, routing_logits=logits) + ) + + x, logits = operands(13 * rows + width + offset) + if ties: + logits = _coarse(logits) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + weights = _stack_window(offset, width) + geometry = (512, 10, 640, offset, width, 1) + call = fp8_block_scale_moe_front(logits, xq, xsf, *weights, *geometry) + assert call is not None + routed = call(None) + fq, fsf = torch.empty_like(xq), torch.empty_like(xsf) + folded = fp8_block_scale_moe_front(logits, fq, fsf, *weights, *geometry)(x) + ref = want(x, logits) + torch.cuda.synchronize() + assert torch.equal(routed, ref), f"{(routed != ref).sum().item()} of {ref.numel()} differ" + assert torch.equal(folded, ref), f"{(folded != ref).sum().item()} of {ref.numel()} differ" + assert torch.equal(fq.view(torch.uint8), xq.view(torch.uint8)) and torch.equal(fsf, xsf) + + static_x, static_logits = x.clone(), logits.clone() + gq, gsf = torch.empty_like(xq), torch.empty_like(xsf) + fp8_block_scale_moe_front(static_logits, gq, gsf, *weights, *geometry)(static_x) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fp8_block_scale_moe_front(static_logits, gq, gsf, *weights, *geometry)(static_x) + for seed in (1, 2): + x2, logits2 = operands(1000 * seed + rows) + static_x.copy_(x2) + static_logits.copy_(logits2) + graph.replay() + ref2 = want(x2, logits2) + torch.cuda.synchronize() + assert torch.equal(out, ref2), f"replay {seed}: {(out != ref2).sum().item()} differ" + + +def _routed_experts_past_the_front_end(rows, offset, ties, monkeypatch): + """Past `_MOE_FRONT_MAX_ROWS` decode routes in the runner front, bitwise the pre-routed call.""" + x, router = _front_operands(rows, 13 * rows + offset) + if ties: + # A router whose logits take three values a row: ties everywhere. + router = (router.float().sign() * (router.float().abs() > 0.03) * 0.001).to(torch.bfloat16) + window = _pad_window(offset, 256) + core = SimpleNamespace(num_experts=512, topk=10, moe_inter=640, dp_size=1, moe_max_rows=16384) + calls = [] + real = M.renorm_moe_routing_op + + def spy(*args, **kwargs): + calls.append(args[0].shape[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(M, "renorm_moe_routing_op", spy) + new = M._routed_experts(core, x, router.t(), window, mm=M._decode_projection) + assert calls == [], "the logits-routed call launched the renorm routing" + ids, w = real(M._decode_projection(x, router.t()), topk=10, output_dtype=x.dtype) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + old = M._window_runner(core, window, xq, xsf, ids, w) + torch.cuda.synchronize() + assert torch.equal(new, old), f"{(new != old).sum().item()} of {new.numel()} elements differ" + + +def _wide_tactic_under_skewed_routing(rows, offset): + """A logits-routed 256-expert window takes its pinned decode tactic for the row count, which + under skewed routing like a real decode batch's is bitwise the fallback and in the gate.""" + x, router = _front_operands(rows, 17 * rows + offset) + logits = (x.float() @ router.float().t()).to(torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + hot = offset + 7 + logits[:, hot] = logits.abs().amax() + 4.0 + logits[: rows // 3, offset + 100] = logits.abs().amax() + 2.0 + inputs = [logits, None, xq, xsf, *_stack_window(offset, 256), None, None] + geometry = (512, 10, 640, offset, 256, 1, 0, None, 8192, False) + _runner_mod._DISPATCH.clear() + runner, tactic = _runner_mod._dispatch(geometry, inputs) + _runner_mod._DISPATCH.clear() + fallback = [int(v) for v in runner.get_fallback_tactic(2560, rows)] + if rows >= WIDE_DECODE_MIN_ROWS: + want = WIDE_DECODE_TACTIC + else: + want = [t for start, t in WIDE_DECODE_SMALL_TACTICS if rows >= start][-1] + assert list(tactic) == list(want), (tactic, want) + out = runner.forward(inputs, tactic=list(tactic)) + ref = runner.forward(inputs, tactic=fallback) + ids, weights = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + assert bool((ids == hot).any(dim=-1).all()), "the hot expert is not in every row's top-10" + exact = fp8_block_scale_moe_runner.reference( + **_runner_kwargs(xq, xsf, offset, 256, topk_weights=weights, topk_ids=ids) + ) + torch.cuda.synchronize() + assert torch.equal(out, ref), "the chosen tactic changes the output" + _moe_compare(out, exact) + + +_FRONT_WINDOWS = ((128, 384), (256, 256), (512, 0)) +#: `(path, rows, window width, window offset, coarse logits)`. +# fmt: off +_FRONT_CASES = ( + [("runner", r, w, o, t) for r in (1, 16) for w, o in _FRONT_WINDOWS for t in (False, True)] + + [("front", r, w, o, t) for r in (1, 129, 576) for w, o in _FRONT_WINDOWS for t in (False, True)] + + [("routed_experts", r, 256, o, t) for r in (96, 576) for o in (0, 256) for t in (False, True)] + + [("wide_tactic", r, 256, o, False) for r in (3, 96, 167, 168, 576) for o in (0, 256)] +) +# fmt: on + + +@pytest.mark.parametrize("path, rows, width, offset, ties", _FRONT_CASES, ids=lambda v: str(v)) +def test_decode_moe_front_routes_as_renorm(path, rows, width, offset, ties, monkeypatch): + """The decode MoE front end routes as `renorm_moe_routing_op` does, along each `path`.""" + if path == "runner": + _runner_routes_as_renorm(rows, width, offset, ties) + elif path == "front": + _front_is_the_logits_routed_runner(rows, width, offset, ties) + elif path == "routed_experts": + _routed_experts_past_the_front_end(rows, offset, ties, monkeypatch) + else: + _wide_tactic_under_skewed_routing(rows, offset) + + +# ------------------------------------------------ contiguous grouped prefill experts --- + + +def _logits_args(tokens: int, offset: int, local: int, seed: int): + """bf16 router logits and the runner's arguments pre-routed from them over a stack window.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(tokens, 2560, device=DEV, generator=g).to(torch.bfloat16) + logits = torch.randn(tokens, 512, device=DEV, generator=g).to(torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + ids, w = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + routing = dict(topk_weights=w.contiguous(), topk_ids=ids.contiguous()) + return logits, _runner_kwargs(xq, xsf, offset, local, **routing) + + +def _reroute(args: dict, logits: torch.Tensor) -> None: + ids, w = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + args["topk_ids"], args["topk_weights"] = ids.contiguous(), w.contiguous() + + +def _grouped_call(args, **kw): + names = ("hidden_states", "hidden_states_scale", "topk_ids", "topk_weights", "gemm1_weights") + names += ("gemm1_weights_scale", "gemm2_weights", "gemm2_weights_scale", "local_expert_offset") + return _grouped.moe_prefill_grouped(*(args[n] for n in names), **kw) + + +def _grouped_matches_the_reference(tokens, offset, local): + """The contiguous grouped path is in the gate, zero outside the window, deterministic despite + atomic slot ranks, and bitwise the same with either FC1 epilogue.""" + _, args = _logits_args(tokens, offset, local, seed=_seed(f"grouped{tokens}-{offset}")) + out = _grouped_call(args) + _moe_check(out, args, _in_window(args)) + assert torch.equal(out, _grouped_call(args)) + epi = _grouped.EPI_QUANT + try: + _grouped.EPI_QUANT = not epi + assert torch.equal(out, _grouped_call(args)) + finally: + _grouped.EPI_QUANT = epi + + +def _grouped_skewed_and_empty(): + """All rows on ten experts (warmup dummy tokens) pass the gate; an empty window gives zeros.""" + logits, args = _logits_args(2048, 0, 256, seed=_seed("grouped-skew")) + hot = torch.tensor([3, 4, 40, 41, 42, 100, 101, 180, 250, 255], device=DEV) + logits = logits.clone() + logits[:, hot] += 30.0 + _reroute(args, logits) + out = _grouped_call(args) + _moe_check(out, args, _in_window(args)) + assert torch.equal(out, _grouped_call(args)) + args["local_expert_offset"] = 256 + assert torch.equal(_grouped_call(args), torch.zeros_like(out)) + + +def _grouped_fin_first(first): + """`fin_first` rows are final when `after_first` gets the output, and the whole output is + bitwise the one-launch finalize's.""" + logits, args = _logits_args(3000, 128, 256, seed=_seed("grouped-finfirst")) + logits = logits.clone() + logits[:, 140:150] += 4.0 + _reroute(args, logits) + ref = _grouped_call(args) + r0, r1 = (0, 1408) if first == "head" else (1500, ref.shape[0]) + seen = [] + + def after_first(out): + torch.cuda.synchronize() + seen.append(out[r0:r1].clone()) + + got = _grouped_call(args, fin_first=(r0, r1), after_first=after_first) + torch.cuda.synchronize() + assert len(seen) == 1 and torch.equal(seen[0], ref[r0:r1]) + assert torch.equal(got, ref) + + +def _grouped_graph_replay(): + """Replays over two steps' inputs are bitwise their eager calls (tiles rebuilt on device).""" + _, args = _logits_args(2048, 256, 256, seed=_seed("groupedcap")) + _, args2 = _logits_args(2048, 256, 256, seed=_seed("groupedcap2")) + eager, eager2 = _grouped_call(args), _grouped_call(args2) + static = {k: (v.clone() if k in STEP_INPUTS else v) for k, v in args.items()} + _warm_on_side_stream(lambda: _grouped_call(static)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = _grouped_call(static) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager) + for k in STEP_INPUTS: + static[k].copy_(args2[k]) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager2) + + +@pytest.mark.parametrize( + "case", + ["2048-0-256", "4096-256-256", "3000-128-128", "skewed", "fin_first-head", "fin_first-tail", + "graph"], +) # fmt: skip +def test_prefill_grouped(case): + """The contiguous grouped prefill experts against the runner's reference, case by case.""" + if case == "skewed": + _grouped_skewed_and_empty() + elif case.startswith("fin_first-"): + _grouped_fin_first(case.split("-")[1]) + elif case == "graph": + _grouped_graph_replay() + else: + _grouped_matches_the_reference(*map(int, case.split("-"))) + + +# ------------------------------------------------------------------ embed_rows__decode --- + + +def _check_embed_rows(cell): + """Bitwise the reference over int32 and int64 ids (-1 clamped); a captured launch replays.""" + rows, copies, clamp = cell.spec["rows"], cell.spec["copies"], cell.spec["clamp"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + table = torch.randn(1000, 2560, device=DEV, generator=g).to(torch.bfloat16) + for dtype in (torch.int32, torch.int64): + ids = torch.randint(0, 1000, (rows,), device=DEV, generator=g).to(dtype) + if clamp: + ids[-1] = -1 + got = embed_rows(ids, table, copies, clamp) + assert torch.equal( + got.view(torch.int16), embed_rows.reference(ids, table, copies, clamp).view(torch.int16) + ) + static = ids.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = embed_rows(static, table, copies, clamp) + static.copy_(torch.randint(0, 1000, (rows,), device=DEV, generator=g).to(static.dtype)) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, embed_rows.reference(static, table, copies, clamp)) + + +# ------------------------------------------------------- mtp_moe_combine_inject__decode --- + + +def _check_mtp_combine(cell): + """Bitwise the unfused ops, within bf16 rounding of the fp32 reference; a capture replays.""" + rows, hc, hidden = cell.spec["rows"], cell.spec["hc_count"], cell.spec["hidden"] + g = torch.Generator(device=DEV).manual_seed(_seed(cell.why)) + + def operands(): + buf = torch.randn(rows, 2 * hc * hidden, device=DEV, generator=g).to(torch.bfloat16) + x = buf[:, : hc * hidden] + routed = (0.5 * torch.randn(rows, hidden, device=DEV, generator=g)).to(torch.bfloat16) + shared = (0.5 * torch.randn(rows, hidden, device=DEV, generator=g)).to(torch.bfloat16) + gug = (2.0 * torch.randn(rows, 1288, device=DEV, generator=g)).to(torch.bfloat16) + gate = gug[:, 1280:1281] if cell.spec["gate"] == "strided" else gug[:, 1280].contiguous() + inject = (2.0 * torch.sigmoid(torch.randn(rows, hc, device=DEV, generator=g))).to( + torch.bfloat16 + ) + return x, routed, gate, shared, inject + + def composed(x, routed, gate, shared, inject): + return qwen4_exp_post_inject( + x, fused_sigmoid_gate_mul_add(routed.clone(), gate, shared), inject + ) + + ops = operands() + got = mtp_moe_combine_inject(*ops) + assert torch.equal(got.view(torch.int16), composed(*ops).view(torch.int16)) + torch.testing.assert_close( + got.float(), mtp_moe_combine_inject.reference(*ops).float(), atol=0.07, rtol=0.02 + ) + static = [t.clone() for t in ops] + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = mtp_moe_combine_inject(*static) + new = operands() + for s, n in zip(static, new): + s.copy_(n) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, composed(*new)) + + +# ------------------------------------------------------------ moe_tpb_decode__decode --- + + +_TPB_BLOCKS: dict = {} + + +def _tpb_dims(rank: int) -> SimpleNamespace: + """The loader's dims of tep4 rank `rank` with the block-major copy on.""" + return SimpleNamespace( + moe_tpb=True, tp_rank=rank, num_experts=512, moe_inter=640, shared_inter=640, + hidden=2560, dtype=torch.bfloat16, + ) # fmt: skip + + +def _loaded(d, fake, names) -> list: + """The loader's parameters `names` of trunk layer 0, filled from `fake` as `load` fills them.""" + entries = {e.name: e for e in M._weights.MODEL_WEIGHTS.WEIGHTS} + out = [] + for name in names: + e = entries[name] + param = torch.empty(e.shape(d), dtype=e.dtype or d.dtype, device=DEV) + sources = [(*s, e.transform) for s in e.sources(d, 0, "")] + M._weights.MODEL_WEIGHTS.fill(param, sources, fake, d) + out.append(param) + return out + + +def _tpb_blocks(rank: int): + """Tep4 rank `rank`'s block-major routed experts as the loader lays them out from a + checkpoint holding the stack `_weights` draws (rows `[up ; gate]`). Cached per rank.""" + if rank not in _TPB_BLOCKS: + w1, w1s, w2, w2s = _weights(2560, 640, 512, seed=2560 + 640 + 512) + p = f"{M._weights._trunk_mlp(0)}.experts" + fake = {} + for e in range(512): + fake.update({ + f"{p}.{e}.up_proj.weight": w1[e, :640], + f"{p}.{e}.gate_proj.weight": w1[e, 640:], + f"{p}.{e}.down_proj.weight": w2[e], + f"{p}.{e}.up_proj.weight_scale_inv": w1s[e, :5], + f"{p}.{e}.gate_proj.weight_scale_inv": w1s[e, 5:], + f"{p}.{e}.down_proj.weight_scale_inv": w2s[e], + }) # fmt: skip + names = ("tpb_w1", "tpb_s1", "tpb_w2", "tpb_s2") + _TPB_BLOCKS[rank] = _tpb.BlockExperts(*_loaded(_tpb_dims(rank), fake, names), rank) + return _TPB_BLOCKS[rank] + + +def _tpb_shared_blocks(gate, up, dn, rank: int): + """Tep4 rank `rank`'s shared-expert slice `(gu, dn)` as the loader lays it out.""" + p = f"{M._weights._trunk_mlp(0)}.shared_expert" + fake = {f"{p}.gate_proj.weight": gate, f"{p}.up_proj.weight": up, f"{p}.down_proj.weight": dn} + return tuple(_loaded(_tpb_dims(rank), fake, ("tpb_sh_gu", "tpb_sh_dn"))) + + +def _tpb_call(fn, logits, args, blocks, **kw): + return fn(logits, args["hidden_states"], args["hidden_states_scale"], blocks, **kw) + + +def _tpb_shared(rows: int, seed: int): + """A MoE site's bf16 rows and a bf16 shared expert: gate / up rows `[640, 2560]`, the + gate-logit row `[2560]` and the down projection `[2560, 640]`.""" + g = torch.Generator(device=DEV).manual_seed(seed) + x = torch.randn(rows, 2560, device=DEV, generator=g) + x = (x / x.abs().amax(-1, keepdim=True)).to(torch.bfloat16).contiguous() + gate = (0.02 * torch.randn(640, 2560, device=DEV, generator=g)).to(torch.bfloat16) + up = (0.02 * torch.randn(640, 2560, device=DEV, generator=g)).to(torch.bfloat16) + gl = (0.02 * torch.randn(2560, device=DEV, generator=g)).to(torch.bfloat16).contiguous() + dn = (0.02 * torch.randn(2560, 640, device=DEV, generator=g)).to(torch.bfloat16) + return x, gate, up, gl, dn + + +def _unfused_shared(x, gate, up, gl, dn): + """`(sigmoid(gate logit), shared)` of the unfused bf16 path, in fp32.""" + with true_fp32_matmul(): + gugate = (x.float() @ torch.cat([gate, up]).float().t()).to(torch.bfloat16).float() + act = ( + (torch.nn.functional.silu(gugate[:, :640]) * gugate[:, 640:]).to(torch.bfloat16).float() + ) + shared = (act @ dn.float().t()).to(torch.bfloat16).float() + logit = (x.float() @ gl.float()).to(torch.bfloat16).float().unsqueeze(1) + return torch.sigmoid(logit), shared + + +def _check_tpb(cell): + """Block-major decode experts in the gate of `reference`, deterministic, and replayable.""" + rows, rank = cell.spec["rows"], cell.spec["rank"] + logits, args = _logits_args(rows, 0, 512, seed=_seed(cell.why)) + blocks = _tpb_blocks(rank) + out = _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks) + _moe_compare(out, _tpb_call(_tpb.moe_tpb_decode.reference, logits, args, blocks)) + assert torch.equal(out, _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks)), "a re-run" + if not cell.spec["capture"]: + return + logits2, args2 = _logits_args(rows, 0, 512, seed=_seed(cell.why) + 1) + eager2 = _tpb_call(_tpb.moe_tpb_decode, logits2, args2, blocks) + slog = logits.clone() + sx = args["hidden_states"].clone() + ssf = args["hidden_states_scale"].clone() + _warm_on_side_stream(lambda: _tpb.moe_tpb_decode(slog, sx, ssf, blocks)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = _tpb.moe_tpb_decode(slog, sx, ssf, blocks) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, out) + slog.copy_(logits2) + sx.copy_(args2["hidden_states"]) + ssf.copy_(args2["hidden_states_scale"]) + graph.replay() + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, eager2) + + +def _tpb_ranks_sum(rows): + """The four round-robin ranks' partials summed in fp32 sit in the MoE gate of the 512-expert + runner reference and of the trtllm-gen runner's four 128-expert windows summed alike.""" + logits, args = _logits_args(rows, 0, 512, seed=_seed(f"tpb-sum-{rows}")) + parts = [_tpb_call(_tpb.moe_tpb_decode, logits, args, _tpb_blocks(r)) for r in range(4)] + total = sum(p.float() for p in parts).to(torch.bfloat16) + with true_fp32_matmul(): + ref = fp8_block_scale_moe_runner.reference(**args) + _moe_compare(total, ref) + xq, xsf = args["hidden_states"], args["hidden_states_scale"] + routing = dict(topk_weights=args["topk_weights"], topk_ids=args["topk_ids"]) + windows = [ + fp8_block_scale_moe_runner(**_runner_kwargs(xq, xsf, 128 * r, 128, **routing)) + for r in range(4) + ] + _moe_compare(total, sum(p.float() for p in windows).to(torch.bfloat16)) + + +def _tpb_shared_slice(rows): + """With the shared fold each rank's partial sits in its reference's gate, and the four summed + sit in the layer's: the routed reference + sigmoid(gate logit) x the unfused shared expert.""" + logits, args = _logits_args(rows, 0, 512, seed=_seed(f"tpb-shared-{rows}")) + x, gate, up, gl, dn = _tpb_shared(rows, _seed(f"tpb-shared-w-{rows}")) + total = torch.zeros(rows, 2560, device=DEV) + for r in range(4): + gu_r, dn_r = _tpb_shared_blocks(gate, up, dn, r) + sh = _tpb.SharedBlocks(x, gu_r, gl, dn_r) + blocks = _tpb_blocks(r) + out = _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks, shared=sh) + with true_fp32_matmul(): + ref = _tpb_call(_tpb.moe_tpb_decode.reference, logits, args, blocks, shared=sh) + _moe_compare(out, ref) + assert torch.equal(out, _tpb_call(_tpb.moe_tpb_decode, logits, args, blocks, shared=sh)) + total += out.float() + with true_fp32_matmul(): + routed = fp8_block_scale_moe_runner.reference(**args).float() + scale, shared = _unfused_shared(x, gate, up, gl, dn) + _moe_compare(total.to(torch.bfloat16), (routed + scale * shared).to(torch.bfloat16)) + + +def _tpb_blocks_dealt(): + """The four ranks' 640 blocks are every (expert, block) of the 512 x 5 exactly once, each rank + holds a block of every expert, and `weights.tpb_blocks` agrees with the op's `block_ids`.""" + seen = set() + for r in range(4): + d = SimpleNamespace(moe_tpb=True, tp_rank=r, num_experts=512, moe_inter=640) + blocks = M._weights.tpb_blocks(d) + e, b = _tpb.block_ids(r) + assert blocks == list(zip(e, b)) + assert len(blocks) == 640 and {x for x, _ in blocks} == set(range(512)) + assert all((x + y) % 4 == r for x, y in blocks) + seen |= set(blocks) + assert seen == {(x, y) for x in range(512) for y in range(5)} + + +def _tpb_window(rank: int, shared) -> "M._weights.ExpertWindow": + """Tep4 rank `rank`'s 128-expert window with its block-major copy and shared-expert slice.""" + bl = _tpb_blocks(rank) + gate, up, gl, dn = shared + sh_gu, sh_dn = _tpb_shared_blocks(gate, up, dn, rank) + bw = M._weights.BlockWindow(bl.w1, bl.s1, bl.w2, bl.s2, rank, sh_gu, gl, sh_dn) + return _pad_window(128 * rank, 128)._replace(blocks=bw) + + +def _moe_block_folds_shared(rows, monkeypatch): + """A deferred tep4 decode step of `_MOE_TPB_SHARED_MIN_ROWS` (5) to 8 rows folds each rank's + shared-expert slice into its block launch, the four partials summing into the gate of the + unfused `routed + sigmoid(gate) * shared`. A 3-row step keeps the side-stream shared expert.""" + x, router = _front_operands(rows, 57 + rows) + x = (x.float() / x.float().abs().amax(dim=-1, keepdim=True)).to(torch.bfloat16).contiguous() + _, gate, up, gl, dn = _tpb_shared(rows, 91 + rows) + gug = torch.zeros(1288, 2560, dtype=torch.bfloat16, device=DEV) + gug[:640], gug[640:1280], gug[1280] = gate, up, gl + folded_calls = [] + real = M.moe_tpb_decode + + def spy(*args, **kwargs): + folded_calls.append( + len(args) > 4 and args[4] is not None or kwargs.get("shared") is not None + ) + return real(*args, **kwargs) + + monkeypatch.setattr(M, "moe_tpb_decode", spy) + + def block(rank): + core = SimpleNamespace( + num_experts=512, topk=10, moe_inter=640, shared_inter=640, dp_size=1, + tp_sum=lambda t: t, _side_stream=torch.cuda.Stream(), + _side_fork=torch.cuda.Event(), _side_join=torch.cuda.Event(), + ) # fmt: skip + L = SimpleNamespace( + router=router.t(), + experts=_tpb_window(rank, (gate, up, gl, dn)), + sh_gug=gug.t(), + sh_dn=dn.t(), + ) + return M._moe_block(core, x, L, None, mm=M._decode_projection, defer=True) + + out = [block(r) for r in range(4)] + torch.cuda.synchronize() + fold = rows >= M._MOE_TPB_SHARED_MIN_ROWS + assert folded_calls == [fold] * 4, folded_calls + # The folded partials already carry sigmoid(gate) * shared; the unfolded block returns it apart. + got = sum(p.float() for p, _, _ in out) + if fold: + assert all(g is None and s is None for _, g, s in out), "a folded block returned a shared" + else: + _, g0, s0 = out[0] + got = got + torch.sigmoid(g0.float()) * s0.float() + scale, shared = _unfused_shared(x, gate, up, gl, dn) + logits = moe_front_router_quant(x, router)[0] + ids, wts = renorm_moe_routing_op(logits, topk=10, output_dtype=torch.bfloat16) + xq, xsf = fp8_quantize_1x128(x, use_ue8m0=False) + args = _runner_kwargs(xq, xsf, 0, 512, topk_ids=ids, topk_weights=wts) + with true_fp32_matmul(): + ref_routed = fp8_block_scale_moe_runner.reference(**args).float() + _moe_compare(got.to(torch.bfloat16), (ref_routed + scale * shared).to(torch.bfloat16)) + + +_TPB_CASES = ( + [("ranks_sum", rows) for rows in (1, 3, 6, 8)] + + [("shared_slice", rows) for rows in (3, 6, 8)] + + [("moe_block", rows) for rows in (3, 5, 8)] + + [("blocks_dealt", 0)] +) + + +@pytest.mark.parametrize("case, rows", _TPB_CASES, ids=lambda v: str(v)) +def test_tpb_decode_moe(case, rows, monkeypatch): + """The TPB decode experts: rank sums, the shared-expert fold, and the block deal.""" + if case == "ranks_sum": + _tpb_ranks_sum(rows) + elif case == "shared_slice": + _tpb_shared_slice(rows) + elif case == "moe_block": + _moe_block_folds_shared(rows, monkeypatch) + else: + _tpb_blocks_dealt() + + +# ------------------------------------------------------------------ certified cells --- + +#: Each entry and its checker, which builds a cell's operands and compares entry and `reference`. +_CELL_CHECKS = ( + (fp8_block_scale_moe_runner, _check_moe_runner), + (low_m_bf16_mm, _check_low_m), + (moe_front_router_quant, _check_front), + (embed_rows, _check_embed_rows), + (mtp_moe_combine_inject, _check_mtp_combine), + (_tpb.moe_tpb_decode, _check_tpb), +) +CASES = [(entry, check, cell) for entry, check in _CELL_CHECKS for cell in entry.CELLS] + + +@pytest.mark.parametrize( + "entry, check, cell", CASES, ids=[f"{type(e).__name__}-{c.why[:40]}" for e, _, c in CASES] +) +def test_certified_cells(entry, check, cell, certifying): # noqa: F811 — fixture by name + certifying(entry) + check(cell) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py new file mode 100644 index 000000000000..743d60c7b5c4 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_routing_and_layout.py @@ -0,0 +1,2006 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Qwen3.8-Flash-Next-FP8 unified target: routing and deployment checks, each rank's share of the +core (on the meta device) and the per-rank KV / recurrent pool layout.""" + +import contextlib +from dataclasses import replace +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + pool as P, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as W, +) +from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight +from tensorrt_llm._torch.speculative.interface import should_use_separate_draft_kv_cache +from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm.llmapi import QSASparseAttentionConfig +from tensorrt_llm.llmapi.llm_args import ( + KvCacheConfig, + MoeConfig, + MTPDecodingConfig, + MultimodalConfig, + NGramDecodingConfig, + TorchLlmArgs, +) +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + +# ------------------------------------------------------------------ the members --- + +NUM_LAYERS = 48 + +#: The seven members; tp2 and tp4 leave the experts to Mapping's default split (the TP size). +MEMBERS = { + "tp1": dict(world_size=1, tp_size=1), + "tp2": dict(world_size=2, tp_size=2), + "tep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2), + "dep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2, enable_attention_dp=True), + "tp4": dict(world_size=4, tp_size=4), + "tep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4), + "dep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4, enable_attention_dp=True), +} + +_RANKS = [(m, r) for m, layout in MEMBERS.items() for r in range(layout["world_size"])] +#: The last rank of one member per attention layout: one rank, TP2, TP4, attention DP. +LAYOUT_RANKS = [("tp1", 0), ("tp2", 1), ("tp4", 3), ("dep4", 3)] + + +def _mapping(member: str, rank: int) -> Mapping: + return Mapping(rank=rank, gpus_per_node=4, pp_size=1, **MEMBERS[member]) + + +def _world(member: str) -> int: + return MEMBERS[member]["world_size"] + + +_needs_gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") + + +# --------------------------------------------------------------------- routing --- + +_DUMMY_MODEL = "/tmp/dummy_model" +_ARCH = "Qwen4ExpForConditionalGeneration" +_TARGET = ( + "models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.modeling." + "ModelingV2Qwen38FlashNextFp8Sm103Unified" +) +_NVFP4_DEFAULT_TARGET = ( + "models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default.modeling." + "ModelingV2Qwen38FlashNextNvfp4Sm103Default" +) + +# The quantization the engine resolves for the FP8 (128x128 blocks) and the NVFP4 exports. +_FP8 = QuantAlgo.FP8_BLOCK_SCALES +_NVFP4 = QuantAlgo.MIXED_PRECISION + + +def _qualified(target) -> str: + return f"{target.__module__.split('.modeling_v2.')[1]}.{target.__name__}" + + +@pytest.fixture +def on_sm103(monkeypatch): + """Route as if on a GB300, with the GDN cached replay at its default (on).""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + for env in ( + "TRTLLM_USE_GDN_FLASHINFER_REPLAY", + M.MOE_COMM_METHOD_ENV, + M.ROWTILE_SCRATCH_GROUPS_ENV, + W.MOE_TPB_ENV, + ple_host_table.PLE_HOST_TABLE_ENV, + ): + monkeypatch.delenv(env, raising=False) + + +def _llm_args(mode="auto", **kwargs) -> TorchLlmArgs: + return TorchLlmArgs(model=_DUMMY_MODEL, modeling_v2=mode, **kwargs) + + +# The checkpoint's `mtp` block. +_MTP_BLOCK = dict( + hybrid=True, + layer_types=["full_attention"], + mtp_use_hidden_state_from_layer=None, + num_hidden_layers=1, + rope_theta=10000000, +) + + +def _qwen4_exp_config(text_overrides=None, composite_overrides=None, **vision_overrides): + """Qwen3.8-Flash-Next's normalized composite config (the FP8 and NVFP4 exports alike).""" + text = dict( + num_hidden_layers=48, + hidden_size=2560, + num_experts=512, + num_attention_heads=24, + linear_num_value_heads=48, + vocab_size=248320, + # The settings the target pins (`modeling._TEXT_SETTINGS`), as the normalized config + # carries them: rope theta and partial factor flattened to the top level. + norm_topk_prob=True, + num_experts_per_tok=10, + hidden_act="silu", + num_key_value_heads=2, + head_dim=256, + attention_bias=False, + output_gate_type="sigmoid", + tie_word_embeddings=False, + rope_theta=10000000, + partial_rotary_factor=0.25, + rope_scaling=dict(type="mrope", mrope_section=[11, 11, 10], mrope_interleaved=True), + max_position_embeddings=262144, + mtp=dict(_MTP_BLOCK), + mtp_num_hidden_layers=1, + mtp_use_dedicated_embeddings=False, + ) + text.update(text_overrides or {}) + vision = dict( + depth=27, + hidden_size=1152, + num_heads=16, + spatial_merge_size=2, + out_hidden_size=2560, + num_position_embeddings=2304, + deepstack_visual_indexes=[], + hidden_act="gelu_pytorch_tanh", + ) + vision.update(vision_overrides) + return PretrainedConfig( + architectures=[_ARCH], + text_config=PretrainedConfig(**text), + vision_config=PretrainedConfig(**vision), + **(composite_overrides or {}), + ) + + +def _routed_config( + member=None, spec_config=None, pretrained_config=None, quant_algo=_FP8, **mapping_kwargs +) -> ModelConfig: + if member is not None: + mapping_kwargs = dict(MEMBERS[member], **mapping_kwargs) + return ModelConfig( + pretrained_config=pretrained_config or _qwen4_exp_config(), + mapping=Mapping(**mapping_kwargs), + spec_config=spec_config, + quant_config=QuantConfig(quant_algo=quant_algo, kv_cache_quant_algo=None), + ) + + +def _admitted(config, **llm_args_kwargs) -> None: + for mode in ("auto", "require"): + target = modeling_v2_resolve(config, _llm_args(mode, **llm_args_kwargs)) + assert target is not None and _qualified(target) == _TARGET, mode + + +def _mtp_with(**attributes): + """Sets post-load MTP fields directly: the constructor would remap or recompute them.""" + config = MTPDecodingConfig(max_draft_len=3) + for name, value in attributes.items(): + setattr(config, name, value) + return config + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, parallel", + [ + ("tp1", dict(tensor_parallel_size=1, moe_expert_parallel_size=1)), + ("tp2", dict(tensor_parallel_size=2, moe_expert_parallel_size=1)), + ("tep2", dict(tensor_parallel_size=2, moe_expert_parallel_size=2)), + ( + "dep2", + dict(tensor_parallel_size=2, moe_expert_parallel_size=2, enable_attention_dp=True), + ), + ("tp4", dict(tensor_parallel_size=4)), + ("tep4", dict(tensor_parallel_size=4, moe_expert_parallel_size=4)), + ( + "dep4", + dict(tensor_parallel_size=4, moe_expert_parallel_size=4, enable_attention_dp=True), + ), + ], +) +def test_each_member_is_admitted_as_the_llm_api_spells_it(on_sm103, member, parallel, monkeypatch): + """Each member's LLM API spelling builds its mapping and is admitted. The node size is pinned: + inferred from one visible GPU, a multi-rank world would span nodes, which `layout` declines.""" + monkeypatch.setattr(torch.cuda, "device_count", lambda: 1) + mapping = _llm_args(gpus_per_node=4, **parallel).parallel_config.to_mapping() + assert routing.layout(mapping, routing._FP8_LAYOUTS) == member + _admitted(_routed_config(member)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, spec_config, llm_args_kwargs", + [ + ("tp1", MTPDecodingConfig(max_draft_len=1), {}), + ("tep2", MTPDecodingConfig(max_draft_len=7), dict(prefill_cuda_graph_backend="breakable")), + ( + "tep4", + MTPDecodingConfig(max_draft_len=3), + dict(kv_cache_config=KvCacheConfig(dtype="fp8")), + ), + ( + "dep4", + MTPDecodingConfig(max_draft_len=3), + dict(kv_cache_config=KvCacheConfig(dtype="nvfp4")), + ), + ("tp4", None, dict(kv_cache_config=KvCacheConfig(mamba_ssm_cache_dtype="bfloat16"))), + # A window of the full length, max_seq_len unset (inferred from max_position_embeddings). + ("tep4", None, dict(kv_cache_config=KvCacheConfig(max_attention_window=[262144]))), + ], + ids=[ + "tp1-mtp1", + "tep2-mtp7-breakable", + "tep4-mtp3-fp8-kv", + "dep4-mtp3-nvfp4-kv", + "tp4-bf16-state", + "tep4-full-window", + ], +) +def test_the_engine_choices_the_target_runs_are_admitted( + on_sm103, member, spec_config, llm_args_kwargs +): + _admitted(_routed_config(member, spec_config), **llm_args_kwargs) + + +_MTP3 = MTPDecodingConfig(max_draft_len=3) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "config_kwargs, llm_args_kwargs, match", + [ + (dict(member="dep2", spec_config=_MTP3), {}, r"attention DP \(dep2\)"), + (dict(member="dep4"), dict(prefill_cuda_graph_backend="breakable"), "breakable"), + (dict(member="tep4"), dict(prefill_cuda_graph_backend="piecewise"), "piecewise"), + ( + dict(member="tep4", spec_config=_MTP3), + dict(kv_cache_config=KvCacheConfig(mamba_ssm_cache_dtype="float32")), + "recurrent-state cache", + ), + (dict(member="tep4"), dict(kv_cache_config=KvCacheConfig(dtype="fp8_ds_mla")), "KV cache"), + ( + dict( + member="tep4", + spec_config=NGramDecodingConfig(max_draft_len=3, max_matching_ngram_size=2), + ), + {}, + "one-model MTP-Eagle", + ), + ( + dict(member="tep4", spec_config=MTPDecodingConfig(max_draft_len=8)), + {}, + "max_draft_len=8", + ), + ( + dict(member="tep4", spec_config=_mtp_with(max_total_draft_tokens=5)), + {}, + "linear chain", + ), + ( + dict(member="tep4", spec_config=MTPDecodingConfig(max_draft_len=3, max_concurrency=8)), + {}, + "draft length", + ), + (dict(world_size=2, tp_size=1, pp_size=2), {}, "tp2 or tep2"), + (dict(MEMBERS["tep4"], gpus_per_node=2), {}, "tp2 or tep2"), + (dict(MEMBERS["dep4"], enable_lm_head_tp_in_adp=True), {}, "tp2 or tep2"), + (dict(world_size=8, tp_size=8, moe_tp_size=1, moe_ep_size=8), {}, "tp2 or tep2"), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(num_hidden_layers=60))), + {}, + "text shape", + ), + ( + dict( + member="tep4", + pretrained_config=_qwen4_exp_config(composite_overrides=dict(dtype="float16")), + ), + {}, + "torch.float16", + ), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(norm_topk_prob=False))), + {}, + "'norm_topk_prob'", + ), + ( + dict(member="tep4", pretrained_config=_qwen4_exp_config(dict(rope_theta=1e6))), + {}, + "'rope_theta'", + ), + ( + dict( + member="tep4", + pretrained_config=_qwen4_exp_config( + dict(mtp=dict(_MTP_BLOCK, layer_types=["linear_attention"])) + ), + ), + {}, + "'mtp.layer_types'", + ), + ( + dict(member="tep4"), + dict(max_seq_len=8192, kv_cache_config=KvCacheConfig(max_attention_window=[4096])), + "sliding window", + ), + ( + dict(member="tep4"), + dict(kv_cache_config=KvCacheConfig(max_attention_window=[32768])), + "sliding window", + ), + ( + dict(member="tep4"), + dict(moe_config=MoeConfig(load_balancer={"num_slots": 512})), + r"load_balancer \(EPLB\)", + ), + ( + dict(member="tep4"), + dict(multimodal_config=MultimodalConfig(vision_gpu_normalize=True)), + "vision_gpu_normalize", + ), + ], + ids=[ + "dep2-mtp", + "dep4-breakable", + "piecewise", + "fp32-state", + "kv-algorithm", + "ngram", + "draft8", + "max-total-draft", + "runtime-draft-length", + "pp2", + "two-nodes", + "dep4-lm-head-tp", + "tep8", + "text-shape", + "fp16-model", + "other-routing-setting", + "other-rope-theta", + "other-mtp-layer", + "sliding-window", + "sliding-window-inferred-max-seq-len", + "eplb", + "vision-gpu-normalize", + ], +) +def test_what_the_target_does_not_run_declines(on_sm103, config_kwargs, llm_args_kwargs, match): + """``auto`` falls back to the built-in model and ``require`` says why, once per reason.""" + config = _routed_config(**config_kwargs) + assert modeling_v2_resolve(config, _llm_args("auto", **llm_args_kwargs)) is None + with pytest.raises(ValueError, match=match): + modeling_v2_resolve(config, _llm_args("require", **llm_args_kwargs)) + + +@pytest.mark.cpu_only +def test_another_moe_backend_or_gpu_or_no_gdn_replay_declines(on_sm103, monkeypatch): + """The engine choices read off the `ModelConfig` or the process rather than `TorchLlmArgs`.""" + config = replace(_routed_config("tep4"), moe_backend_requested="CUTLASS") + with pytest.raises(ValueError, match="moe_config.backend"): + modeling_v2_resolve(config, _llm_args("require")) + monkeypatch.setenv("TRTLLM_USE_GDN_REPLAY", "0") + with pytest.raises(ValueError, match="TRTLLM_USE_GDN_REPLAY=0"): + modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 0)) + with pytest.raises(ValueError, match=r"compute capability \(10, 0\)"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("require")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY") + monkeypatch.setenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY", "1") + with pytest.raises(ValueError, match="TRTLLM_USE_GDN_FLASHINFER_REPLAY"): + modeling_v2_resolve(_routed_config("tep4", _MTP3), _llm_args("require")) + monkeypatch.delenv("TRTLLM_USE_GDN_FLASHINFER_REPLAY") + # Attention TP has no MoE round trip: the stock override is ignored there, as the stock + # factory ignores it. + for method in ("nvlink_one_sided", "NVLINK_TWO_SIDED"): + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, method) + for member in ("tp2", "tep4"): + _admitted(_routed_config(member)) + # An attention-DP all-to-all MoE the deployment cannot run: no MNNVL. + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "nvlink_one_sided") + with mock.patch( + "tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided.NVLinkOneSided" + ".is_platform_supported", + return_value=False, + ): + config = _routed_config("dep4") + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_ONE_SIDED at dep4"): + modeling_v2_resolve(config, _llm_args("require")) + # A stock method the target does not run declines rather than routing as the default. + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "NVLINK_TWO_SIDED") + with pytest.raises(ValueError, match=f"{M.MOE_COMM_METHOD_ENV}=NVLINK_TWO_SIDED at dep4"): + modeling_v2_resolve(_routed_config("dep4"), _llm_args("require")) + # `ALLGATHER`, in any case, is the default round trip. + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "allgather") + _admitted(_routed_config("dep4")) + monkeypatch.delenv(M.MOE_COMM_METHOD_ENV) + monkeypatch.setenv(W.MOE_TPB_ENV, "1") + with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}='1': expected rr"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) + monkeypatch.delenv(W.MOE_TPB_ENV) + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "nodes") + with pytest.raises(ValueError, match=f"{ple_host_table.PLE_HOST_TABLE_ENV}='nodes': expected"): + modeling_v2_resolve(_routed_config("tep4"), _llm_args("auto")) + monkeypatch.delenv(ple_host_table.PLE_HOST_TABLE_ENV) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("strategy", ["UB", "LOWPRECISION"]) +def test_a_strategy_only_the_stock_module_sets_up_declines(on_sm103, strategy): + """The target calls the all-reduce op directly: under attention TP a strategy whose buffers + the stock `AllReduce` module sets up declines; with no TP sum (one rank, attention DP) it is + admitted.""" + for member in ("tp2", "tep2", "tp4", "tep4"): + config = replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy]) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"allreduce_strategy={strategy} at {member}"): + modeling_v2_resolve(config, _llm_args("require")) + for member in ("tp1", "dep2", "dep4"): + _admitted(replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy])) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy", ["NCCL", "NCCL_SYMMETRIC", "MIN_LATENCY", "ONESHOT", "TWOSHOT", "MNNVL", "SYMM_MEM"] +) +def test_a_strategy_the_target_runs_is_admitted(on_sm103, strategy): + """The raw all-reduce op runs these; MNNVL takes the target's MNNVL sum and the raw op at + AUTO, SYMM_MEM the raw op at AUTO.""" + for member in ("tp2", "tep4"): + _admitted(replace(_routed_config(member), allreduce_strategy=AllReduceStrategy[strategy])) + + +@pytest.mark.cpu_only +def test_a_switch_that_would_do_nothing_declines(on_sm103, monkeypatch): + """`MOE_TPB_ENV=rr` outside tp4 / tep4 and `ROWTILE_SCRATCH_GROUPS_ENV` without an FP8 KV + cache decline, so a run that sets one never measures the target without it.""" + monkeypatch.setenv(W.MOE_TPB_ENV, "rr") + for member in ("tp1", "tp2", "tep2", "dep2", "dep4"): + config = _routed_config(member) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{W.MOE_TPB_ENV}=rr at {member}"): + modeling_v2_resolve(config, _llm_args("require")) + for member in ("tp4", "tep4"): + _admitted(_routed_config(member)) + monkeypatch.delenv(W.MOE_TPB_ENV) + monkeypatch.setenv(M.ROWTILE_SCRATCH_GROUPS_ENV, "1") + for kv, stored in ((KvCacheConfig(), "bf16"), (KvCacheConfig(dtype="nvfp4"), "NVFP4")): + config = _routed_config("tep4") + assert modeling_v2_resolve(config, _llm_args("auto", kv_cache_config=kv)) is None + with pytest.raises(ValueError, match=f"{M.ROWTILE_SCRATCH_GROUPS_ENV} with a {stored} KV"): + modeling_v2_resolve(config, _llm_args("require", kv_cache_config=kv)) + _admitted(_routed_config("tep4"), kv_cache_config=KvCacheConfig(dtype="fp8")) + + +@pytest.mark.cpu_only +def test_the_quantization_picks_the_checkpoint(on_sm103): + """MIXED_PRECISION routes to the NVFP4 target (its own checks mocked), FP8_BLOCK_SCALES here.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 + modeling as nvfp4_modeling, + ) + + with mock.patch.object( + nvfp4_modeling.ModelingV2Qwen38FlashNextNvfp4Sm103Default, "check_deployment" + ): + target = modeling_v2_resolve( + _routed_config("tep4", quant_algo=_NVFP4), _llm_args("require") + ) + assert _qualified(target) == _NVFP4_DEFAULT_TARGET + _admitted(_routed_config("tep4")) + + +# ------------------------------------------------------------------------ the core --- + +# Qwen3.8-Flash-Next's text config (the FP8 export's `text_config`). +# fmt: off +_TEXT = dict( + architectures=["Qwen4ExpForCausalLM"], attention_bias=False, attention_dropout=0.0, + bos_token_id=248044, dtype="bfloat16", eos_token_id=248044, full_attention_interval=4, + hc_count=4, hc_lowrank=320, head_dim=256, heads_per_ngram=8, hidden_act="silu", + hidden_size=2560, indexer_budget=2048, indexer_compress_ratio=4, indexer_head_dim=128, + indexer_kv_heads=1, indexer_n_heads=4, + layer_types=["full_attention" if i % 4 == 3 else "linear_attention" for i in range(NUM_LAYERS)], + linear_conv_kernel_dim=4, linear_key_head_dim=128, linear_num_key_heads=16, + linear_num_value_heads=48, linear_value_head_dim=128, make_ngram_vocab_size_divisible_by=128, + mamba_ssm_dtype="float32", max_position_embeddings=262144, moe_intermediate_size=640, + mtp=dict(hybrid=True, layer_types=["full_attention"], mtp_use_hidden_state_from_layer=None, + num_hidden_layers=1, rope_theta=10000000), + mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=False, ngram_size=3, + ngram_vocab_size_base=20000000, num_attention_heads=24, num_experts=512, + num_experts_per_tok=10, num_hidden_layers=NUM_LAYERS, num_key_value_heads=2, + output_gate_type="sigmoid", partial_rotary_factor=0.25, ple_conv_kernel_size=4, + ple_embed_dim=2560, ple_layer_ids=[2], rms_norm_eps=1e-06, + rope_parameters=dict(mrope_interleaved=True, mrope_section=[11, 11, 10], + partial_rotary_factor=0.25, rope_theta=10000000, rope_type="default"), + shared_expert_intermediate_size=640, split_ngram_parts=128, tie_word_embeddings=False, + vocab_size=248320, +) +_VISION = dict( + deepstack_visual_indexes=[], depth=27, dtype="bfloat16", hidden_act="gelu_pytorch_tanh", + hidden_size=1152, in_channels=3, intermediate_size=4304, num_heads=16, + num_position_embeddings=2304, out_hidden_size=2560, patch_size=16, spatial_merge_size=2, + temporal_patch_size=2, +) +# fmt: on + +HIDDEN = 2560 +NUM_EXPERTS = 512 +TOPK = 10 + +# Per member: (q heads, KV heads, attn row, rope "v" heads, GDN k / v heads, in_proj split, o K). +_GEOMETRY = { + "tp1": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), + "tp2": (12, 1, 7424, 16, 8, 24, (5120, 3072, 24, 24), 3072), + "tep2": (12, 1, 7424, 16, 8, 24, (5120, 3072, 24, 24), 3072), + "dep2": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), + "tp4": (6, 1, 4352, 10, 4, 12, (2560, 1536, 12, 12), 1536), + "tep4": (6, 1, 4352, 10, 4, 12, (2560, 1536, 12, 12), 1536), + "dep4": (24, 2, 14080, 29, 16, 48, (10240, 6144, 48, 48), 6144), +} + + +def _mtp(max_draft_len: int = 3) -> MTPDecodingConfig: + return MTPDecodingConfig(max_draft_len=max_draft_len) + + +def _core(member: str, rank: int, spec_config=None, **config): + model_config = ModelConfig( + pretrained_config=Qwen4ExpTextConfig(**_TEXT), + mapping=_mapping(member, rank), + quant_config=QuantConfig(quant_algo=QuantAlgo.FP8_BLOCK_SCALES), + moe_backend="TRTLLM", + moe_backend_requested="TRTLLM", + sparse_attention_config=QSASparseAttentionConfig(), + spec_config=spec_config, + **config, + ) + composite = SimpleNamespace( + vision_config=Qwen4ExpVisionConfig(**_VISION), image_token_id=248056, video_token_id=248057 + ) + # The core asserts the device it runs on; the weights it declares stay on meta. + with ( + mock.patch.object(torch.cuda, "get_device_capability", return_value=M._SM), + torch.device("meta"), + ): + return M.Qwen4ExpUnifiedModelingV2(model_config, composite) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("strategy", ["AUTO", "MNNVL", "SYMM_MEM", "NCCL"]) +def test_the_tp_sum_runs_at_the_engines_allreduce_strategy(strategy, monkeypatch) -> None: + """At AUTO or MNNVL, sums up to `_TP_SUM_MNNVL_MAX_ROWS` rows take the MNNVL kernel (asked + for by name, so one node qualifies) and wider ones the raw op at AUTO, which has no MNNVL + branch. SYMM_MEM, which the raw op has no branch for either, reaches it at AUTO at every + width. Another strategy reaches the raw op as it is; NCCL reads no IPC workspace.""" + import tensorrt_llm._utils as utils + from tensorrt_llm._torch.distributed import ops as dist_ops + + core = _core("tep4", 0, allreduce_strategy=AllReduceStrategy[strategy]) + asked = [] + + def is_mnnvl(mapping, dtype, explicitly_requested=False): + asked.append(explicitly_requested) + return True + + monkeypatch.setattr(M.MNNVLAllReduce, "is_mnnvl", staticmethod(is_mnnvl)) + monkeypatch.setattr(dist_ops, "get_allreduce_workspace", lambda mapping: "ipc workspace") + monkeypatch.setattr(utils, "mpi_disabled", lambda: False) + monkeypatch.setattr(M, "mnnvl_workspace_args", lambda *args: dict(buffers="mnnvl workspace")) + monkeypatch.setattr(M, "mnnvl_tp_sum", lambda x, buffers: ("mnnvl", buffers)) + monkeypatch.setattr( + M, "allreduce", lambda x, *args, **kwargs: ("raw", kwargs["strategy"], kwargs["workspace"]) + ) + tp_sum = core._tp_sum_op() + narrow = torch.zeros(M._TP_SUM_MNNVL_MAX_ROWS, 8) + wide = torch.zeros(M._TP_SUM_MNNVL_MAX_ROWS + 1, 8) + if strategy == "NCCL": + assert asked == [] + assert tp_sum(narrow) == tp_sum(wide) == ("raw", int(AllReduceStrategy.NCCL), None) + return + if strategy == "SYMM_MEM": + assert asked == [] + assert ( + tp_sum(narrow) == tp_sum(wide) == ("raw", int(AllReduceStrategy.AUTO), "ipc workspace") + ) + return + assert asked == [True] + assert tp_sum(narrow) == ("mnnvl", "mnnvl workspace") + assert tp_sum(wide) == ("raw", int(AllReduceStrategy.AUTO), "ipc workspace") + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy, own", [("AUTO", True), ("MNNVL", True), ("NCCL", False), ("ONESHOT", False)] +) +def test_the_target_opens_its_own_ipc_buffers_at_auto_or_mnnvl_only( + strategy, own, monkeypatch +) -> None: + """`_own_ipc` gates the split index gather's Lamport buffers and the two-rank copy-engine + exchange; under another strategy neither is built and both gather over NCCL.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm.ce_exchange__sp_pair import ( # noqa: E501 + CopyEngineExchange, + ) + + built = [] + monkeypatch.setattr(M, "can_access_peer", lambda mapping: True) + monkeypatch.setattr(M, "get_env_enable_pdl", lambda: True) + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + monkeypatch.setattr( + M, "_selection_lamport_build", lambda mapping, size: built.append("lamport") or "lamport" + ) + monkeypatch.setattr( + CopyEngineExchange, + "create", + classmethod(lambda cls, mapping, capacity: built.append("ce") or "ce"), + ) + core = SimpleNamespace( + attn_tp_size=2, + dp_size=1, + mapping=_mapping("tep2", 0), + allreduce_strategy=int(AllReduceStrategy[strategy]), + token_topk=2048, + compress_ratio=4, + max_num_tokens=8192, + hidden=HIDDEN, + model_config=SimpleNamespace(cuda_graph_batch_sizes=()), + _side_stream=SimpleNamespace(device="cpu"), + tp_sum=None, + ) + assert M._own_ipc(core) is own + assert (M._index_split_lamport(core) is not None) is own + # The prefill target's construction, its views and graphs stubbed out. + monkeypatch.setattr(M, "_layer_views", lambda core: None) + monkeypatch.setattr(M, "_stack_views", lambda core: None) + monkeypatch.setattr(M, "_keyed_eager_on_graph", lambda *args, **kwargs: None) + monkeypatch.setattr(M._KvRangeProbe, "for_target", classmethod(lambda cls, core, on: None)) + monkeypatch.setattr(torch.cuda, "Stream", lambda **kwargs: SimpleNamespace(**kwargs)) + monkeypatch.setattr(torch.cuda, "Event", lambda: None) + assert (M.PrefillTarget(core)._ce_exchange is not None) is own + assert built == (["lamport", "ce"] if own else []) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "strategy, fp32_acc, fused", + [ + ("AUTO", None, True), + ("AUTO", "1", True), + ("AUTO", "0", False), + ("MNNVL", None, False), + ("NCCL", None, False), + ], +) +def test_the_fused_decode_sum_runs_at_auto_with_fp32_accumulation( + strategy, fp32_acc, fused, monkeypatch +) -> None: + """`hc_allreduce_inject_norm` sums in fp32 only: under the stock all-reduce's opt-out + (`AR_FP32_ACC_ENV=0`) and at a strategy other than AUTO the decode sites sum through + `tp_sum`.""" + monkeypatch.setattr(M, "can_access_peer", lambda mapping: True) + monkeypatch.setattr(M, "allreduce_group_args", lambda mapping, s: dict(workspace="ipc")) + if fp32_acc is None: + monkeypatch.delenv(M.AR_FP32_ACC_ENV, raising=False) + else: + monkeypatch.setenv(M.AR_FP32_ACC_ENV, fp32_acc) + state = _core("tep4", 0, allreduce_strategy=AllReduceStrategy[strategy])._hc_ar_inject() + assert (state is not None) is fused + if fused: + assert (state.workspace, state.rank, state.nranks) == ("ipc", 0, 4) + + +#: Every rank; MTP with 7 drafts at a member's last rank, 1 elsewhere, and none at dep2. +_CORE_CASES = [(m, r, None if m == "dep2" else 7 if r == _world(m) - 1 else 1) for m, r in _RANKS] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, rank, max_draft_len", + _CORE_CASES, + ids=[f"{m}-rank{r}-mtp{d}" for m, r, d in _CORE_CASES], +) +def test_each_rank_builds_its_share_of_the_core(member, rank, max_draft_len) -> None: + """This rank's heads (rank `r` of two holds KV head `r`, of four `r // 2`) and one + whole-expert window at the full intermediate, whatever the MoE TP/EP split; with MTP the draft + layer 48 is cut like a trunk layer and a verify step carries `max_draft_len + 1` rows.""" + hq, hkv, row, tail, hk, hv, split, out_k = _GEOMETRY[member] + world, adp = _world(member), MEMBERS[member].get("enable_attention_dp", False) + attn_tp = 1 if adp else world + core = _core(member, rank) + assert (core.heads_q, core.heads_kv) == (hq, hkv) + assert core.kv_head_start == (rank * 2 // attn_tp if attn_tp > 1 else 0) + assert (core.attn_in_width, core.attn_in_tail_heads) == (row, tail) + assert core.index_col == core.qkv_width + core.q_width == 2 * (hq + hkv) * 256 + assert row == 2 * hq * 256 + 2 * hkv * 256 + 640 + 128 + assert tuple(core.w["l3_attn_in_w"].shape) == (row, HIDDEN) + assert tuple(core.w["l3_o_w"].shape) == (HIDDEN, out_k) + assert (core.num_k_heads, core.num_v_heads) == (hk, hv) + assert tuple(core.in_proj_split) == split + # Padded with zero rows to a multiple of 16, the decode GEMM's whole-tile width. + assert core.in_proj_rows == -(-sum(split) // 16) * 16 + assert tuple(core.w["l0_in_proj_w"].shape) == (core.in_proj_rows, HIDDEN) + assert tuple(core.w["l0_out_w"].shape) == (HIDDEN, out_k) + assert (core.attn_tp_size, core.dp_size) == (attn_tp, world if adp else 1) + + count = NUM_EXPERTS // world + assert (core.local_experts, core.expert_offset, core.moe_inter) == (count, count * rank, 640) + window = W.expert_window(core.w, core.dims, 0) + assert (window.offset, window.count) == (count * rank, count) + assert tuple(window.fc1_w.shape) == (count, 2 * 640, HIDDEN) + assert window.fc1_w.dtype == window.fc2_w.dtype == torch.float8_e4m3fn + assert tuple(window.fc1_s.shape) == (count, 2 * 640 // 128, HIDDEN // 128) + assert window.fc1_s.dtype == torch.float32 + assert tuple(window.fc2_w.shape) == (count, HIDDEN, 640) + assert tuple(window.fc2_s.shape) == (count, HIDDEN // 128, 640 // 128) + assert tuple(core.w["l0_router"].shape) == (NUM_EXPERTS, HIDDEN) + assert not core.mtp_enabled and core.spec_rollback is None + assert not any("_mtp_" in k for k in core.w) + if max_draft_len is None: + return + + mtp = _core(member, rank, _mtp(max_draft_len)) + assert mtp.mtp_enabled and mtp.mtp_layer_idx == NUM_LAYERS and mtp.spec_rollback is not None + assert max_draft_len + 1 <= M._MAX_VERIFY_WIDTH + assert mtp.mtp_theta == mtp.theta == 1e7 + draft = W.expert_window(mtp.w, mtp.dims, NUM_LAYERS) + assert (draft.offset, draft.count) == (window.offset, window.count) + assert draft.fc1_w.shape == window.fc1_w.shape and draft.fc2_s.shape == window.fc2_s.shape + assert draft.fc1_w.dtype == draft.fc2_w.dtype == torch.float8_e4m3fn + p = f"l{NUM_LAYERS}_mtp_" + assert tuple(mtp.w[f"{p}attn_in_w"].shape) == (row, HIDDEN) + assert tuple(mtp.w[f"{p}o_w"].shape) == (HIDDEN, out_k) + assert tuple(mtp.w[f"{p}router"].shape) == (NUM_EXPERTS, HIDDEN) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "rows, dp_size, step, grouped_min, calls", + [ + (1, 1, 8192, 0, [("runner", 1)]), + (20000, 1, 8192, 0, [("runner", 8192), ("runner", 8192), ("runner", 3616)]), + (4095, 1, 8192, 4096, [("runner", 4095)]), + (16384, 1, 16384, 4096, [("grouped", 16384)]), + # An attention-DP window: up to four 8192-row chunks a runner call, never grouped. + (40000, 4, 8192, 4096, [("runner", 32768), ("runner", 7232)]), + ], + ids=["one-row", "tp-chunks", "tp-below-grouped", "tp-context-grouped", "adp-window"], +) +def test_a_moe_layer_enters_the_fp8_runner_over_its_window( + rows, dp_size, step, grouped_min, calls, monkeypatch +) -> None: + """`_routed_experts` enters the FP8 runner pre-routed over this rank's window in row-ordered + chunks of `core.moe_max_rows` rows, or `moe_prefill_grouped` for a large TP context chunk.""" + core = SimpleNamespace( + topk=TOPK, num_experts=NUM_EXPERTS, moe_inter=640, dp_size=dp_size, moe_max_rows=step + ) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384, 128) + x = torch.arange(rows, dtype=torch.float32)[:, None].expand(rows, 4).contiguous() + seen = [] + + def router(chunk, mat_b): + assert mat_b == "router" + return chunk[:, :1] + + def routing(logits, topk, output_dtype): + assert topk == TOPK + return logits.int().expand(-1, topk).contiguous(), logits.expand(-1, topk).contiguous() + + def quantize(chunk, use_ue8m0): + assert use_ue8m0 is False + return chunk + 0.5, chunk[:, :2].t().contiguous() + + def runner(**kw): + seen.append(("runner", kw["hidden_states"], kw["hidden_states_scale"], kw)) + assert kw["routing_logits"] is None and kw["routing_bias"] is None + assert (kw["gemm1_weights"], kw["gemm1_weights_scale"]) == ("fc1_w", "fc1_s") + assert (kw["gemm2_weights"], kw["gemm2_weights_scale"]) == ("fc2_w", "fc2_s") + assert (kw["num_experts"], kw["top_k"], kw["intermediate_size"]) == (512, 10, 640) + assert (kw["local_expert_offset"], kw["local_num_experts"]) == (384, 128) + assert kw["routing_method_type"] == 1 and kw["act_type"] == 0 + return kw["hidden_states"] * 2 + + def grouped(xq, xsf, ids, w, *weights): + assert weights == ("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384) + seen.append(("grouped", xq, xsf, dict(topk_ids=ids, topk_weights=w))) + return xq * 2 + + monkeypatch.setattr(M, "cublas_mm", router) + monkeypatch.setattr(M, "renorm_moe_routing_op", routing) + monkeypatch.setattr(M, "fp8_quantize_1x128", quantize) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", runner) + monkeypatch.setattr(M, "moe_prefill_grouped", grouped) + monkeypatch.setattr(M, "prefill_grouped_ready", lambda local: local == 128) + monkeypatch.setattr(M, "_MOE_PREFILL_GROUPED_MIN_ROWS", grouped_min) + out = M._routed_experts(core, x, "router", experts) + assert [(kind, xq.shape[0]) for kind, xq, *_ in seen] == calls + torch.testing.assert_close(out, (x + 0.5) * 2, rtol=0, atol=0) + lo = 0 + for _, xq, xsf, kw in seen: + n = xq.shape[0] + torch.testing.assert_close(xsf, x[lo : lo + n, :2].t(), rtol=0, atol=0) + torch.testing.assert_close(kw["topk_ids"][:, 0], x[lo : lo + n, 0].int(), rtol=0, atol=0) + torch.testing.assert_close(kw["topk_weights"][:, 0], x[lo : lo + n, 0], rtol=0, atol=0) + lo += n + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "dp_size, graphed, want", + [(1, True, "front"), (1, False, "front"), (2, True, "front"), (4, False, "cublas")], + ids=["tp-graphed", "tp-eager", "adp-graphed", "adp-eager"], +) +def test_a_decode_router_under_attention_dp_follows_the_graphed_step( + dp_size, graphed, want, monkeypatch +) -> None: + """Under attention DP a graphed decode step keeps the single-launch decode router (every rank + runs the same graph) and an eager one takes cuBLAS, the router of another rank's prefill.""" + core = SimpleNamespace( + topk=TOPK, + num_experts=NUM_EXPERTS, + moe_inter=640, + dp_size=dp_size, + moe_max_rows=8192, + _decode_graph_step=graphed, + ) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 384, 128) + x = torch.zeros(4, 8) + seen = [] + + def front(core_, rows, router, experts_): + seen.append("front") + return rows + + def cublas(chunk, mat_b): + seen.append("cublas") + return chunk[:, :1] + + monkeypatch.setattr(M, "_routed_experts_front", front) + monkeypatch.setattr(M, "cublas_mm", cublas) + monkeypatch.setattr( + M, + "renorm_moe_routing_op", + lambda logits, topk, output_dtype: ( + logits.int().expand(-1, topk).contiguous(), + logits.expand(-1, topk).contiguous(), + ), + ) + monkeypatch.setattr( + M, "fp8_quantize_1x128", lambda chunk, use_ue8m0: (chunk, chunk[:, :2].t().contiguous()) + ) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", lambda **kw: kw["hidden_states"]) + M._routed_experts(core, x, "router", experts, mm=M._decode_projection) + assert seen == [want] + assert M._step_projection(core) is (M._decode_projection if graphed else cublas) + + +def _pad(x: torch.Tensor, rows: int) -> torch.Tensor: + return torch.cat([x, x.new_zeros(rows - x.shape[0], x.shape[1])], 0) + + +def _expert_out(x, ids, wts, start, count): + """A stand-in MoE over experts `[start, start + count)`: whole-expert windows sum to all 512.""" + keep = ((ids >= start) & (ids < start + count)).to(x.dtype) + return (wts * keep * (ids + 1).to(x.dtype)).sum(-1, keepdim=True) * x + + +#: The core's side stream on the host, where there is nothing to order. +_HOST_STREAM = SimpleNamespace(wait_stream=lambda other: None) + + +class _HostMoe: + """A MoE layer's ops in float64 over a fixed router: top-10 softmax routing, `_expert_out` + over the window, chunks of `chunk_rows`, two chunks a runner call on an attention-DP window.""" + + def __init__(self, monkeypatch, chunk_rows: int): + gen = torch.Generator().manual_seed(0) + self.router = torch.randn(HIDDEN, NUM_EXPERTS, generator=gen, dtype=torch.float64) + self.chunk_rows = chunk_rows + self.calls, self.windows = [], [] + monkeypatch.setattr(M, "_MOE_MAX_ROWS", chunk_rows) + monkeypatch.setattr(M, "_MOE_MAX_ROWS_TP", chunk_rows) + monkeypatch.setattr(M, "_MOE_PREFILL_GROUPED_MIN_ROWS", 0) # the runner path + monkeypatch.setattr(M, "_ADP_WINDOW_MAX_ROWS", 2 * chunk_rows) + monkeypatch.setattr(M, "cublas_mm", lambda a, b, *args, **kwargs: a @ b) + monkeypatch.setattr(M, "renorm_moe_routing_op", self.routing) + # Scales in the `[hidden / 128, rows]` layout, which grouped chunks concatenate. + monkeypatch.setattr( + M, "fp8_quantize_1x128", lambda t, use_ue8m0: (t, t.new_zeros(HIDDEN // 128, len(t))) + ) + monkeypatch.setattr(M, "fp8_block_scale_moe_runner", self.runner) + monkeypatch.setattr(M, "silu_and_mul_rows", lambda gug, inter: gug[:, :inter]) + # The shared expert and its gate are replicated and checked elsewhere. + monkeypatch.setattr(M, "fused_sigmoid_gate_mul_add", lambda routed, gate, shared: routed) + # `_mtp_moe_block` forks the shared expert onto a side stream; the host runs it in order. + monkeypatch.setattr(torch.cuda, "current_stream", lambda *args, **kwargs: _HOST_STREAM) + monkeypatch.setattr(torch.cuda, "stream", lambda stream: contextlib.nullcontext()) + + def route(self, x): + top_logits, top_ids = torch.topk(x @ self.router, TOPK, dim=-1) + return top_ids, torch.softmax(top_logits, dim=-1) + + def routing(self, logits, topk, output_dtype): + assert topk == TOPK and logits.shape[1] == NUM_EXPERTS + top_logits, top_ids = torch.topk(logits, topk, dim=-1) + return top_ids, torch.softmax(top_logits, dim=-1) + + def runner(self, **kw): + assert kw["num_experts"] == NUM_EXPERTS and kw["intermediate_size"] == 640 + x = kw["hidden_states"] + assert 0 < x.shape[0] <= 2 * self.chunk_rows + self.calls.append(x.shape[0]) + start, count = kw["local_expert_offset"], kw["local_num_experts"] + self.windows.append((start, count)) + return _expert_out(x, kw["topk_ids"], kw["topk_weights"], start, count) + + def views(self, core): + core._side_stream = _HOST_STREAM + return SimpleNamespace( + router=self.router, + experts=W.expert_window(core.w, core.dims, NUM_LAYERS), + sh_gug=torch.zeros(HIDDEN, 2 * 640 + 8, dtype=torch.float64), + sh_dn=torch.zeros(640, HIDDEN, dtype=torch.float64), + ) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "counts, draft", + [ + ([3, 7], False), + ([0, 0, 5, 3], False), + ([4, 4, 4, 4], False), + ([3, 7, 1, 4], True), + ([1, 2, 1, 3], True), + ([4, 4, 4, 4], True), + ], + ids=[ + "dep2-trunk", + "dep4-trunk-two-ranks-without-rows", + "dep4-trunk-uniform", + "dep4-draft-extend", + "dep4-draft-decode-with-a-dummy-rank", + "dep4-draft-captured-batch", + ], +) +def test_the_attention_dp_round_trip_returns_each_ranks_rows(counts, draft, monkeypatch) -> None: + """Each rank gathers the group's rows, runs its window over the real rows only (a padding + row's zero logits would route to the lowest experts) and reduce-scatters, with the same + collectives on every rank (ragged when counts differ), getting back its own rows' MoE.""" + world = len(counts) + moe = _HostMoe(monkeypatch, chunk_rows=5) + gen = torch.Generator().manual_seed(1) + xs = [torch.randn(n, HIDDEN, generator=gen, dtype=torch.float64) for n in counts] + rows, n = max(counts), sum(counts) + split = min(counts) != rows + if split: + gathered = torch.cat(xs, 0) + starts = [sum(counts[:r]) for r in range(world)] + real = torch.arange(n) + else: + gathered = torch.cat([_pad(x, rows) for x in xs], 0) + starts = [r * rows for r in range(world)] + real = torch.cat([torch.arange(r * rows, r * rows + c) for r, c in enumerate(counts)]) + real_rows = gathered[real] + full = _expert_out(gathered, *moe.route(gathered), 0, NUM_EXPERTS) + state = {} + + def fake_allgather(t, sizes, group, process_group): + assert sizes == (list(counts) if split else None) + assert group == list(range(world)) and process_group is None + state["collectives"].append(("allgather", tuple(t.shape))) + x = xs[state["rank"]] + torch.testing.assert_close(t, x if split else _pad(x, rows), rtol=0, atol=0) + return gathered + + def fake_reducescatter(t, sizes, group, process_group): + assert sizes == (list(counts) if split else None) + assert group == list(range(world)) and process_group is None + state["collectives"].append(("reducescatter", tuple(t.shape))) + state["partial"] = t + r = state["rank"] + return full[starts[r] : starts[r] + (counts[r] if split else rows)] + + def no_tp_sum(t): + raise AssertionError("the MoE under attention DP went through the TP sum") + + monkeypatch.setattr(M, "allgather", fake_allgather) + monkeypatch.setattr(M, "reducescatter", fake_reducescatter) + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + per_rank = [] + for rank in range(world): + core = _core(f"dep{world}", rank, _mtp() if draft else None) + core.expert_sum, core.tp_sum = core._expert_sum_op(), no_tp_sum + state.update(rank=rank, collectives=[]) + moe.calls, moe.windows = [], [] + if draft: + pad_rows = M._draft_pad_rows(core, counts, counts[rank]) + out = M._mtp_moe_block(core, moe.views(core), xs[rank], pad_rows) + else: + md = SimpleNamespace(all_rank_num_tokens=counts) + pad_rows = core._dp_pad_rows(md, counts[rank]) + trunk = W.expert_window(core.w, core.dims, 0) + window = lambda t: M._routed_experts(core, t, moe.router, trunk) # noqa: E731 + out = core.expert_sum(window, xs[rank], pad_rows) + count = NUM_EXPERTS // world + assert moe.calls == [min(10, n - lo) for lo in range(0, n, 10)] + assert set(moe.windows) == {(count * rank, count)} + torch.testing.assert_close( + state["partial"][real], + _expert_out(real_rows, *moe.route(real_rows), count * rank, count), + ) + assert tuple(out.shape) == (counts[rank], HIDDEN) + torch.testing.assert_close(out, _expert_out(xs[rank], *moe.route(xs[rank]), 0, NUM_EXPERTS)) + # Each rank hands the gather its own rows: the ragged form's shapes differ by rank. + assert state["collectives"][0] == ("allgather", (counts[rank] if split else rows, HIDDEN)) + per_rank.append(state["collectives"][1:]) + assert all(c == per_rank[0] for c in per_rank), per_rank + assert per_rank[0] == [("reducescatter", (n if split else world * rows, HIDDEN))] + + +class _HostA2a: + """The NVLink MoE all-to-all at one rank: `dispatch` hands it each source's rows routed into + its window (padded with -1 routes); `combine` checks its partials and returns its rows' MoE.""" + + def __init__(self, xs, routes, rank): + self.xs, self.routes, self.rank = xs, routes, rank + self.world, self.cap = len(xs), max(len(x) for x in xs) + self.max_num_tokens_per_rank = 1 << 20 + self.calls = [] + + def dispatch(self, x, sf, ids, w, counts): + self.calls.append("dispatch") + assert sf is None and counts == [len(t) for t in self.xs] + own_ids, own_w = self.routes[self.rank] + torch.testing.assert_close(x, self.xs[self.rank], rtol=0, atol=0) + # The router runs over chunks of the rows, so the weights match to rounding. + assert torch.equal(ids, own_ids), "not routed on its own rows" + torch.testing.assert_close(w, own_w.float()) # fp32 on the wire, as the stock MoE's + lo, count = self.rank * (NUM_EXPERTS // self.world), NUM_EXPERTS // self.world + rows, rids, rw = [], [], [] + for src, (sx, (sids, sw)) in enumerate(zip(self.xs, self.routes)): + hit = ((sids >= lo) & (sids < lo + count)).any(1) + pad = self.cap - int(hit.sum()) + rows.append(torch.cat([sx[hit], sx.new_zeros(pad, HIDDEN)])) + rids.append(torch.cat([sids[hit], sids.new_full((pad, TOPK), -1)])) + rw.append(torch.cat([sw[hit], sw.new_zeros(pad, TOPK)])) + self.got = torch.cat(rows), torch.cat(rids), torch.cat(rw), lo, count + return self.got[0], None, self.got[1], self.got[2] + + def combine(self, partials): + self.calls.append("combine") + rows, ids, w, lo, count = self.got + torch.testing.assert_close(partials, _expert_out(rows, ids, w, lo, count)) + own = self.xs[self.rank] + return _expert_out(own, *self.routes[self.rank], 0, NUM_EXPERTS) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "counts, draft", + [([3, 7], False), ([0, 5, 2, 4], False), ([4, 4, 4, 4], True)], + ids=["dep2-trunk-ragged", "dep4-trunk-a-rank-without-rows", "dep4-draft-captured-batch"], +) +def test_the_attention_dp_alltoall_round_trip(counts, draft, monkeypatch) -> None: + """`TRTLLM_FORCE_COMM_METHOD=NVLINK_ONE_SIDED`: each rank routes its own rows through the + all-to-all and returns what the combine sums; no all-gather, reduce-scatter or TP sum.""" + world = len(counts) + moe = _HostMoe(monkeypatch, chunk_rows=5) + gen = torch.Generator().manual_seed(2) + xs = [torch.randn(n, HIDDEN, generator=gen, dtype=torch.float64) for n in counts] + routes = [moe.route(x) for x in xs] + + def forbidden(*args, **kwargs): + raise AssertionError("the all-to-all round trip took the default collectives") + + monkeypatch.setenv(M.MOE_COMM_METHOD_ENV, "NVLINK_ONE_SIDED") + monkeypatch.setattr(M, "mpi_disabled", lambda: False) + for name in ("allgather", "reducescatter"): + monkeypatch.setattr(M, name, forbidden) + for rank in range(world): + core = _core(f"dep{world}", rank, _mtp() if draft else None) + a2a = _HostA2a(xs, routes, rank) + monkeypatch.setattr(M, "_build_moe_a2a", lambda mapping, max_tokens: a2a) + core.expert_sum, core.tp_sum = core._expert_sum_op(), forbidden + assert core.moe_a2a is a2a + if draft: + pad_rows = M._draft_pad_rows(core, counts, counts[rank]) + out = M._mtp_moe_block(core, moe.views(core), xs[rank], pad_rows) + else: + md = SimpleNamespace(all_rank_num_tokens=counts) + run = M._WindowRun(core, moe.router, W.expert_window(core.w, core.dims, 0)) + out = core.expert_sum(run, xs[rank], core._dp_pad_rows(md, counts[rank])) + assert a2a.calls == ["dispatch", "combine"] + torch.testing.assert_close(out, _expert_out(xs[rank], *routes[rank], 0, NUM_EXPERTS)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "accepted, rows_per_request, index_share, dp_size, want", + [ + ([1, 3, 2, 1], 3, False, 1, [0, 5, 7, 9]), + ([1, 3, 2, 1] * 4, 3, False, 1, [r * 3 + a - 1 for r, a in enumerate([1, 3, 2, 1] * 4)]), + ([1, 3, 2, 1], 1, False, 1, None), + ([1, 3, 2, 1], 3, True, 1, None), # IndexShare's capture keeps every row + ([1, 3, 2, 1], 3, False, 2, None), # attention DP pads by the worker's counts + (None, 3, False, 1, None), + ], + ids=["draft-extend", "ht-batch", "not-extend", "index-share", "attention-dp", "no-counts"], +) +def test_a_draft_extend_step_continues_from_each_requests_last_accepted_row( + accepted, rows_per_request, index_share, dp_size, want +) -> None: + """Request r's draft-extend outputs are read from row `num_accepted[r] - 1` of its + `rows_per_request` (the golden token counts as accepted); None means every row.""" + counts = None if accepted is None else torch.tensor(accepted, dtype=torch.int32) + md = SimpleNamespace(qsa_mtp_num_accepted=counts) + requests = 4 if accepted is None else len(accepted) + got = M._draft_continue_rows( + SimpleNamespace(dp_size=dp_size), md, requests, rows_per_request, index_share + ) + assert (None if got is None else got.tolist()) == want + + +# ------------------------------------------- sequence-parallel prefill at attention TP --- + + +def _sp_target(rank: int, world: int, comm: bool = False, ce=None): + target = object.__new__(M.PrefillTarget) + target._sequence_parallel = (rank, world, list(range(world)), None) + target._ce_exchange = ce + target._comm_stream = target._comm_fork = target._comm_join = target._sp_rows = None + if comm: + target._comm_stream = torch.cuda.Stream(priority=-1) + target._comm_fork, target._comm_join = torch.cuda.Event(), torch.cuda.Event() + return target + + +def _sp_share(monkeypatch, num_tokens, world, rank, want, configured=True): + """From `_SP_MIN_ROWS` rows that divide evenly a rank keeps its consecutive `T / world` rows, + gathered and scattered uniformly over the attention-TP group; otherwise there is no split.""" + target = _sp_target(rank, world) + if not configured: + target._sequence_parallel = None + calls = [] + monkeypatch.setattr( + M, "allgather", lambda t, sizes, group, pg: calls.append(("ag", sizes, group)) or t + ) + monkeypatch.setattr( + M, "reducescatter", lambda t, sizes, group, pg: calls.append(("rs", sizes, group)) or t + ) + sp = target._sp_split(num_tokens) + if want is None: + assert sp is None + return + lo, hi, gather, scatter = sp + assert (lo, hi) == want + gather(torch.zeros(1)) + scatter(torch.zeros(1)) + assert calls == [("ag", None, list(range(world))), ("rs", None, list(range(world)))] + + +def _sp_moe_own_rows(monkeypatch, rows, step, padded): + """`_routed_experts_sp` routes (padding rows to `_NO_EXPERT`) and quantizes only this rank's + rows; the gathered codes, scales, ids and weights reach the runner exactly, in row order.""" + hidden, topk = 384, 10 + core = SimpleNamespace(topk=topk, num_experts=512, moe_inter=640, moe_max_rows=step) + experts = W.ExpertWindow("fc1_w", "fc1_s", "fc2_w", "fc2_s", 0, 256) + gen = torch.Generator().manual_seed(rows) + x = torch.randn(rows, hidden, generator=gen).to(torch.bfloat16) + ids = torch.randint(0, 512, (rows, topk), generator=gen, dtype=torch.int32) + weights = torch.rand(rows, topk, generator=gen).to(torch.bfloat16) + codes = (torch.randn(rows, hidden, generator=gen) * 8).to(torch.float8_e4m3fn) + scales = torch.rand(hidden // 128, rows, generator=gen) + monkeypatch.setattr(M, "cublas_mm", lambda a, b, *args: ("logits", a.shape[0])) + monkeypatch.setattr( + M, "renorm_moe_routing_op", lambda logits, topk, output_dtype: (ids.clone(), weights) + ) + monkeypatch.setattr(M, "fp8_quantize_1x128", lambda t, use_ue8m0: (codes, scales)) + calls = [] + monkeypatch.setattr( + M, + "fp8_block_scale_moe_runner", + lambda **kw: calls.append(kw) or torch.zeros(kw["hidden_states"].shape[0], 4), + ) + mask = (torch.arange(rows) >= rows - padded)[:, None] if padded else None + want_ids = ids.clone() + want_ids[rows - padded :] = M._NO_EXPERT + gathers = [] + out = M._routed_experts_sp( + core, x, "router", experts, lambda t: gathers.append(t.dtype) or t.clone(), mask + ) + assert gathers == [torch.bfloat16, torch.bfloat16] + assert tuple(out.shape) == (rows, 4) + assert [c["hidden_states"].shape[0] for c in calls] == [ + min(step, rows - lo) for lo in range(0, rows, step) + ] + got_codes = torch.cat([c["hidden_states"] for c in calls]) + assert torch.equal(got_codes.view(torch.uint8), codes.view(torch.uint8)) + assert torch.equal(torch.cat([c["hidden_states_scale"] for c in calls], 1), scales) + assert all(c["hidden_states_scale"].is_contiguous() for c in calls) + assert torch.equal(torch.cat([c["topk_ids"] for c in calls]), want_ids) + assert torch.equal(torch.cat([c["topk_weights"] for c in calls]), weights) + assert all(c["topk_ids"].dtype == torch.int32 for c in calls) + + +def _sp_ce_scatter(monkeypatch, rank): + """A copy-engine two-rank scatter sends the peer's rows and adds its partial (the same bits in + either order) without NCCL; `CopyEngineExchange.create` builds only a one-node pair.""" + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.comm.ce_exchange__sp_pair import ( # noqa: E501 + CopyEngineExchange, + ) + + num_tokens, hidden = 4096, 64 + rows = num_tokens // 2 + gen = torch.Generator().manual_seed(3) + partial = [torch.randn(num_tokens, hidden, generator=gen).to(torch.bfloat16) for _ in range(2)] + sent = [] + + class Exchange: + def exchange(self, x): + sent.append(x) + return partial[1 - rank][rank * rows : (rank + 1) * rows].clone() + + monkeypatch.setattr(M, "reducescatter", lambda *a: pytest.fail("NCCL reduce-scatter called")) + lo, hi, _, scatter = _sp_target(rank, 2, ce=Exchange())._sp_split(num_tokens) + got = scatter(partial[rank]) + assert (lo, hi) == (rank * rows, (rank + 1) * rows) + assert torch.equal(sent[0], partial[rank][(1 - rank) * rows : (2 - rank) * rows]) + assert torch.equal(got, partial[0][lo:hi] + partial[1][lo:hi]) + assert torch.equal(got, partial[1][lo:hi] + partial[0][lo:hi]) + four = SimpleNamespace(tp_size=4, gpus_per_node=4, tp_rank=0) + across = SimpleNamespace(tp_size=2, gpus_per_node=1, tp_rank=0) + assert CopyEngineExchange.create(four, 1 << 20) is None + assert CopyEngineExchange.create(across, 1 << 20) is None + + +def _sp_project(monkeypatch, rank, overlap): + """`_project_overlapped` projects this rank's rows while the comm stream gathers the others', + each into its own rows of one output; without the comm stream the block gathers first.""" + dev = torch.device("cuda") + gen = torch.Generator(device=dev).manual_seed(11 + rank) + rows, hidden, width = 512, 256, 384 + shares = [ + torch.randn(rows, hidden, generator=gen, device=dev).to(torch.bfloat16) for _ in range(2) + ] + w = (torch.randn(width, hidden, generator=gen, device=dev) * 0.1).to(torch.bfloat16).t() + target = _sp_target(rank, 2, comm=overlap) + gathered = [] + + def gather(t): + assert t.data_ptr() == shares[rank].data_ptr() + gathered.append(torch.cuda.current_stream()) + return torch.cat(shares) + + target._sp_rows = (rank * rows, (rank + 1) * rows, gather) + assert target._overlap_projection() is overlap + if overlap: + out = target._project_overlapped(shares[rank], w) + torch.cuda.synchronize() + assert gathered == [target._comm_stream], "the gather runs on the comm stream" + assert tuple(out.shape) == (2 * rows, width) + want = torch.cat(shares).float() @ w.float() + torch.testing.assert_close(out.float(), want, rtol=2e-2, atol=2e-2) + else: + assert torch.equal(target._block_rows(shares[rank]), torch.cat(shares)) + target._sp_rows = None + assert not target._overlap_projection() + assert target._block_rows(shares[rank]) is shares[rank] + + +def _sp_ce_project_reduce(monkeypatch, rank): + """With the copy-engine pair exchange, `_project_overlapped` and `_reduce_overlapped` exchange + on the comm stream and match NCCL bitwise; the reduction sums both ranks' products.""" + dev = torch.device("cuda") + gen = torch.Generator(device=dev).manual_seed(23) + rows, hidden, width = 384, 256, 320 + shares = [ + torch.randn(rows, hidden, generator=gen, device=dev).to(torch.bfloat16) for _ in range(2) + ] + a = [ + torch.randn(2 * rows, hidden, generator=gen, device=dev).to(torch.bfloat16) + for _ in range(2) + ] + w = (torch.randn(width, hidden, generator=gen, device=dev) * 0.1).to(torch.bfloat16).t() + sent = [torch.mm(a[r][(1 - r) * rows : (2 - r) * rows], w) for r in range(2)] + calls = [] + + class Exchange: + def exchange(self, x): + assert torch.cuda.current_stream() == with_ce._comm_stream + if torch.equal(x, shares[rank]): + calls.append("rows") + return shares[1 - rank].clone() + assert torch.equal(x, sent[rank]) + calls.append("partials") + return sent[1 - rank].clone() + + def gather(t): + return torch.cat(shares) if torch.equal(t, shares[rank]) else torch.cat(sent) + + with_ce, nccl = _sp_target(rank, 2, comm=True, ce=Exchange()), _sp_target(rank, 2, comm=True) + for t in (with_ce, nccl): + t._sp_rows = (rank * rows, (rank + 1) * rows, gather) + assert with_ce._overlap_reduce() + got = with_ce._project_overlapped(shares[rank], w) + want = nccl._project_overlapped(shares[rank], w) + red = with_ce._reduce_overlapped(a[rank], w) + red_want = nccl._reduce_overlapped(a[rank], w) + torch.cuda.synchronize() + assert calls == ["rows", "partials"] + assert torch.equal(got, want) and torch.equal(red, red_want) + full = sum(torch.mm(a[r], w).float() for r in range(2)) + torch.testing.assert_close( + red.float(), full[rank * rows : (rank + 1) * rows], rtol=2e-2, atol=2e-2 + ) + for world, comm in ((4, True), (2, False)): # four ranks, or no comm stream: one GEMM + target = _sp_target(rank, world, comm=comm) + target._sp_rows = with_ce._sp_rows + assert not target._overlap_reduce() + + +_cpu = pytest.mark.cpu_only +_SP_CASES = [ + *( + pytest.param(_sp_share, dict(num_tokens=t, world=w, rank=r, want=want, configured=c), + marks=_cpu, id=f"share-{t}-tp{w}-rank{r}{'' if c else '-off'}") + for t, w, r, want, c in [ + (16384, 2, 1, (8192, 16384), True), + (4096, 2, 0, (0, 2048), True), + (4095, 2, 0, None, True), # an odd eager step keeps every row + (2048, 2, 0, None, True), # below `_SP_MIN_ROWS` + (8192, 4, 3, (6144, 8192), True), + (16384, 2, 0, None, False), # one rank, attention DP, or the switch off + ] + ), + *( + pytest.param(_sp_moe_own_rows, dict(rows=rows, step=step, padded=padded), marks=_cpu, + id=f"moe-own-rows-{rows}-step{step}-pad{padded}") + for rows, step, padded in [(96, 8192, 0), (96, 40, 5), (7, 8192, 3)] + ), + *( + pytest.param(_sp_ce_scatter, dict(rank=r), marks=_cpu, id=f"ce-scatter-rank{r}") + for r in (0, 1) + ), + *( + pytest.param(_sp_project, dict(rank=r, overlap=o), marks=_needs_gpu, + id=f"project-rank{r}-{'overlap' if o else 'gather'}") + for r, o in [(0, True), (1, True), (1, False)] + ), + *( + pytest.param(_sp_ce_project_reduce, dict(rank=r), marks=_needs_gpu, + id=f"ce-project-reduce-rank{r}") + for r in (0, 1) + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _SP_CASES) +def test_a_sequence_parallel_prefill_step(check, kwargs, monkeypatch) -> None: + """Sequence-parallel prefill at TP2 / TP4: row shares, own-row MoE, overlapped projections.""" + check(monkeypatch, **kwargs) + + +# ------------------------------------------------- the split context selection --- + + +def _selection_balance(monkeypatch, chunks, starts, world): + """`_selection_cuts` ascend from 0 to the step's rows, interior cuts on the index scores' + 64-row tile, each rank's summed row cost (position + 1) within one tile of the even share.""" + cuts = M._selection_cuts(chunks, starts, world) + rows = sum(chunks) + assert len(cuts) == world + 1 and cuts[0] == 0 and cuts[-1] == rows + assert all(a <= b for a, b in zip(cuts, cuts[1:])) + assert all(c % M._SELECTION_CUT_ALIGN == 0 for c in cuts[1:-1]) + cost = [s0 + j + 1 for c, s0 in zip(chunks, starts) for j in range(c)] + parts = [sum(cost[a:b]) for a, b in zip(cuts, cuts[1:])] + even = sum(cost) / world + assert max(abs(x - even) for x in parts) <= M._SELECTION_CUT_ALIGN * max(cost) + 1, parts + + +def _selection_lands_rows(monkeypatch, chunks, starts, world, balanced, pieces): + """`_split_context_selection`: each rank selects only its share of the context rows, gathered + in `pieces` on the comm stream, and every rank ends bitwise as one rank selecting them all.""" + dev = torch.device("cuda" if pieces else "cpu") + lo, width = 5, 2051 + rows = sum(chunks) + cuts = M._selection_cuts(chunks, starts, world) if balanced else None + bounds = cuts or [min(r * -(-rows // world), rows) for r in range(world + 1)] + pipeline = None + if pieces: + pipeline = (pieces, torch.cuda.Stream(priority=-1), torch.cuda.Event(), torch.cuda.Event()) + + def select_rows(a, b, context_rows, out): + assert context_rows + out.copy_( + torch.arange(a, b, dtype=torch.int32, device=dev)[:, None] * 4099 + + torch.arange(width, dtype=torch.int32, device=dev) + ) + + want = torch.full((lo + rows + 3, width), -1, dtype=torch.int32, device=dev) + select_rows(lo, lo + rows, True, want[lo : lo + rows]) + buffers = {} # (rank, piece) -> the rank's buffer + + def run(rank, gather_of): + scored, count = [], [0] + + def gather(mine, sizes, group, pg): + if pipeline is not None: + assert torch.cuda.current_stream() == pipeline[1], "a piece gathers on comm" + count[0] += 1 + return gather_of(rank, count[0] - 1, mine) + + def record(a, b, context_rows, out): + scored.append((a, b)) + select_rows(a, b, context_rows, out) + + monkeypatch.setattr(M, "allgather", gather) + sel = torch.full_like(want, -1) + M._split_context_selection( + (rank, world, list(range(world)), None), record, sel, lo, lo + rows, pipeline, cuts + ) + if pieces: + torch.cuda.synchronize() + return sel, scored + + for rank in range(world): # each rank's own buffers (the stand-in returns a gather's shape) + _, scored = run( + rank, lambda r, j, mine: torch.cat([buffers.setdefault((r, j), mine.clone())] * world) + ) + assert scored[0][0] == lo + bounds[rank] and scored[-1][1] == lo + bounds[rank + 1] + assert all(b == a2 for (_, b), (a2, _) in zip(scored, scored[1:])), scored + assert all(b.dtype == torch.bfloat16 and b.shape[1] == 2 * width for b in buffers.values()) + if not pieces: + per = max(b - a for a, b in zip(bounds, bounds[1:])) + assert all(b.shape[0] == per for b in buffers.values()) + for rank in range(world): + sel, _ = run(rank, lambda r, j, mine: torch.cat([buffers[(q, j)] for q in range(world)])) + assert torch.equal(sel, want), rank + + +_SELECTION_CASES = [ + *( + pytest.param(_selection_balance, dict(chunks=c, starts=s, world=w), marks=_cpu, + id=f"balance-{len(c)}x{c[0]}-tp{w}") + for c, s, w in [ + ([32766], [88064], 2), + ([2728] * 12, [30000 + 9000 * i for i in range(12)], 2), + ([2046] * 16, [88064] * 16, 2), + ([32766], [88064], 4), + ([5000, 3, 7000], [100000, 5, 0], 2), + ([1024], [0], 2), + ] + ), + *( + pytest.param(_selection_lands_rows, dict(chunks=c, starts=s, world=w, balanced=b, + pieces=0), marks=_cpu, id=f"lands-{sum(c)}-tp{w}-{'balanced' if b else 'even'}") + for c, s, w, b in [ + ([16384], [0], 2, False), + ([4093], [0], 2, False), + ([1025], [0], 4, False), + ([9000, 4000], [60000, 120], 2, True), + ([9000, 4000], [60000, 120], 4, True), + ] + ), + *( + pytest.param(_selection_lands_rows, dict(chunks=c, starts=s, world=2, balanced=b, + pieces=p), marks=_needs_gpu, id=f"pipelined-{sum(c)}-{p}-pieces") + for c, s, b, p in [([32766], [88064], True, 4), ([16382], [0], False, 4)] + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _SELECTION_CASES) +def test_a_tp_ranks_context_selection_is_its_balanced_share(check, kwargs, monkeypatch) -> None: + """An attention-TP rank selects its cost-balanced share of the context rows, then gathers.""" + check(monkeypatch, **kwargs) + + +# ----------------------------------------------------------------------- the pools --- + +ATTN_LAYERS = [i for i in range(NUM_LAYERS) if i % 4 == 3] +GDN_LAYERS = [i for i in range(NUM_LAYERS) if i % 4 != 3] +PLE_LAYER = 1 # `ple_layer_ids: [2]` is one-based +MTP_LAYER = NUM_LAYERS +TOKENS_PER_BLOCK = 32 # `kv_cache_config.tokens_per_block`'s default, which the runtime keeps +MAX_BATCH_SIZE = 4 + + +_ALL = dict(heads_kv=2, kv_head_start=0, num_k_heads=16, k_head_start=0, num_v_heads=48, + v_head_start=0, conv_dim=10240) # fmt: skip + + +def _tp2(r): + return dict(heads_kv=1, kv_head_start=r, num_k_heads=8, k_head_start=8 * r, num_v_heads=24, + v_head_start=24 * r, conv_dim=5120) # fmt: skip + + +def _tp4(r): + return dict(heads_kv=1, kv_head_start=r // 2, num_k_heads=4, k_head_start=4 * r, + num_v_heads=12, v_head_start=12 * r, conv_dim=2560) # fmt: skip + + +#: Per member and rank: KV heads and the first's global index, GDN key and value heads and their +#: first, and the conv channels. TP4 repeats each KV head on two neighbouring ranks. +LAYOUT = { + **{(m, r): _ALL for m in ("tp1", "dep2", "dep4") for r in range(_world(m))}, + **{(m, r): _tp2(r) for m in ("tp2", "tep2") for r in range(2)}, + **{(m, r): _tp4(r) for m in ("tp4", "tep4") for r in range(4)}, +} + +COMMON = dict( + tokens_per_block=TOKENS_PER_BLOCK, + head_dim=256, + index_kv_heads=1, + index_head_dim=128, + conv_kernel=4, + head_k_dim=128, + head_v_dim=128, + stream=4 * 2560, + ple_state_len=(4 - 1) * 3, + ngram_context_len=3 - 1, + dtype=torch.bfloat16, +) + +#: K/V storage per KV-cache algorithm: (pool dtype, innermost extent, whether the QSA index and +#: position pages share one side page). A 32-token bf16 index page is 8192 bytes: an FP8 K page +#: of one head, an NVFP4 one of two, never a bf16 K/V page (16384 bytes a head). +KV_STORAGE = { + None: (torch.bfloat16, 256, False), + QuantAlgo.FP8: (torch.float8_e4m3fn, 256, True), + QuantAlgo.NVFP4: (torch.uint8, 128, True), +} +KV_IDS = {None: "kv_bf16", QuantAlgo.FP8: "kv_fp8", QuantAlgo.NVFP4: "kv_nvfp4"} +_ALGOS = list(KV_STORAGE) + + +def _geometry(member: str, rank: int, kv_cache_quant_algo=None) -> P.PoolGeometry: + return P.pool_geometry( + Qwen4ExpTextConfig(**_TEXT), + _mapping(member, rank), + tokens_per_block=TOKENS_PER_BLOCK, + kv_cache_quant_algo=kv_cache_quant_algo, + dtype=torch.bfloat16, + ) + + +def _meta(shape, dtype): + return torch.empty(shape, dtype=dtype, device="meta") + + +class _MetaManager: + """The stock manager's views on meta, sized by a geometry, with the QSA planes as own roles or + slices of one packed side page. Empty `gdn_layers` stands for a separate draft manager.""" + + PAGES = 64 + + def __init__(self, g: P.PoolGeometry, mapping: Mapping, layers, gdn_layers=GDN_LAYERS): + self.g, self.mapping, self.max_batch_size = g, mapping, MAX_BATCH_SIZE + self.attn_layers, self.gdn_layers = list(layers), list(gdn_layers) + mapped = sorted([*self.attn_layers, *self.gdn_layers]) + self.layer_offsets = {layer: local for local, layer in enumerate(mapped)} + self.conv_section_dims = list(g.conv_sections) + self.slots = MAX_BATCH_SIZE + P.reserved_dummy_slots(mapping) + self.ssm_dtype, self.scale_pages_per_slot = torch.bfloat16, 1 + tpb = g.tokens_per_block + if g.qsa_side_planes: + index, position = g.index_page_bytes, g.position_page_bytes + side = _meta([self.PAGES, len(self.attn_layers) * index + position + 16], torch.uint8) + self._index = { + layer: side[:, k * index : (k + 1) * index] + .view(torch.bfloat16) + .view(self.PAGES, tpb, g.index_kv_heads, g.index_head_dim) + for k, layer in enumerate(self.attn_layers) + } + start = len(self.attn_layers) * index + self._position = ( + side[:, start : start + position].view(torch.int32).view(self.PAGES, tpb, 3) + ) + else: + self._index = { + layer: _meta([self.PAGES, tpb, g.index_kv_heads, g.index_head_dim], torch.bfloat16) + for layer in self.attn_layers + } + self._position = _meta([self.PAGES, tpb, 3], torch.int32) + + def get_buffers(self, layer, kv_layout="NHD"): + assert kv_layout == "HND" + g = self.g + return _meta([self.PAGES, 2, g.heads_kv, g.tokens_per_block, g.kv_width], g.kv_dtype) + + def get_block_scale_buffers(self, layer, kv_layout="NHD"): + assert kv_layout == "HND" + g = self.g + page = [g.heads_kv, g.tokens_per_block, g.head_dim // P.NVFP4_BLOCK] + return _meta([self.PAGES, 2, *page], torch.uint8) + + def get_index_k_buffer(self, layer): + return self._index[layer] + + def get_qsa_position_buffer(self): + return self._position + + def get_conv_states(self, layer): + return _meta([self.slots, self.g.conv_dim, self.g.conv_kernel - 1], torch.bfloat16) + + def get_ssm_states(self, layer): + g = self.g + return _meta([self.slots, g.num_v_heads, g.head_v_dim, g.head_k_dim], self.ssm_dtype) + + def ple_layer_cache(self, layer): + if not self.gdn_layers: + return None + g = self.g + return ( + _meta([self.slots, g.stream, g.ple_state_len], torch.bfloat16), + _meta([self.slots, g.ngram_context_len], torch.int64), + ) + + +#: Every rank under bf16 K/V; each quantized K/V storage at one member per attention layout (the +#: storage sets the pool's dtype, width and side-page packing alone, the rank its heads alone). +_POOL_CASES = [(m, r, None) for m, r in _RANKS] + [ + (m, r, a) for m, r in LAYOUT_RANKS for a in (QuantAlgo.FP8, QuantAlgo.NVFP4) +] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, rank, algo", _POOL_CASES, ids=[f"{m}-rank{r}-{KV_IDS[a]}" for m, r, a in _POOL_CASES] +) +def test_each_rank_binds_the_pools_of_its_layout(member, rank, algo): + """`pool_geometry` is `LAYOUT` and a rank's KV head matches the stock loader's split. + `take_pools` binds a stock-like manager; `take_draft_pools` takes the draft-only manager under + attention TP (unless a cache transceiver forbids it), else the trunk's, refusing the other.""" + g, mapping = _geometry(member, rank, algo), _mapping(member, rank) + kv_dtype, kv_width, side_planes = KV_STORAGE[algo] + assert g._asdict() == dict( + COMMON, + **LAYOUT[member, rank], + kv_dtype=kv_dtype, + kv_width=kv_width, + qsa_side_planes=side_planes, + ) + assert g.conv_sections == (g.num_k_heads * 128, g.num_k_heads * 128, g.num_v_heads * 128) + assert g.kv_block_scales == (algo == QuantAlgo.NVFP4) + heads, rows_per_head = 2, 4 + weight = torch.arange(heads * rows_per_head, dtype=torch.float32).reshape(-1, 1) + tp = P.attention_tp_size(mapping) + local = duplicate_kv_weight(weight, heads, tp).chunk(tp)[rank if tp > 1 else 0] + start, stop = g.kv_head_start * rows_per_head, (g.kv_head_start + g.heads_kv) * rows_per_head + assert torch.equal(local, weight[start:stop]) + + trunk = _MetaManager(g, mapping, ATTN_LAYERS) + pools = P.take_pools(trunk, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert sorted(pools.kv) == ATTN_LAYERS and sorted(pools.conv) == GDN_LAYERS + first = pools.kv[ATTN_LAYERS[0]] + assert tuple(first.k.shape[1:]) == (g.heads_kv, TOKENS_PER_BLOCK, g.kv_width) + assert (first.k_scale is not None) == (algo == QuantAlgo.NVFP4) + + spec = MTPDecodingConfig(max_draft_len=3, num_nextn_predict_layers=1) + separate = P.draft_kv_cache_is_separate(mapping) + assert separate == ( + not mapping.enable_attention_dp and should_use_separate_draft_kv_cache(spec) + ) + assert P.draft_kv_cache_is_separate(mapping, spec) == separate + spec._allow_separate_draft_kv_cache = False # what a cache transceiver makes the executor do + assert not should_use_separate_draft_kv_cache(spec) + assert not P.draft_kv_cache_is_separate(mapping, spec) + draft_alone = _MetaManager(g, mapping, [MTP_LAYER], []) + trunk_and_draft = _MetaManager(g, mapping, [*ATTN_LAYERS, MTP_LAYER]) + for sep in sorted({separate, False}): # the engine's choice, and a disaggregated engine's + cm, wrong = (draft_alone, trunk_and_draft) if sep else (trunk_and_draft, draft_alone) + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=sep) + assert draft.pool_idx == cm.layer_offsets[MTP_LAYER] + assert tuple(draft.kv.v.shape[1:]) == (g.heads_kv, TOKENS_PER_BLOCK, g.kv_width) + with pytest.raises(AssertionError, match="draft layer"): + P.take_draft_pools(wrong, g, MTP_LAYER, separate=sep) + with pytest.raises(AssertionError, match="does not map"): + P.take_draft_pools(trunk, g, MTP_LAYER, separate=sep) + + +# ------------------------------------------------ the stock manager, on GPU --- + + +@contextlib.contextmanager +def _manager(member: str, rank: int, kv_cache_quant_algo=None, spec_config=None, is_draft=False): + """The trunk manager (or with `is_draft` the separate draft KV cache) the engine builds on + this rank, with the core's cache config: bf16 recurrent state, no block reuse, manager V2.""" + # Imported here: the stock factory pulls in the executor, which the cpu_only cases skip. + from tensorrt_llm._torch.pyexecutor._util import ( + _create_kv_cache_manager, + get_kv_cache_manager_cls, + ) + from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype + + mapping = _mapping(member, rank) + model_config = ModelConfig( + pretrained_config=Qwen4ExpTextConfig(**_TEXT), + mapping=mapping, + quant_config=QuantConfig( + quant_algo=QuantAlgo.FP8_BLOCK_SCALES, kv_cache_quant_algo=kv_cache_quant_algo + ), + sparse_attention_config=QSASparseAttentionConfig(), + spec_config=spec_config, + ) + # What the loader does with `kv_cache_config.mamba_ssm_cache_dtype: bfloat16`. + validate_and_set_mamba_ssm_cache_dtype(model_config, "bfloat16") + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + mamba_ssm_cache_dtype="bfloat16", + max_tokens=4096, + use_kv_cache_manager_v2=True, + ) + cls = get_kv_cache_manager_cls(model_config, kv_cache_config) + cm = _create_kv_cache_manager( + None, + cls, + mapping, + kv_cache_config, + tokens_per_block=TOKENS_PER_BLOCK, + max_seq_len=2048, + max_batch_size=MAX_BATCH_SIZE, + spec_config=spec_config, + sparse_attention_config=model_config.sparse_attention_config, + max_num_tokens=2048, + max_beam_width=1, + kv_connector_manager=None, + model_config=model_config, + dtype=torch.bfloat16, + is_draft=is_draft, + max_cuda_graph_batch_size=MAX_BATCH_SIZE, + ) + try: + yield cm + finally: + cm.shutdown() + + +def _stock_trunk(member, rank, algo): + """Each trunk view passes the contract; the manager's per-rank numbers are the geometry's.""" + from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import Qwen4ExpHybridCacheManager + from tensorrt_llm.bindings import DataType + + binding = {None: DataType.BF16, QuantAlgo.FP8: DataType.FP8, QuantAlgo.NVFP4: DataType.NVFP4} + g, mapping = _geometry(member, rank, algo), _mapping(member, rank) + with _manager(member, rank, algo) as cm: + assert type(cm) is Qwen4ExpHybridCacheManager + assert cm.dtype == binding[algo] + assert cm.ssm_state_dtype == torch.bfloat16 + pools = P.take_pools(cm, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert {cm.num_kv_heads_per_layer[cm.layer_offsets[i]] for i in ATTN_LAYERS} == {g.heads_kv} + assert (cm.qsa_index_kv_heads, cm.qsa_index_dim) == (g.index_kv_heads, g.index_head_dim) + assert list(cm.conv_state_shape) == [g.conv_dim, g.conv_kernel - 1] + assert list(cm.ssm_state_shape) == [g.num_v_heads, g.head_v_dim, g.head_k_dim] + assert (cm.qsa_side_planes_size is not None) == g.qsa_side_planes + assert cm._num_reserved_dummy_slots == P.reserved_dummy_slots(mapping) + assert len(set(pools.pool_idx.values())) == len(ATTN_LAYERS) + # One slot space for every QSA view of the layers, one for the recurrent and PLE state. + slots = {view.shape[0] for view in (pools.position_cache, *pools.index_cache.values())} + slots |= {planes.k.shape[0] for planes in pools.kv.values()} + assert len(slots) == 1 + assert {t.shape[0] for t in (*pools.conv.values(), *pools.ssm.values())} == { + pools.ple_state[0].shape[0] + } + assert MTP_LAYER not in cm.layer_offsets + + +def _stock_mtp(member, rank, max_draft_len): + """With MTP the verify buffers (conv windows, GDN replay history) hold this rank's heads at + width `max_draft_len + 1`; the trunk manager maps the draft layer only under attention DP.""" + g, mapping = _geometry(member, rank), _mapping(member, rank) + spec = MTPDecodingConfig(max_draft_len=max_draft_len, num_nextn_predict_layers=1) + with _manager(member, rank, spec_config=spec) as cm: + pools = P.take_pools(cm, g, ATTN_LAYERS, GDN_LAYERS, PLE_LAYER) + assert cm.is_speculative() + assert cm._num_reserved_dummy_slots == P.reserved_dummy_slots(mapping) + ssm0 = pools.ssm[GDN_LAYERS[0]] + caches = {i: cm.mamba_layer_cache(i) for i in GDN_LAYERS} + for i, cache in caches.items(): + P.check_step_buffers( + g, cache.intermediate_conv_window, pools.conv[i], cm.intermediate_state_indices + ) + assert cache.intermediate_ssm is None, "the cached replay keeps no per-step state" + assert not cm.use_gdn_flashinfer_ring_replay + meta = cm.get_replay_state_update_metadata() + # The history and its bookkeeping have replay slots of their own. + replay_slots = meta.prev_num_accepted_tokens.shape[0] + P.check_replay_state( + meta.replay_history_size, + meta.replay_step_width, + meta.prev_num_accepted_tokens, + meta.cache_buf_idx, + ssm0, + replay_slots, + ) + assert meta.replay_step_width == max_draft_len + 1 + P.check_verify_width(meta.replay_step_width) + for i, cache in caches.items(): + P.check_replay_history( + g, + i, + meta.replay_history_size, + replay_slots, + cache.old_x, + cache.old_B, + cache.old_dt, + ) + separate = P.draft_kv_cache_is_separate(mapping) + assert (MTP_LAYER in cm.layer_offsets) == (not separate) + if not separate: + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=False) + assert draft.position_cache.data_ptr() == pools.position_cache.data_ptr() + + +def _stock_draft(member, rank, algo): + """Under attention TP the separate draft manager holds the draft layer alone at trunk shapes.""" + g = _geometry(member, rank, algo) + spec = MTPDecodingConfig(max_draft_len=3, num_nextn_predict_layers=1) + with _manager(member, rank, algo, spec_config=spec, is_draft=True) as cm: + draft = P.take_draft_pools(cm, g, MTP_LAYER, separate=True) + assert cm.ple_layer_cache(PLE_LAYER) is None + assert cm.local_num_mamba_layers == 0 + assert cm.num_kv_heads_per_layer[draft.pool_idx] == g.heads_kv + assert (cm.qsa_side_planes_size is not None) == g.qsa_side_planes + assert draft.kv.k.shape[0] == draft.index_cache.shape[0] == draft.position_cache.shape[0] + + +_STOCK_CASES = [ + *( + pytest.param(_stock_trunk, m, r, dict(algo=a), id=f"trunk-{m}-rank{r}-{KV_IDS[a]}") + for m, r in LAYOUT_RANKS + for a in _ALGOS + ), + # Each attention layout at three drafts, and the widest verify under attention TP and DP. + *( + pytest.param(_stock_mtp, m, r, dict(max_draft_len=d), id=f"mtp{d}-{m}-rank{r}") + for m, r, d in [*((m, r, 3) for m, r in LAYOUT_RANKS), ("tp4", 3, 7), ("dep4", 3, 7)] + ), + *( + pytest.param(_stock_draft, m, r, dict(algo=a), id=f"draft-kv-{m}-rank{r}-{KV_IDS[a]}") + for m, r in LAYOUT_RANKS + if m != "dep4" + for a in _ALGOS + ), +] + + +@_needs_gpu +@pytest.mark.parametrize("check, member, rank, kwargs", _STOCK_CASES) +def test_the_stock_manager_serves_the_layout(check, member, rank, kwargs, monkeypatch): + """The stock `Qwen4ExpHybridCacheManager`, built as the engine does, passes the contract.""" + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + check(member, rank, **kwargs) + + +# ------------------------------------------------------ the decode step's runtime paths --- + + +def _runtime_hooks(monkeypatch): + from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks + + hooks = M._runtime_hooks() + assert isinstance(hooks.decode_prologue, M.SpecStepMetadataPrologue) + assert replace(hooks, decode_prologue=None) == RuntimeHooks(loop_on_execution_stream=True) + + +def _decode_stream(monkeypatch): + """A generation-only step of up to `_DECODE_HIGH_PRIORITY_MAX_ROWS` rows runs on the core's + high-priority decode stream, ordered after the caller's work and before its next, eager and + captured; a larger step, a prefill step or a core without the stream stays on the caller's.""" + hi = torch.cuda.Stream(priority=-100) + assert hi.priority < torch.cuda.Stream().priority + core = SimpleNamespace(_decode_stream=hi) + caller = torch.cuda.current_stream() + seen = [] + src = torch.full((1 << 20,), 1.0, device="cuda") + + def step(scale): + seen.append(torch.cuda.current_stream()) + return src * scale + + src.fill_(3.0) # the caller's pending write is ordered before the step + out = M._on_decode_stream(core, 3, step, 2.0) + assert seen[-1] == hi and torch.cuda.current_stream() == caller + assert torch.equal(out, torch.full_like(src, 6.0)) + # Captured: the waits become graph edges and a replay reads the refilled source. + side = torch.cuda.Stream() + side.wait_stream(caller) + with torch.cuda.stream(side): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=side): + captured = M._on_decode_stream(core, 3, step, 5.0) + caller.wait_stream(side) + src.fill_(1.5) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, torch.full_like(src, 7.5)) + target = M.DecodeTarget.__new__(M.DecodeTarget) + target.forward = lambda *a, **k: seen.append(torch.cuda.current_stream()) + target.mtp_forward = lambda *a, **k: seen.append(torch.cuda.current_stream()) + prefill = SimpleNamespace(forward=target.forward, mtp_forward=target.mtp_forward) + rows = M._DECODE_HIGH_PRIORITY_MAX_ROWS + for chosen, n, want in ((target, rows, hi), (target, rows + 1, caller), (prefill, 3, caller)): + ids = torch.zeros(n, dtype=torch.int32, device="cuda") + core.__dict__.update(_step_begun=True, _select_target=lambda md, t=chosen: t) + M.Qwen4ExpUnifiedModelingV2.forward(core, None, input_ids=ids) + assert seen[-1] == want, (n, chosen) + M.MTPLayer.forward(SimpleNamespace(core=core), None, None, ids, None, None, None) + assert seen[-1] == want, (n, chosen) + M._on_decode_stream(SimpleNamespace(), 3, step, 1.0) + assert seen[-1] == caller, "a core without the stream runs on the caller's" + + +class _ShardHead: + """A COLUMN-parallel `lm_head` stand-in recording each call's `gather_output`.""" + + def __init__(self, padding_size: int = 0) -> None: + from tensorrt_llm._torch.modules.linear import TensorParallelMode + + self.tp_mode = TensorParallelMode.COLUMN + self.gather_output = True + self.padding_size = padding_size + self.calls = [] + + def __call__(self, x, gather_output=True): + self.calls.append(gather_output) + return "shard" + + +def _draft_head_shard(monkeypatch, case, sharded): + """With the worker's TP key exchange a greedy draft pass keeps this rank's vocab shard of the + logits; rejection sampling, guidance, excess rows, a padded vocab or no exchange gather them.""" + monkeypatch.setattr(M, "_hc_site", lambda *args: (None, torch.zeros(3, 8), None)) + core = SimpleNamespace(_mtp=SimpleNamespace(fhc_norm=None, fhc_first=None, fhc_up=None)) + head = _ShardHead(padding_size=16 if case == "padded_vocab" else 0) + worker = SimpleNamespace(key_exchange=SimpleNamespace(MAX_ROWS=8), guided_decoder=None) + if case == "no_exchange": + worker.key_exchange = None + if case == "guided": + worker.guided_decoder = object() + if case == "rows_past_exchange": + worker.key_exchange.MAX_ROWS = 2 + layer = M.MTPLayer(core, logits_processor=SimpleNamespace(forward=lambda *args: "gathered")) + layer.worker = worker + layer.spec_metadata = SimpleNamespace( + wants_advanced_draft_sampling=case == "rejection_sampling" + ) + out = layer.shared_head(torch.zeros(3, 32), head, None, True) + assert out == ("shard" if sharded else "gathered") + assert head.calls == ([False] if sharded else []) + + +_RUNTIME_CASES = [ + pytest.param(_runtime_hooks, {}, marks=_cpu, id="runtime-hooks"), + pytest.param(_decode_stream, {}, marks=_needs_gpu, id="decode-stream"), + *( + pytest.param(_draft_head_shard, dict(case=c, sharded=c == "greedy"), marks=_cpu, + id=f"draft-head-{c}") + for c in ("greedy", "rejection_sampling", "no_exchange", "guided", "rows_past_exchange", + "padded_vocab") + ), +] # fmt: skip + + +@pytest.mark.parametrize("check, kwargs", _RUNTIME_CASES) +def test_the_decode_steps_runtime_paths(check, kwargs, monkeypatch) -> None: + """The runtime hooks, the high-priority decode stream and the draft head's vocab-shard path.""" + check(monkeypatch, **kwargs) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py new file mode 100644 index 000000000000..a090c410605e --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_spec_rollback.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU test for the FP8 unified target's `SpeculativeRollback`: after a simulated verify step, +every pool and plane is bitwise that of a run over the accepted prefixes only, compared in full +so a write to a slot the step does not own fails too.""" + +from __future__ import annotations + +import itertools + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.attention.qsa_rollback_planes import ( # noqa: E501 + plane_table as _plane_table, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ple.ngram_token_history import ( # noqa: E501 + ngram_token_history, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.modeling import ( # noqa: E501 + SpeculativeRollback, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.qsa_paged_index_store import ( # noqa: E501 + qsa_paged_index_store, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.spec_rollback_begin__verify import ( # noqa: E501 + spec_rollback_begin, +) + +__extra_import_path__ = ["../../.."] +from _validating import certifying # noqa: E402,F401 — path above; fixture by name + +assert torch.cuda.is_available(), "the rollback handler test requires a CUDA device" + +DEV = "cuda" +EOS, VOCAB, NGRAM = 248044, 248320, 3 +KERNEL, DILATION, CHANNELS = 4, 3, 64 +STATE_LEN = (KERNEL - 1) * DILATION +TPB, INDEX_DIM, LAYERS, COMPRESS = 32, 128, 3, 4 +SLOTS, PAGES, BLOCKS_PER_SEQ = 24, 48, 4 + + +class _Batch: + """One verify step: fresh context requests of `ctx_lens` rows, then generation requests of + `p` rows at `gen_starts` cached tokens, in `slots`. Generation request `s` reuses request + `dup_of[s]`'s block-table row, as the CUDA-graph padding dummy does.""" + + def __init__(self, seed, ctx_lens, gen_starts, p, slots, dup_of=None): + g = torch.Generator().manual_seed(seed) + self.ctx_lens, self.num_contexts, self.p = list(ctx_lens), len(ctx_lens), p + self.n = len(gen_starts) + self.slots = torch.tensor(slots, dtype=torch.int32, device=DEV) + assert self.slots.shape[0] == self.num_contexts + self.n + lengths = self.ctx_lens + [p] * self.n + starts = [0] * self.num_contexts + list(gen_starts) + perm = torch.randperm(PAGES, generator=g) + table = torch.reshape(perm[: len(lengths) * BLOCKS_PER_SEQ], [len(lengths), -1]) + for dup, original in (dup_of or {}).items(): + table[self.num_contexts + dup] = table[self.num_contexts + original] + self.block_table = table.to(DEV, torch.int32) + req, logical = [], [] + for r, (length, start) in enumerate(zip(lengths, starts)): + req += [r] * length + logical += range(start, start + length) + self.rows = len(req) + self.prefix = sum(self.ctx_lens) + self.req_idx = torch.tensor(req, dtype=torch.int32, device=DEV) + self.logical = torch.tensor(logical, dtype=torch.int64, device=DEV) + self.carry = torch.tensor(_carried(self), dtype=torch.bool, device=DEV) + self.lengths = lengths + self.qsl = _qsl(lengths) + + # Every pre-step pool and plane is random, so a stray write shows. + self.ngram0 = torch.randint(0, VOCAB, (SLOTS, NGRAM - 1), generator=g).to(DEV) + self.conv0 = torch.randn(SLOTS, CHANNELS, STATE_LEN, generator=g).to(DEV, torch.bfloat16) + self.index0 = [ + torch.randn(PAGES, TPB, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.position0 = torch.randint(0, 1 << 20, (PAGES, TPB, 3), generator=g).to( + DEV, torch.int32 + ) + self.ids = torch.randint(0, VOCAB, (self.rows,), generator=g).to(DEV, torch.int32) + self.x = torch.randn(self.rows, CHANNELS, generator=g).to(DEV, torch.bfloat16) + self.keys = [ + torch.randn(self.rows, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.anchor_keys = [ + torch.randn(self.rows, 1, INDEX_DIM, generator=g).to(DEV, torch.bfloat16) + for _ in range(LAYERS) + ] + self.coords = torch.randint(0, 1 << 20, (self.rows, 3), generator=g).to(DEV, torch.int32) + self.ngram, self.conv = self.ngram0.clone(), self.conv0.clone() + self.index, self.position = [t.clone() for t in self.index0], self.position0.clone() + self.accepted = torch.ones(self.num_contexts + self.n, dtype=torch.int32, device=DEV) + + def reset(self): + self.ngram.copy_(self.ngram0) + self.conv.copy_(self.conv0) + for live, pre in zip(self.index, self.index0): + live.copy_(pre) + self.position.copy_(self.position0) + + def set_accepted(self, gen_accepted): + assert len(gen_accepted) == self.n + assert all(1 <= a <= self.p for a in gen_accepted) + self.gen_accepted = list(gen_accepted) + values = [1] * self.num_contexts + list(gen_accepted) + self.accepted.copy_(torch.tensor(values, dtype=torch.int32)) + + +def _qsl(lengths): + return torch.tensor([0, *itertools.accumulate(lengths)], dtype=torch.int64, device=DEV) + + +def _conv_state(conv, x, lengths, slots, carry, publish_end=None): + """Stand-in for `ple_gate_conv_inject_norm`'s conv state: per sequence below `publish_end`, the + last `STATE_LEN` columns of `[carried state or zeros | its input]`, by capturable slices.""" + start = 0 + for s, length in enumerate(lengths): + if publish_end is not None and s >= publish_end: + break + slot = slots[s : s + 1].long() + prev = conv[slot][0] if carry[s] else torch.zeros_like(conv[0]) + window = torch.cat([prev, x[start : start + length].t()], 1) + conv[slot] = window[:, -STATE_LEN:].unsqueeze(0) + start += length + + +def _carried(b): + """Per sequence, whether it carries a state (generation) or starts fresh (context).""" + return [False] * b.num_contexts + [True] * b.n + + +def _write_planes(b, layer, plane, position, keep=slice(None)): + """The index store over rows `keep`, then a compression stand-in overwriting anchor slots.""" + req_idx, logical, anchor_keys = b.req_idx[keep], b.logical[keep], b.anchor_keys[layer][keep] + keys, coords = b.keys[layer][keep], b.coords[keep] + qsa_paged_index_store(keys, coords, plane, position, req_idx, logical, b.block_table, TPB) + pages = b.block_table[req_idx.long(), logical // TPB].long() + within = logical % TPB + anchor = ((logical + 1) % COMPRESS == 0).reshape(-1, 1, 1) + plane[pages, within] = torch.where(anchor, anchor_keys, plane[pages, within]) + + +def _verify_step(b, rollback, fused, begin_position=False): + """The forward's writes over all `p` rows, staged at its call sites, then the commit. Under + `fused` the generation conv rows stay unpublished and the commit rebuilds them from the pool.""" + rollback.discard() + rollback.begin( + num_contexts=b.num_contexts, + prefix_tokens=b.prefix, + num_gens=b.n, + rows_per_gen=b.p, + slots=b.slots[b.num_contexts :], + request_indices=b.req_idx[b.prefix :], + logical_positions=b.logical[b.prefix :], + block_table=b.block_table, + tokens_per_block=TPB, + fused=fused, + **(dict(position_cache=b.position) if begin_position else {}), + ) + rollback.stage_ple_verify(b.ngram, b.conv, b.ids) + ngram_token_history(b.ids, b.ngram, b.slots, b.carry, b.qsl, EOS, NGRAM) + rollback.stage_ple_conv_input(b.x) + _conv_state(b.conv, b.x, b.lengths, b.slots, _carried(b), b.num_contexts if fused else None) + for layer in range(LAYERS): + rollback.stage_index(b.position) + _write_planes(b, layer, b.index[layer], b.position) + rollback.commit_speculative_states(b.accepted, b.slots, b.num_contexts) + + +def _reference(b): + """The same writes from the pre-step state over context rows and accepted prefixes only.""" + keep = list(range(b.prefix)) + lengths = list(b.ctx_lens) + for s, a in enumerate(b.gen_accepted): + first = b.prefix + s * b.p + keep += range(first, first + a) + lengths.append(a) + keep = torch.tensor(keep, dtype=torch.long, device=DEV) + ngram, conv = b.ngram0.clone(), b.conv0.clone() + index = [t.clone() for t in b.index0] + position = b.position0.clone() + ngram_token_history(b.ids[keep], ngram, b.slots, b.carry, _qsl(lengths), EOS, NGRAM) + _conv_state(conv, b.x[keep], lengths, b.slots, _carried(b)) + for layer in range(LAYERS): + _write_planes(b, layer, index[layer], position, keep) + return ngram, conv, index, position + + +def _bits(t): + return t.view(torch.int16) if t.dtype == torch.bfloat16 else t + + +def _assert_state(b, expected, what): + ngram, conv, index, position = expected + assert torch.equal(b.ngram, ngram), f"{what}: n-gram pool" + assert torch.equal(_bits(b.conv), _bits(conv)), f"{what}: conv pool" + for layer in range(LAYERS): + assert torch.equal(_bits(b.index[layer]), _bits(index[layer])), f"{what}: index {layer}" + assert torch.equal(b.position, position), f"{what}: position plane" + + +# (ctx_lens, gen_starts, p, gen_accepted): positions straddle pages and land on compression +# anchors; p is 4 (MTP-3) behind contexts, and 2 and 8 (max_draft_len 1 and 7) in decode alone. +STEPS = { + "mixed": ((7, 3), (12, 29, 95), 4, (2, 1, 3)), + "decode_p2": ((), (3, 31, 64), 2, (1, 2, 1)), + "decode_p8": ((), (0, 27, 60, 90, 101, 5, 44, 119), 8, (1, 2, 3, 4, 5, 6, 7, 8)), +} +#: Acceptance counts one captured decode_p8 step is replayed with (bonus token included). +_REPLAYS = [(1,) * 8, (8, 7, 6, 5, 4, 3, 2, 1), (2, 8, 1, 7, 3, 6, 4, 5), (8,) * 8] + + +def _slots(num): + return list(range(3, 3 + 2 * num, 2)) + + +#: Handler forms: torch staging with batched plane gather; `fused` (one-launch stage and commit); +#: `fused_begin`, opened with the position plane so one launch stages all, as in a captured decode. +MODES = ["batched", "fused", "fused_begin"] + + +def _mode_args(mode): + """`(fused, begin_position)` of a handler form.""" + return mode != "batched", mode == "fused_begin" + + +def _rollback(b): + rollback = SpeculativeRollback() + rollback.set_index_planes(b.index) + return rollback + + +def _eager(case, mode): + ctx_lens, gen_starts, p, gen_accepted = STEPS[case] + b = _Batch(11, ctx_lens, gen_starts, p, _slots(len(ctx_lens) + len(gen_starts))) + b.set_accepted(gen_accepted) + _verify_step(b, _rollback(b), *_mode_args(mode)) + _assert_state(b, _reference(b), case) + + +def _captured(case, mode): + """The step and commit captured once, replayed with new acceptances over reset state.""" + ctx_lens, gen_starts, p, _ = STEPS[case] + b = _Batch(18, ctx_lens, gen_starts, p, _slots(len(gen_starts))) + rollback = _rollback(b) + b.set_accepted((p,) * len(gen_starts)) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + _verify_step(b, rollback, *_mode_args(mode)) + torch.cuda.current_stream().wait_stream(side) + b.reset() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _verify_step(b, rollback, *_mode_args(mode)) + for accepted in _REPLAYS: + b.reset() + b.set_accepted(accepted) + graph.replay() + torch.cuda.synchronize() + _assert_state(b, _reference(b), f"replay {accepted}") + + +def _padded(case, mode): + """Requests 2 and 3 are the CUDA-graph padding dummy request 1 is: one slot, one block-table + row, written concurrently with no defined winner, so only the unique request is compared.""" + gen_starts, p = (5, 40, 40, 40), 4 + b = _Batch(17, (), gen_starts, p, [3, 7, 7, 7], dup_of={2: 1, 3: 1}) + b.set_accepted((2, 3, 3, 3)) + _verify_step(b, _rollback(b), *_mode_args(mode)) + pages = b.block_table[0].long() + expected = _reference(_single(b, 0)) + assert torch.equal(b.ngram[3], expected[0][3]) + assert torch.equal(_bits(b.conv[3]), _bits(expected[1][3])) + for layer in range(LAYERS): + assert torch.equal(_bits(b.index[layer][pages]), _bits(expected[2][layer][pages])), ( + f"index {layer}" + ) + assert torch.equal(b.position[pages], expected[3][pages]) + + +def _single(b, s): + """`b` restricted to its generation request `s`, same pre-step state.""" + one = _Batch.__new__(_Batch) + one.__dict__.update(b.__dict__) + rows = torch.arange(b.prefix + s * b.p, b.prefix + (s + 1) * b.p, device=DEV) + one.ctx_lens, one.num_contexts, one.n, one.prefix = [], 0, 1, 0 + one.lengths = [b.p] + one.slots = b.slots[b.num_contexts + s : b.num_contexts + s + 1] + one.carry = b.carry[b.num_contexts + s : b.num_contexts + s + 1] + one.qsl = _qsl(one.lengths) + one.ids, one.x = b.ids[rows], b.x[rows] + one.keys = [k[rows] for k in b.keys] + one.anchor_keys = [k[rows] for k in b.anchor_keys] + one.coords = b.coords[rows] + one.logical = b.logical[rows] + one.req_idx = torch.zeros_like(b.req_idx[rows]) + one.block_table = b.block_table[b.num_contexts + s : b.num_contexts + s + 1] + one.gen_accepted = [b.gen_accepted[s]] + return one + + +_RUNS = ( + [(_eager, case, mode) for case in STEPS for mode in MODES] + + [(_captured, "decode_p8", mode) for mode in ("fused", "fused_begin")] + + [(_padded, "dup", "fused")] +) + + +@pytest.mark.parametrize( + "run,case,mode", _RUNS, ids=[f"{r.__name__[1:]}-{c}-{m}" for r, c, m in _RUNS] +) +def test_commit_leaves_the_accepted_prefix_state(run, case, mode): + """After partial acceptance, the n-gram and conv pools and every index and position plane + are bitwise those of a run over the context rows and the accepted prefixes only.""" + run(case, mode) + + +@pytest.mark.parametrize( + "cell", spec_rollback_begin.CELLS, ids=[c.why[:40] for c in spec_rollback_begin.CELLS] +) +def test_certified_cells(cell, certifying): # noqa: F811 — fixture by name + """Bitwise the composed torch reference, eagerly and in a capture replayed at new slots.""" + certifying(spec_rollback_begin) + n, p = cell.spec["requests"], cell.spec["p"] + starts = [(37 * s + 5) % 100 for s in range(n)] + b = _Batch(23 + n * p, (), starts, p, _slots(n)) + planes = [torch.randn_like(t) for t in b.index0] * (cell.spec["planes"] // LAYERS) + ptrs, strides = _plane_table(planes) + args = (b.req_idx, b.logical, b.block_table, TPB) + got = spec_rollback_begin(*args, ptrs, strides, b.position0) + want = spec_rollback_begin.reference(*args, planes, b.position0) + for g, w in zip(got, want): + assert torch.equal(_bits(g), _bits(w)) + if not cell.spec["capture"]: + return + logical = b.logical.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outs = spec_rollback_begin( + b.req_idx, logical, b.block_table, TPB, ptrs, strides, b.position0 + ) + logical.add_(3) + graph.replay() + torch.cuda.synchronize() + want = spec_rollback_begin.reference( + b.req_idx, logical, b.block_table, TPB, planes, b.position0 + ) + for g, w in zip(outs, want): + assert torch.equal(_bits(g), _bits(w)), "replay at new slots" diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py new file mode 100644 index 000000000000..3947f9d64df6 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_ssm_ops.py @@ -0,0 +1,1764 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the FP8 unified target's Gated-DeltaNet entries over their certified cells, and +for launches no cell reaches. Engine-sized state pools are reserved virtual ranges mapped under +the touched slots only (`_SparsePage`); every other byte of their pages is checked unchanged.""" + +from __future__ import annotations + +import math +import types +import warnings +import zlib + +import cuda.bindings.driver as cuda_drv +import pytest +import torch +import torch.nn.functional as F +from flashinfer.gdn_kernels.delta_rule_dsl.varlen_helper import should_use_cp_host +from flashinfer.utils import get_compute_capability, get_device_name, get_device_sm_count + +import tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.causal_conv1d_token_major__verify as ctm # noqa: E501 +import tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_cached_replay_cute__verify as cr # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.norm.rms_norm_gated_token_major import ( # noqa: E501 + rms_norm_gated_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.gdn_prefill_conv_post import ( # noqa: E501 + HEAD_K_DIM, + HEAD_V_DIM, + NUM_K_HEADS, + NUM_V_HEADS, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as _m, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.gemm.low_m_bf16_mm__decode import ( # noqa: E501 + low_m_bf16_mm, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.causal_conv1d_token_major__verify import ( # noqa: E501 + causal_conv1d_token_major, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.flashinfer_chunk_gated_delta_rule import ( # noqa: E501 + INPLACE_MAX_TOKENS, + flashinfer_chunk_gated_delta_rule, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.ssm.gdn_in_proj_conv__verify import ( # noqa: E501 + fused_tactic, + gdn_in_proj_conv, +) +from tensorrt_llm._torch.modules.fla.cache_manager import ( + MIN_REPLAY_HISTORY_SIZE, + ReplayStateUpdateMetadata, + _advance_replay_state, +) +from tensorrt_llm._torch.modules.fla.cached_replay import ( + CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + commit_gdn_cached_replay_history_layers, +) +from tensorrt_llm._torch.modules.fla.cached_replay import ( + fused_recurrent_gated_delta_rule_cached_replay_update as stock_replay, +) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import ( + REPLAY_WORK_ITEM_WIDTH, + _build_replay_work_items_triton, +) + +__extra_import_path__ = ["../../.."] +from _validating import ULP, certifying, validating # noqa: E402,F401 — path above; fixture by name + +assert torch.cuda.is_available(), "the Gated-DeltaNet entries require a CUDA device" + +DEV = "cuda" +#: Key heads, value heads, key and value head dims of the whole layer (tp1, a dep4 rank). +H, HV, K, V = NUM_K_HEADS, NUM_V_HEADS, HEAD_K_DIM, HEAD_V_DIM + + +# ------------------------------------------------- reserved pages, mapped where touched --- + + +def _scattered(n: int, slots: int) -> list[int]: + """`n` distinct slots: both ends, then a stride-37 walk (coprime to every `slots - 2` here).""" + rest = [(i * 37 + 4) % (slots - 2) + 1 for i in range(max(n - 2, 0))] + picked = ([slots - 1, 0] + rest)[:n] + assert len(set(picked)) == n, (n, slots) + return picked + + +def _check(result): + err, *value = result + assert err == cuda_drv.CUresult.CUDA_SUCCESS, err + return value[0] if len(value) == 1 else value + + +#: Array-interface type of each pool dtype; bf16 has none, so it is read as int16 and viewed. +_TYPESTR = {torch.float32: (" torch.Tensor: + typestr, carrier = _TYPESTR[self.dtype] + es = self.es + t = torch.as_tensor( + _ArrayInterface(self.va + es * offset, shape, [es * s for s in strides], typestr), + device=DEV, + ) + assert t.dtype == carrier + return t.view(self.dtype) + + def close(self) -> None: + torch.cuda.synchronize() + for ptr, size, handle in self.maps: + _check(cuda_drv.cuMemUnmap(ptr, size)) + _check(cuda_drv.cuMemRelease(handle)) + _check(cuda_drv.cuMemAddressFree(cuda_drv.CUdeviceptr(self.va), self.size)) + + +class _EnginePool: + """The stock manager's coalesced state page (`layers` `[v_heads, head_v, head_k]` states a + slot), viewed at the last layer's sub-page. Mapped under each touched slot's page plus the next + slot's first layer, and under the view's base (torch reads the device off it).""" + + def __init__(self, slots, layers, used, v_heads, head_v, head_k, dtype, init): + layer = v_heads * head_v * head_k + stride = layers * layer + total = slots * stride + base = (layers - 1) * layer + self.used = sorted(set(used)) + self.spans = [(s * stride, min((s + 1) * stride + layer, total)) for s in self.used] + self.page = _SparsePage(total, [*self.spans, (base, base + layer)], dtype) + self.own, self.layer = base, layer + self.pool = self.page.view( + base, (slots, v_heads, head_v, head_k), (stride, head_v * head_k, head_k, 1) + ) + self.flat = [self.page.view(lo, (hi - lo,), (1,)) for lo, hi in self.spans] + for f in self.flat: + init(f) + torch.cuda.synchronize() + self.before = [f.clone() for f in self.flat] + + def dense(self, slots: torch.Tensor): + """A dense copy of the touched slots and `slots` renumbered into it (-1 stays).""" + index = {s: i for i, s in enumerate(self.used)} + compact = torch.stack([self.pool[s] for s in self.used]).contiguous() + renamed = [index[s] if s >= 0 else -1 for s in slots.tolist()] + return compact, torch.tensor(renamed, dtype=slots.dtype, device=DEV) + + def touched(self) -> torch.Tensor: + return torch.stack([self.pool[s] for s in self.used]) + + def assert_rest_unchanged(self) -> None: + """Every mapped byte outside this layer's state of each touched slot is unchanged.""" + torch.cuda.synchronize() + lo, hi = self.own, self.own + self.layer + for f, b, s in zip(self.flat, self.before, self.used): + assert torch.equal(f[:lo], b[:lo]) and torch.equal(f[hi:], b[hi:]), ( + f"slot {s}: the call wrote outside its state" + ) + + def close(self) -> None: + del self.pool, self.flat + self.page.close() + + +# ------------------------------------------------ flashinfer_chunk_gated_delta_rule --- + +#: The Triton chunk op's band (rtol bf16 default; atol 2e-3 on `o`, 2e-2 on states): this op is +#: the same class of sum against fp64; a value head reading the wrong key head lands far outside. +CHUNK_RTOL, CHUNK_ATOL_OUTPUT, CHUNK_ATOL_STATE = 1.6e-2, 2e-3, 2e-2 +#: Rows past `out[:, :num_tokens]` in a `slice` cell's wider output buffer; they must not change. +_CHUNK_OUTPUT_TAIL = 16 + + +def _chunk_compare(out: tuple, ref: tuple) -> None: + """`(o, final_state, pool)` against `reference`'s triple; `None` must match.""" + for got, want, atol in zip(out, ref, (CHUNK_ATOL_OUTPUT, CHUNK_ATOL_STATE, CHUNK_ATOL_STATE)): + assert (got is None) == (want is None), (got is None, want is None) + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape, ( + (got.dtype, got.shape), + (want.dtype, want.shape), + ) + assert torch.isfinite(got).all() + torch.testing.assert_close(got.float(), want.float(), rtol=CHUNK_RTOL, atol=atol) + + +def _flashinfer_path(spec) -> str: + """The FlashInfer kernel the call reaches: `inplace` for a `has_initial_state` call of at most + `INPLACE_MAX_TOKENS` tokens, else `cp` when FlashInfer's heuristic picks it, else `chunked`.""" + if "initial" in spec and sum(spec["seqlens"]) <= INPLACE_MAX_TOKENS: + return "inplace" + device = torch.device(DEV) + cp = should_use_cp_host( + len(spec["seqlens"]) * spec["geometry"][1], + get_device_sm_count(device), + get_device_name(device), + device_capability=get_compute_capability(device), + ) + return "cp" if cp else "chunked" + + +def _chunk_build(spec, seed): + """`(args, engine)`: inputs in a GDN post-conv step's domain (unit-norm q/k, g < 0 in log + space, beta a sigmoid), and the `_EnginePool` behind the pool or None.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + num_k_heads, num_v_heads, head_k_dim, head_v_dim = spec["geometry"] + seqlens = spec["seqlens"] + total = sum(seqlens) + q = torch.randn(1, total, num_k_heads, head_k_dim, generator=gen, device=DEV) + k = torch.randn(1, total, num_k_heads, head_k_dim, generator=gen, device=DEV) + q = q / torch.sqrt(q.pow(2).sum(-1, keepdim=True) + 1e-6) + k = k / torch.sqrt(k.pow(2).sum(-1, keepdim=True) + 1e-6) + q, k = q.bfloat16(), k.bfloat16() + v = torch.randn( + 1, total, num_v_heads, head_v_dim, generator=gen, device=DEV, dtype=torch.bfloat16 + ) + a_log = torch.log(torch.rand(num_v_heads, generator=gen, device=DEV) * 15 + 1) + x = torch.randn(1, total, num_v_heads, generator=gen, device=DEV) + torch.randn( + num_v_heads, generator=gen, device=DEV + ) + g = (-torch.exp(a_log) * torch.nn.functional.softplus(x)).float() + beta = torch.rand(1, total, num_v_heads, generator=gen, device=DEV).sigmoid().float() + engine = None + if spec.get("page_layers"): + engine = _EnginePool( + spec["slots"], + spec["page_layers"], + spec["indices"], + num_v_heads, + head_v_dim, + head_k_dim, + torch.bfloat16, + lambda f: f.copy_(torch.randn(f.numel(), generator=gen, device=DEV) * 0.1), + ) + pool = engine.pool + else: + pool = ( + torch.randn( + spec["slots"], num_v_heads, head_v_dim, head_k_dim, generator=gen, device=DEV + ) + * 0.1 + ).to(spec.get("pool_dtype", torch.float32)) + cu = torch.tensor([0] + torch.tensor(seqlens).cumsum(0).tolist(), dtype=torch.long, device=DEV) + indices = torch.tensor(spec["indices"], dtype=torch.int32, device=DEV) + output = None + if spec.get("output") == "buffer": + output = torch.full( + (1, total, num_v_heads, head_v_dim), -7.0, device=DEV, dtype=torch.bfloat16 + ) + elif spec.get("output") == "slice": + output = torch.full( + (1, total + _CHUNK_OUTPUT_TAIL, num_v_heads, head_v_dim), + -7.0, + device=DEV, + dtype=torch.bfloat16, + )[:, :total] + extra = {} + if "initial" in spec: + extra["has_initial_state"] = torch.tensor(spec["initial"], dtype=torch.bool, device=DEV) + if spec.get("linear_gate"): + extra["linear_gate"] = True + g = torch.exp(g) + args = dict( + **extra, + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=None, + initial_state=pool, + initial_state_indices=indices, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=output, + state_workspace=None, + ) + return args, engine + + +def _check_flashinfer_chunk_gated_delta_rule(cell) -> None: + args, engine = _chunk_build(cell.spec, seed=zlib.crc32(cell.why.encode())) + try: + if engine is None: + pool = args["initial_state"] + pool_before = pool.clone() + ref = flashinfer_chunk_gated_delta_rule.reference(**args) + else: + compact, renamed = engine.dense(args["initial_state_indices"]) + ref = flashinfer_chunk_gated_delta_rule.reference( + **dict(args, initial_state=compact, initial_state_indices=renamed) + ) + output = args["output"] + full = None if output is None else output._base + tail_before = None if full is None else full[:, output.shape[1] :].clone() + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + o, final_state = flashinfer_chunk_gated_delta_rule(**args) + # FlashInfer falls back from a CP heuristic match with a RuntimeWarning, so a CP + # cell certifies the CP pipeline only if none was raised. + fallback = [str(w.message) for w in caught if "falling back to non-CP" in str(w.message)] + path = "chunked" if fallback else _flashinfer_path(cell.spec) + + got_pool = pool if engine is None else engine.touched() + _chunk_compare((o, final_state, got_pool), ref) + if "fi_path" in cell.spec: + assert path == cell.spec["fi_path"], (path, fallback) + if output is not None: + assert o.data_ptr() == output.data_ptr(), "output= must be written in place" + if tail_before is not None: + assert torch.equal(full[:, output.shape[1] :], tail_before), "rows past output written" + if engine is None: + named = set(args["initial_state_indices"].tolist()) + unnamed = [s for s in range(pool.shape[0]) if s not in named] + assert torch.equal(pool[unnamed], pool_before[unnamed]), "unnamed slots written" + else: + engine.assert_rest_unchanged() + finally: + if engine is not None: + engine.close() + + +# ------------------------------------------------------- the cached-replay test harness --- + +#: `replay_history_size` = max(MIN_REPLAY_HISTORY_SIZE, T): the minimum at every T <= 8. +HIST = MIN_REPLAY_HISTORY_SIZE +BF16_ULP = ULP[torch.bfloat16] + +#: This checkpoint's GDN `A_log` / `dt_bias` statistics: mean, standard deviation, range. +A_LOG_STATS = (1.006, 1.487, (-7.125, 5.0625)) +DT_BIAS_STATS = (-4.411, 1.732, (-11.4375, 2.53125)) + +#: Element gate of the bf16 members (output, old_u, old_k), in bf16 ulps of their block's absmax: +#: the kernel rounds q, k and the history-mixing coefficients to bf16 before its dots. +_ELEMENT_ULPS = 4.0 +#: Relative-RMS gate of the output, in bf16 ulps. +_OUTPUT_RMS_ULPS = 1.5 +#: Element gate of an fp32 checkpoint state, relative to its (slot, head) absmax: the commit +#: splits each history term into bf16 hi and lo parts (~16 bits), up to 16 terms within 2^-17. +_STATE_TOL = 2.0**-13 +#: Element gate of a bf16 checkpoint state: that fp32 commit rounded once, as the reference's is, +#: so the two are equal or adjacent, at most 2 ulps of the head's absmax apart. +_BF16_STATE_ULPS = 2.0 +#: Gate of the log-decays, |dG| <= atol + rtol * |G|: G enters only as exp(G_t - G_j), atol is an +#: eighth of a bf16 ulp of the u it scales, rtol covers the cumsum where |G| reaches thousands. +_G_ATOL, _G_RTOL = 2.0**-11, 2.0**-18 + + +def _gate(name, got, want, element, rms=None): + """Every element of `got` within `element` of its block's absmax in `want` (a block is the + last two dims, so a near-silent element's rounding does not read as relative noise).""" + block = got.shape[-2] * got.shape[-1] + g, w = got.float().reshape(-1, block), want.float().reshape(-1, block) + keep = ~torch.isnan(w).any(1) + g, w = g[keep], w[keep] + scale = w.abs().amax(1, keepdim=True).clamp_min(torch.finfo(torch.float32).tiny) + ratio = ((g - w).abs() / scale).nan_to_num(nan=float("inf")) + worst = ratio.max().item() if ratio.numel() else 0.0 + assert worst <= element, ( + f"{name}: an element is {worst:.3e} of its block's absmax from the reference " + f"(gate {element:.3e})" + ) + if rms is not None and w.numel(): + rel = ((g - w).pow(2).mean().sqrt() / w.pow(2).mean().sqrt().clamp_min(1e-30)).item() + assert rel <= rms, f"{name}: relative RMS {rel:.3e} exceeds {rms:.3e}" + + +def _replay_compare(out: tuple, ref: tuple) -> None: + """`(returned, ssm_states, old_u, old_k, old_G)` against `reference`'s five by the gates above; + a member `None` on both sides is skipped (the exact twin compares the output alone).""" + names = ("output", "ssm_states", "old_u", "old_k", "old_G") + assert len(out) == len(ref) == len(names) + for name, got, want in zip(names, out, ref): + assert (got is None) == (want is None), f"{name}: present on one side only" + if got is None: + continue + assert got.dtype == want.dtype and got.shape == want.shape, ( + name, + (got.dtype, tuple(got.shape)), + (want.dtype, tuple(want.shape)), + ) + if name == "old_G": + torch.testing.assert_close(got, want, rtol=_G_RTOL, atol=_G_ATOL, msg=name) + elif name == "ssm_states": + bf16 = got.dtype == torch.bfloat16 + _gate(name, got, want, _BF16_STATE_ULPS * BF16_ULP if bf16 else _STATE_TOL) + else: + rms = _OUTPUT_RMS_ULPS * BF16_ULP if name == "output" else None + _gate(name, got, want, _ELEMENT_ULPS * BF16_ULP, rms) + + +def _seed(why): + return zlib.crc32(why.encode()) + + +def _gating(gen, layers, hv=HV): + """Per-layer A_log / dt_bias around the checkpoint's statistics; head 0 pinned to the corner + that forgets everything every token (g ~ -400), head 1 to the one that hardly decays.""" + (a_mean, a_std, a_range), (d_mean, d_std, d_range) = A_LOG_STATS, DT_BIAS_STATS + a_log = torch.randn(layers, hv, generator=gen, device=DEV) * a_std + a_mean + dt_bias = torch.randn(layers, hv, generator=gen, device=DEV) * d_std + d_mean + a_log, dt_bias = a_log.clamp(*a_range), dt_bias.clamp(*d_range) + a_log[:, 0], dt_bias[:, 0] = a_range[1], d_range[1] + a_log[:, 1], dt_bias[:, 1] = a_range[0], d_range[0] + return a_log.contiguous(), dt_bias.contiguous() + + +def _unit(x): + return (x / x.norm(dim=-1, keepdim=True)).to(torch.bfloat16) + + +class _ReplayEnginePage: + """The stock page with the bench's `layers` on top, mapped under each touched slot's page, the + next slot's first sub-page (where an overrun lands) and slot 0's page (the views' base).""" + + def __init__(self, slots, page_layers, layers, used, hv, dtype, gen): + state = hv * V * K + stride = page_layers * state + total = slots * stride + self.state, self.stride, self.layers = state, stride, layers + self.top = (page_layers - layers) * state + self.used = sorted(set(used)) + self.mapped = sorted(set(self.used) | {0}) + self.spans = [(s * stride, min((s + 1) * stride + state, total)) for s in self.mapped] + self.page = _SparsePage(total, self.spans, dtype) + self.pools = [ + self.page.view(self.top + j * state, (slots, hv, V, K), (stride, V * K, K, 1)) + for j in range(layers) + ] + self.flat = [self.page.view(lo, (hi - lo,), (1,)) for lo, hi in self.spans] + for f in self.flat: + f.normal_(0.0, 0.5, generator=gen) + torch.cuda.synchronize() + self.before = [f.clone() for f in self.flat] + + def assert_rest_unchanged(self) -> None: + """Every mapped byte outside the touched slots' own sub-pages is what it was.""" + torch.cuda.synchronize() + lo, hi = self.top, self.top + self.layers * self.state + for s, f, b in zip(self.mapped, self.flat, self.before): + if s in self.used: + same = torch.equal(f[:lo], b[:lo]) and torch.equal(f[hi:], b[hi:]) + else: + same = torch.equal(f, b) + assert same, f"slot {s}: the page was written outside the cell's sub-pages" + + def close(self) -> None: + del self.pools, self.flat, self.before + self.page.close() + + +class _ReplayBench: + """`layers` GDN layers' state as `MambaHybridCacheManagerV2` lays it out, driven as the engine + drives it (work items, op per layer, all-layer commit, `_advance_replay_state`). `exact=True` + adds the exact twin: fp32 `reference` over a never-wrapping history advanced by `p += accepted` + alone. Over an engine page both see a dense copy of the touched slots (`position` maps them).""" + + def __init__(self, spec, seed, exact=True): + self.spec = spec + self.path, self.n, self.t = spec["path"], spec["n"], spec["t"] + self.h, self.hv = spec.get("H", H), spec.get("HV", HV) + assert spec.get("history_size", HIST) == HIST, spec + self.width = 2 * self.h * K + self.hv * V + self.layers = spec.get("layers", 2) + self.slots = spec.get("slots", 64) + self.steps = spec.get("steps", 8) + self.gen = torch.Generator(device=DEV).manual_seed(seed) + self.cpu = torch.Generator().manual_seed(seed) + n, t, L, S = self.n, self.t, self.layers, self.slots + h, hv = self.h, self.hv + page_layers = spec.get("page_layers") + + if page_layers: + chosen = torch.tensor(_scattered(n, S)) + else: + chosen = torch.randperm(S, generator=self.cpu)[:n] + chosen = chosen.to(torch.int32) + live_rows = torch.ones(n, dtype=torch.bool) + dup = tuple(spec.get("dup", ())) + if dup: + chosen[list(dup)] = int(chosen[dup[0]]) + # The padding rows' shared slot is nobody's state: the bookkeeping never names it. + live_rows[list(dup)] = False + self.state_indices = chosen.to(DEV) + self.live = self.state_indices[live_rows.to(DEV)].clone() + + state_dtype = spec.get("state_dtype", torch.float32) + self.page = self.idx = self.position = None + if page_layers: + self.page = _ReplayEnginePage( + S, page_layers, L, chosen.tolist(), hv, state_dtype, self.gen + ) + self.pool = self.page.pools + self.layer_stride, self.slot_stride = self.page.state, self.page.stride + self.idx = torch.tensor(self.page.used, dtype=torch.long, device=DEV) + self.position = torch.full((S,), -1, dtype=torch.long, device=DEV) + self.position[self.idx] = torch.arange(self.idx.numel(), device=DEV) + else: + # [slots, layers, HV, V, K]: the affine layout whose strides the all-layer commit takes. + self.backing = (self._randn(S, L, hv, V, K) * 0.5).to(state_dtype) + self.pool = [self.backing[:, i] for i in range(L)] + self.layer_stride, self.slot_stride = self.backing.stride(1), self.backing.stride(0) + self.old_x = torch.zeros(L, S, 2, HIST, hv, V, device=DEV, dtype=torch.bfloat16) + self.old_B = torch.zeros(L, S, 2, HIST, h, K, device=DEV, dtype=torch.bfloat16) + self.old_dt = torch.zeros(L, S, 2, hv, HIST, device=DEV) + self.old_dA = self._randn(L, S, 2, hv, HIST) # handed over as old_beta; never read + self.pnat = torch.zeros(S, dtype=torch.int32, device=DEV) + self.buf = torch.zeros(S, dtype=torch.int32, device=DEV) + self.a_log, self.dt_bias = _gating(self.gen, L, hv) + if spec.get("start", "warm") == "warm": + self.old_x.copy_(self._randn(L, S, 2, HIST, hv, V) * 0.5) + self.old_B.copy_(_unit(self._randn(L, S, 2, HIST, h, K))) + raw_a = self._randn(L, S, 2, hv, HIST) + g = -torch.exp(self.a_log)[:, None, None, :, None] * F.softplus( + raw_a + self.dt_bias[:, None, None, :, None] + ) + self.old_dt.copy_(g.cumsum(-1)) + pnat = torch.randint(0, HIST + 1, (S,), generator=self.cpu, dtype=torch.int32) + live = self.live.long().cpu() + pnat[live[0]] = HIST # a wrap at step 0 + if live.numel() > 1: + pnat[live[1]] = 0 # an append at step 0 + self.pnat.copy_(pnat) + self.buf.copy_(torch.randint(0, 2, (S,), generator=self.cpu, dtype=torch.int32)) + self.meta = ReplayStateUpdateMetadata(self.pnat, self.buf, t, HIST) + self.work_items = self.n_writes = None + if self.path == "all_layer": + # The engine hands out work items exactly from the all-layer commit's minimum batch. + assert n >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, n + self.work_items = torch.zeros(n, REPLAY_WORK_ITEM_WIDTH, dtype=torch.int32, device=DEV) + self.n_writes = torch.zeros(1, dtype=torch.int32, device=DEV) + + self.exact = exact + if exact: + self.depth = HIST + (self.steps + 1) * t + self.x_pool = [self.compact(p).clone() for p in self.pool] + rows = self.x_pool[0].shape[0] + self.x_u = torch.zeros(L, rows, 1, self.depth, hv, V, device=DEV) + self.x_k = torch.zeros(L, rows, 1, self.depth, h, K, device=DEV) + self.x_G = torch.zeros(L, rows, 1, hv, self.depth, device=DEV) + self.x_pnat = self.compact(self.pnat).clone() + self.x_buf = torch.zeros_like(self.x_pnat) + for s in self.live.tolist(): + p, b = int(self.pnat[s]), int(self.buf[s]) + c = s if self.position is None else int(self.position[s]) + self.x_u[:, c, 0, :p] = self.old_x[:, s, b, :p].float() + self.x_k[:, c, 0, :p] = self.old_B[:, s, b, :p].float() + self.x_G[:, c, 0, :, :p] = self.old_dt[:, s, b, :, :p] + + def _randn(self, *shape, dtype=torch.float32): + return torch.randn(*shape, generator=self.gen, device=DEV, dtype=dtype) + + def compact(self, per_slot: torch.Tensor) -> torch.Tensor: + """A per-slot tensor as the reference sees it: over a page, the touched slots' rows.""" + return per_slot if self.idx is None else per_slot[self.idx] + + def compact_indices(self, slots: torch.Tensor) -> torch.Tensor: + """Slot ids as rows of `compact`'s output (negative ids stay negative).""" + if self.position is None: + return slots + renamed = torch.where(slots >= 0, self.position[slots.long().clamp(min=0)], -1) + return renamed.to(slots.dtype) + + def close(self) -> None: + if self.page is not None: + self.pool = [] + self.page.close() + + def inputs(self): + """One layer's post-conv `[q|k|v]` rows and raw `a`/`b`, as the verify step passes them.""" + n, t, h, hv = self.n, self.t, self.h, self.hv + packed = self._randn(n * t, self.width, dtype=torch.bfloat16) + rows = packed.view(n, t, self.width) + q = rows[..., : h * K].view(n, t, h, K) + k = rows[..., h * K : 2 * h * K].view(n, t, h, K) + v = rows[..., 2 * h * K :].view(n, t, hv, V) + a = self._randn(n, t, hv, dtype=torch.bfloat16) + b = self._randn(n, t, hv, dtype=torch.bfloat16) + return dict(q=q, k=k, v=v, a=a, b=b, packed=packed) + + def kwargs(self, layer, act): + return dict( + q=act["q"], + k=act["k"], + v=act["v"], + g=act["a"], + beta=act["b"], + ssm_states=self.pool[layer], + state_indices=self.state_indices, + old_u=self.old_x[layer], + old_k=self.old_B[layer], + old_G=self.old_dt[layer], + old_beta=self.old_dA[layer], + cache_buf_idx=self.buf, + prev_num_accepted_tokens=self.pnat, + history_size=HIST, + scale=None, + use_qk_l2norm_in_kernel=True, + A_log=self.a_log[layer], + dt_bias=self.dt_bias[layer], + launch_with_pdl=False, + replay_work_items=self.work_items, + n_writes=self.n_writes, + packed_qkv=act["packed"], + use_all_layer_commit=self.path == "all_layer", + output=None, + ) + + def exact_kwargs(self, layer, kw): + return dict( + kw, + ssm_states=self.x_pool[layer], + state_indices=self.compact_indices(kw["state_indices"]), + old_u=self.x_u[layer], + old_k=self.x_k[layer], + old_G=self.x_G[layer], + cache_buf_idx=self.x_buf, + prev_num_accepted_tokens=self.x_pnat, + history_size=self.depth, + replay_work_items=None, + n_writes=None, + use_all_layer_commit=False, + output=None, + ) + + def adopt_exact(self, layer, ref): + assert torch.equal(ref[1], self.x_pool[layer]), "the twin must never wrap" + self.x_u[layer].copy_(ref[2]) + self.x_k[layer].copy_(ref[3]) + self.x_G[layer].copy_(ref[4]) + + def state(self, layer): + return self.pool[layer], self.old_x[layer], self.old_B[layer], self.old_dt[layer] + + def observed(self, layer): + """`state(layer)` through `compact`, as `reference`'s results are laid out.""" + return tuple(self.compact(x) for x in self.state(layer)) + + def prepare(self): + if self.path == "all_layer": + self.work_items.zero_() + _build_replay_work_items_triton( + self.live, self.pnat, self.buf, self.work_items, self.n_writes, self.t, HIST + ) + + def commit(self): + if self.path == "all_layer": + commit_gdn_cached_replay_history_layers( + ssm_states=self.pool[0], + ssm_state_num_layers=self.layers, + ssm_state_layer_stride=self.layer_stride, + ssm_state_slot_stride=self.slot_stride, + old_u=self.old_x, + old_k=self.old_B, + old_G=self.old_dt, + replay_work_items=self.work_items, + n_writes=self.n_writes, + history_size=HIST, + ) + + def draw_acceptance(self): + """Accepted tokens per live row, bonus token included, by `spec["accept"]`.""" + mode = self.spec.get("accept", "random") + rows = self.live.numel() + if mode == "full": + return torch.full((rows,), self.t, dtype=torch.int32, device=DEV) + assert mode == "random", mode + return torch.randint(1, self.t + 1, (rows,), generator=self.cpu, dtype=torch.int32).to(DEV) + + def accept(self, accepted): + _advance_replay_state(self.meta, self.live, accepted) + if self.exact: + self.x_pnat[self.compact_indices(self.live).long()] += accepted + + def writes(self): + """This step's append mask `[slots, 2, HIST]`, wrapping slot ids, and exact-fill count.""" + s = self.live.long() + p, b = self.pnat[s].long(), self.buf[s].long() + wrap = p + self.t > HIST + half, start = torch.where(wrap, 1 - b, b), torch.where(wrap, 0, p) + mask = torch.zeros(self.slots, 2, HIST, dtype=torch.bool, device=DEV) + for t in range(self.t): + mask[s, half, start + t] = True + return mask, s[wrap], int((p + self.t == HIST).sum()) + + +# -------------------------------------------------------------- gdn_cached_replay_cute__verify --- +# Each cell drives consecutive verify steps through `_ReplayBench`, holding every call to the +# entry's `reference` on its own inputs and every output to the exact twin. + + +_grc_op = cr.gdn_cached_replay_cute +GRC_K, GRC_V = cr.K, cr.V + + +def _grc_gates(bench, act, strided): + """The raw a / b `[N, T, HV]`: contiguous, or the same values as column views of the rank's + `[N*T, conv + z + 2 HV]` input projection, as the target hands them over.""" + if not strided: + return act["a"], act["b"] + n, t, hv = bench.n, bench.t, bench.hv + conv, z = 2 * bench.h * GRC_K + hv * GRC_V, hv * GRC_V + proj = torch.randn(n * t, conv + z + 2 * hv, device=DEV, dtype=torch.bfloat16) + a = proj[:, conv + z : conv + z + hv] + b = proj[:, conv + z + hv :] + a.copy_(act["a"].reshape(n * t, hv)) + b.copy_(act["b"].reshape(n * t, hv)) + a, b = a.reshape(n, t, hv), b.reshape(n, t, hv) + assert a.stride() == (t * proj.shape[1], proj.shape[1], 1), a.stride() + return a, b + + +def _grc_reference(bench, kw): + """The entry's `reference` on a call's arguments, every per-slot buffer through `compact`.""" + if bench.idx is None: + return _grc_op.reference(**kw) + per_slot = ( + "ssm_states", + "old_u", + "old_k", + "old_G", + "old_beta", + "cache_buf_idx", + "prev_num_accepted_tokens", + ) + compact = {name: bench.compact(kw[name]) for name in per_slot} + return _grc_op.reference( + **dict(kw, **compact, state_indices=bench.compact_indices(kw["state_indices"])) + ) + + +def _grc_drive(cell, bench) -> None: + strided = cell.spec.get("strided_gates", False) + wraps = appends = 0 + for _ in range(bench.steps): + mask, wrapped, _ = bench.writes() + wraps += wrapped.numel() + appends += bench.live.numel() - wrapped.numel() + bench.prepare() + after_commit = [] + for layer in range(bench.layers): + act = bench.inputs() + kw = bench.kwargs(layer, act) + kw["g"], kw["beta"] = _grc_gates(bench, act, strided) + _, u, k, G = bench.state(layer) + before = dict(u=u.clone(), k=k.clone(), G=G.clone(), dA=bench.old_dA[layer].clone()) + ref = _grc_reference(bench, kw) + exact = _grc_op.reference(**bench.exact_kwargs(layer, kw)) + committed = None + if bench.path == "all_layer": + committed = _grc_reference( + bench, + dict(kw, replay_work_items=None, n_writes=None, use_all_layer_commit=False), + ) + got = _grc_op(**kw) + _replay_compare((got, *bench.observed(layer)), ref) + _, u, k, G = bench.state(layer) + assert torch.equal(bench.old_dA[layer], before["dA"]), "old_beta is never written" + assert torch.equal(u[~mask], before["u"][~mask]), "old_u written outside the append" + assert torch.equal(k[~mask], before["k"][~mask]), "old_k written outside the append" + Gt, Gt0 = G.transpose(-1, -2), before["G"].transpose(-1, -2) + assert torch.equal(Gt[~mask], Gt0[~mask]), "old_G written outside the append" + bench.adopt_exact(layer, exact) + after_commit.append((got, exact, committed)) + bench.commit() + for layer, (got, exact, committed) in enumerate(after_commit): + if committed is not None: + # The partitioned call plus the manager's commit equals the in-kernel commit. + _replay_compare((got, *bench.observed(layer)), committed) + _replay_compare((got, None, None, None, None), (exact[0], None, None, None, None)) + if bench.page is not None: + bench.page.assert_rest_unchanged() + bench.accept(bench.draw_acceptance()) + assert wraps > 0 and appends > 0, f"the cell never wrapped or appended ({wraps}, {appends})" + + +def _grc_drive_padding(cell) -> None: + """Trailing rows sharing one slot, as CUDA-graph padding does: the unique rows and every + other slot match the reference run on the unique rows alone.""" + spec = {k_: v_ for k_, v_ in cell.spec.items() if k_ not in ("page_layers",)} + spec["slots"] = 24 + bench = _ReplayBench(spec, seed=_seed(cell.why), exact=False) + dup = spec["dup"] + unique = bench.n - len(dup) + shared = int(bench.state_indices[unique]) + others = torch.ones(bench.slots, dtype=torch.bool, device=DEV) + others[shared] = False + for _ in range(bench.steps): + for layer in range(bench.layers): + act = bench.inputs() + kw = bench.kwargs(layer, act) + kw["g"], kw["beta"] = _grc_gates(bench, act, spec.get("strided_gates", False)) + alone = dict( + kw, + q=kw["q"][:unique], + k=kw["k"][:unique], + v=kw["v"][:unique], + g=kw["g"][:unique], + beta=kw["beta"][:unique], + state_indices=kw["state_indices"][:unique], + packed_qkv=act["packed"][: unique * bench.t], + ) + ref = _grc_op.reference(**alone) + got = _grc_op(**kw) + pool, u, k, G = bench.state(layer) + _replay_compare( + (got[:unique], pool[others], u[others], k[others], G[others]), + (ref[0], *(x[others] for x in ref[1:])), + ) + bench.accept(bench.draw_acceptance()) + + +def _check_gdn_cached_replay_cute(cell) -> None: + if "dup" in cell.spec: + _grc_drive_padding(cell) + else: + bench = _ReplayBench(cell.spec, seed=_seed(cell.why)) + try: + _grc_drive(cell, bench) + finally: + bench.close() + + +#: The geometries the target calls the entry at: a tp4/tep4 rank's heads, a dep4 rank's. +_GRC_GEOMETRIES = {"tep4": (cr.RANK_K_HEADS, cr.RANK_V_HEADS), "dep4": (16, 48)} + + +def _grc_rank_spec(geometry, **spec): + h, hv = _GRC_GEOMETRIES[geometry] + return dict(state_dtype=torch.bfloat16, H=h, HV=hv, **spec) + + +def _variant_replay_graph(n, path, geometry, pdl) -> None: + """Two replay layers (and the all-layer commit when partitioned) captured once and replayed on + new activations with acceptance between: bitwise the same steps run eagerly on twin state.""" + t = 4 + spec = _grc_rank_spec(geometry, path=path, n=n, t=t, layers=2, slots=40) + seed = 9100 + n + eager, graph, warm = (_ReplayBench(spec, seed, exact=False) for _ in range(3)) + acts = [warm.inputs() for _ in range(2)] + gates = [_grc_gates(warm, act, True) for act in acts] + + def step(b): + b.prepare() + outs = [] + for layer in range(2): + kw = b.kwargs(layer, acts[layer]) + kw["g"], kw["beta"] = gates[layer] + kw["launch_with_pdl"] = pdl + outs.append(_grc_op(**kw)) + b.commit() + return outs + + step(warm) + torch.cuda.synchronize() + captured = torch.cuda.CUDAGraph() + with torch.cuda.graph(captured): + outs_graph = step(graph) + refill = torch.Generator(device=DEV).manual_seed(seed + 1) + for _ in range(3): + for act, (a, b) in zip(acts, gates): + for x in (act["packed"], a, b): + x.copy_(torch.randn(x.shape, generator=refill, device=DEV, dtype=x.dtype)) + outs_eager = step(eager) + captured.replay() + torch.cuda.synchronize() + for e, g in zip(outs_eager, outs_graph): + assert torch.equal(e, g), "replayed output differs from eager" + for name in ("backing", "old_x", "old_B", "old_dt", "pnat", "buf"): + assert torch.equal(getattr(eager, name), getattr(graph, name)), f"{name} differs" + accepted = eager.draw_acceptance() + eager.accept(accepted) + graph.accept(accepted) + + +def _variant_ht_verify(n) -> None: + """A tp2/tep2 rank's HT verify step as `_gdn_verify` launches it (`_GDN_REPLAY_HT_LAUNCH`, + gated norm folded, the conv too at few requests) bitwise the plain chain on twin state. A + folded conv leaves the pool to the acceptance promote (the plain advance: the last window).""" + core = types.SimpleNamespace(num_k_heads=8, num_v_heads=24) + t, h, hv = 3, 8, 24 + assert _m._gdn_verify_folds_norm(core, n, t) + conv_fold = _m._gdn_verify_folds_conv(core, n) + pdl, mapping = _m._gdn_replay_launch(core, n) + assert (pdl, mapping) != (False, None), "every HT bucket changes the launch" + spec = dict( + state_dtype=torch.bfloat16, H=h, HV=hv, path="all_layer", n=n, t=t, layers=1, slots=n + 8 + ) + width = 2 * h * GRC_K + hv * GRC_V + gen = torch.Generator(device=DEV).manual_seed(5100 + n) + proj = torch.randn(n * t, width + hv * GRC_V + 2 * hv, generator=gen, device=DEV).to( + torch.bfloat16 + ) + x = proj[:, :width] + z = proj[:, width : width + hv * GRC_V].reshape(n * t, hv, GRC_V) + weight = (torch.randn(width, 4, generator=gen, device=DEV) * 0.3).to(torch.bfloat16) + norm_w = (1 + 0.1 * torch.randn(GRC_V, generator=gen, device=DEV)).to(torch.bfloat16) + pool0 = (torch.randn(n + 8, width, 3, generator=gen, device=DEV) * 0.5).to(torch.bfloat16) + rows = torch.arange(n, dtype=torch.int32, device=DEV).flip(0).contiguous() + results = [] + for target in (False, True): + bench = _ReplayBench(spec, 7700 + n, exact=False) + act = bench.inputs() + a, b = _grc_gates(bench, act, True) + pool = pool0.clone() + window = torch.zeros(n, t, width, 3, dtype=torch.bfloat16, device=DEV) + bench.prepare() + kw = bench.kwargs(0, act) + kw.update(g=a, beta=b) + if target and conv_fold: + fold = dict(x=x, weight=weight, pool=pool, window=window, window_rows=rows) + qkv = x + kw.update(packed_qkv=None, conv_fold=fold) + else: + qkv = causal_conv1d_token_major( + x, n, pool, weight, bench.state_indices, window, rows, -1 + ) + kw.update(packed_qkv=qkv) + q, k, v = torch.split(qkv, [h * GRC_K, h * GRC_K, hv * GRC_V], 1) + kw.update( + q=q.reshape(n, t, h, GRC_K), k=k.reshape(n, t, h, GRC_K), v=v.reshape(n, t, hv, GRC_V) + ) + if target: + kw.update(launch_with_pdl=pdl, gated_norm=dict(z=z, weight=norm_w, eps=1e-6)) + if mapping is not None: + kw["config"] = mapping + out = _grc_op(**kw) + if not target: + out = rms_norm_gated_token_major( + torch.reshape(out, [-1, GRC_V]), + z, + weight=norm_w, + eps=1e-6, + fp8_scale=None, + gate_activation="sigmoid", + ) + bench.commit() + torch.cuda.synchronize() + out = torch.reshape(out, [n * t, hv * GRC_V]) + results.append(([out.clone()] + [y.clone() for y in bench.state(0)], window, pool, bench)) + (ref, win_ref, pool_ref, bench_ref), (got, win_got, pool_got, bench_got) = results + for name, y, w in zip(("normed out", "state", "u", "k", "G"), got, ref): + assert torch.equal(y, w), f"{name}: n={n} launch {pdl, mapping}, conv fold {conv_fold}" + assert torch.equal(win_got, win_ref), "snapshot windows" + if conv_fold: + assert torch.equal(pool_got, pool0), "the fold must not advance the pool" + live = bench_ref.state_indices.long() + assert torch.equal(pool_ref[live], win_ref[rows.long(), t - 1]), "advance = last window" + else: + assert torch.equal(pool_got, pool_ref), "the conv pool" + for bench in (bench_ref, bench_got): + bench.close() + + +# ------------------------------------------ DecodeTarget._gdn_verify's replay by rank geometry --- + + +GVR_K = GVR_V = 128 +GVR_HIST, GVR_T, GVR_CONV_W, GVR_PAGE_LAYERS, GVR_LAYERS = 16, 4, 4, 36, 2 +#: A rank's GDN (key, value) heads per member. +GVR_GEOM = {"tep4": (4, 12), "tep2": (8, 24), "dep4": (16, 48)} + + +def _gvr_build(member: str, n: int, seed: int): + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + h, hv = GVR_GEOM[member] + key_dim, value_dim = h * GVR_K, hv * GVR_V + conv_dim = 2 * key_dim + value_dim + split = [conv_dim, value_dim, hv, hv] + slots = n + 3 + page = ( + torch.randn(slots, GVR_PAGE_LAYERS, hv, GVR_V, GVR_K, generator=g, device=DEV) * 0.05 + ).to(bf) + conv_page = torch.randn( + slots, GVR_PAGE_LAYERS, conv_dim, GVR_CONV_W - 1, generator=g, device=DEV + ) + conv_page = conv_page.to(bf) + old, int_conv, layers, projs = {}, {}, [], [] + for i in range(GVR_LAYERS): + u = (torch.randn(slots, 2, GVR_HIST, hv, GVR_V, generator=g, device=DEV) * 0.1).to(bf) + k = torch.randn(slots, 2, GVR_HIST, h, GVR_K, generator=g, device=DEV) + k = (k / k.norm(dim=-1, keepdim=True)).to(bf) + G = -torch.cumsum(torch.rand(slots, 2, hv, GVR_HIST, generator=g, device=DEV) * 0.3, -1) + old[i] = (u, k, G.contiguous(), torch.zeros_like(G)) + int_conv[i] = torch.zeros(n, GVR_T, conv_dim, GVR_CONV_W - 1, device=DEV, dtype=bf) + layers.append( + types.SimpleNamespace( + conv_w=(torch.randn(conv_dim, GVR_CONV_W, generator=g, device=DEV) * 0.3).to(bf), + a_log=(torch.randn(hv, generator=g, device=DEV) * 1.5 + 1.0).clamp(-7, 5), + dt_bias=(torch.randn(hv, generator=g, device=DEV) * 1.7 - 4.4).clamp(-11, 2.5), + ) + ) + projs.append(torch.randn(n * GVR_T, sum(split), generator=g, device=DEV).to(bf)) + gc = torch.Generator().manual_seed(seed) + cache_buf_idx = torch.randint(0, 2, (slots,), generator=gc, dtype=torch.int32).to(DEV) + # Odd slots hold 13..15 pending tokens (they wrap: p + 4 > 16), even ones 1..8 (they + # append); the step's rows start at slot 1, so every batch has a wrapping row. + pending = torch.tensor( + [(13 + (s // 2) % 3) if s % 2 else (1 + s % 8) for s in range(slots)], dtype=torch.int32 + ).to(DEV) + replay = _m._Replay(old, cache_buf_idx, pending, GVR_HIST, GVR_T) + pools = types.SimpleNamespace( + ssm={i: page[:, i] for i in range(GVR_LAYERS)}, + conv={i: conv_page[:, i] for i in range(GVR_LAYERS)}, + ) + core = types.SimpleNamespace( + conv_dim=conv_dim, + key_dim=key_dim, + value_dim=value_dim, + num_k_heads=h, + head_k_dim=GVR_K, + num_v_heads=hv, + head_v_dim=GVR_V, + _bound=types.SimpleNamespace(pools=pools, int_conv=int_conv, replay=replay), + ) + rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) # slot 0 untouched + verify = (torch.arange(n, dtype=torch.int32, device=DEV), None, None, False, None) + + def run(): + outs = [] + for i in range(GVR_LAYERS): + mq, _, a, b = torch.split(projs[i], split, 1) + outs.append(_m._gdn_verify(core, mq, a, b, i, layers[i], rows, n, GVR_T, verify)) + torch.cuda.synchronize() + return outs + + state = dict(page=page, conv=conv_page, old=old, int_conv=int_conv) + return run, state + + +def _gvr_rel(x, y): + x, y = x.float(), y.float() + return float((x - y).norm() / y.norm().clamp_min(1e-30)) + + +def _gvr_stock_replay(**kw): + """The engine's stock Triton cached replay in the CuTe entry's place: contiguous gate + columns, no CuTe launch mapping (`_gdn_verify` folds nothing at these batches).""" + assert "conv_fold" not in kw and "gated_norm" not in kw + kw.pop("config", None) + return stock_replay(**dict(kw, g=kw["g"].contiguous(), beta=kw["beta"].contiguous())) + + +@pytest.mark.parametrize("member,n", [("tep4", 1), ("tep2", 4), ("dep4", 8)]) +def test_gdn_verify_agrees_with_the_engines_stock_replay(member, n, monkeypatch): + """`_gdn_verify` over two MTP-3 layers at every rank geometry against the same layers over the + engine's stock Triton replay on twin state: outputs, checkpoint page, history and conv agree + to the replays' arithmetic (one fp32 recurrence, another dot order); other slots stay put.""" + run_c, st_c = _gvr_build(member, n, seed=500 + n) + page0 = st_c["page"].clone() + outs_c = run_c() + monkeypatch.setattr(_m, "gdn_cached_replay_cute", _gvr_stock_replay) + run_t, st_t = _gvr_build(member, n, seed=500 + n) + outs_t = run_t() + for oc, o_t in zip(outs_c, outs_t): + assert torch.isfinite(oc).all() + assert _gvr_rel(oc, o_t) < 4e-3, _gvr_rel(oc, o_t) + live = slice(1, n + 1) + assert _gvr_rel(st_c["page"][live], st_t["page"][live]) < 4e-3 + assert torch.equal(st_c["page"][0], page0[0]) and torch.equal( + st_c["page"][n + 1 :], page0[n + 1 :] + ) + assert not torch.equal(st_c["page"][live], page0[live]), "no row wrapped and committed" + assert torch.equal(st_c["conv"], st_t["conv"]), "the conv is the same kernel on both paths" + for i in range(GVR_LAYERS): + assert torch.equal(st_c["int_conv"][i], st_t["int_conv"][i]) + for x, y in zip(st_c["old"][i][:3], st_t["old"][i][:3]): + assert _gvr_rel(x[live], y[live]) < 4e-3 + assert torch.equal(x[0], y[0]) + + +#: Requests the precompile test walks: past every `_GDN_REPLAY_HT_LAUNCH` bucket and the partition. +GVR_MAX_REQUESTS = 96 + + +@pytest.mark.parametrize("separate", [False, True], ids=["coupled", "replay_slots"]) +@pytest.mark.parametrize("pdl", [True, False], ids=["pdl", "no_pdl"]) +@pytest.mark.parametrize("member", list(GVR_GEOM)) +def test_gdn_replay_precompile_is_every_key_a_verify_step_launches( + member, pdl, separate, monkeypatch +): + """The CuTe replay kernels the bind precompiles (`_gdn_replay_launches`) are exactly those + `_gdn_verify` launches over 1..`GVR_MAX_REQUESTS` requests, from a generation-only step + (the gated norm folded where `_gdn_verify_folds_norm` says) and from a mixed step's verify + rows (never folded), the partitioned ones with work items of the metadata's width. With + `separate` the manager keeps the history at replay slots of its own: precompiled with + `separate_replay_slots`, every step passes `replay_indices`. Kernels are recorded, not + compiled or run.""" + monkeypatch.setattr(_m, "get_env_enable_pdl", lambda: pdl) + monkeypatch.setattr( + _m, + "_GDN_REPLAY_HT_LAUNCH", + tuple((bound, p and pdl, mapping) for bound, p, mapping in _m._GDN_REPLAY_HT_LAUNCH), + ) + launched = [] + # An empty cache: every launch reaches `_compile`, which records the key instead. + monkeypatch.setattr(cr, "_COMPILED", {}) + monkeypatch.setattr(cr, "_compile", lambda key, kern: launched.append(key) or (lambda *a: None)) + # The conv runs as its own kernel unless folded; the fold's operands are the engine's layout. + monkeypatch.setattr(_m, "causal_conv1d_token_major", lambda mq, *a, **k: mq.contiguous()) + monkeypatch.setattr(_m, "gdn_conv_fold_supported", lambda *a: True) + h, hv = GVR_GEOM[member] + key_dim, value_dim = h * GVR_K, hv * GVR_V + conv_dim = 2 * key_dim + value_dim + bf, nmax, t = torch.bfloat16, GVR_MAX_REQUESTS, GVR_T + slots = nmax + 1 + old = { + 0: ( + torch.zeros(slots, 2, GVR_HIST, hv, GVR_V, dtype=bf, device=DEV), + torch.zeros(slots, 2, GVR_HIST, h, GVR_K, dtype=bf, device=DEV), + torch.zeros(slots, 2, hv, GVR_HIST, device=DEV), + torch.zeros(slots, 2, hv, GVR_HIST, device=DEV), + ) + } + zeros_i32 = torch.zeros(slots, dtype=torch.int32, device=DEV) + work_items = torch.zeros(slots, REPLAY_WORK_ITEM_WIDTH, dtype=torch.int32, device=DEV) + replay_slots = torch.arange(slots, 2 * slots, dtype=torch.int32, device=DEV) + pools = types.SimpleNamespace( + ssm={0: torch.zeros(slots, hv, GVR_V, GVR_K, dtype=bf, device=DEV)}, + conv={0: torch.zeros(slots, conv_dim, GVR_CONV_W - 1, dtype=bf, device=DEV)}, + ) + core = types.SimpleNamespace( + conv_dim=conv_dim, + key_dim=key_dim, + value_dim=value_dim, + num_k_heads=h, + head_k_dim=GVR_K, + num_v_heads=hv, + head_v_dim=GVR_V, + eps=1e-6, + _bound=types.SimpleNamespace( + pools=pools, + int_conv={0: torch.zeros(nmax, t, conv_dim, GVR_CONV_W - 1, dtype=bf, device=DEV)}, + replay=_m._Replay(old, zeros_i32, zeros_i32, GVR_HIST, t), + ), + ) + layer = types.SimpleNamespace( + conv_w=torch.zeros(conv_dim, GVR_CONV_W, dtype=bf, device=DEV), + a_log=torch.zeros(hv, device=DEV), + dt_bias=torch.zeros(hv, device=DEV), + gnorm=torch.ones(GVR_V, dtype=bf, device=DEV), + ) + proj = torch.zeros(nmax * t, conv_dim + value_dim + 2 * hv, dtype=bf, device=DEV) + for n in range(1, nmax + 1): + mq, z, a, b = torch.split(proj[: n * t], [conv_dim, value_dim, hv, hv], 1) + rows = torch.arange(1, n + 1, dtype=torch.int32, device=DEV) + partitioned = n >= _m.CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + work = work_items[:n] if partitioned else None + n_writes = zeros_i32[:1] if partitioned else None + replay_indices = replay_slots[:n] if separate else None + int_idx = torch.arange(n, dtype=torch.int32, device=DEV) + verify = (int_idx, work, n_writes, partitioned, replay_indices) + _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify) # a mixed step's rows + fold_norm = _m._gdn_verify_folds_norm(core, n, t) + _m._gdn_verify(core, mq, a, b, 0, layer, rows, n, t, verify, z=z if fold_norm else None) + want = set(launched) + got = cr.precompile_keys( + t, + h, + hv, + nmax, + **_m._gdn_replay_launches(core, t), + work_item_width=REPLAY_WORK_ITEM_WIDTH, + separate_replay_slots=separate, + ) + assert all(dict(key)["rsep"] == separate for key in got) + assert {dict(key)["wi_w"] for key in got if dict(key)["use_wi"]} == {REPLAY_WORK_ITEM_WIDTH} + assert got == want, ( + f"precompiled but never launched: {len(got - want)}, launched but not precompiled: " + f"{len(want - got)}" + ) + if pdl and hv * 2 <= _m._GDN_REPLAY_PDL_MAX_HEAD_ROWS: + assert not any(not dict(key)["pdl"] and dict(key)["tpc"] == 2 for key in got), ( + "with PDL on, a batch of up to two requests never launches without it" + ) + + +# ------------------------------------------------ the replay history at slots of its own --- + + +def _replay_pages(slots, layers, h, hv): + """The history and its bookkeeping in pages of their own, as a manager keeps them in its + KVCM pool: one page per replay slot holding every layer's `old_u`, then every layer's + `old_k`, then every layer's `old_G`, then the pending count and the half (int32), and 8 + bytes nobody writes; every byte 0xFF (NaN history, counts of -1) until written. The + `[layers, slots, ...]` history views have the page as their slot stride, the bookkeeping + columns the page as their element stride; `data` is the `[slots, page]` bytes, `tail` + the offset of the unwritten bytes.""" + regions = ( + (torch.bfloat16, (2, HIST, hv, V)), + (torch.bfloat16, (2, HIST, h, K)), + (torch.float32, (2, hv, HIST)), + ) + sizes = [math.prod(shape) * dtype.itemsize for dtype, shape in regions] + page = layers * sum(sizes) + 16 + data = torch.full((slots, page), 0xFF, dtype=torch.uint8, device=DEV) + flat = data.view(-1) + views, offset = [], 0 + for (dtype, shape), size in zip(regions, sizes): + es = dtype.itemsize + rows = torch.empty(shape, device="meta").stride() + views.append( + torch.as_strided( + flat.view(dtype), + (layers, slots, *shape), + (size // es, page // es, *rows), + offset // es, + ) + ) + offset += layers * size + pnat, buf = ( + torch.as_strided(flat.view(torch.int32), (slots,), (page // 4,), offset // 4 + field) + for field in (0, 1) + ) + return types.SimpleNamespace( + data=data, u=views[0], k=views[1], G=views[2], pnat=pnat, buf=buf, tail=offset + 8 + ) + + +class _ReplaySlotBench(_ReplayBench): + """`_ReplayBench` (the same draws for the same seed) with the history and its bookkeeping + moved to replay slots of their own (`_replay_pages`): row `i`'s at `replay_indices[i]`, a + permutation of slots past the checkpoint pool's, so no row's replay slot is a checkpoint + slot. The work items carry the replay slot as their fifth field; the kernel is handed their + first `width` fields (4: it reads the replay slot from `replay_indices`), the manager's + all-layer commit the full items. `writes` is the coupled bench's alone (it reads the + bookkeeping at the checkpoint slots).""" + + def __init__(self, spec, seed, width): + super().__init__(spec, seed, exact=False) + assert "dup" not in spec and "page_layers" not in spec, spec + n, slots = self.n, self.state_indices.long() + cpu = torch.Generator().manual_seed(seed + 1) + replay = torch.randperm(2 * n, generator=cpu)[:n] + self.slots + self.replay_indices = replay.to(DEV, torch.int32) + rows = self.replay_indices.long() + self.pages = _replay_pages(self.slots + 2 * n, self.layers, self.h, self.hv) + pages = self.pages + for paged, coupled in zip( + (pages.u, pages.k, pages.G), (self.old_x, self.old_B, self.old_dt) + ): + paged[:, rows] = coupled[:, slots] + pages.pnat[rows] = self.pnat[slots] + pages.buf[rows] = self.buf[slots] + self.old_x, self.old_B, self.old_dt = pages.u, pages.k, pages.G + self.pnat, self.buf = pages.pnat, pages.buf + self.meta = ReplayStateUpdateMetadata(self.pnat, self.buf, self.t, HIST) + self.item_width = width + self.kernel_items = None + + def prepare(self): + if self.path == "all_layer": + self.work_items.zero_() + _build_replay_work_items_triton( + self.live, + self.pnat, + self.buf, + self.work_items, + self.n_writes, + self.t, + HIST, + self.replay_indices, + ) + full = self.work_items.shape[1] + self.kernel_items = ( + self.work_items + if self.item_width == full + else self.work_items[:, : self.item_width].contiguous() + ) + + def kwargs(self, layer, act): + kw = super().kwargs(layer, act) + kw["replay_indices"] = self.replay_indices + if self.kernel_items is not None: + kw["replay_work_items"] = self.kernel_items + return kw + + def accept(self, accepted): + _advance_replay_state(self.meta, self.replay_indices, accepted) + + +_SLOT_STATE = ("checkpoint", "old_u", "old_k", "old_G", "pending count", "half") + + +def _slot_state(bench): + """A bench's checkpoints, and its rows' history and bookkeeping at their history slots + (the replay slots, when it has them).""" + rows = getattr(bench, "replay_indices", bench.state_indices).long() + return ( + bench.backing, + bench.old_x[:, rows], + bench.old_B[:, rows], + bench.old_dt[:, rows], + bench.pnat[rows], + bench.buf[rows], + ) + + +def _rs_spec(heads, path, n, t): + h, hv = heads + return dict( + state_dtype=torch.bfloat16, + H=h, + HV=hv, + path=path, + n=n, + t=t, + layers=2, + steps=HIST // t + 3, + ) + + +#: Replay-slot runs: requests and the work-item width the kernel is handed (None: no work +#: items, the small path, the kernel commits wrapping rows itself). +_REPLAY_SLOT_RUNS = { + "small_n1": (1, None), + "small_n4": (4, None), + "w4_all_layer": (CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, 4), + "w5_all_layer": (32, REPLAY_WORK_ITEM_WIDTH), +} + + +@pytest.mark.parametrize("heads", cr.GEOMETRIES, ids=[f"{h}x{hv}" for h, hv in cr.GEOMETRIES]) +@pytest.mark.parametrize("n,width", list(_REPLAY_SLOT_RUNS.values()), ids=list(_REPLAY_SLOT_RUNS)) +def test_separate_replay_slots_match_coupled(n, width, heads, certifying) -> None: # noqa: F811 + """Replay slots that differ from the checkpoint slots give, bit for bit, the coupled run. + + The coupled side is `_ReplayBench`: history and bookkeeping at the checkpoint slots. The + other (`_ReplaySlotBench`) keeps the same history and bookkeeping at a permuted set of + replay slots past the checkpoint pool's, in pages holding every layer's rows (a slot stride + of the whole page, the bookkeeping as strided int32 fields of it), 0xFF elsewhere. Both run + verify steps through this entry -- strided a / b and PDL, as the target hands them over -- + with checkpoint wraps committed in-kernel (no work items) or by the manager's all-layer + commit, the kernel handed the work items at `width` fields (4: the replay slot from + `replay_indices`; 5: from the items' fifth field), then advance their bookkeeping with the + same acceptance. Each replay-slot call is also held to this entry's `reference` given the + same `replay_indices`, and no byte of the pages outside the rows' replay slots is written. + """ + certifying(_grc_op) + t = 4 + path = "small" if width is None else "all_layer" + spec = _rs_spec(heads, path, n, t) + seed = 31 + n + coupled = _ReplayBench(spec, seed, exact=False) + paged = _ReplaySlotBench(spec, seed, width or REPLAY_WORK_ITEM_WIDTH) + sides = (coupled, paged) + rows = paged.replay_indices.long() + assert not torch.isin(rows, coupled.state_indices.long()).any() + wraps = appends = 0 + for step in range(coupled.steps): + wrapped = coupled.writes()[1].numel() + wraps += wrapped + appends += n - wrapped + for side in sides: + side.prepare() + if width is not None: + assert torch.equal(coupled.n_writes, paged.n_writes) + for layer in range(coupled.layers): + act = coupled.inputs() + a, b = _grc_gates(coupled, act, True) + outs = [] + for side in sides: + kw = dict(side.kwargs(layer, act), g=a, beta=b, launch_with_pdl=True) + if side is paged: + ref = _grc_op.reference(**kw) + outs.append(_grc_op(**kw)) + pool, u, k, G = paged.state(layer) + _replay_compare( + (outs[1], pool, u[rows], k[rows], G[rows]), + (ref[0], ref[1], ref[2][rows], ref[3][rows], ref[4][rows]), + ) + assert torch.equal(outs[0], outs[1]), f"step {step} layer {layer}: outputs differ" + for side in sides: + side.commit() + accepted = coupled.draw_acceptance() + for side in sides: + side.accept(accepted) + for name, x, y in zip(_SLOT_STATE, _slot_state(coupled), _slot_state(paged)): + assert torch.equal(x, y), f"step {step}: {name} differs" + assert wraps > 0 and appends > 0, f"the run never wrapped or appended ({wraps}, {appends})" + pages = paged.pages + elsewhere = torch.ones(pages.data.shape[0], dtype=torch.bool, device=DEV) + elsewhere[rows] = False + assert (pages.data[elsewhere] == 0xFF).all(), "a page outside the rows' replay slots moved" + assert (pages.data[:, pages.tail :] == 0xFF).all(), "a page's bytes past its rows moved" + + +#: Context rows ahead of the generation rows in the dispatch test's mixed-step metadata. +_DISPATCH_PREFILLS = 3 + + +def _dispatch_core(bench, separate): + """What `_verify_state` and `_gdn_verify` read, over `bench`'s pool, history and + bookkeeping: the GDN core's geometry and bind (the pools, the replay buffers, a manager + with the all-layer commit on and, with `separate`, replay slots of its own), and the mamba + metadata of a mixed step whose generation rows follow `_DISPATCH_PREFILLS` context rows + (`bench.prepare` fills the work items).""" + replay = _m._Replay( + { + layer: ( + bench.old_x[layer], + bench.old_B[layer], + bench.old_dt[layer], + bench.old_dA[layer], + ) + for layer in range(bench.layers) + }, + bench.buf, + bench.pnat, + HIST, + bench.t, + ) + cm = types.SimpleNamespace( + use_gdn_cached_replay_all_layer_commit=True, + replay_layer_group_id=1 if separate else None, + ) + core = types.SimpleNamespace( + conv_dim=bench.width, + key_dim=bench.h * K, + value_dim=bench.hv * V, + num_k_heads=bench.h, + head_k_dim=K, + num_v_heads=bench.hv, + head_v_dim=V, + _bound=types.SimpleNamespace( + pools=types.SimpleNamespace(ssm=dict(enumerate(bench.pool))), + replay=replay, + cm=cm, + int_idx=torch.arange(bench.n, dtype=torch.int32, device=DEV), + ), + ) + context = torch.full((_DISPATCH_PREFILLS,), -1, dtype=torch.int32, device=DEV) + rows = bench.replay_indices if separate else bench.state_indices + mm = types.SimpleNamespace( + replay_work_items=bench.work_items, + replay_n_writes=bench.n_writes, + replay_state_indices=torch.cat([context, rows]), + ) + return core, mm + + +@pytest.mark.parametrize("heads", cr.GEOMETRIES, ids=[f"{h}x{hv}" for h, hv in cr.GEOMETRIES]) +@pytest.mark.parametrize("separate", [False, True], ids=["coupled", "replay_slots"]) +@pytest.mark.parametrize("n,t", [(1, 4), (4, 4), (32, 4), (5, 3)]) +def test_target_dispatch_matches_the_plain_call(n, t, separate, heads) -> None: + """The target's verify dispatch over a bf16 pool (`_verify_state` for a mixed step's + generation rows, then `_gdn_verify` per layer: `a` / `b` column views of the input + projection, the conv output as `packed_qkv`, the launch `_gdn_replay_launch` picks) against + the call it replaces (contiguous `a` / `b`, no PDL, the default mapping), each on its own + copy of the pool, the history and the bookkeeping, over consecutive verify steps with + random partial acceptance in between: outputs, checkpoints, history and bookkeeping bitwise + equal at every step, with the entry's guard on. With `separate` the dispatched side's + manager keeps the history and its bookkeeping at replay slots of its own + (`_ReplaySlotBench`), which `_verify_state` takes from the metadata's `replay_state_indices` + behind the context rows and `_gdn_verify` hands this entry as `replay_indices`. From + `CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE` requests both sides take the work items and the + manager's all-layer commit; below it the kernel commits wraps.""" + path = "all_layer" if n >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE else "small" + spec = _rs_spec(heads, path, n, t) + seed = 77 + n + plain = _ReplayBench(spec, seed, exact=False) + disp = ( + _ReplaySlotBench(spec, seed, REPLAY_WORK_ITEM_WIDTH) + if separate + else _ReplayBench(spec, seed, exact=False) + ) + core, mm = _dispatch_core(disp, separate) + wraps = 0 + with validating(_grc_op): + for step in range(plain.steps): + wraps += plain.writes()[1].numel() + plain.prepare() + disp.prepare() + verify = _m._verify_state(core, mm, n, _DISPATCH_PREFILLS) + assert verify[3] == (path == "all_layer"), ( + "the all-layer commit exactly when partitioned" + ) + if separate: + assert torch.equal(verify[4], disp.replay_indices), "the rows' replay slots" + else: + assert verify[4] is None, "no replay slots of its own: the checkpoint slots" + for layer in range(plain.layers): + act = plain.inputs() + a, b = _grc_gates(plain, act, True) + want = _grc_op(**plain.kwargs(layer, act)) + gating = types.SimpleNamespace(a_log=disp.a_log[layer], dt_bias=disp.dt_bias[layer]) + got = _m._gdn_verify( + core, + None, + a.reshape(n * t, -1), + b.reshape(n * t, -1), + layer, + gating, + disp.state_indices, + n, + t, + verify, + mq_out=act["packed"], + ) + assert torch.equal(got.view_as(want), want), f"step {step} layer {layer}" + plain.commit() + disp.commit() + accepted = plain.draw_acceptance() + plain.accept(accepted) + disp.accept(accepted) + for name, x, y in zip(_SLOT_STATE, _slot_state(disp), _slot_state(plain)): + assert torch.equal(x, y), f"step {step}: {name} differs" + assert wraps > 0, "no history wrapped: no checkpoint commit was exercised" + + +# ------------------------------------------- causal_conv1d_token_major__verify --- + + +def _ctm_build(rows, p, seed): + """A tep4 rank's verify conv operands: the q|k|v column view of a [rows * p, 4128] + projection, one layer's view of a 36-layer conv page, the snapshot buffer.""" + g = torch.Generator(device=DEV).manual_seed(seed) + bf = torch.bfloat16 + conv_dim, width, slots = 2560, 4, rows + 5 + proj = torch.randn(rows * p, 4128, generator=g, device=DEV).to(bf) + page = torch.randn(slots, 36, conv_dim, width - 1, generator=g, device=DEV).to(bf) + weight = (torch.randn(conv_dim, width, generator=g, device=DEV) * 0.3).to(bf) + window = torch.zeros(rows + 2, p, conv_dim, width - 1, device=DEV, dtype=bf) + slot_ids = torch.arange(2, rows + 2, dtype=torch.int32, device=DEV) + window_rows = torch.arange(rows, dtype=torch.int32, device=DEV) + return proj, page, weight, window, slot_ids, window_rows + + +def _check_causal_conv1d_token_major(cell) -> None: + rows, p, wait = cell.spec["rows"], cell.spec["p"], cell.spec.get("wait", False) + proj, page, weight, window, slot_ids, window_rows = _ctm_build(rows, p, _seed(cell.why)) + page_ref, window_ref = page.clone(), window.clone() + x = proj[:, :2560] + causal_conv1d_token_major.is_valid(x, rows, page[:, 7], weight, slot_ids, window, window_rows) + out = causal_conv1d_token_major( + x, rows, page[:, 7], weight, slot_ids, window, window_rows, -1, True, wait + ) + ref = causal_conv1d_token_major.reference( + x, rows, page_ref[:, 7], weight, slot_ids, window_ref, window_rows, -1 + ) + torch.cuda.synchronize() + assert out.is_contiguous() and out.shape == (rows * p, 2560) + assert torch.equal(out, ref), "output" + assert torch.equal(page, page_ref), "the advanced pool" + assert torch.equal(window, window_ref), "the snapshots" + + +def _variant_conv_seqs(rows, p, monkeypatch) -> None: + """The verify conv's multi-sequence kernel (`_AUTO_SEQS` sequences a program, a tep2 rank's + large batches) bitwise the one-sequence launch, with a pad slot and a ragged sequence count.""" + seqs = ctm._AUTO_SEQS + g = torch.Generator(device=DEV).manual_seed(rows * 31 + p * 7 + seqs) + bf = torch.bfloat16 + conv_dim, slots_n = 5120, rows + 7 + proj = torch.randn(rows * p, 8240, generator=g, device=DEV).to(bf) + page = torch.randn(slots_n, 12, conv_dim, 3, generator=g, device=DEV).to(bf) + weight = (torch.randn(conv_dim, 4, generator=g, device=DEV) * 0.3).to(bf) + window = torch.randn(rows + 3, p, conv_dim, 3, generator=g, device=DEV).to(bf) + slot_ids = torch.randperm(slots_n, generator=g, device=DEV)[:rows].to(torch.int32) + slot_ids[rows // 2] = -1 # a padded sequence: nothing of it is read or written + window_rows = torch.randperm(rows + 3, generator=g, device=DEV)[:rows].to(torch.int32) + x = proj[:, :conv_dim] + + def run(n_seqs, pg, win): + monkeypatch.setattr(ctm, "_SEQS", n_seqs) + return ctm.causal_conv1d_token_major( + x, rows, pg[:, 5], weight, slot_ids, win, window_rows, -1, True, _m._GDN_CONV_PDL_WAIT + ) + + # a padded sequence's output rows are never written (torch.empty): compare the live ones + keep = torch.repeat_interleave(slot_ids != -1, p) + page_ref, win_ref = page.clone(), window.clone() + ref = run(1, page_ref, win_ref) + page_new, win_new = page.clone(), window.clone() + out = run(seqs, page_new, win_new) + torch.cuda.synchronize() + assert torch.equal(out[keep], ref[keep]), "output" + assert torch.equal(page_new, page_ref), "the advanced pool" + assert torch.equal(win_new, win_ref), "the snapshots" + + +# ------------------------------------------------- gdn_in_proj_conv__verify / norm_out_proj --- + + +def _ipc_operands(n: int, p: int, seed: int): + """A tep4 verify step's rows, input projection, conv taps, strided conv pool and snapshots.""" + g = torch.Generator(device=DEV).manual_seed(seed) + rows = n * p + x = (0.5 * torch.randn(rows, 2560, device=DEV, generator=g)).to(torch.bfloat16) + w = (0.03 * torch.randn(4128, 2560, device=DEV, generator=g)).to(torch.bfloat16) + cw = (0.4 * torch.randn(2560, 4, device=DEV, generator=g)).to(torch.bfloat16) + page = torch.randn(9, 3, 2560, 3, device=DEV, generator=g).to(torch.bfloat16) + slots = torch.tensor([7 - 2 * i for i in range(n)], dtype=torch.int32, device=DEV) + win = torch.randn(8, p, 2560, 3, device=DEV, generator=g).to(torch.bfloat16) + wrows = torch.tensor([i + 1 for i in range(n)], dtype=torch.int32, device=DEV) + return x, w, cw, page, slots, win, wrows + + +def _check_gdn_in_proj_conv(cell) -> None: + """Projection, conv, snapshots and pool bitwise the two-launch chain and within the entry's + fp32 reference; a captured launch replays on new rows.""" + n, p = cell.spec["requests"], cell.spec["p"] + x, w, cw, page, slots, win, wrows = _ipc_operands(n, p, zlib.crc32(cell.why.encode())) + tac = fused_tactic(n * p) + assert tac is not None + + def two_launch(x, page, win): + proj = low_m_bf16_mm(x, w.t(), True) + conv = causal_conv1d_token_major( + proj[:, :2560], n, page[:, 1], cw, slots, win, wrows, -1, True, True + ) + return proj, conv + + page_ref, win_ref = page.clone(), win.clone() + proj_ref, conv_ref = two_launch(x, page_ref, win_ref) + page_new, win_new = page.clone(), win.clone() + proj, conv = gdn_in_proj_conv( + x, w.t(), cw, page_new[:, 1], slots, win_new, wrows, -1, p, tac, True + ) + torch.cuda.synchronize() + for got, want, what in ( + (proj, proj_ref, "projection"), + (conv, conv_ref, "conv"), + (win_new, win_ref, "snapshots"), + (page_new, page_ref, "pool page"), + ): + assert torch.equal(got.view(torch.int16), want.view(torch.int16)), what + # The entry's own fp32 reference: the projection within bf16 rounding of a 2560-term + # sum, the conv within a bf16 ulp of the activations it reads. + rp, rc = gdn_in_proj_conv.reference( + x, w.t(), cw, page.clone()[:, 1], slots, win.clone(), wrows, -1, p, tac, True + ) + torch.testing.assert_close(proj.float(), rp.float(), atol=0.06, rtol=0.02) + torch.testing.assert_close(conv.float(), rc.float(), atol=0.06, rtol=0.02) + if not cell.spec["capture"]: + return + sx = x.clone() + graph = torch.cuda.CUDAGraph() + page_g, win_g = page.clone(), win.clone() + with torch.cuda.graph(graph): + gp, gc = gdn_in_proj_conv( + sx, w.t(), cw, page_g[:, 1], slots, win_g, wrows, -1, p, tac, True + ) + x2 = (0.5 * torch.randn_like(x.float())).to(torch.bfloat16) + sx.copy_(x2) + page_g.copy_(page) + win_g.copy_(win) + graph.replay() + torch.cuda.synchronize() + page_r, win_r = page.clone(), win.clone() + rp, rc = two_launch(x2, page_r, win_r) + assert torch.equal(gp, rp) and torch.equal(gc, rc) and torch.equal(page_g, page_r) + + +# ------------------------------------------------------------------------------ the tests --- + +#: Every Gated-DeltaNet entry of the target with its checker: `CASES` is each (entry, cell). +_ENTRIES = ( + (flashinfer_chunk_gated_delta_rule, _check_flashinfer_chunk_gated_delta_rule), + (_grc_op, _check_gdn_cached_replay_cute), + (causal_conv1d_token_major, _check_causal_conv1d_token_major), + (gdn_in_proj_conv, _check_gdn_in_proj_conv), +) +CASES = [(op, check, cell) for op, check in _ENTRIES for cell in op.CELLS] + + +@pytest.mark.parametrize( + "entry,check,cell", + CASES, + ids=[f"{type(op).__name__}-{i}" for op, _ in _ENTRIES for i in range(len(op.CELLS))], +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + """Every certified cell of every entry against the entry's own reference.""" + certifying(entry) + check(cell) + + +@pytest.mark.parametrize("t", range(2, _m._MAX_VERIFY_WIDTH + 1), ids=lambda t: f"T{t}") +def test_gdn_cached_replay_every_verify_width_is_the_recurrence(t) -> None: + """The cached replay at every verify width over consecutive steps from warm histories with + random acceptance, against its reference and the exact twin.""" + spec = _grc_rank_spec( + "tep4", path="small", n=2, t=t, steps=6, layers=1, slots=24, strided_gates=True + ) + cell = types.SimpleNamespace(why=f"tep4 small n=2 T={t}", spec=spec) + bench = _ReplayBench(spec, seed=_seed(cell.why)) + try: + _grc_drive(cell, bench) + finally: + bench.close() + + +#: `(id, helper, args)`: a launch the cells do not reach, against its plain launch. +_VARIANTS = ( + [("replay_graph-tep4-n1-pdl", _variant_replay_graph, (1, "small", "tep4", True))] + + [("replay_graph-dep4-n16", _variant_replay_graph, (16, "all_layer", "dep4", False))] + + [(f"ht_verify-n{n}", _variant_ht_verify, (n,)) for n in (16, 24, 32, 64, 128)] + + [(f"conv_seqs-{r}x{p}", _variant_conv_seqs, (r, p)) for r, p in ((37, 4), (128, 3))] +) + + +@pytest.mark.parametrize("helper,args", [v[1:] for v in _VARIANTS], ids=[v[0] for v in _VARIANTS]) +def test_launch_variants_are_bitwise_the_plain_launch(helper, args, monkeypatch) -> None: + """Launches no cell reaches, each bitwise its plain counterpart.""" + if helper is _variant_conv_seqs: + helper(*args, monkeypatch) + else: + helper(*args) diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py new file mode 100644 index 000000000000..ee9145103fb5 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_vision.py @@ -0,0 +1,895 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The FP8 unified target's vision path: the tower's two entries over their certified cells, the +tower against HuggingFace on the checkpoint's weights, request admission and the multimodal splice. +""" + +from __future__ import annotations + +import contextlib +import functools +import json +import zlib +from types import SimpleNamespace +from typing import List, Sequence, Tuple + +import numpy as np +import pytest +import safetensors +import torch +import torch.nn.functional as F +from utils.llm_data import llm_models_root + +from tensorrt_llm._torch.configs import Qwen4ExpVisionConfig +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + modeling as M, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + vision as V, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as W, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.attention.vision_rope_2d_packed_qk import ( # noqa: E501 + HEAD_DIM, + NUM_HEADS, + vision_rope_2d_packed_qk, +) +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified.ops.vision.triton_pos_embed_interpolate import ( # noqa: E501 + triton_pos_embed_interpolate, +) +from tensorrt_llm.inputs.multimodal import ( + MULTIMODAL_ENCODER_ITEM_METADATA_KEY, + MultimodalItemMetadata, + MultimodalParams, +) + +__extra_import_path__ = ["../../.."] +from _validating import ( # noqa: E402,F401 — path above; fixture by name + ULP, + assert_within_ulp, + certifying, +) + +CHECKPOINT = "Qwen3.8-Flash-Next-FP8" + +Shell = M.ModelingV2Qwen38FlashNextFp8Sm103Unified + +_VISION = dict( + deepstack_visual_indexes=[], depth=27, dtype="bfloat16", hidden_act="gelu_pytorch_tanh", + hidden_size=1152, in_channels=3, intermediate_size=4304, num_heads=16, + num_position_embeddings=2304, out_hidden_size=2560, patch_size=16, spatial_merge_size=2, + temporal_patch_size=2, +) # fmt: skip +HIDDEN_OUT = 2560 + +_needs_gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device") + + +def _dims(vcfg) -> SimpleNamespace: + """The vision widths of `weights.rank_dims` (the tower is replicated: every rank's).""" + return SimpleNamespace( + hidden=HIDDEN_OUT, dtype=torch.bfloat16, v_depth=vcfg.depth, v_hidden=vcfg.hidden_size, + v_inter=vcfg.intermediate_size, v_channels=vcfg.in_channels, + v_tpatch=vcfg.temporal_patch_size, v_patch=vcfg.patch_size, + v_pos_rows=vcfg.num_position_embeddings, + v_merge_dim=vcfg.hidden_size * vcfg.spatial_merge_size**2, + ) # fmt: skip + + +def _vision_entries(): + return [entry for entry in W.MODEL_WEIGHTS.WEIGHTS if entry.name.startswith("vis_")] + + +def _tower_over_the_weight_table() -> V.VisionTower: + """The tower over the table's `vis_*` weights on meta; `embed` only supplies the dtype.""" + vcfg = Qwen4ExpVisionConfig(**_VISION) + d = _dims(vcfg) + w = {"embed": torch.empty(8, HIDDEN_OUT, dtype=torch.bfloat16, device="meta")} + for entry in _vision_entries(): + for key, _, _ in W.MODEL_WEIGHTS._targets(entry, d): + w[key] = torch.empty(*entry.shape(d), dtype=entry.dtype or d.dtype, device="meta") + return V.VisionTower(w, vcfg) + + +MERGE = 2 +THETA = 10000.0 + + +def _num_tokens(grid) -> int: + return sum(t * h * w for t, h, w in grid) + + +def _spatial_merge_positions(grid: Sequence[Sequence[int]], merge: int) -> List[Tuple[int, int]]: + """Per-token `(row, col)` in spatial-merge order, by plain nested loops.""" + positions: List[Tuple[int, int]] = [] + for frames, height, width in grid: + frame = [ + (block_row * merge + r, block_col * merge + c) + for block_row in range(height // merge) + for block_col in range(width // merge) + for r in range(merge) + for c in range(merge) + ] + positions.extend(frame * frames) + return positions + + +def _angle_table(grid_thw: torch.Tensor, merge: int, theta: float, head_dim: int, device="cuda"): + """fp32 `cos` / `sin` `[T, head_dim // 2]`, height then width, rounded once from fp64.""" + rope_dim = head_dim // 2 + inv_freq = 1.0 / (theta ** (torch.arange(0, rope_dim, 2, dtype=torch.float64) / rope_dim)) + pos = torch.tensor(_spatial_merge_positions(grid_thw.tolist(), merge), dtype=torch.float64) + angles = (pos[:, :, None] * inv_freq).flatten(1).to(device=device, dtype=torch.float32) + return angles.cos().contiguous(), angles.sin().contiguous() + + +HIDDEN = 3 + +IMAGE, VIDEO = 1, 2 + + +class _Tower: + """Stub `VisionTower`: a merged row carries its first patch's pixel value (`_media`).""" + + merge = MERGE + + def __init__(self): + self.calls = 0 + self.patches = 0 + + def __call__(self, pixel_values, grid_thw): + self.calls += 1 + self.patches += pixel_values.shape[0] + assert sum(t * h * w for t, h, w in grid_thw.tolist()) == pixel_values.shape[0] + return pixel_values[:: MERGE * MERGE].expand(-1, HIDDEN) + + +def _media(modality, grids): + """One `multimodal_data` entry; every patch row of item `i` holds `100 * modality + i`.""" + key = "pixel_values" if modality == IMAGE else "pixel_values_videos" + grid_key = "image_grid_thw" if modality == IMAGE else "video_grid_thw" + pixels = torch.cat( + [ + torch.full((t * h * w, 1), float(100 * modality + item)) + for item, (t, h, w) in enumerate(grids) + ] + ) + return {key: pixels, grid_key: torch.tensor(grids)} + + +def _params(image=None, video=None, order=None, runtime=None, mrope=True): + data = {} + if mrope: + data["mrope_config"] = {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)} + if image is not None: + data["image"] = _media(IMAGE, image) + if video is not None: + data["video"] = _media(VIDEO, video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + return MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + + +# Two videos of 4 and 8 rows, two images of 4 and 1. +VIDEOS = [(2, 2, 4), (2, 4, 4)] +IMAGES = [(1, 4, 4), (1, 2, 2)] +MIXED_ORDER = [("video", 0), ("image", 0), ("video", 1), ("image", 1)] +MIXED_ROWS = [200.0] * 4 + [100.0] * 4 + [201.0] * 8 + [101.0] + + +class _StepBegun(Exception): + """Raised by the stub core's `begin_step`, so the shell's forward stops there.""" + + +def _forward_to_begin_step(tower, kwargs): + """The `vision` argument the shell's forward hands `begin_step` for a step with `kwargs`.""" + seen = {} + + def begin_step(attn_metadata, input_ids, position_ids, **step): + seen.update(step) + raise _StepBegun + + shell = SimpleNamespace(model=SimpleNamespace(begin_step=begin_step), vision_tower=tower) + with pytest.raises(_StepBegun): + Shell.forward(shell, None, torch.zeros(1, dtype=torch.int32), None, **kwargs) + return seen["vision"] + + +TEXT_TOKENS = 5 + + +@functools.cache +def _assembled_tower() -> V.VisionTower: + return _tower_over_the_weight_table() + + +def _request_media(modality, grids): + """A payload whose pixel tensor has the assembled tower's patch width and dtype and no + storage.""" + patches = sum(t * h * w for t, h, w in grids) + key = "pixel_values" if modality == "image" else "pixel_values_videos" + grid_key = "image_grid_thw" if modality == "image" else "video_grid_thw" + tower = _assembled_tower() + return { + key: torch.empty(patches, tower.pixel_width, dtype=tower.dtype, device="meta"), + grid_key: torch.tensor(grids), + } + + +def _request(image=None, video=None, order=None, **overrides): + """Placeholders plus `TEXT_TOKENS` text tokens, with the mrope ids and placeholder cumsum.""" + rows = sum(t * h * w for t, h, w in (image or []) + (video or [])) // (MERGE * MERGE) + prompt_len = rows + TEXT_TOKENS + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, prompt_len, dtype=torch.int32)}, + "multimodal_embed_mask_cumsum": torch.cat( + [torch.arange(1, rows + 1), torch.full((TEXT_TOKENS,), rows)] + ), + } + if image is not None: + data["image"] = _request_media("image", image) + if video is not None: + data["video"] = _request_media("video", video) + if order is not None: + data[MULTIMODAL_ENCODER_ITEM_METADATA_KEY] = MultimodalItemMetadata(item_refs=order) + fields = {"py_multimodal_data": data, "prompt_len": prompt_len} + fields.update(overrides) + return SimpleNamespace(**fields) + + +def _whole_op_reference(q, k, grid_thw, spatial_merge_size, theta=THETA): + """`VisionRope2d.reference` verbatim: the whole op from the grid, as fresh `(q_out, k_out)`.""" + rope_dim = q.shape[-1] // 2 + axis_dim = rope_dim // 2 + positions = _spatial_merge_positions(grid_thw.tolist(), spatial_merge_size) + inverse = torch.tensor( + [theta ** (-(2.0 * j) / rope_dim) for j in range(axis_dim)], dtype=torch.float64 + ) + pos = torch.tensor(positions, dtype=torch.float64).reshape(-1, 2) + angles = (pos[:, :, None] * inverse).flatten(1).to(device=q.device, dtype=torch.float32) + rotor = torch.polar(torch.ones_like(angles), angles).unsqueeze(-2) + + def rotate(x: torch.Tensor) -> torch.Tensor: + fp32 = x.float() + rotated = torch.complex(fp32[..., :rope_dim], fp32[..., rope_dim:]) * rotor + return torch.cat((rotated.real, rotated.imag), dim=-1).to(x.dtype) + + return rotate(q), rotate(k) + + +def _packed_qkv(tokens: int, dtype: torch.dtype, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randn(tokens, 3 * NUM_HEADS, HEAD_DIM, generator=g, device="cuda").to(dtype) + + +def _qk(qkv: torch.Tensor) -> torch.Tensor: + qk = qkv[:, : 2 * NUM_HEADS] + assert qk.stride() == (3 * NUM_HEADS * HEAD_DIM, HEAD_DIM, 1) and not qk.is_contiguous() + return qk + + +def _mismatch(got: torch.Tensor, want: torch.Tensor) -> Tuple[int, float]: + """(elements differing in their bits, largest absolute difference).""" + differing = (got.reshape(-1).view(torch.int16) != want.reshape(-1).view(torch.int16)).sum() + return int(differing.item()), (got.float() - want.float()).abs().max().item() + + +def _check_rope(cell) -> None: + spec = cell.spec + grid_thw = torch.tensor([list(row) for row in spec["grid"]], dtype=spec["grid_dtype"]) + tokens = _num_tokens(spec["grid"]) + qkv = _packed_qkv(tokens, spec["dtype"], seed=zlib.crc32(cell.why.encode()) % 2**31) + before = qkv.clone() + cos, sin = _angle_table(grid_thw, MERGE, THETA, HEAD_DIM) + qk = _qk(qkv) + + want = vision_rope_2d_packed_qk.reference(qk, cos, sin) + assert torch.equal(qkv, before), "reference must not mutate its input" + out = vision_rope_2d_packed_qk(qk, cos, sin) + torch.cuda.synchronize() + + assert out is qk, "the op returns the view it rotated" + assert torch.equal(qkv[:, 2 * NUM_HEADS :], before[:, 2 * NUM_HEADS :]), "v was written" + assert not torch.equal(qk, before[:, : 2 * NUM_HEADS]), "q|k was not rotated" + + differing, max_dev = _mismatch(qk, want) + assert differing == 0, f"{differing} elements differ from reference (max {max_dev:.3e})" + + q_whole, k_whole = _whole_op_reference( + before[:, :NUM_HEADS], before[:, NUM_HEADS : 2 * NUM_HEADS], grid_thw, MERGE, THETA + ) + for name, got, ref in ( + ("q", qkv[:, :NUM_HEADS], q_whole), + ("k", qkv[:, NUM_HEADS : 2 * NUM_HEADS], k_whole), + ): + differing, max_dev = _mismatch(got, ref) + assert differing == 0, ( + f"{name}: {differing} elements differ from the whole op (max {max_dev:.3e})" + ) + + +#: (height, width) in pixels of the two test images: grids 1x28x42 and 1x14x20. +_IMAGE_SIZES = ((448, 672), (224, 320)) +#: Four 256x384 frames: one video item, grid 2x16x24 (two temporal groups). +_VIDEO_FRAMES, _VIDEO_SIZE = 4, (256, 384) +#: SDPA backends of the bf16 HF forward; their spread defines the bf16 error envelope. +_HF_BACKENDS = ("MATH", "EFFICIENT_ATTENTION", "FLASH_ATTENTION", "CUDNN_ATTENTION") +#: Allowed factor over the worst bf16 HF forward: the backends differ among themselves by up to +#: 1.5x on one item, and the tower is one more sample of that spread. +_BAND = 1.25 + + +def _picture(height: int, width: int, seed: int) -> np.ndarray: + """A deterministic `[H, W, 3]` uint8 picture with both low and high spatial frequencies.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:height, 0:width] / max(height, width) + rgb = np.stack( + [np.sin(6 * x + seed), np.cos(5 * y - seed), np.sin(3 * (x + y) + 0.5 * seed)], -1 + ) + rgb = 0.5 + 0.5 * rgb + for _ in range(4): + top, left = rng.integers(0, height // 2), rng.integers(0, width // 2) + rgb[top : top + height // 4, left : left + width // 4] = rng.random(3) + rgb += 0.04 * rng.standard_normal(rgb.shape) + return (np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8) + + +@contextlib.contextmanager +def _ieee_fp32(): + """IEEE fp32 for the reference: GEMMs and the patch conv off TF32, SDPA on its math backend.""" + from torch.nn.attention import SDPBackend, sdpa_kernel + + matmul, conv = torch.backends.cuda.matmul, torch.backends.cudnn.conv + previous = (matmul.fp32_precision, conv.fp32_precision) + matmul.fp32_precision = "ieee" + conv.fp32_precision = "ieee" + try: + with sdpa_kernel(SDPBackend.MATH): + yield + finally: + matmul.fp32_precision, conv.fp32_precision = previous + + +def _stats(out: torch.Tensor, ref: torch.Tensor) -> dict: + """Max abs error, max abs error over max |ref| ("max rel"), relative L2 and cosine.""" + o, r = out.double().flatten(), ref.double().flatten() + diff = (o - r).abs() + return dict( + max_abs=diff.max().item(), + max_rel=(diff.max() / r.abs().max()).item(), + rel_l2=(torch.linalg.vector_norm(o - r) / torch.linalg.vector_norm(r)).item(), + cos=torch.nn.functional.cosine_similarity(o, r, dim=0).item(), + ) + + +@pytest.fixture(scope="module") +def checkpoint(): + root = llm_models_root() + path = None if root is None else root / CHECKPOINT + if path is None or not (path / "model.safetensors.index.json").is_file(): + pytest.skip(f"{CHECKPOINT} is not under LLM_MODELS_ROOT") + return path + + +@pytest.fixture(scope="module") +def models(checkpoint): + if not torch.cuda.is_available(): + pytest.skip("needs a CUDA device") + return _load_models(checkpoint) + + +@pytest.fixture(scope="module") +def media(checkpoint): + return _load_media(checkpoint) + + +@pytest.fixture(scope="module") +def references(models, media): + return _reference_outputs(models, media) + + +def _load_models(checkpoint): + """`(tower, hf32, hf16)`: this target's tower over the table's `vis_*` sources and HF's + `Qwen3_5VisionModel` in fp32 and bf16, all on the checkpoint's same vision tensors.""" + from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5VisionConfig + from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5VisionModel + + vision = json.loads((checkpoint / "config.json").read_text())["vision_config"] + assert vision["deepstack_visual_indexes"] == [], "the reference has no deepstack mergers" + vcfg = Qwen4ExpVisionConfig(**{k: v for k, v in vision.items() if k != "model_type"}) + d = _dims(vcfg) + index = json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + + # Each `vis_*` parameter is one whole checkpoint tensor: the tower is replicated and bf16. + sources = {} + for entry in _vision_entries(): + assert entry.transform is None and entry.cut is None, entry.name + for key, i, prefix in W.MODEL_WEIGHTS._targets(entry, d): + [(ckpt_key, at, cut)] = entry.sources(d, i, prefix) + assert at is None and cut is None, (key, ckpt_key) + sources[key] = (ckpt_key, entry.shape(d), entry.dtype or d.dtype) + visual = {k for k in index if k.startswith("model.visual.")} + assert {ckpt_key for ckpt_key, _, _ in sources.values()} == visual, ( + "the table's vis_* sources are not exactly the checkpoint's model.visual.* tensors" + ) + assert len(sources) == len(visual) + + tensors = {} + for shard in sorted({index[k] for k in visual}): + with safetensors.safe_open(str(checkpoint / shard), framework="pt", device="cpu") as f: + for key in f.keys(): + if key in visual: + tensors[key] = f.get_tensor(key) + w = {} + for key, (ckpt_key, shape, dtype) in sources.items(): + tensor = tensors[ckpt_key] + assert tuple(tensor.shape) == tuple(shape) and tensor.dtype == dtype, key + w[key] = tensor.cuda() + w["embed"] = torch.empty(1, d.hidden, dtype=torch.bfloat16, device="cuda") + tower = V.VisionTower(w, vcfg) + + fields = ( + "depth hidden_size hidden_act intermediate_size num_heads in_channels patch_size " + "spatial_merge_size temporal_patch_size out_hidden_size num_position_embeddings" + ).split() + hf_cfg = Qwen3_5VisionConfig(**{k: vision[k] for k in fields}) + hf_cfg._attn_implementation = "sdpa" + state = {k.removeprefix("model.visual."): t for k, t in tensors.items()} + out = [] + for dtype in (torch.float32, torch.bfloat16): + with torch.device("cuda"): + hf = Qwen3_5VisionModel(hf_cfg) + hf.load_state_dict(state, strict=True) + # Keep the rotary's `inv_freq` fp32 as `from_pretrained` does; `.to(bf16)` rounds it. + inv_freq = hf.rotary_pos_emb.inv_freq.clone() + hf = hf.to(dtype).eval() + hf.rotary_pos_emb.inv_freq = inv_freq + out.append(hf) + return tower, out[0], out[1] + + +def _load_media(checkpoint): + """`{name: (pixel_values, grid_thw)}` from the checkpoint's processors for two images and a + four-frame video, the pixels rounded to bf16 once so every model reads the same values.""" + from PIL import Image + from transformers import AutoImageProcessor, AutoVideoProcessor + + images = AutoImageProcessor.from_pretrained(str(checkpoint)) + videos = AutoVideoProcessor.from_pretrained(str(checkpoint)) + out = {} + for n, (height, width) in enumerate(_IMAGE_SIZES): + batch = images(images=[Image.fromarray(_picture(height, width, n))], return_tensors="pt") + out[f"image{n}"] = (batch["pixel_values"], batch["image_grid_thw"]) + frames = np.stack([_picture(*_VIDEO_SIZE, 10 + f) for f in range(_VIDEO_FRAMES)]) + batch = videos(videos=[frames], return_tensors="pt", do_sample_frames=False) + out["video"] = (batch["pixel_values_videos"], batch["video_grid_thw"]) + assert [grid.tolist() for _, grid in out.values()] == [ + [[1, 28, 42]], + [[1, 14, 20]], + [[2, 16, 24]], + ] + return { + name: (px.to(device="cuda", dtype=torch.bfloat16), grid) for name, (px, grid) in out.items() + } + + +def _reference_outputs(models, media): + """Per media item `(ref, envelope)`: the fp32 HF reference `[rows, 2560]`, and the worst + of each statistic (`_stats`) the bf16 HF forward reaches against it over `_HF_BACKENDS`.""" + from torch.nn.attention import SDPBackend, sdpa_kernel + + _, hf32, hf16 = models + out = {} + with torch.inference_mode(): + for name, (px, grid) in media.items(): + with _ieee_fp32(): + ref = hf32(px.float(), grid_thw=grid).pooler_output + floors = {} + for backend in _HF_BACKENDS: + with sdpa_kernel(getattr(SDPBackend, backend)): + floors[backend] = _stats(hf16(px, grid_thw=grid).pooler_output, ref) + envelope = { + key: max(st[key] for st in floors.values()) for key in ("max_abs", "rel_l2") + } + out[name] = (ref, envelope) + return out + + +def _check(name: str, out: torch.Tensor, ref: torch.Tensor, envelope: dict) -> dict: + """`out` within `_BAND` of the worst bf16 HF forward on every gated statistic.""" + tower = _stats(out, ref) + assert out.dtype == torch.bfloat16 and out.shape == ref.shape + assert torch.isfinite(out).all() + for key in ("rel_l2", "max_abs"): + assert tower[key] <= _BAND * envelope[key], (name, key, tower, envelope) + return tower + + +def _hf_single(name): + def run(tower, media, references, monkeypatch): + px, grid = media[name] + with torch.inference_mode(): + out = tower(px, grid) + assert out.shape == (int(grid.prod(-1).sum()) // tower.merge**2, HIDDEN_OUT) + _check(name, out, *references[name]) + + return run + + +def _hf_pair(cap=None): + """Both images in one call (or one per call under `cap`), each its own attention segment.""" + + def run(tower, media, references, monkeypatch): + (px0, grid0), (px1, grid1) = media["image0"], media["image1"] + if cap is not None: + assert V._frame_calls([(1, 28, 42), (1, 14, 20)], cap) == [[(1, 28, 42)], [(1, 14, 20)]] + monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", cap) + with torch.inference_mode(): + out = tower(torch.cat([px0, px1]), torch.cat([grid0, grid1])) + split = int(grid0.prod()) // tower.merge**2 + _check("image0 in a pair", out[:split], *references["image0"]) + _check("image1 in a pair", out[split:], *references["image1"]) + + return run + + +def _hf_video_in_two_calls(tower, media, references, monkeypatch): + """The per-call ceiling lowered to one 384-patch frame: the video runs as two calls.""" + px, grid = media["video"] + assert V._frame_calls([(2, 16, 24)], 384) == [[(1, 16, 24)], [(1, 16, 24)]] + monkeypatch.setattr(V, "VISION_CALL_MAX_PATCHES", 384) + with torch.inference_mode(): + out = tower(px, grid) + _check("video in two calls", out, *references["video"]) + + +def _counted_runs(tower, monkeypatch) -> list: + """The patch rows of each `_run` call `tower` makes from here on.""" + runs = [] + run = tower._run + + def counted(pixel_values, grid): + runs.append(pixel_values.shape[0]) + return run(pixel_values, grid) + + monkeypatch.setattr(tower, "_run", counted) + return runs + + +def _spliced_in_chunks(tower, data, bounds): + """The rows `multimodal_embeds` splices for each `(start, end)` chunk of one request whose + `multimodal_data` is `data`, the one dict the executor hands all of a request's chunks.""" + total = bounds[-1][1] + chunks = [] + for start, end in bounds: + runtime = SimpleNamespace( + num_cached_mm_tokens=start, + num_mm_tokens_in_chunk=end - start, + total_embeds_in_request=total, + ) + params = MultimodalParams(multimodal_data=data, multimodal_runtime=runtime) + kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(end - start)} + chunks.append(V.multimodal_embeds(tower, kwargs)[0]) + assert V._TOWER_ROWS_KEY not in data, "the chunk with the last row drops the kept rows" + return torch.cat(chunks) + + +def _hf_image_in_chunks(tower, media, references, monkeypatch): + """One image spliced over three chunked-prefill chunks: the tower runs once, in the first + chunk, and the later chunks read its kept rows, bitwise the one-pass rows.""" + px, grid = media["image0"] + total = int(grid.prod(-1).sum()) // tower.merge**2 + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)}, + "image": {"pixel_values": px, "image_grid_thw": grid}, + } + with torch.inference_mode(): + whole = tower(px, grid) + runs = _counted_runs(tower, monkeypatch) + out = _spliced_in_chunks(tower, data, ((0, 100), (100, 200), (200, total))) + assert runs == [px.shape[0]], runs + assert torch.equal(out, whole), _mismatch(out, whole) + _check("image in three chunks", out, *references["image0"]) + + +def _hf_video_in_chunks(tower, media, references, monkeypatch): + """The video spliced over three chunked-prefill chunks whose boundaries cut its two 96-row + frames: each frame runs once, in the first chunk holding its rows, and together the chunks + are the one-pass rows bitwise.""" + px, grid = media["video"] + total = int(grid.prod(-1).sum()) // tower.merge**2 + data = { + "mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8, dtype=torch.int32)}, + "video": {"pixel_values_videos": px, "video_grid_thw": grid}, + } + with torch.inference_mode(): + whole = tower(px, grid) + runs = _counted_runs(tower, monkeypatch) + out = _spliced_in_chunks(tower, data, ((0, 50), (50, 150), (150, total))) + frame = px.shape[0] // 2 + assert runs == [frame, frame], runs + assert torch.equal(out, whole), _mismatch(out, whole) + _check("video in three chunks", out, *references["video"]) + + +_HF_CASES = { + "image0": _hf_single("image0"), + "image1": _hf_single("image1"), + "video": _hf_single("video"), + "image_pair": _hf_pair(), + "image_pair_two_calls": _hf_pair(cap=1176), + "video_two_calls": _hf_video_in_two_calls, + "image_chunks": _hf_image_in_chunks, + "video_chunks": _hf_video_in_chunks, +} + + +@_needs_gpu +@pytest.mark.parametrize("case", list(_HF_CASES)) +def test_the_tower_is_huggingfaces_vision_model(models, media, references, case, monkeypatch): + """Each case lands within the bf16 HF forward's own error of the fp32 HF reference.""" + _HF_CASES[case](models[0], media, references, monkeypatch) + + +NUM_GRID = 48 +VISION_HIDDEN = 1152 + + +def _pos_embed_compare(out: torch.Tensor, ref: torch.Tensor) -> None: + """4 bf16 ulp per element, 1 ulp relative RMS: the op rounds its corner weights to bf16 and + multiply-adds in bf16, against the exact value (measured worst 3.5 / 0.75).""" + assert out.dtype == torch.bfloat16, out.dtype + assert_within_ulp(out.float(), ref, element_ulp=4.0, rms_ulp=1.0, ulp=ULP[torch.bfloat16]) + + +def _pos_table(seed: int, num_grid: int = NUM_GRID, hidden: int = VISION_HIDDEN) -> torch.Tensor: + """A smooth `[num_grid**2, hidden]` bf16 table, as a learned 2-D position table is.""" + g = torch.Generator(device="cuda").manual_seed(seed) + coarse = torch.randn(1, hidden, 12, 12, generator=g, device="cuda") + up = F.interpolate(coarse, size=(num_grid, num_grid), mode="bicubic", align_corners=True) + up = up[0].permute(1, 2, 0) + 0.1 * torch.randn( + num_grid, num_grid, hidden, generator=g, device="cuda" + ) + return up.reshape(num_grid * num_grid, hidden).to(torch.bfloat16).contiguous() + + +def _resize64(table, t, h, w, m_size, *, align=True, raster=False, swap=False): + """torch's bilinear resize in fp64, in merge order unless `raster`.""" + num_grid = int(round(table.shape[0] ** 0.5)) + grid = table.double().view(num_grid, num_grid, -1) + if swap: + grid = grid.transpose(0, 1) + src = grid.permute(2, 0, 1).unsqueeze(0).contiguous() + out = F.interpolate(src, size=(h, w), mode="bilinear", align_corners=align)[0].permute(1, 2, 0) + if raster: + rows = out.reshape(h * w, -1) + else: + rows = ( + out.reshape(h // m_size, m_size, w // m_size, m_size, -1) + .permute(0, 2, 1, 3, 4) + .reshape(h * w, -1) + ) + return rows.repeat(t, 1) if t > 1 else rows + + +def _pos_embed_args(spec, table): + return (table, *(spec[k] for k in ("t", "h", "w", "num_grid_per_side", "m_size", "dtype"))) + + +def _check_pos_embed(cell) -> None: + spec = cell.spec + t, h, w, m_size = spec["t"], spec["h"], spec["w"], spec["m_size"] + table = _pos_table( + zlib.crc32(cell.why.encode()) % 2**31, spec["num_grid_per_side"], spec["hidden"] + ) + before = table.clone() + + out = triton_pos_embed_interpolate(*_pos_embed_args(spec, table)) + assert out.shape == (t * h * w, spec["hidden"]) and out.dtype == spec["dtype"] + assert torch.equal(table, before) and out.data_ptr() != table.data_ptr() + + ref = triton_pos_embed_interpolate.reference(*_pos_embed_args(spec, table)) + _pos_embed_compare(out, ref) + + # An independent fp64 implementation confirms the reference. + torch.testing.assert_close(ref, _resize64(table, t, h, w, m_size).float()) + + # The band refuses each wrong variant of the function. + for wrong in (dict(align=False), dict(raster=True), dict(swap=True)): + with pytest.raises(AssertionError): + _pos_embed_compare(out, _resize64(table, t, h, w, m_size, **wrong).float()) + + +_ENTRIES = ( + (vision_rope_2d_packed_qk, _check_rope), + (triton_pos_embed_interpolate, _check_pos_embed), +) +CASES = [(op, check, cell) for op, check in _ENTRIES for cell in op.CELLS] + + +@_needs_gpu +@pytest.mark.parametrize( + "entry,check,cell", + CASES, + ids=[f"{type(op).__name__}-{i}" for op, _ in _ENTRIES for i in range(len(op.CELLS))], +) +def test_certified_cells(entry, check, cell, certifying) -> None: # noqa: F811 — fixture by name + """Every certified cell of the tower's two entries against the entry's own reference.""" + certifying(entry) + check(cell) + + +def _text(data): + return lambda: SimpleNamespace(py_multimodal_data=data) + + +def _req(edit=None, **media): + """`_request(**media)`, then `edit` on its `py_multimodal_data`.""" + + def build(): + request = _request(**media) + if edit is not None: + edit(request.py_multimodal_data) + return request + + return build + + +def _grid(value): + return lambda d: d["image"].update(image_grid_thw=torch.tensor(value)) + + +def _pixels(rows, extra=0, dtype=None): + def edit(d): + tower = _assembled_tower() + d["video"]["pixel_values_videos"] = torch.empty( + rows, tower.pixel_width + extra, dtype=dtype or tower.dtype, device="meta" + ) + + return edit + + +def _positions(*shape): + return lambda d: d["mrope_config"].update(mrope_position_ids=torch.zeros(*shape)) + + +def _cumsum(d): + d["multimodal_embed_mask_cumsum"] = torch.arange(1, 14) + + +_MIXED = dict(image=IMAGES, video=VIDEOS, order=MIXED_ORDER) +_TILE = [(1, 4, 4)] +#: Served. The per-call ceiling of 32768 patches bounds a frame, not a whole video. +_SERVED = [ + ("image", _req(image=IMAGES)), + ("video", _req(video=VIDEOS)), + ("mixed", _req(**_MIXED)), + ("frame_at_ceiling", _req(image=[(1, 128, 256)])), + ("long_video", _req(video=[(33, 32, 32)])), + ("no_cumsum", _req(lambda d: d.pop("multimodal_embed_mask_cumsum"), **_MIXED)), +] +#: Text, and position ids without media (a warmup request's): admitted without the tower. +_POSITIONS_ONLY = {"mrope_config": {"mrope_position_ids": torch.zeros(3, 1, 8)}} +_TEXT = [("text", _text(None)), ("positions_only", _text(_POSITIONS_ONLY))] +#: Refused: what the tower or the splice would assert on, or read wrongly. +_REFUSED = [ + ("frame_past_ceiling", _req(video=[(1, 192, 192)]), + r"video 0 has frames of 192x192 = 36864 patches \(3072x3072 pixels\).*at most 32768"), + ("grid_not_merge_blocks", _req(_grid([(1, 4, 5)]), image=_TILE), "spatial_merge_size 2"), + ("grid_zero", _req(_grid([(0, 4, 4)]), image=_TILE), "every entry must be positive"), + ("grid_1d", _req(_grid([1, 4, 4]), image=_TILE), r"one \(t, h, w\) row per image"), + ("pixel_rows", _req(_pixels(5), video=VIDEOS), "describe 48 patch rows"), + ("pixel_width", _req(_pixels(48, 1), video=VIDEOS), "48 patch rows of"), + ("pixel_dtype", _req(_pixels(48, dtype=torch.uint8), video=VIDEOS), + "torch.uint8; the vision tower runs at torch.bfloat16"), + ("precomputed_embedding", + _req(lambda d: d.update(multimodal_embedding=[torch.zeros(4, 8)]), image=_TILE), + "precomputed image embeddings"), + ("no_grid", _req(lambda d: d["video"].pop("video_grid_thw"), video=VIDEOS), + "this target reads 'pixel_values_videos'"), + ("positions_short", _req(_positions(3, 1, 40), video=VIDEOS), "one position per prompt token"), + ("positions_2d", _req(_positions(1, 17), video=VIDEOS), r"got \(1, 17\)"), + ("no_positions", _req(lambda d: d.pop("mrope_config"), video=VIDEOS), + "mrope_position_ids.*got None"), + ("no_manifest", _req(image=IMAGES, video=VIDEOS), "needs item metadata"), + ("short_manifest", _req(**dict(_MIXED, order=MIXED_ORDER[:3])), + r"lists image items \[0\]; the payload carries 2"), + ("unknown_item", _req(**dict(_MIXED, order=MIXED_ORDER + [("audio", 0)])), + "is not an item of this request"), + ("placeholder_count", _req(_cumsum, **_MIXED), "holds 13 image/video placeholder tokens.*17"), +] # fmt: skip +_ADMISSIONS = ( + [(i, b, None, True) for i, b in _SERVED] + + [(i, b, None, False) for i, b in _TEXT] + + [(i, b, r, True) for i, b, r in _REFUSED] +) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "build,refusal,reads_tower", [a[1:] for a in _ADMISSIONS], ids=[a[0] for a in _ADMISSIONS] +) +def test_admission_serves_what_the_tower_serves(build, refusal, reads_tower): + """`validate_request` admits text without reading the tower and the media it serves, and + refuses what the tower or the splice would assert on, naming the item and the bound.""" + shell = SimpleNamespace(vision_tower=_assembled_tower() if reads_tower else None) + if refusal is None: + Shell.validate_request(shell, build()) + else: + with pytest.raises(ValueError, match=refusal): + Shell.validate_request(shell, build()) + + +def _chunk(cached, in_chunk, total=len(MIXED_ROWS)): + return SimpleNamespace( + num_cached_mm_tokens=cached, + num_mm_tokens_in_chunk=in_chunk, + total_embeds_in_request=total, + ) + + +#: `(id, _params kwargs, placeholder rows, rows that land there (None: no media), tower calls, +#: patch rows the tower runs)`. The mixed chunk takes prompt-order rows 3..9: the second frame of +#: video 0, image 0 and the first frame of video 1 (40 of the request's 68 patch rows); the video +#: chunk rows 5..7, inside video 1's first frame. +_SPLICES = [ + ("text", dict(), 0, None, 0, 0), + ("image", dict(image=IMAGES), 5, [100.0] * 4 + [101.0], 1, 20), + ("video", dict(video=VIDEOS), 12, [200.0] * 4 + [201.0] * 8, 1, 48), + ("mixed", _MIXED, len(MIXED_ROWS), MIXED_ROWS, 1, 68), + ("chunk", dict(_MIXED, runtime=_chunk(3, 6)), 6, MIXED_ROWS[3:9], 1, 40), + ("video_chunk", dict(video=VIDEOS, runtime=_chunk(5, 2, 12)), 2, [201.0] * 2, 1, 16), + ("chunk_without_placeholders", dict(_MIXED, runtime=_chunk(len(MIXED_ROWS), 0)), 0, None, 0, 0), +] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "params,num_rows,want,calls,patches", [s[1:] for s in _SPLICES], ids=[s[0] for s in _SPLICES] +) +def test_the_splice_puts_each_items_rows_on_its_placeholders( + params, num_rows, want, calls, patches +): + """The tower's rows reach `begin_step` with the placeholder indices, and the splice writes them + over the placeholders in prompt order and zeroes the padding rows after the step. A chunk runs + the tower on only the frames that hold its rows.""" + tower = _Tower() + indices = torch.arange(1, 1 + num_rows, dtype=torch.int64) + kwargs = { + "multimodal_params": [_params(**params)], + "mm_token_indices": indices, + "resource_manager": object(), + } + vision = _forward_to_begin_step(tower, kwargs) + assert (tower.calls, tower.patches) == (calls, patches) + total = num_rows + 2 + h = torch.full((total + 3, HIDDEN), -1.0) + M.PrefillTarget._splice( + SimpleNamespace(_step=SimpleNamespace(vision=vision, num_tokens=total)), h + ) + assert torch.count_nonzero(h[total:]) == 0, "padding rows must be zeroed" + if want is None: + assert vision is None and torch.all(h[:total] == -1.0) + return + assert vision[1] is indices + assert h[1 : 1 + num_rows, 0].tolist() == want + assert torch.all(h[0] == -1.0) and torch.all(h[total - 1] == -1.0) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("chunk", range(1, len(MIXED_ROWS) + 1)) +def test_a_chunked_request_runs_each_frame_once(chunk): + """Over all of one request's chunks the tower runs each frame once (the one-pass 68 patch + rows) and the chunks splice the one-pass rows; the last chunk drops the kept rows.""" + tower = _Tower() + params = _params(**_MIXED) + spliced = [] + for start in range(0, len(MIXED_ROWS), chunk): + count = min(chunk, len(MIXED_ROWS) - start) + params.multimodal_runtime = _chunk(start, count) + kwargs = {"multimodal_params": [params], "mm_token_indices": torch.arange(count)} + spliced += V.multimodal_embeds(tower, kwargs)[0][:, 0].tolist() + assert spliced == MIXED_ROWS + assert tower.patches == 68, (tower.calls, tower.patches) + assert V._TOWER_ROWS_KEY not in params.multimodal_data diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py new file mode 100644 index 000000000000..be23081f81a1 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified/test_weights.py @@ -0,0 +1,1043 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The unified target's per-rank weight table against the FP8 checkpoint, on the CPU from its +index and safetensors headers: each rank's shard, each member's coverage and `load`'s operands.""" + +from __future__ import annotations + +import collections +import json +import re +import struct +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from utils.llm_data import llm_models_root + +from tensorrt_llm._torch.configs import Qwen4ExpTextConfig, Qwen4ExpVisionConfig +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_fp8__sm_103__unified import ( # noqa: E501 + weights as WT, +) +from tensorrt_llm._torch.models.modeling_utils import duplicate_kv_weight +from tensorrt_llm.mapping import Mapping + +CHECKPOINT = "Qwen3.8-Flash-Next-FP8" + +#: Each member's mapping as the LLM API's parallel arguments build it. +MEMBERS = { + "tp1": dict(world_size=1, tp_size=1), + "tp2": dict(world_size=2, tp_size=2), + "tep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2), + "dep2": dict(world_size=2, tp_size=2, moe_tp_size=1, moe_ep_size=2, enable_attention_dp=True), + "tp4": dict(world_size=4, tp_size=4), + "tep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4), + "dep4": dict(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4, enable_attention_dp=True), +} +NUM_EXPERTS = 512 +MTP_LAYER = 48 + +_STORED_DTYPES = {"BF16": torch.bfloat16, "F32": torch.float32, "F8_E4M3": torch.float8_e4m3fn, + "I64": torch.int64} # fmt: skip + +#: A routed (trunk) or draft (MTP) expert's tensors. +_EXPERT_TENSOR = re.compile(r"\.mlp\.experts\.\d+\.(gate|up|down)_proj\.(weight|weight_scale_inv)$") + + +@pytest.fixture(scope="module") +def checkpoint() -> Path: + root = llm_models_root() + path = None if root is None else root / CHECKPOINT + if path is None or not (path / "model.safetensors.index.json").is_file(): + pytest.skip(f"{CHECKPOINT} is not under LLM_MODELS_ROOT") + return path + + +@pytest.fixture(scope="module") +def index(checkpoint) -> dict[str, str]: + return json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + + +@pytest.fixture(scope="module") +def stored(checkpoint, index) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: + """Every checkpoint tensor's stored shape and dtype, from the index and the headers.""" + specs = {} + for file in sorted(set(index.values())): + with open(checkpoint / file, "rb") as fh: + (length,) = struct.unpack(" dict[str, torch.Tensor]: + """The n-gram id space's tensors (per-head offsets and vocabulary sizes), read whole.""" + from safetensors import safe_open + + tensors = {} + for key, file in index.items(): + if key.endswith((".ngram_heads_offsets", ".ngram_heads_vocab_sizes")): + with safe_open(str(checkpoint / file), framework="pt") as fh: + tensors[key] = fh.get_tensor(key) + return tensors + + +@pytest.fixture(scope="module") +def configs(checkpoint): + raw = json.loads((checkpoint / "config.json").read_text()) + return Qwen4ExpTextConfig(**raw["text_config"]), Qwen4ExpVisionConfig(**raw["vision_config"]) + + +def _mapping(member: str, rank: int) -> Mapping: + return Mapping(rank=rank, gpus_per_node=4, pp_size=1, **MEMBERS[member]) + + +def _dims(configs, member: str, rank: int, mtp: bool = True): + text, vision = configs + return WT.rank_dims(text, vision, _mapping(member, rank), mtp_enabled=mtp, dtype=torch.bfloat16) + + +def _world(member: str) -> int: + return MEMBERS[member]["world_size"] + + +def _attention_tp(member: str) -> int: + layout = MEMBERS[member] + return 1 if layout.get("enable_attention_dp", False) else layout["tp_size"] + + +def _lm_head_rows(member: str, vocab: int) -> int: + """The shell's lm_head rows: vocab-parallel under attention TP, whole otherwise.""" + return vocab // _attention_tp(member) + + +def _rows_per_shard(stored, d) -> int: + shapes = {stored[key][0] for key in WT.ngram_shard_keys(d)} + assert len(shapes) == 1, shapes + return next(iter(shapes))[0] + + +def _rank_manifest(stored, d, member: str) -> dict[str, list[tuple]]: + """The rank's manifest as `load` builds it, with the n-gram table and the shell's lm_head.""" + manifest = WT.MODEL_WEIGHTS.manifest(d) + manifest["ple_table"] = WT.ngram_table_sources(d, _rows_per_shard(stored, d)) + cut, pad = WT.lm_head_share(_lm_head_rows(member, d.vocab), d.vocab, d.tp_rank) + manifest["lm_head.weight"] = [("lm_head.weight", None, cut, pad)] + return manifest + + +def _keys_and_cuts(ckpt_key, cut): + if isinstance(ckpt_key, tuple): + return list(zip(ckpt_key, cut if cut is not None else (None,) * len(ckpt_key))) + return [(ckpt_key, cut)] + + +def _cut_numel(shape: tuple[int, ...], cut) -> int: + """Elements of a `shape` tensor `cut` reads, after checking each slice lies inside it.""" + if cut is None: + return int(torch.Size(shape).numel()) + extents = list(shape) + for axis, s in enumerate(cut): + assert isinstance(s, slice) and s.step in (None, 1), (shape, cut) + start = 0 if s.start is None else s.start + stop = shape[axis] if s.stop is None else s.stop + assert 0 <= start < stop <= shape[axis], f"cut {cut} leaves the stored {shape}" + extents[axis] = stop - start + return int(torch.Size(extents).numel()) + + +class _HeaderEntry: + """A checkpoint tensor known by its header: indexing returns a meta tensor, nothing is read.""" + + def __init__(self, shape, dtype): + self.shape, self.dtype = torch.Size(shape), dtype + self.ndim = len(shape) + + def __getitem__(self, index): + return torch.empty(self.shape, dtype=self.dtype, device="meta")[index] + + +def _share(member: str, key: str) -> float: + """The fraction of `key` a rank of `member` reads: attention and Gated-DeltaNet tensors by + head over the attention TP ranks (two KV heads), the lm_head by vocabulary, the rest whole.""" + tp = _attention_tp(member) + if re.search(r"\.self_attn\.(q_proj|o_proj)\.weight$", key) or key == "lm_head.weight": + return 1 / tp + if re.search(r"\.self_attn\.(k_proj|v_proj)\.weight$", key): + return 1 / min(tp, 2) + if re.search( + r"\.linear_attn\.((in_proj_qkv|in_proj_z|in_proj_a|in_proj_b|conv1d|out_proj)\.weight" + r"|A_log|dt_bias)$", + key, + ): + return 1 / tp + return 1.0 + + +def _window(member: str, rank: int) -> range: + """The `512 / world` consecutive whole experts a rank holds, whatever the MoE split.""" + count = NUM_EXPERTS // _world(member) + return range(count * rank, count * (rank + 1)) + + +def _check_operand_shapes(stored, d, member: str) -> dict: + """Each source fills its parameter exactly; each tensor is read at the rank's share, once.""" + with torch.device("meta"): + params = dict(WT.MODEL_WEIGHTS.declare(d)) + # What `load` adds beside the table: the n-gram table and the shell's lm_head. + params["ple_table"] = torch.empty( + d.ple_shards * _rows_per_shard(stored, d), d.ple_head_dim, dtype=d.ple_table_dtype + ) + params["lm_head.weight"] = torch.empty( + _lm_head_rows(member, d.vocab), d.hidden, dtype=torch.bfloat16 + ) + entries = {key: _HeaderEntry(*spec) for key, spec in stored.items()} + manifest = _rank_manifest(stored, d, member) + assert manifest.keys() == params.keys() + read = collections.Counter() + results: dict = {} + for param_key, sources in manifest.items(): + param = params[param_key] + for ckpt_key, index, cut, transform in sources: + key_cuts = _keys_and_cuts(ckpt_key, cut) + for key, key_cut in key_cuts: + read[key] += _cut_numel(stored[key][0], key_cut) + dst = param if index is None else param[index] + # Equal stored tensors, cuts and transform give equal results: read each once. + signature = (transform, tuple((stored[k], c) for k, c in key_cuts)) + if signature not in results: + src = WT.MODEL_WEIGHTS.read(entries, ckpt_key, cut, transform, d) + results[signature] = (tuple(src.shape), src.dtype) + got = (tuple(dst.shape), dst.dtype) + assert results[signature] == got, (param_key, ckpt_key, results[signature], got) + wrong = { + key: (n, _share(member, key)) + for key, n in read.items() + if n != round(torch.Size(stored[key][0]).numel() * _share(member, key)) + } + assert not wrong, f"tensors not read at the rank's share, once: {sorted(wrong.items())[:8]}" + return params + + +_HIDDEN = 2 # the projections' input axis, shrunk: no cut touches it +_FAKE_OFFSETS = dict(q=1, k=2, v=3, idx=4, o=5, qkv=10, z=11, a=12, b=13, conv=14, a_log=15, + dtb=16, out=17) # fmt: skip + + +def _ids(tag: str, rows: int, *trailing: int) -> torch.Tensor: + """`rows` rows whose every value is `tag`'s offset plus the row's index.""" + ids = torch.arange(rows, dtype=torch.int64) + _FAKE_OFFSETS[tag] * 10**7 + return ids.view(rows, *([1] * len(trailing))).expand(rows, *trailing).contiguous() + + +def _col_ids(tag: str, cols: int) -> torch.Tensor: + return _ids(tag, cols, _HIDDEN).t().contiguous() + + +def _fake_checkpoint(stored) -> dict[str, torch.Tensor]: + """Attention layer 3, Gated-DeltaNet layer 0 and MTP attention at stored rows, 2-wide input.""" + fake = {} + + def rows_of(key): + return stored[key][0][0] + + for p in ("model.language_model.layers.3.self_attn", "mtp.layers.0.self_attn"): + for tag, name in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj")): + fake[f"{p}.{name}.weight"] = _ids(tag, rows_of(f"{p}.{name}.weight"), _HIDDEN) + key = f"{p}.indexer.index_qk_proj.weight" + fake[key] = _ids("idx", rows_of(key), _HIDDEN) + fake[f"{p}.o_proj.weight"] = _col_ids("o", stored[f"{p}.o_proj.weight"][0][1]) + p = "model.language_model.layers.0.linear_attn" + for tag, name in (("qkv", "in_proj_qkv"), ("z", "in_proj_z"), ("a", "in_proj_a"), + ("b", "in_proj_b")): # fmt: skip + fake[f"{p}.{name}.weight"] = _ids(tag, rows_of(f"{p}.{name}.weight"), _HIDDEN) + fake[f"{p}.conv1d.weight"] = _ids("conv", rows_of(f"{p}.conv1d.weight"), 1, 4) + fake[f"{p}.A_log"] = _ids("a_log", rows_of(f"{p}.A_log")) + fake[f"{p}.dt_bias"] = _ids("dtb", rows_of(f"{p}.dt_bias")) + fake[f"{p}.out_proj.weight"] = _col_ids("out", stored[f"{p}.out_proj.weight"][0][1]) + return fake + + +def _assemble(d, fake, sources) -> torch.Tensor: + """A parameter filled from its sources the way `load` fills it.""" + if len(sources) == 1 and sources[0][1] is None: + ckpt_key, _, cut, transform = sources[0] + return WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d) + parts = [ + (index, WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d)) + for ckpt_key, index, cut, transform in sources + ] + rows = max(index[0].stop for index, _ in parts) + out = torch.full((rows, *parts[0][1].shape[1:]), -1, dtype=parts[0][1].dtype) + for index, part in parts: + out[index] = part + return out + + +def _expected_rows(tag: str, starts) -> torch.Tensor: + return torch.tensor([_FAKE_OFFSETS[tag] * 10**7 + r for r in starts], dtype=torch.int64) + + +def _check_heads(stored, d, member: str, rank: int, params: dict) -> None: + """The cuts select the rank's heads in fused-operand order, the heads the stock pools hold.""" + fake = _fake_checkpoint(stored) + manifest = WT.MODEL_WEIGHTS.manifest(d) + tp = _attention_tp(member) + # Under attention DP and at tp1 each rank holds every head: the split is over one rank. + r = rank if tp > 1 else 0 + assert (d.heads_q, d.q_head_start) == (24 // tp, r * 24 // tp) + assert (d.heads_kv, d.kv_head_start) == (max(2 // tp, 1), r * 2 // tp) + assert (d.num_k_heads, d.k_head_start) == (16 // tp, r * 16 // tp) + assert (d.num_v_heads, d.v_head_start) == (48 // tp, r * 48 // tp) + assert params["l0_conv_w"].shape[0] == 2 * d.key_dim + d.value_dim + q_heads, k_heads, v_heads = (range(r * n // tp, (r + 1) * n // tp) for n in (24, 16, 48)) + kv_heads = [r * 2 // tp] if tp > 1 else [0, 1] + + def rows(heads, width, base=0): + return [base + h * width + j for h in heads for j in range(width)] + + q_rows = [h * 512 + j for h in q_heads for j in range(256)] + gate_rows = [h * 512 + 256 + j for h in q_heads for j in range(256)] + # fmt: off + expected = torch.cat(( + _expected_rows("q", q_rows), _expected_rows("k", rows(kv_heads, 256)), + _expected_rows("v", rows(kv_heads, 256)), _expected_rows("q", gate_rows), + _expected_rows("idx", range(640)), torch.zeros(128, dtype=torch.int64), + )) + # fmt: on + attn_in = { + key: _assemble(d, fake, manifest[key]) for key in ("l3_attn_in_w", "l48_mtp_attn_in_w") + } + for key, got in attn_in.items(): + assert torch.equal(got[:, 0], expected) and torch.equal(got[:, 1], expected), key + # The KV head a rank holds is the one the built-in path's `duplicate_kv_weight` gives it. + k_key = "mtp.layers.0.self_attn.k_proj.weight" + builtin_k = duplicate_kv_weight(fake[k_key], 2, tp).chunk(tp)[r] if tp > 1 else fake[k_key] + q_width = len(q_rows) + mtp_k = attn_in["l48_mtp_attn_in_w"][q_width : q_width + builtin_k.shape[0]] + assert torch.equal(mtp_k, builtin_k) + o_cols = _expected_rows("o", rows(q_heads, 256)) + for key in ("l3_o_w", "l48_mtp_o_w"): + assert torch.equal(_assemble(d, fake, manifest[key])[0], o_cols), key + + def qkv(tag): + k, v = rows(k_heads, 128), rows(v_heads, 128, 4096) + return torch.cat([_expected_rows(tag, r) for r in (k, rows(k_heads, 128, 2048), v)]) + + z, a, b = rows(v_heads, 128), v_heads, v_heads + in_proj = torch.cat([qkv("qkv")] + [_expected_rows(t, r) for t, r in zip("zab", (z, a, b))]) + assert torch.equal(_assemble(d, fake, manifest["l0_in_proj_w"])[:, 0], in_proj) + assert torch.equal(_assemble(d, fake, manifest["l0_conv_w"])[:, 0, 0], qkv("conv")) + assert torch.equal(_assemble(d, fake, manifest["l0_A_log"]), _expected_rows("a_log", v_heads)) + assert torch.equal(_assemble(d, fake, manifest["l0_dt_bias"]), _expected_rows("dtb", v_heads)) + assert torch.equal(_assemble(d, fake, manifest["l0_out_w"])[0], + _expected_rows("out", rows(v_heads, 128))) # fmt: skip + + +#: Per-expert operand (shape, dtype); block scales hold one fp32 value per 128x128 block. +_EXPERT_SHAPES = { + "fc1_w": ((1280, 2560), torch.float8_e4m3fn), + "fc1_s": ((10, 20), torch.float32), + "fc2_w": ((2560, 640), torch.float8_e4m3fn), + "fc2_s": ((20, 5), torch.float32), +} + + +def _check_expert_window(d, member: str, rank: int, params: dict) -> None: + """The rank holds `512 / world` uncut experts, trunk and draft layer alike (tp2 and tp4 too), + and `expert_window` hands the runner exactly those parameters.""" + window = _window(member, rank) + assert (d.expert_offset, d.local_experts, d.moe_inter) == (window.start, len(window), 640) + manifest = WT.MODEL_WEIGHTS.manifest(d) + for prefix, module in (("l0_", "model.language_model.layers.0.mlp"), + ("l48_mtp_", "mtp.layers.0.mlp")): # fmt: skip + for name, (shape, dtype) in _EXPERT_SHAPES.items(): + param, rows = params[f"{prefix}{name}"], manifest[f"{prefix}{name}"] + assert (tuple(param.shape), param.dtype) == ((len(window), *shape), dtype), name + assert [index for _, index, _, _ in rows] == [(i,) for i in range(len(rows))] + assert all(cut is None for _, _, cut, _ in rows), (prefix, name) + experts = [s[0] for s in manifest[f"{prefix}fc2_w"]] + assert experts == [f"{module}.experts.{e}.down_proj.weight" for e in window] + win = WT.expert_window(params, d, MTP_LAYER if prefix == "l48_mtp_" else 0) + assert (win.offset, win.count) == (window.start, len(window)) + assert win.fc1_w is params[f"{prefix}fc1_w"] and win.fc2_s is params[f"{prefix}fc2_s"] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("member, rank", [("tp1", 0), ("tp2", 1), ("tp4", 2), ("dep4", 3)]) +def test_rank_shard(stored, configs, member, rank): + """The heads each cut selects, every operand's shape and the expert window. tep is left out: + it differs from tp only in the `Mapping`'s MoE split, which no cut reads.""" + d = _dims(configs, member, rank) + params = _check_operand_shapes(stored, d, member) + _check_heads(stored, d, member, rank, params) + _check_expert_window(d, member, rank, params) + + +def _stub_model(configs, member: str, rank: int, mtp: bool): + """The shell and core that `load` reads, on meta: configs, mapping, parameters, lm_head.""" + text, vision = configs + core = SimpleNamespace( + model_config=SimpleNamespace( + pretrained_config=text, + mapping=_mapping(member, rank), + spec_config=object() if mtp else None, + torch_dtype=torch.bfloat16, + ), + vision_config=vision, + ) + d = WT.MODEL_WEIGHTS.dims(core) + with torch.device("meta"): + core.w = WT.MODEL_WEIGHTS.declare(d) + lm_head = torch.nn.Parameter( + torch.empty(_lm_head_rows(member, d.vocab), d.hidden, dtype=torch.bfloat16), + requires_grad=False, + ) + + def named_parameters(): + yield from ((f"model.w.{k}", v) for k, v in core.w.items()) + yield "lm_head.weight", lm_head + + return SimpleNamespace( + model=core, lm_head=SimpleNamespace(weight=lm_head), named_parameters=named_parameters + ) + + +#: The rank `load` runs at, end to end on meta tensors, for these members. +_LOAD_RANK = {("tp1", True): 0, ("tp2", True): 1, ("tep4", False): 3, ("dep4", True): 0} + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "member, mtp", + [(member, True) for member in MEMBERS] + [("tep4", False)], + ids=[f"{member}-mtp" for member in MEMBERS] + ["tep4-no-mtp"], +) +def test_member_covers_the_checkpoint(stored, configs, id_space, member, mtp, monkeypatch): + """Each tensor is read or predicted unread, never both; each expert's tensors by one rank and + the rest by every rank; the windows tile the 512 experts; and `load` fills every parameter.""" + everything = set(stored) + read_by = collections.Counter() + seen = [] + for rank in range(_world(member)): + d = _dims(configs, member, rank, mtp) + manifest = _rank_manifest(stored, d, member) + claimed = { + key + for sources in manifest.values() + for ckpt_key, _, cut, _ in sources + for key, _ in _keys_and_cuts(ckpt_key, cut) + } + unread = WT.MODEL_WEIGHTS.expected_unconsumed(d, stored.keys()) + assert claimed <= everything, ( + f"rank {rank} reads unknown {sorted(claimed - everything)[:8]}" + ) + assert not claimed & unread, f"rank {rank}: read and unread {sorted(claimed & unread)[:8]}" + assert claimed | unread == everything, ( + f"rank {rank}: neither read nor predicted {sorted(everything - claimed - unread)[:8]}" + ) + read_by.update(claimed) + seen.extend(range(d.expert_offset, d.expert_offset + d.local_experts)) + mtp_keys = set() if mtp else {k for k in everything if k.startswith("mtp.")} + assert set(read_by) == everything - mtp_keys, "tensors no rank reads: " + str( + sorted(everything - mtp_keys - set(read_by))[:8] + ) + wrong = sorted( + k for k, n in read_by.items() if n != (1 if _EXPERT_TENSOR.search(k) else _world(member)) + ) + assert not wrong, f"tensors not read by their expected rank count: {wrong[:8]}" + assert seen == list(range(NUM_EXPERTS)) + if member in ("tp2", "tp4"): + mapping = _mapping(member, 0) + assert (mapping.moe_tp_size, mapping.moe_ep_size) == (mapping.tp_size, 1) + + if (member, mtp) not in _LOAD_RANK: + return + monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None) + monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None) + model = _stub_model(configs, member, _LOAD_RANK[member, mtp], mtp) + WT.MODEL_WEIGHTS.load(model, _header_entries(stored, id_space)) + d = WT.MODEL_WEIGHTS.dims(model.model) + assert tuple(model.model.w["ple_table"].shape) == (128 * _rows_per_shard(stored, d), 160) + assert model.lm_head.weight.shape[0] == _lm_head_rows(member, 248320) + + +def _header_entries(stored, id_space) -> dict: + """Every checkpoint tensor by its header, but the n-gram id space's, which `load` reads.""" + return { + key: id_space[key] if key in id_space else _HeaderEntry(*spec) + for key, spec in stored.items() + } + + +@pytest.mark.cpu_only +def test_load_rejects_a_per_shard_padded_ngram_table(stored, configs, id_space, monkeypatch): + """Shards padded one by one hold more padding than one divisor past the id space the + per-head offsets and sizes address; `load` raises before it declares the table.""" + d = _dims(configs, "tp1", 0) + rows_per_shard = _rows_per_shard(stored, d) + prefix = f"{WT._LM}.layers.{d.ple_layer}.ple.ple_embedding" + offsets = id_space[f"{prefix}.ngram_heads_offsets"] + vocab = id_space[f"{prefix}.ngram_heads_vocab_sizes"].clone() + # The ids of each shard end `pad` rows before it does. + pad = -(-d.ngram_vocab_divisor // d.ple_shards) + vocab[-1] = d.ple_shards * (rows_per_shard - pad) - int(offsets[-1]) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None) + monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None) + model = _stub_model(configs, "tp1", 0, True) + entries = _header_entries(stored, {**id_space, f"{prefix}.ngram_heads_vocab_sizes": vocab}) + with pytest.raises(ValueError, match="Shards padded one by one"): + WT.MODEL_WEIGHTS.load(model, entries) + assert "ple_table" not in model.model.w + + +def _random_bytes(gen, *shape) -> torch.Tensor: + return torch.randint(0, 256, shape, generator=gen, dtype=torch.uint8) + + +def _random_scales(gen, *shape) -> torch.Tensor: + return (0.001 + torch.rand(shape, generator=gen)).to(torch.bfloat16) + + +def _expert_operands(configs, which: str) -> None: + """An expert's operands are the checkpoint's bytes in the runner's layout: FC1 `[up ; gate]` + with its block scales in that order widened exactly to fp32, FC2 as stored, trunk and MTP.""" + d = _dims(configs, "tp4", 1) + window = _window("tp4", 1) + expert = window[0] if which == "first" else window[-1] + local = expert - window.start + gen = torch.Generator().manual_seed(expert * 7 + 1) + manifest = WT.MODEL_WEIGHTS.manifest(d) + for prefix, module in (("l5_", "model.language_model.layers.5.mlp"), + ("l48_mtp_", "mtp.layers.0.mlp")): # fmt: skip + p = f"{module}.experts.{expert}" + fake = { + f"{p}.up_proj.weight": _random_bytes(gen, 640, 2560).view(torch.float8_e4m3fn), + f"{p}.gate_proj.weight": _random_bytes(gen, 640, 2560).view(torch.float8_e4m3fn), + f"{p}.down_proj.weight": _random_bytes(gen, 2560, 640).view(torch.float8_e4m3fn), + f"{p}.up_proj.weight_scale_inv": _random_scales(gen, 5, 20), + f"{p}.gate_proj.weight_scale_inv": _random_scales(gen, 5, 20), + f"{p}.down_proj.weight_scale_inv": _random_scales(gen, 20, 5), + } + + def operand(name): + (row,) = [s for s in manifest[f"{prefix}{name}"] if s[1] == (local,)] + ckpt_key, _, cut, transform = row + return WT.MODEL_WEIGHTS.read(fake, ckpt_key, cut, transform, d) + + def u8(name): + return fake[f"{p}.{name}"].view(torch.uint8) + + fc1_w, fc1_s, fc2_w, fc2_s = (operand(n) for n in ("fc1_w", "fc1_s", "fc2_w", "fc2_s")) + assert fc1_w.dtype == fc2_w.dtype == torch.float8_e4m3fn + assert fc1_s.dtype == fc2_s.dtype == torch.float32 + assert torch.equal(fc1_w.view(torch.uint8), + torch.cat((u8("up_proj.weight"), u8("gate_proj.weight")))) # fmt: skip + assert torch.equal(fc2_w.view(torch.uint8), u8("down_proj.weight")) + up_s, gate_s = (fake[f"{p}.{n}_proj.weight_scale_inv"] for n in ("up", "gate")) + assert torch.equal(fc1_s, torch.cat((up_s, gate_s)).float()) + assert torch.equal(fc2_s, fake[f"{p}.down_proj.weight_scale_inv"].float()) + assert torch.equal(fc1_s.to(torch.bfloat16), torch.cat((up_s, gate_s))) + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("expert-first", marks=pytest.mark.cpu_only), + pytest.param("expert-last", marks=pytest.mark.cpu_only), + ], +) +def test_load_operands(case, request): + """What `load` fills the routed-expert operands from: codes and scales.""" + _expert_operands(request.getfixturevalue("configs"), case.split("-")[1]) + + +# ------------------------------------------------------------ the host n-gram table -- + + +def _host_table_module(): + from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table + + return ple_host_table + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("rows, cols", [(320001536, 160), (1000, 160), (37, 4)]) +def test_host_table_chunks_cover_the_rows(rows, cols): + """Chunks are whole multiples of the alignment, tile the rows in order, and every server size + that divides `CHUNKS` splits them into disjoint contiguous shares.""" + PH = _host_table_module() + align = PH.CHUNK_ALIGN if cols == 160 else cols + per = PH.chunk_rows(rows, cols, align) + assert (per * cols) % align == 0 and per * PH.CHUNKS >= rows + ranges = [PH.chunk_range(k, rows, cols, align) for k in range(PH.CHUNKS)] + assert ranges[0][0] == 0 and ranges[-1][1] == rows + assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:])) + for world in (1, 2, 4, 8): + shares = [PH.owned_rows(r, world, rows, cols, align) for r in range(world)] + assert shares[0][0] == 0 and shares[-1][1] == rows + assert all(a[1] == b[0] for a, b in zip(shares, shares[1:])), shares + + +def _broker_role(path, key, queue, go, done, die_on_go): + """A child process: the broker loaded from its file, without importing tensorrt_llm. An owner + publishes (or exits, `die_on_go`) when the test sets `go` and stays until it sets `done`.""" + import importlib.util + import mmap + import os + + spec = importlib.util.spec_from_file_location("chunk_broker", path) + broker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(broker) + queue.put("ready") + got = broker.acquire(key, 300) + if got[0] == broker.OWNER: + queue.put("owner") + go.wait(300) + if die_on_go: + os._exit(0) + fd = os.memfd_create("t") + os.ftruncate(fd, 4096) + mmap.mmap(fd, 4096)[:5] = b"bytes" + got[1].publish(fd, {"n": 4096}) + done.wait(300) + got[1].close() + else: + queue.put(("attached", bytes(mmap.mmap(got[1], 4096)[:5]), got[2])) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("owner_dies", [False, True], ids=["publish", "owner-exits"]) +def test_chunk_broker_hands_over(owner_dies): + """The first process owns a key; a waiter that is already waiting attaches to what it + publishes (and so does a late one), or owns the key itself when the owner exits before + publishing. No files are left.""" + import multiprocessing + import os + import time + import uuid + + PH = _host_table_module() + path = PH._broker.__file__ + key = uuid.uuid4().hex + ctx = multiprocessing.get_context("spawn") + go, done = ctx.Event(), ctx.Event() + procs = [] + + def start(die_on_go=False): + queue = ctx.Queue() + proc = ctx.Process(target=_broker_role, args=(path, key, queue, go, done, die_on_go)) + proc.start() + procs.append(proc) + assert queue.get(timeout=300) == "ready" + return queue + + try: + assert start(owner_dies).get(timeout=60) == "owner" + waiter = start() + time.sleep(1.0) # the waiter is connected and blocked on the owner + go.set() + if owner_dies: + assert waiter.get(timeout=60) == "owner" + else: + assert waiter.get(timeout=60) == ("attached", b"bytes", {"n": 4096}) + assert start().get(timeout=60) == ("attached", b"bytes", {"n": 4096}) + finally: + go.set() + done.set() + for proc in procs: + proc.join(60) + assert all(proc.exitcode == 0 for proc in procs), [proc.exitcode for proc in procs] + assert not [f for f in os.listdir("/dev/shm") if key in f] + + +@pytest.mark.cpu_only +def test_chunk_broker_refuses_a_foreign_owner(monkeypatch): + """A waiter whose key's name is held by a socket of another uid raises `PermissionError` (an + `OSError`, so `node` falls back) at once instead of waiting for it to publish.""" + import os + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + owner = broker.try_own(key) + assert owner is not None + monkeypatch.setattr(broker, "_peer_uid", lambda conn: os.getuid() + 1) + try: + start = time.monotonic() + with pytest.raises(PermissionError): + broker.acquire(key, 60) + assert time.monotonic() - start < 5 + finally: + owner.close() + + +@pytest.mark.cpu_only +def test_chunk_broker_times_out_on_a_name_nobody_listens_on(): + """A key's name bound by a socket that never listens refuses every connection; the waiter + raises `TimeoutError` at its deadline.""" + import socket + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + squatter = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + squatter.bind(broker.socket_name(key)) + try: + start = time.monotonic() + with pytest.raises(TimeoutError): + broker.acquire(key, 2.0) + assert 2.0 <= time.monotonic() - start < 10 + finally: + squatter.close() + + +@pytest.mark.cpu_only +def test_chunk_broker_frees_the_name_when_accept_fails(monkeypatch): + """An owner whose `accept` fails (EMFILE here) closes its listener: a waiter in its backlog + sees the connection reset and owns the key itself at once, instead of waiting out its + timeout on a name nobody accepts on.""" + import errno + import socket + import threading + import time + import uuid + + broker = _host_table_module()._broker + key = uuid.uuid4().hex + gate = threading.Event() + + def failing_accept(self): + gate.wait(30) + raise OSError(errno.EMFILE, "Too many open files") + + monkeypatch.setattr(socket.socket, "accept", failing_accept) + owner = broker.try_own(key) + got = [] + waiter = threading.Thread(target=lambda: got.append(broker.acquire(key, 60))) + waiter.start() + time.sleep(0.5) # the waiter is connected, in the owner's backlog + start = time.monotonic() + gate.set() + waiter.join(30) + try: + assert got and got[0][0] == broker.OWNER, got + assert time.monotonic() - start < 10 + assert not owner._thread.is_alive() + finally: + owner.close() + for result in got: + result[1].close() + + +@pytest.mark.cpu_only +def test_chunk_broker_reads_a_uid_above_2_31(): + """`SO_PEERCRED` is `struct ucred` (pid_t, uid_t, gid_t): an LDAP-mapped uid of 2**31 or + more reads back unsigned.""" + import struct + + broker = _host_table_module()._broker + uid = 2**31 + 5 + + class _Conn: + def getsockopt(self, level, option, size): + assert size == broker._CRED.size == 12 + return struct.pack("iII", 4242, uid, 2**31 + 7) + + assert broker._peer_uid(_Conn()) == uid + + +def _fake_shards(rows_per_shard: int, shards: int, cols: int, seed: int) -> tuple: + g = torch.Generator().manual_seed(seed) + data = torch.randint(0, 256, (shards * rows_per_shard, cols), generator=g, dtype=torch.uint8) + keys = [f"s{i}" for i in range(shards)] + weights = { + k: data[i * rows_per_shard : (i + 1) * rows_per_shard].view(WT.PLE_TABLE_DTYPE) + for i, k in enumerate(keys) + } + return data, keys, weights + + +def _host_tables(monkeypatch, mode, world, data, keys, weights, rows_per_shard, identity=None): + """`world` ranks' tables in one process: threads, so `node` ranks wait on each other, and a + barrier in place of the server's collective agreement. `weights` is every rank's checkpoint, + or a list of one per rank.""" + import threading + + PH = _host_table_module() + barrier = threading.Barrier(world) + votes = [] + + def agree(ok): + votes.append(ok) + barrier.wait() + return all(votes) + + monkeypatch.setattr(PH, "_all_agree", agree) + rows, cols = data.shape + checkpoints = weights if isinstance(weights, list) else [weights] * world + sources = [PH.shard_rows(w, keys, rows_per_shard, WT._read) for w in checkpoints] + tables = [ + PH.PleHostTable( + mode, + rows, + cols, + WT.PLE_TABLE_DTYPE, + torch.device("cuda"), + rank=r, + world=world, + dp=1, + private=WT._host_resident_empty, + identity=identity, + ) # fmt: skip + for r in range(world) + ] + threads = [threading.Thread(target=t.fill, args=sources[r]) for r, t in enumerate(tables)] + for t in threads: + t.start() + for t in threads: + t.join(300) + return tables + + +def _vmm_host_numa() -> bool: + PH = _host_table_module() + try: + PH._VmmChunks(torch.cuda.current_device(), PH.CHUNK_ALIGN, 1, 0).close() + except (RuntimeError, AttributeError): + return False + return True + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +@pytest.mark.parametrize("mode", ["rank", "server", "node", "node-fallback"]) +def test_host_table_modes(mode, monkeypatch): + """`rank` holds every row, `server` this rank's rows with its window, `node` every row in + chunks one rank creates and the other attaches to; `node` without a table identity runs + `server`. Each holds exactly the checkpoint's bytes, in host memory.""" + import uuid + + PH = _host_table_module() + if mode == "node" and not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=5) + want = {"rank": PH.RANK, "server": PH.SERVER, "node": PH.NODE, "node-fallback": PH.SERVER} + requested = PH.NODE if mode.startswith("node") else mode + identity = uuid.uuid4().hex if mode == "node" else None + tables = _host_tables(monkeypatch, requested, 2, data, keys, weights, rows_per_shard, identity) + _check_tables(tables, data, want[mode]) + if mode == "node": + created = sorted(len(t._segments) for t in tables) + assert created == [PH.CHUNKS // 2, PH.CHUNKS // 2], [t.describe() for t in tables] + for t in tables: + t.close() + + +def _check_tables(tables, data, mode) -> None: + """Every rank runs `mode` and holds exactly `data`'s bytes (its rows under `server`), in + host memory.""" + PH = _host_table_module() + rows, cols = data.shape + expected = data.cuda() + for rank, t in enumerate(tables): + assert t.mode == mode, (rank, t.mode, t.describe()) + assert t.resident_on_host(), t.describe() + if t.mode == PH.SERVER: + lo, hi, base = t.window + assert (lo, hi) == PH.owned_rows(rank, len(tables), rows, cols) + got = t.table[lo - base : hi - base] + assert torch.equal(got.view(torch.uint8), expected[lo:hi]) + assert t.needs_sum + else: + assert t.window is None and not t.needs_sum + assert torch.equal(t.table.view(torch.uint8), expected) + + +def _fallback_warnings(monkeypatch) -> list: + """The warnings `ple_host_table` emits from here on, with none emitted yet this process.""" + PH = _host_table_module() + warnings = [] + monkeypatch.setattr(PH, "_WARNED", set()) + monkeypatch.setattr(PH.logger, "warning", warnings.append) + return warnings + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +def test_host_table_owner_that_cannot_create_a_chunk_hands_it_over(monkeypatch): + """A rank whose chunk allocation fails closes the chunk it just owned: the other rank owns + it at once instead of waiting out the attach timeout, and both run `server`, with one + warning naming the requested mode, the mode that runs and why.""" + import time + import uuid + + PH = _host_table_module() + if not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + monkeypatch.setattr(PH, "_ATTACH_TIMEOUT_S", 60.0) + warnings = _fallback_warnings(monkeypatch) + create, failed = PH._VmmChunks.create, [] + + def create_once(self, k): + if k == 0 and not failed: + failed.append(k) + raise RuntimeError("CUDA driver call failed: out of host memory") + create(self, k) + + monkeypatch.setattr(PH._VmmChunks, "create", create_once) + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=7) + start = time.monotonic() + tables = _host_tables( + monkeypatch, PH.NODE, 2, data, keys, weights, rows_per_shard, uuid.uuid4().hex + ) + elapsed = time.monotonic() - start + try: + assert failed == [0] and "out of host memory" in tables[0]._node_error + assert elapsed < 20, f"the other rank waited {elapsed:.0f} s for the failed chunk" + _check_tables(tables, data, PH.SERVER) + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=node requested, running server" in warnings[0] + finally: + for t in tables: + t.close() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="host memory the GPU maps") +def test_host_table_refuses_a_chunk_that_differs_from_its_checkpoint(monkeypatch): + """Two ranks whose keys agree but whose checkpoints differ at one row of an attached chunk's + sample: the attaching rank raises on it, and both run `server` from their own bytes.""" + import uuid + + PH = _host_table_module() + if not _vmm_host_numa(): + pytest.skip("no CUDA VMM host NUMA allocation on this platform") + warnings = _fallback_warnings(monkeypatch) + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, weights = _fake_shards(rows_per_shard, shards, cols, seed=9) + other = data.clone() + row = PH.sample_blocks(0, *data.shape, phase=1)[3][0] + other[row] ^= 1 + other_weights = { + k: other[i * rows_per_shard : (i + 1) * rows_per_shard].view(WT.PLE_TABLE_DTYPE) + for i, k in enumerate(keys) + } + tables = _host_tables( + monkeypatch, + PH.NODE, + 2, + data, + keys, + [weights, other_weights], + rows_per_shard, + uuid.uuid4().hex, + ) + try: + # Row `row` is rank 0's (chunk 0): each rank's own rows are its checkpoint's. + _check_tables(tables, data, PH.SERVER) + assert tables[1]._node_error == f"shared chunk 0 differs from the checkpoint at row {row}" + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=node requested, running server" in warnings[0] + finally: + for t in tables: + t.close() + + +@pytest.mark.cpu_only +def test_table_identity_is_the_sampled_rows(): + """The `node` key is the table's bytes at every chunk's phase-0 sample and nothing about + where they were read from; phase 1, the rows an attaching rank compares, is disjoint.""" + PH = _host_table_module() + rows_per_shard, shards, cols = 40960, 8, 160 + data, keys, _ = _fake_shards(rows_per_shard, shards, cols, seed=11) + rows = data.shape[0] + + def identity(table) -> str: + parts = { + k: table[i * rows_per_shard : (i + 1) * rows_per_shard] for i, k in enumerate(keys) + } + _, read = PH.shard_rows(parts, keys, rows_per_shard, WT._read) + return PH.table_identity(read, rows, cols, "layer 3") + + def sample(phase: int) -> set: + return { + r + for k in range(PH.CHUNKS) + for lo, hi in PH.sample_blocks(k, rows, cols, phase=phase) + for r in range(lo, hi) + } + + key = identity(data) + assert identity(data.clone()) == key + sampled, checked = sample(0), sample(1) + assert sampled and checked and not sampled & checked + assert all(0 <= r < rows for r in sampled | checked) + changed = data.clone() + changed[max(sampled)] ^= 1 + assert identity(changed) != key + # The key hashes its sample only; the attaching rank's check covers this row. + changed = data.clone() + changed[min(checked)] ^= 1 + assert identity(changed) == key + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "ids, rows_per_shard, error", + [ + (800, 208, None), + (800, 256, "Shards padded one by one"), + (800, 200, "not padded to a multiple"), + (900, 208, "address 900"), + ], + ids=["padded-once", "padded-per-shard", "unpadded", "short"], +) +def test_ngram_table_height_matches_the_id_space(ids, rows_per_shard, error): + """The table's height is the id space its offsets and sizes address, padded to the divisor by + less than one divisor; otherwise the declaration raises.""" + d = SimpleNamespace( + ple_layer=3, ple_shards=4, ple_head_dim=4, ngram_heads=2, ple_embed_dim=8, + ple_table_dtype=WT.PLE_TABLE_DTYPE, ngram_vocab_divisor=64, + ) # fmt: skip + prefix = f"{WT._LM}.layers.3.ple.ple_embedding" + weights = { + key: torch.zeros(rows_per_shard, 4, dtype=torch.uint8).view(WT.PLE_TABLE_DTYPE) + for key in WT.ngram_shard_keys(d) + } + weights[f"{prefix}.ngram_heads_offsets"] = torch.tensor([0, ids // 2]) + weights[f"{prefix}.ngram_heads_vocab_sizes"] = torch.tensor([ids // 2, ids - ids // 2]) + core = SimpleNamespace(w={"embed": torch.empty(0)}) + if error is None: + assert WT._declare_ngram_table(core, d, weights) == rows_per_shard + assert tuple(core.w["ple_table"].shape) == (4 * rows_per_shard, 4) + else: + with pytest.raises(ValueError, match=error): + WT._declare_ngram_table(core, d, weights) + + +@pytest.mark.cpu_only +def test_host_table_warns_once_when_server_runs_as_rank(monkeypatch): + """`server` under attention DP keeps a copy per rank: one warning per process says so.""" + PH = _host_table_module() + warnings = _fallback_warnings(monkeypatch) + + def private(rows, cols, dtype, device): + return torch.empty(rows, cols, dtype=dtype), None + + for _ in range(2): + t = PH.PleHostTable( + PH.SERVER, 64, 4, WT.PLE_TABLE_DTYPE, torch.device("cuda"), + rank=0, world=2, dp=2, private=private, + ) # fmt: skip + assert t.mode == PH.RANK + assert len(warnings) == 1, warnings + assert f"{PH.PLE_HOST_TABLE_ENV}=server requested, running rank" in warnings[0] + assert "attention DP 2" in warnings[0] diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py index 48b65e311d00..d320852b53c3 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/test_routing.py @@ -17,7 +17,7 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modeling_v2._router_index import modeling_v2_resolve -from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import routing +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp import ple_host_table, routing from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.qwen3_8_flash_next_nvfp4__sm_103__default import ( # noqa: E501 modeling as M, ) @@ -74,6 +74,7 @@ def _on_sm103(monkeypatch): monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (10, 3)) monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + monkeypatch.delenv(ple_host_table.PLE_HOST_TABLE_ENV, raising=False) def _llm_args(mode="auto", **kwargs) -> TorchLlmArgs: @@ -535,3 +536,13 @@ def test_mtp_needs_the_cached_replay(monkeypatch, max_draft_len): _claimed(_model_config(**_LAYOUTS["tep4"])) monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY") _claimed(config) + + +def test_a_misspelled_ple_host_table_mode_fails_at_routing(monkeypatch): + """The n-gram table's storage mode is read at routing, so a misspelled value fails there, + under `auto` too, rather than at the build.""" + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "nodes") + with pytest.raises(ValueError, match=f"{ple_host_table.PLE_HOST_TABLE_ENV}='nodes': expected"): + modeling_v2_resolve(_model_config(**_LAYOUTS["tep4"]), _llm_args("auto")) + monkeypatch.setenv(ple_host_table.PLE_HOST_TABLE_ENV, "Node") + _claimed(_model_config(**_LAYOUTS["tep4"]))