diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md b/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md new file mode 100644 index 000000000000..2eef809c8ce4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md @@ -0,0 +1,51 @@ + + + +# Inkling 2 + +`Inkling2ForConditionalGeneration` (`thinkingmachines/Inkling-2-Small`): a 48-layer MoE text decoder +(hidden 4608, 384 routed experts, top 8) with a vision tower. The family implements the text model. + +## What the architecture needs + +* **Attention without rotary embeddings.** Position enters through a relative bias each token mixes + per head from 16 bias-vs-distance profiles (`rel_logits_proj`), over distances below 1024. 40 + sliding-window layers attend the last 1024 tokens; 8 global layers (5, 11, ..., 47) attend the + whole prefix and scale q and the relative states by `tau = 1 + 0.1 ln(max((p + 1) / 8192, 1))`. + Each token and head has a sink logit, and the output is gated by `sigmoid(gate)`. q, k and the + relative states are L2-clipped; v is RMS-normalized without a weight. +* **Short convolutions.** Width-4 causal convolutions with a residual run on k and v (dynamic + per-token taps on the global layers) and on each sublayer's output. Their windows are + per-request state. +* **Block attention residuals.** Every 6 layers the residual stream closes a block; each + sublayer reads a learned softmax mix of the normalized blocks and the current partial block. +* **Experts.** fp32 sigmoid router with a selection bias; SiTU activation + (`4 tanh(g / 4) sigmoid(g) * 25 tanh(u / 25)`). The checkpoint stores layers 0-39 in NVFP4 (with + per-token 4/6 NVFP4 activations) and layers 40-47 in MXFP8. + +## Layout + +```text +inkling2/ +├── config.py # InklingDims: the decoder's shape and per-layer expert formats +├── kernels/ # Triton: short convolution, paged attention with the relative bias +├── ops/ # OpWrappers: conv, attention prologue and core, residual mix, +│ # router, NVFP4 experts (FlashInfer CuTe DSL), MXFP8 experts +└── inkling_2_small_nvfp4__sm_103__tep4/ + ├── weights.py # weight table: TP4 heads, EP4 experts, kernel layouts + ├── modeling.py # the 48-layer forward and the engine shell + └── cache.py # KVCacheManagerV2 with the convolution windows +``` + +## Target `inkling_2_small_nvfp4__sm_103__tep4` + +The released checkpoint on one GB300 tray: attention TP4 (12 query and 2 KV heads per rank), MoE +EP4 (96 experts per rank), BF16 KV cache. The baseline runs eagerly, without CUDA graphs, the +overlap scheduler, chunked prefill, KV cache block reuse or speculative decoding. + +The family is not routed yet: these are the operators and the target's components; the route that +serves the target through `LLM()` and `trtllm-serve` follows separately. + +The NVFP4 experts run FlashInfer's CuTe DSL fused MoE with SiTU and per-token 4/6 activations. The +MXFP8 experts run a grouped Triton MoE that expands the E4M3 weights to bf16 in registers (exactly) +and accumulates in fp32. diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py new file mode 100644 index 000000000000..49fdf3f909aa --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2 (`Inkling2ForConditionalGeneration`): ops, kernels and targets of the family.""" diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py new file mode 100644 index 000000000000..f4f97bc41f58 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The Inkling 2 text decoder's shape, as the family's ops and targets read it. + +`InklingDims` is built once from the checkpoint's `config.json` text config and its +`hf_quant_config.json`, and is the only place a target reads model geometry from. It accepts both +`tensorrt_llm._torch.configs.Inkling2TextConfig` and a native Transformers `Inkling2TextConfig`, +which name a few fields differently (`conv_kernel_size` for `sconv_kernel_size`, +`moe_intermediate_size` for `intermediate_size`) and mark local layers by `layer_types` value +`hybrid_sliding` instead of listing `local_layer_ids`. + +Expert precision is per layer and comes from `hf_quant_config.json`, whose `parameter_groups` name +every expert tensor with the format it is stored in. The released checkpoint stores layers 0-39 +in NVFP4 and layers 40-47 in MXFP8; nothing here assumes that split. +""" + +from __future__ import annotations + +import enum +import json +import os +import re +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +#: The expert weight tensor `hf_quant_config.json` names for each MoE layer. +_EXPERT_W13 = re.compile(r"^model\.llm\.layers\.(\d+)\.mlp\.experts\.w13_weight$") + + +class ExpertFormat(str, enum.Enum): + """How a layer's routed experts are stored and how their activations are quantized.""" + + #: E2M1 values in packed uint8, an FP8 E4M3 scale per 16 values along K, and a per-expert FP32 + #: global scale `amax / outer_scale_denominator`. Activations are quantized per token with the + #: 4/6 block-scale rule. + NVFP4 = "nvfp4" + #: FP8 E4M3 values and a UE8M0 scale per 32 values along K; activations are MXFP8 too. + MXFP8 = "mxfp8" + #: BF16 experts, as in a dequantized or randomly initialized test checkpoint. + BF16 = "bfloat16" + + +def _first(config: Any, *names: str, default: Any = None) -> Any: + for name in names: + value = getattr(config, name, None) + if value is not None: + return value + return default + + +def expert_formats_from_quant_config( + hf_quant_config: Optional[Mapping[str, Any]], num_layers: int +) -> tuple[ExpertFormat, ...]: + """Each layer's expert format, from an Inkling `hf_quant_config.json` payload. + + A layer that no parameter group names keeps BF16 experts; a layer that two groups name is a + malformed file. + """ + formats: dict[int, ExpertFormat] = {} + groups = ((hf_quant_config or {}).get("model") or {}).get("parameter_groups") or [] + for group in groups: + compute = str(group.get("compute_dtype", "")).lower() + fmt = {"nvfp4": ExpertFormat.NVFP4, "mxfp8": ExpertFormat.MXFP8}.get(compute) + if fmt is None: + continue + for tensor in group.get("tensors", []): + match = _EXPERT_W13.match(tensor) + if match is None: + continue + layer = int(match.group(1)) + if layer in formats and formats[layer] is not fmt: + raise ValueError(f"hf_quant_config.json gives layer {layer} two expert formats") + formats[layer] = fmt + out_of_range = sorted(i for i in formats if not 0 <= i < num_layers) + if out_of_range: + raise ValueError( + f"hf_quant_config.json names experts of layers {out_of_range}, " + f"outside the {num_layers} decoder layers" + ) + return tuple(formats.get(i, ExpertFormat.BF16) for i in range(num_layers)) + + +def read_hf_quant_config(checkpoint_dir: Optional[str]) -> Optional[dict]: + """The checkpoint's `hf_quant_config.json`, or None when it has none.""" + if not checkpoint_dir: + return None + path = os.path.join(checkpoint_dir, "hf_quant_config.json") + if not os.path.isfile(path): + return None + with open(path) as f: + return json.load(f) + + +@dataclass(frozen=True) +class InklingDims: + """Geometry of the Inkling 2 text decoder, whole-model (not per rank).""" + + hidden_size: int + num_layers: int + vocab_size: int + unpadded_vocab_size: int + num_heads: int + num_kv_heads: int + head_dim: int + d_rel: int + rel_extent: int + sliding_window: int + global_layers: tuple[int, ...] + conv_kernel_size: int + attn_res_block_size: int + num_experts: int + top_k: int + expert_intermediate_size: int + route_scale: float + gate_softcap: float + up_softcap: float + logit_softcap: float + qk_clip_threshold: float + rel_clip_threshold: float + log_scaling_alpha: float + log_scaling_n_floor: float + rms_norm_eps: float + expert_formats: tuple[ExpertFormat, ...] + + @classmethod + def from_config( + cls, text_config: Any, hf_quant_config: Optional[Mapping[str, Any]] = None + ) -> "InklingDims": + """Read the text config (either config class) and the quant config payload.""" + num_layers = int(text_config.num_hidden_layers) + local = getattr(text_config, "local_layer_ids", None) + if local is not None: + local = set(int(i) for i in local) + global_layers = tuple(i for i in range(num_layers) if i not in local) + else: + layer_types = list(text_config.layer_types) + global_layers = tuple( + i for i, kind in enumerate(layer_types) if "sliding" not in str(kind).lower() + ) + swa = ( + _first(text_config, "swa_num_attention_heads"), + _first(text_config, "swa_num_key_value_heads"), + _first(text_config, "swa_head_dim"), + ) + full = ( + text_config.num_attention_heads, + text_config.num_key_value_heads, + text_config.head_dim, + ) + if any(s is not None and s != f for s, f in zip(swa, full)): + raise ValueError( + f"sliding-window layers with a different attention shape {swa} from " + f"the global layers {full} are not supported" + ) + return cls( + hidden_size=int(text_config.hidden_size), + num_layers=num_layers, + vocab_size=int(text_config.vocab_size), + unpadded_vocab_size=int( + _first(text_config, "unpadded_vocab_size", default=text_config.vocab_size) + ), + num_heads=int(text_config.num_attention_heads), + num_kv_heads=int(text_config.num_key_value_heads), + head_dim=int(text_config.head_dim), + d_rel=int(text_config.d_rel), + rel_extent=int(text_config.rel_extent), + sliding_window=int(_first(text_config, "sliding_window_size", "sliding_window")), + global_layers=global_layers, + conv_kernel_size=int(_first(text_config, "sconv_kernel_size", "conv_kernel_size")), + attn_res_block_size=int(text_config.n_layers_per_attn_res_block), + num_experts=int(text_config.n_routed_experts), + top_k=int(text_config.num_experts_per_tok), + expert_intermediate_size=int( + _first(text_config, "moe_intermediate_size", "intermediate_size") + ), + route_scale=float(_first(text_config, "route_scale", "routed_scaling_factor")), + gate_softcap=float(text_config.mlp_gate_softcap), + up_softcap=float(text_config.mlp_up_softcap), + logit_softcap=float(text_config.final_logit_softcapping), + qk_clip_threshold=float(text_config.qk_l2_clip_threshold), + rel_clip_threshold=float(text_config.rel_l2_clip_threshold), + log_scaling_alpha=float(text_config.log_scaling_alpha), + log_scaling_n_floor=float(text_config.log_scaling_n_floor), + rms_norm_eps=float(text_config.rms_norm_eps), + expert_formats=expert_formats_from_quant_config(hf_quant_config, num_layers), + ) + + def is_global(self, layer: int) -> bool: + """Whether `layer` attends over the whole context rather than the sliding window.""" + return layer in self.global_layers + + def rel_extent_of(self, layer: int) -> int: + """Relative distances the layer's position bias covers: the window in a sliding layer.""" + return self.rel_extent if self.is_global(layer) else self.sliding_window + + def is_block_boundary(self, layer: int) -> bool: + """Whether `layer` starts a new attention-residual block.""" + return layer % self.attn_res_block_size == 0 + + @property + def num_block_states(self) -> int: + """Completed blocks the final output mix reads: the embedding plus one per later boundary.""" + return len([i for i in range(self.num_layers) if self.is_block_boundary(i)]) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py new file mode 100644 index 000000000000..5e384e7e3b11 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""`KVCacheManagerV2` with Inkling 2's short-convolution windows. + +Every decoder layer convolves four streams over the last 4 tokens: its keys and values (this +rank's KV heads) and its attention and MoE outputs (the full hidden width). A request's step +needs the 3 inputs of each stream before its first new token, so the manager keeps them, per +layer and request, in a slot of four tensors `[slots, 3, C]` (bf16): `k`, `v`, `attn`, `mlp`. + +A request takes a slot the first time a step names it and keeps it until the engine frees its +KV cache. The slot pool holds `max_batch_size` requests plus the engine's padding and warmup +requests. The windows are not part of the KV cache's prefix reuse, so block reuse must be off. +""" + +from __future__ import annotations + +from typing import Iterable + +import torch + +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 + +from ..config import InklingDims + +#: Slots beyond the resident requests: CUDA-graph padding and warmup requests. +SPARE_SLOTS = 8 +#: Inputs a width-4 window keeps. +WINDOW = 3 + + +class ConvSlots: + """Which short-convolution slot each live request owns. + + A request keeps its slot, and so its windows, from its first step until it is released, also + while it is paused; a released slot goes to the next new request. + """ + + def __init__(self, num_slots: int): + self.num_slots = num_slots + self._slot_of: dict[int, int] = {} + self._free = list(range(num_slots - 1, -1, -1)) + + def acquire(self, request_ids: Iterable[int]) -> list[int]: + """Each request's slot; a request seen for the first time gets a free one.""" + slots = [] + for request_id in request_ids: + slot = self._slot_of.get(request_id) + if slot is None: + if not self._free: + raise RuntimeError( + f"all {self.num_slots} short-convolution slots are in use; " + "more requests are live than the KV cache manager admits" + ) + slot = self._free.pop() + self._slot_of[request_id] = slot + slots.append(slot) + return slots + + def release(self, request_id: int) -> None: + """Free the request's slot; a request that never got one is ignored.""" + slot = self._slot_of.pop(request_id, None) + if slot is not None: + self._free.append(slot) + + +class Inkling2KVCacheManager(KVCacheManagerV2): + """The KV cache plus each request's short-convolution windows, for every rank-local layer. + + Block reuse is refused: a reused prefix would come without the convolution windows of the + tokens it covers. + """ + + def __init__( + self, *args, pretrained_config=None, mapping=None, max_batch_size: int = 1, **kwargs + ): + super().__init__( + *args, + pretrained_config=pretrained_config, + mapping=mapping, + max_batch_size=max_batch_size, + **kwargs, + ) + if self.enable_block_reuse: + raise ValueError( + "Inkling2KVCacheManager does not support KV cache block reuse: a reused prefix " + "comes without its short-convolution windows. Set " + "kv_cache_config.enable_block_reuse to false." + ) + text = getattr(pretrained_config, "text_config", None) or pretrained_config + dims = InklingDims.from_config(text) + tp = mapping.tp_size if mapping is not None else 1 + kv_width = dims.num_kv_heads * dims.head_dim // tp + # Every sequence the manager admits holds its windows: a retiring request keeps its slot + # one more iteration under the overlap scheduler, and a request paused by GEN preemption + # keeps its slot, so the base manager's own count sizes the pool. + self.num_conv_slots = self.max_admissible_sequences + SPARE_SLOTS + widths = {"k": kv_width, "v": kv_width, "attn": dims.hidden_size, "mlp": dims.hidden_size} + device = torch.device("cuda", torch.cuda.current_device()) + self._conv_state = [ + { + name: torch.zeros( + self.num_conv_slots, WINDOW, width, dtype=torch.bfloat16, device=device + ) + for name, width in widths.items() + } + for _ in range(dims.num_layers) + ] + self._conv_slots = ConvSlots(self.num_conv_slots) + + def conv_state(self, layer: int) -> dict: + """Layer `layer`'s windows: `k`, `v`, `attn`, `mlp`, each `[slots, 3, C]` bf16.""" + return self._conv_state[layer] + + def conv_slots(self, request_ids: Iterable[int]) -> torch.Tensor: + """Each request's slot, as int32 on the device; a request seen for the first time gets one.""" + return torch.tensor(self._conv_slots.acquire(request_ids), dtype=torch.int32).to( + self._conv_state[0]["k"].device, non_blocking=True + ) + + def free_resources(self, request, pin_on_release: bool = False): + self._conv_slots.release(request.py_request_id) + return super().free_resources(request, pin_on_release) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py new file mode 100644 index 000000000000..f71f660f60c6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py @@ -0,0 +1,441 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling-2-Small, NVFP4/MXFP8 experts / sm_103 / attention TP4 + MoE EP4: token ids to logits. + +One engine step through the 48 decoder layers of the text model. Per layer, with `P` the +partial residual block (bf16 between layers) and `blocks` the normalized completed blocks: + + attention inputs mix(blocks, P) through the input norm(s): one input on a sliding layer, + q, k and v inputs on a global layer (layer 0 reads the embedding block) + projections q, r (16 per head), gate, sink, k and v; a global layer's k and v also + give their convolution's per-token taps + k, v short convolution (dynamic on a global layer), then the prologue: q and r + scaled and clipped (times tau on a global layer), k clipped, v normalized + attention paged, relative bias from r, sink and output gate (window 1024 on a + sliding layer); output projection; all-reduce over TP; short convolution + residual a block boundary (layer % 6 == 0) closes P into blocks and restarts the + partial block from the attention output; otherwise P' = P + a + MoE mix(blocks, P') through the MLP norm; fp32 sigmoid router, top-8 of 384; + this rank's 96 experts (NVFP4 or MXFP8); all-reduce over EP; short conv + output P'' = P' + m, rounded to bf16 + +then the final mix over the 8 blocks and the last partial block through the final norm. The +shell's vocab-parallel lm_head GEMM writes bf16 logits, which the shell upcasts to fp32 (as the +Hugging Face implementation does); they are soft-capped in fp32 (30 tanh(x / 30)) and cut to +the tokenizer's 200058 entries (the padded rows are set to -inf). + +The KV cache is the engine's `KVCacheManagerV2` (BF16, 2 KV heads of 128 per rank, a 1024-token +window on the 40 sliding layers) extended with each request's short-convolution windows +(`Inkling2KVCacheManager`, `cache.py`). The forward runs eagerly: no CUDA graphs, no chunked +prefill, no KV cache reuse, no speculative decoding; routing admits only that. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2.catalog._op import true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import allreduce, allreduce_group_args +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM +from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks +from tensorrt_llm.functional import AllReduceFusionOp +from tensorrt_llm.logger import logger + +from ..config import ExpertFormat, InklingDims, read_hf_quant_config +from ..kernels.paged_attention import block_tokens, tile_plan +from ..ops.attention.inkling2_attention_prologue import inkling2_attention_prologue, log_scaling +from ..ops.attention.inkling2_paged_attention import SM_SCALE, inkling2_paged_attention +from ..ops.conv.inkling2_short_conv import inkling2_dynamic_short_conv, inkling2_short_conv +from ..ops.moe.inkling2_mxfp8_experts import inkling2_mxfp8_experts +from ..ops.moe.inkling2_nvfp4_experts import fc2_input_scale, inkling2_nvfp4_experts +from ..ops.moe.inkling2_router import inkling2_router +from ..ops.residual.inkling2_residual_mix import ( + inkling2_residual_mix, + normalize_block, + rmsnorm_to_bf16, +) +from . import weights as _weights +from .cache import Inkling2KVCacheManager + +_NO_FUSION = int(AllReduceFusionOp.NONE) + + +def text_config_of(pretrained_config: PretrainedConfig) -> PretrainedConfig: + return getattr(pretrained_config, "text_config", None) or pretrained_config + + +@dataclass +class _Step: + """Per-step views of the attention metadata, shared by every layer.""" + + num_tokens: int + query_start: torch.Tensor # [R + 1] int32 + kv_len: torch.Tensor # [R] int32, cached + new + token_request: torch.Tensor # [T] int32 + has_state: torch.Tensor # [R] int32: the request has convolution history + conv_slots: torch.Tensor # [R] int32 + tile_request: torch.Tensor + tile_start: torch.Tensor + block_t: int + page_tables: dict # layer -> [R, W] int32 + tau: torch.Tensor # [T] fp32 + + +class Inkling2SmallCore(DecoderModel): + """The 48-layer text decoder on one rank of attention TP4 + MoE EP4.""" + + def __init__(self, model_config: ModelConfig): + super().__init__(model_config) + text = text_config_of(model_config.pretrained_config) + checkpoint_dir = getattr(model_config.pretrained_config, "_name_or_path", None) + hf_quant_config = read_hf_quant_config(checkpoint_dir) + if hf_quant_config is None: + # Without it every layer would read as BF16 experts, which this target does not serve. + raise ValueError( + f"Inkling2: found no hf_quant_config.json in {checkpoint_dir!r}, the directory the " + "config was loaded from (`_name_or_path`); this target reads each layer's expert " + "format from that file, so load the config from the checkpoint directory" + ) + self.dims = InklingDims.from_config(text, hf_quant_config) + self.shares = _weights.shares_of(self.dims, model_config.mapping) + self.eps = self.dims.rms_norm_eps + self.w = _weights.declare(self.shares) + self._layers: Optional[list] = None + self._allreduce: Optional[dict] = None + self._fc2_scale: Optional[torch.Tensor] = None + self._shared_tables: Optional[bool] = None + + def post_load_weights(self) -> None: + """Per-layer weight views and the TP communicator, once the weights are real.""" + self._layers = [_weights.layer_view(self.w, layer) for layer in range(self.dims.num_layers)] + self._allreduce = allreduce_group_args(self.model_config.mapping) + self._fc2_scale = fc2_input_scale(self.w["embed"].device) + + # ------------------------------------------------------------------------------- step plan + + def _plan(self, md: AttentionMetadata, position_ids: torch.Tensor, num_tokens: int) -> _Step: + manager: Inkling2KVCacheManager = md.kv_cache_manager + num_seqs = md.num_seqs + lengths = [int(n) for n in md.seq_lens[:num_seqs].tolist()] + cached = [int(c) for c in md.kv_cache_params.num_cached_tokens_per_seq[:num_seqs]] + request_ids = list(md.request_ids[:num_seqs]) + assert sum(lengths) == num_tokens, (lengths, num_tokens) + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + device = position_ids.device + + def i32(values): + return torch.tensor(values, dtype=torch.int32).to(device, non_blocking=True) + + block_t = block_tokens(max(lengths)) + tile_request, tile_start = tile_plan(lengths, block_t, device) + kv_lens = [c + n for c, n in zip(cached, lengths)] + blocks = [-(-length // manager.tokens_per_block) for length in kv_lens] + page_tables = self._page_tables(manager, request_ids, blocks, cached, device) + return _Step( + num_tokens=num_tokens, + query_start=i32(bounds), + kv_len=i32(kv_lens), + token_request=i32([r for r, n in enumerate(lengths) for _ in range(n)]), + has_state=i32([int(c > 0) for c in cached]), + conv_slots=manager.conv_slots(request_ids), + tile_request=tile_request, + tile_start=tile_start, + block_t=block_t, + page_tables=page_tables, + tau=log_scaling( + position_ids.reshape(-1), self.dims.log_scaling_alpha, self.dims.log_scaling_n_floor + ), + ) + + def _page_tables(self, manager, request_ids, blocks, cached, device) -> dict: + """Each layer's page table `[R, W]` int32 (-1 past a request's pages). + + A layer's indices address its own `get_buffers` view. The manager gives the layers of one + window the same indices; the first step checks that once, after which one table per window + is built and shared. Every table is checked to have a live page for each block a request + reads this step (see `_check_live_pages`). + """ + + def build(layer): + rows = manager.get_batch_cache_indices( + request_ids, layer_idx=layer, num_blocks_per_seq=blocks + ) + # As wide as the longest read, so a row the manager returns short shows as -1. + table = torch.full((len(rows), max(blocks)), -1, dtype=torch.int32) + for i, r in enumerate(rows): + table[i, : len(r)] = torch.tensor(r, dtype=torch.int32) + return table + + layers = range(self.dims.num_layers) + window = {layer: self.dims.is_global(layer) for layer in layers} + if self._shared_tables is None: + tables = {layer: build(layer) for layer in layers} + first = {} + for layer in layers: + first.setdefault(window[layer], layer) + self._shared_tables = all( + torch.equal(tables[layer], tables[first[window[layer]]]) for layer in layers + ) + if not self._shared_tables: + logger.warning( + "Inkling2: layers of one window have different page tables; building one per layer" + ) + elif self._shared_tables: + per_window = {} + for layer in layers: + if window[layer] not in per_window: + per_window[window[layer]] = build(layer) + tables = {layer: per_window[window[layer]] for layer in layers} + else: + tables = {layer: build(layer) for layer in layers} + # One check and one upload per distinct table: the layers of a window share theirs. + tokens_per_block = manager.tokens_per_block + read_end = torch.tensor(blocks) + sliding_start = torch.tensor( + [max(0, c - self.dims.sliding_window + 1) // tokens_per_block for c in cached] + ) + uploaded = {} + for layer, t in tables.items(): + if id(t) not in uploaded: + start = torch.zeros_like(read_end) if window[layer] else sliding_start + self._check_live_pages(t, start, read_end, layer, request_ids) + uploaded[id(t)] = t.to(device, non_blocking=True) + return {layer: uploaded[id(t)] for layer, t in tables.items()} + + @staticmethod + def _check_live_pages(table, start, end, layer, request_ids) -> None: + """Fail if a block a request reads this step has no page (-1) in `table`. + + Request `r` reads blocks `[start[r], end[r])`: from the window of its first new token on a + sliding layer (from block 0 on a global one) to its last token. `KVCacheManagerV2` keeps + a sliding layer's pages from `history + 1 - window` on, which covers that range, so a hole + there is a broken retention contract, reported here instead of read by the kernels. + """ + blocks = torch.arange(table.shape[1]) + read = (blocks[None, :] >= start[:, None]) & (blocks[None, :] < end[:, None]) + holes = (table < 0) & read + if bool(holes.any()): + row, block = (int(i) for i in holes.nonzero()[0]) + raise RuntimeError( + f"Inkling2: layer {layer}: request {request_ids[row]} reads KV cache block " + f"{block}, which has no page; the KV cache manager did not keep the window " + "the attention kernels read" + ) + + # ------------------------------------------------------------------------------- forward + + def _all_reduce(self, x: torch.Tensor) -> torch.Tensor: + if self.model_config.mapping.tp_size == 1: + return x + return allreduce( + x, + None, + None, + None, + None, + op=_NO_FUSION, + eps=self.eps, + trigger_completion_at_end=True, + **self._allreduce, + ) + + def _conv(self, x, taps, state, step: _Step): + return inkling2_short_conv( + x, taps, state, step.conv_slots, step.query_start, step.has_state, step.token_request + ) + + def _attention( + self, layer: int, inputs: torch.Tensor, step: _Step, md: AttentionMetadata + ) -> torch.Tensor: + d, s, w = self.dims, self.shares, self._layers[layer] + manager: Inkling2KVCacheManager = md.kv_cache_manager + state = manager.conv_state(layer) + t = step.num_tokens + if d.is_global(layer): + yq = cublas_mm(inputs[0], w.w_q.t()) + q, r, gate, sink = yq.split([s.q_width, s.r_width, s.q_width, s.heads], dim=1) + yk = cublas_mm(inputs[1], w.w_k.t()) + yv = cublas_mm(inputs[2], w.w_v.t()) + k_lin, k_taps = yk.split([s.kv_width, s.kv_heads * 4], dim=1) + v_lin, v_taps = yv.split([s.kv_width, s.kv_heads * 4], dim=1) + conv_args = ( + step.conv_slots, + step.query_start, + step.has_state, + step.token_request, + d.head_dim, + ) + k = inkling2_dynamic_short_conv( + k_lin.contiguous(), w.k_conv, k_taps.contiguous(), state["k"], *conv_args + ) + v = inkling2_dynamic_short_conv( + v_lin.contiguous(), w.v_conv, v_taps.contiguous(), state["v"], *conv_args + ) + tau, window = step.tau, 0 + else: + y = cublas_mm(inputs[0], w.w_in.t()) + q, r, gate, sink, k_lin, v_lin = y.split( + [s.q_width, s.r_width, s.q_width, s.heads, s.kv_width, s.kv_width], dim=1 + ) + k = self._conv(k_lin.contiguous(), w.k_conv, state["k"], step) + v = self._conv(v_lin.contiguous(), w.v_conv, state["v"], step) + tau, window = None, d.sliding_window + q, k, v, r = inkling2_attention_prologue( + q.reshape(t, s.heads, d.head_dim), + k.view(t, s.kv_heads, d.head_dim), + v.view(t, s.kv_heads, d.head_dim), + r.reshape(t, s.heads, d.d_rel), + w.q_scale, + w.r_scale, + tau, + d.qk_clip_threshold, + d.rel_clip_threshold, + d.rms_norm_eps, + ) + # The bias profiles mix 16 values per head; TF32 would round them, so a true fp32 product. + with true_fp32_matmul(): + rel_logits = torch.matmul(r.float(), w.rel_proj.float()) # [T, heads, rel_extent] + o = inkling2_paged_attention( + q, + k, + v, + manager.get_buffers(layer, kv_layout="HND"), + step.page_tables[layer], + step.query_start, + step.kv_len, + step.token_request, + step.tile_request, + step.tile_start, + step.block_t, + rel_logits, + sink.contiguous(), + gate.reshape(t, s.heads, d.head_dim), + window, + SM_SCALE, + ) + a = self._all_reduce(cublas_mm(o.view(t, s.q_width), w.wo.t())) + return self._conv(a, w.attn_conv, state["attn"], step) + + def _moe(self, layer: int, x: torch.Tensor, step: _Step, md: AttentionMetadata) -> torch.Tensor: + d, s, w = self.dims, self.shares, self._layers[layer] + ids, weights = inkling2_router(x, w.router, w.router_bias, d.top_k, d.route_scale) + if d.expert_formats[layer] is ExpertFormat.NVFP4: + m = inkling2_nvfp4_experts( + x, ids, weights, w.w13, w.w13_sf, w.w13_alpha, w.w2, w.w2_sf, w.w2_alpha, self._fc2_scale, + d.num_experts, s.expert_offset, d.gate_softcap, d.up_softcap, + ) # fmt: skip + else: + m = inkling2_mxfp8_experts( + x, + ids, + weights, + w.w13, + w.w13_s, + w.w2, + w.w2_s, + s.expert_offset, + d.gate_softcap, + d.up_softcap, + ) + m = self._all_reduce(m) + return self._conv(m, w.mlp_conv, md.kv_cache_manager.conv_state(layer)["mlp"], step) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + assert inputs_embeds is None, "the text-only target embeds token ids itself" + d, w = self.dims, self.w + ids = input_ids.reshape(-1) + num_tokens = ids.shape[0] + step = self._plan(attn_metadata, position_ids, num_tokens) + + partial = rmsnorm_to_bf16( + torch.nn.functional.embedding(ids, w.embed), w.embed_norm, self.eps + ) + blocks = torch.empty( + d.num_block_states, num_tokens, d.hidden_size, dtype=torch.float32, device=ids.device + ) + num_blocks = 0 + for layer in range(d.num_layers): + lw = self._layers[layer] + norms = (lw.q_norm, lw.k_norm, lw.v_norm) if d.is_global(layer) else (lw.attn_norm,) + if layer == 0: + inputs = [rmsnorm_to_bf16(partial, n, self.eps) for n in norms] + else: + inputs = inkling2_residual_mix( + blocks[:num_blocks], partial, lw.attn_query, norms, self.eps + ) + a = self._attention(layer, inputs, step, attn_metadata) + if d.is_block_boundary(layer): + blocks[num_blocks] = normalize_block(partial) + num_blocks += 1 + residual = a.float() + else: + residual = partial.float() + a.float() + x = inkling2_residual_mix( + blocks[:num_blocks], residual, lw.mlp_query, (lw.mlp_norm,), self.eps + )[0] + m = self._moe(layer, x, step, attn_metadata) + partial = (residual + m.float()).to(torch.bfloat16) + assert num_blocks == d.num_block_states + return inkling2_residual_mix(blocks, partial, w.final_query, (w.final_norm,), self.eps)[0] + + +class ModelingV2Inkling2SmallNvfp4Sm103Tep4( + DecoderModelForCausalLM[Inkling2SmallCore, PretrainedConfig] +): + """Inkling-2-Small on GB300 with attention TP4 and MoE EP4: the engine-facing shell.""" + + def __init__(self, model_config: ModelConfig): + cfg = model_config.pretrained_config + text = text_config_of(cfg) + # lm_head is built in the config's dtype, which `ModelConfig.from_pretrained` takes from + # the text config when config.json declares none at the top level. + if getattr(cfg, "torch_dtype", None) is None: + raise ValueError( + "Inkling2: the config declares no dtype (`torch_dtype`, or `dtype` in its text " + "config) to build lm_head in" + ) + super().__init__( + Inkling2SmallCore(model_config), + config=model_config, + hidden_size=text.hidden_size, + vocab_size=text.vocab_size, + ) + dims = self.model.dims + self.logit_softcap = dims.logit_softcap + self.unpadded_vocab_size = dims.unpadded_vocab_size + + #: The engine builds the KV cache manager with the short-convolution windows. + runtime_hooks = RuntimeHooks(kv_cache_manager_cls=Inkling2KVCacheManager) + + def load_weights(self, weights, *args, **kwargs): + _weights.load(self, weights, self.model.shares) + + def post_load_weights(self): + super().post_load_weights() + self.model.post_load_weights() + + def forward(self, attn_metadata: AttentionMetadata, *args, **kwargs) -> torch.Tensor: + logits = super().forward(attn_metadata, *args, **kwargs) + logits = torch.tanh(logits / self.logit_softcap) * self.logit_softcap + if logits.shape[-1] > self.unpadded_vocab_size: + logits[..., self.unpadded_vocab_size :] = float("-inf") + return logits + + +__all__ = ["Inkling2SmallCore", "ModelingV2Inkling2SmallNvfp4Sm103Tep4"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py new file mode 100644 index 000000000000..b9194d7d7b32 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py @@ -0,0 +1,430 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Weight table: Inkling-2-Small / sm_103 / attention TP4 + MoE EP4. + +Every weight the target holds, at this rank's share, with the checkpoint tensors it comes from +and the transform on the way. `declare` allocates them (the engine materializes the storage), +`load` fills them from the engine's lazy checkpoint view. + +* **Attention heads** are split over TP: a rank holds 12 of the 48 query heads and 2 of the 8 KV + heads, the matching rows of the q, k, v, relative-state (16 per head), gate and sink + projections, of the q and relative clip scales and of the k/v short convolutions (their + channels are the KV heads' 128-wide slices, and a global layer's dynamic taps are 4 per KV + head), and the matching input columns of the output projection, whose partial sums meet in an + all-reduce. A sliding layer's six projections read one input and are stacked into one weight; + a global layer reads three inputs (q, k and v each have their own residual mix and norm), so it + stacks q, r, gate and sink, and appends each dynamic convolution's tap projection to its k or v + projection. + +* **Experts** are split over EP: a rank holds the 96 experts `[96 * rank, 96 * rank + 96)`, and + reads only those rows of the checkpoint. NVFP4 layers (0-39 in the release) are stored for + FlashInfer's CuTe DSL MoE: `w13` rows reordered from the checkpoint's per-row gate/up + interleave (gate `j` at row `2j`, up `j` at `2j + 1`) into the kernel's order, block scales + reordered the same way and swizzled into 128x4 atoms, and the per-expert global scale + `amax / outer_scale_denominator` as the GEMM alpha. MXFP8 layers keep the checkpoint's bytes. + +* **Short convolutions** are stored as lag-major fp32 taps `[4, C]` (`taps[0]` weights the + current token): a checkpoint `nn.Conv1d` weight `[C, 1, 4]` is reversed along its last axis; a + dynamic convolution's `static_w` is lag-major already. + +* **Everything else** is replicated: the embedding (whole, so no all-reduce follows the lookup), + the norms, the block-residual queries (fp32), the relative-logit profiles and the router (bf16 + weight, fp32 logits; fp32 bias). `lm_head` is the shell's vocab-parallel module; `load` fills + its rows of `unembed`. + +Only the text decoder's tensors (`model.llm.*`) are loaded: the target is text only and drafts nothing. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Callable, Optional + +import torch +import torch.nn as nn + +from ..config import ExpertFormat, InklingDims +from ..ops.moe.inkling2_nvfp4_experts import fc1_row_order, swizzle_block_scales + +#: Prefix of the text decoder's tensors in the checkpoint. The target loads every one of them and +#: nothing else: not the vision tower (`model.visual.*`) nor the DSpark drafter (`mtp.safetensors`). +LLM = "model.llm" + + +@dataclass(frozen=True) +class Shares: + """This rank's share of every split dimension, from the config and the engine's Mapping.""" + + model: InklingDims + tp_size: int + tp_rank: int + ep_size: int + ep_rank: int + + @property + def heads(self) -> int: + return self.model.num_heads // self.tp_size + + @property + def kv_heads(self) -> int: + return self.model.num_kv_heads // self.tp_size + + @property + def q_width(self) -> int: + return self.heads * self.model.head_dim + + @property + def kv_width(self) -> int: + return self.kv_heads * self.model.head_dim + + @property + def r_width(self) -> int: + return self.heads * self.model.d_rel + + @property + def local_experts(self) -> int: + return self.model.num_experts // self.ep_size + + @property + def expert_offset(self) -> int: + return self.ep_rank * self.local_experts + + def rows(self, width_total: int) -> slice: + """This rank's rows of a projection whose output is split evenly over TP.""" + width = width_total // self.tp_size + return slice(self.tp_rank * width, (self.tp_rank + 1) * width) + + def experts(self) -> slice: + return slice(self.expert_offset, self.expert_offset + self.local_experts) + + +def shares_of(model: InklingDims, mapping) -> Shares: + assert mapping.pp_size == 1 and mapping.cp_size == 1, "no pipeline or context parallelism" + assert not mapping.enable_attention_dp, "attention DP is not implemented" + assert mapping.moe_tp_size == 1, "the experts are split by EP only" + assert mapping.moe_ep_size == mapping.tp_size, "EP must span the TP group" + s = Shares(model, mapping.tp_size, mapping.tp_rank, mapping.moe_ep_size, mapping.moe_ep_rank) + assert model.num_heads % s.tp_size == 0 and model.num_kv_heads % s.tp_size == 0 + assert model.num_experts % s.ep_size == 0 + return s + + +# --------------------------------------------------------------------------------------------- +# Transforms: (sources, shares) -> the tensor to copy in. Sources are the checkpoint tensors in +# the order the entry names them, already on the device. +# --------------------------------------------------------------------------------------------- + + +def _cat_rows(srcs, s: Shares) -> torch.Tensor: + """Each source's TP rows, stacked in source order (projections that read one input).""" + return torch.cat([t[s.rows(t.shape[0])] for t in srcs], dim=0).contiguous() + + +def _rows(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[s.rows(t.shape[0])].contiguous() + + +def _o_cols(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[:, s.rows(t.shape[1])].contiguous() + + +def _conv_taps(srcs, s: Shares) -> torch.Tensor: + """`nn.Conv1d` weight `[C, 1, 4]` (oldest tap first) -> lag-major fp32 `[4, C]`.""" + (t,) = srcs + return t[:, 0, :].flip(-1).t().float().contiguous() + + +def _kv_conv_taps(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return _conv_taps((t[s.rows(t.shape[0])],), s) + + +def _static_taps(srcs, s: Shares) -> torch.Tensor: + """A dynamic convolution's `static_w` `[4, C]` (lag-major): this rank's channels, fp32.""" + (t,) = srcs + return t[:, s.rows(t.shape[1])].float().contiguous() + + +def _kv_with_taps(srcs, s: Shares) -> torch.Tensor: + """k or v projection rows, then the dynamic convolution's tap projection rows (4 per head).""" + proj, taps = srcs + return torch.cat([proj[s.rows(proj.shape[0])], taps[s.rows(taps.shape[0])]], dim=0).contiguous() + + +def _fp32(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.float().reshape(-1, t.shape[-1]).contiguous() + + +def _as_is(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.contiguous() + + +def _nvfp4_w13(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[:, fc1_row_order(t.shape[1] // 2, t.device)].contiguous() + + +def _nvfp4_w13_scale(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + rows = t.view(torch.uint8)[:, fc1_row_order(t.shape[1] // 2, t.device)] + return swizzle_block_scales(rows) + + +def _nvfp4_scale(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return swizzle_block_scales(t.view(torch.uint8)) + + +def _nvfp4_alpha(srcs, s: Shares) -> torch.Tensor: + """Per local expert: `amax / outer_scale_denominator`, fp32.""" + amax, denominator = srcs + return (amax[s.experts()].float() / denominator.float().reshape(())).contiguous() + + +def _raw_u8(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.view(torch.uint8).contiguous() + + +@dataclass(frozen=True) +class Entry: + """One weight of one layer (or of the model): shape, dtype, checkpoint sources, transform. + + `sources` are checkpoint names; `expert_rows` marks sources read only at this rank's expert + rows (the lazy view reads just those bytes). + """ + + name: str + shape: tuple + dtype: torch.dtype + sources: tuple[str, ...] + transform: Callable[[Any, Shares], torch.Tensor] + expert_rows: tuple[bool, ...] = () + + +def layer_entries(s: Shares, layer: int) -> list[Entry]: + d = s.model + p = f"{LLM}.layers.{layer}" + a = f"{p}.attn" + h, bf16, f32, u8 = d.hidden_size, torch.bfloat16, torch.float32, torch.uint8 + entries = [ + Entry("mlp_norm", (h,), bf16, (f"{p}.mlp_norm.weight",), _as_is), + Entry("wo", (h, s.q_width), bf16, (f"{a}.wo_ud.weight",), _o_cols), + Entry("q_scale", (s.heads, d.head_dim), bf16, (f"{a}.q_l2_clip.scale",), _rows), + Entry("r_scale", (s.heads, d.d_rel), bf16, (f"{a}.rel_l2_clip.scale",), _rows), + Entry( + "rel_proj", + (d.d_rel, d.rel_extent_of(layer)), + bf16, + (f"{a}.rel_logits_proj.proj",), + _as_is, + ), + Entry("attn_conv", (4, h), f32, (f"{p}.attn_sconv.weight",), _conv_taps), + Entry("mlp_conv", (4, h), f32, (f"{p}.mlp_sconv.weight",), _conv_taps), + Entry("router", (d.num_experts, h), bf16, (f"{p}.mlp.gate.weight",), _as_is), + Entry("router_bias", (d.num_experts,), f32, (f"{p}.mlp.gate.bias",), _as_is), + Entry("mlp_query", (1, h), f32, (f"{LLM}.attn_res.mlp_queries.{layer}",), _fp32), + ] + stacked_q = s.q_width + s.r_width + s.q_width + s.heads + if d.is_global(layer): + dyn = s.kv_heads * 4 + entries += [ + Entry("q_norm", (h,), bf16, (f"{p}.q_input_norm.weight",), _as_is), + Entry("k_norm", (h,), bf16, (f"{p}.k_input_norm.weight",), _as_is), + Entry("v_norm", (h,), bf16, (f"{p}.v_input_norm.weight",), _as_is), + Entry( + "w_q", + (stacked_q, h), + bf16, + ( + f"{a}.wq_du.weight", + f"{a}.wr_du.weight", + f"{a}.gate_attn.weight", + f"{a}.attn_sink_proj.weight", + ), + _cat_rows, + ), + Entry( + "w_k", + (s.kv_width + dyn, h), + bf16, + (f"{a}.wk_dv.weight", f"{a}.k_sconv.weight_proj"), + _kv_with_taps, + ), + Entry( + "w_v", + (s.kv_width + dyn, h), + bf16, + (f"{a}.wv_dv.weight", f"{a}.v_sconv.weight_proj"), + _kv_with_taps, + ), + Entry("k_conv", (4, s.kv_width), f32, (f"{a}.k_sconv.static_w",), _static_taps), + Entry("v_conv", (4, s.kv_width), f32, (f"{a}.v_sconv.static_w",), _static_taps), + ] + else: + entries += [ + Entry("attn_norm", (h,), bf16, (f"{p}.attn_norm.weight",), _as_is), + Entry( + "w_in", + (stacked_q + 2 * s.kv_width, h), + bf16, + ( + f"{a}.wq_du.weight", + f"{a}.wr_du.weight", + f"{a}.gate_attn.weight", + f"{a}.attn_sink_proj.weight", + f"{a}.wk_dv.weight", + f"{a}.wv_dv.weight", + ), + _cat_rows, + ), + Entry("k_conv", (4, s.kv_width), f32, (f"{a}.k_sconv.weight",), _kv_conv_taps), + Entry("v_conv", (4, s.kv_width), f32, (f"{a}.v_sconv.weight",), _kv_conv_taps), + ] + if layer > 0: + queries = 3 if d.is_global(layer) else 1 + entries.append( + Entry( + "attn_query", + (queries, h), + f32, + (f"{LLM}.attn_res.attn_queries.{layer - 1}",), + _fp32, + ) + ) + + e, i = s.local_experts, d.expert_intermediate_size + x = f"{p}.mlp.experts" + fmt = d.expert_formats[layer] + if fmt is ExpertFormat.NVFP4: + entries += [ + Entry("w13", (e, 2 * i, h // 2), u8, (f"{x}.w13_weight",), _nvfp4_w13, (True,)), + Entry( + "w13_sf", + (e, 2 * i, h // 16), + u8, + (f"{x}.w13_weight_scale",), + _nvfp4_w13_scale, + (True,), + ), + Entry( + "w13_alpha", + (e,), + f32, + (f"{x}.w13_weight_amax", f"{x}.w13_weight_outer_scale_denominator"), + _nvfp4_alpha, + ), + Entry("w2", (e, h, i // 2), u8, (f"{x}.w2_weight",), _as_is, (True,)), + Entry("w2_sf", (e, h, i // 16), u8, (f"{x}.w2_weight_scale",), _nvfp4_scale, (True,)), + Entry( + "w2_alpha", + (e,), + f32, + (f"{x}.w2_weight_amax", f"{x}.w2_weight_outer_scale_denominator"), + _nvfp4_alpha, + ), + ] + elif fmt is ExpertFormat.MXFP8: + e4m3 = torch.float8_e4m3fn + entries += [ + Entry("w13", (e, 2 * i, h), e4m3, (f"{x}.w13_weight",), _as_is, (True,)), + Entry("w13_s", (e, 2 * i, h // 32), u8, (f"{x}.w13_weight_scale",), _raw_u8, (True,)), + Entry("w2", (e, h, i), e4m3, (f"{x}.w2_weight",), _as_is, (True,)), + Entry("w2_s", (e, h, i // 32), u8, (f"{x}.w2_weight_scale",), _raw_u8, (True,)), + ] + else: + raise NotImplementedError( + f"layer {layer}: {fmt.value} experts are not supported by this target" + ) + return entries + + +def model_entries(s: Shares) -> list[Entry]: + d = s.model + h, bf16, f32 = d.hidden_size, torch.bfloat16, torch.float32 + return [ + Entry("embed", (d.vocab_size, h), bf16, (f"{LLM}.embed.weight",), _as_is), + Entry("embed_norm", (h,), bf16, (f"{LLM}.embed_norm.weight",), _as_is), + Entry("final_query", (1, h), f32, (f"{LLM}.attn_res.output_query",), _fp32), + Entry("final_norm", (h,), bf16, (f"{LLM}.norm.weight",), _as_is), + ] + + +def _key(layer: Optional[int], name: str) -> str: + return name if layer is None else f"l{layer}_{name}" + + +def all_entries(s: Shares) -> list[tuple[Optional[int], Entry]]: + rows = [(None, entry) for entry in model_entries(s)] + for layer in range(s.model.num_layers): + rows += [(layer, entry) for entry in layer_entries(s, layer)] + return rows + + +def declare(s: Shares) -> nn.ParameterDict: + """Every weight at this rank's share; the engine's meta init gives them storage later.""" + w = nn.ParameterDict() + for layer, entry in all_entries(s): + w[_key(layer, entry.name)] = nn.Parameter( + torch.empty(entry.shape, dtype=entry.dtype), requires_grad=False + ) + return w + + +def layer_view(w: nn.ParameterDict, layer: int) -> SimpleNamespace: + """Layer `layer`'s weights by name, `w.`.""" + prefix = f"l{layer}_" + return SimpleNamespace(**{k[len(prefix) :]: v for k, v in w.items() if k.startswith(prefix)}) + + +def load(model, weights, s: Shares) -> None: + """Fill every declared weight and the shell's lm_head rows from the checkpoint. + + `weights` is the engine's lazy view (`weights[name][a:b]` reads only those rows). Every + text-decoder tensor of the checkpoint must be consumed; other tensors are not read. + """ + core = model.model + consumed: set[str] = set() + expert_rows = s.experts() + + def read(name: str, rows: Optional[slice], device: torch.device) -> torch.Tensor: + assert name in weights, f"checkpoint tensor missing: {name}" + consumed.add(name) + t = weights[name][rows] if rows is not None else weights[name][:] + return t.to(device, non_blocking=True) + + for layer, entry in all_entries(s): + dst = core.w[_key(layer, entry.name)].data + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = tuple( + read(n, expert_rows if f else None, dst.device) for n, f in zip(entry.sources, flags) + ) + src = entry.transform(srcs, s) + if src.dtype != dst.dtype and src.dtype == torch.uint8 and dst.dtype.itemsize == 1: + src = src.view(dst.dtype) + assert src.shape == dst.shape, (layer, entry.name, tuple(src.shape), tuple(dst.shape)) + assert src.dtype == dst.dtype, (layer, entry.name, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + if entry.expert_rows: + del srcs, src + torch.cuda.empty_cache() + + lm_head = model.lm_head.weight + vocab = s.model.vocab_size + rows = lm_head.shape[0] + start = 0 if rows == vocab else s.tp_rank * rows + unembed = read(f"{LLM}.unembed.weight", slice(start, min(start + rows, vocab)), lm_head.device) + if unembed.shape[0] < rows: + lm_head.data.zero_() + lm_head.data[: unembed.shape[0]].copy_(unembed) + torch.cuda.synchronize() + + leftover = [ + name for name in weights.keys() if name.startswith(f"{LLM}.") and name not in consumed + ] + assert not leftover, f"text-decoder tensors the table does not name: {sorted(leftover)[:8]}" diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py new file mode 100644 index 000000000000..899a8b324345 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton grouped MoE for Inkling 2's MXFP8 experts: one launch per GEMM for all local experts. + +The routes `(token, slot)` that land on this rank's experts are grouped by expert into blocks of +`BLOCK_M` rows (`align_routes`, on the device, without a host synchronization; the block count is +bounded by `T * top_k / BLOCK_M + E`). Then: + +* `gemm1`: per block and 128 intermediate columns, `x^ @ gate^T` and `x^ @ up^T` (gate rows `2j`, + up rows `2j + 1` of `w13`), with the E4M3 weights and their UE8M0 scales expanded to bf16 in the + kernel (exactly: an E4M3 value times a power of two is a bf16 value), fp32 accumulation, then + SiTU and the MXFP8 rounding of the result per 32 columns; writes `h^` `[routes, I]` bf16. +* `gemm2`: per block and 128 hidden columns, `h^ @ down^T` in fp32, times the route's weight, + added atomically into the fp32 output row of its token. + +`x^`, the MXFP8-rounded input, is computed once per call by the caller. + +Both activation roundings, `x^` (the caller's `mxfp8_quantize`) and `h^`, take the block scale +`2^ceil(log2(amax / 448))`, as SGLang's Inkling 2 MoE does (`_mxfp8_block` in +`sglang/kernels/ops/moe/inkling2_moe.py`), so that no value of a block saturates. The OCP MX rule +`2^(floor(log2(amax)) - 8)` would instead clip a block whose `amax` lies in +`(448 * 2^k, 512 * 2^k)`. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + +BLOCK_M = 32 +#: The K tile of both GEMMs: the hidden and intermediate sizes must be multiples of it. +K_TILE = 128 + + +def align_routes( + topk_ids: torch.Tensor, local_expert_offset: int, num_local: int, block_m: int = BLOCK_M +): + """Group this rank's routes by expert into blocks of `block_m` rows, on the device and without + a host synchronization. + + Returns `route_of_row` `[max_blocks * block_m]` int32 (the flat route `token * top_k + slot` of + each row, -1 for padding), `expert_of_block` `[max_blocks]` int32 (-1 past the last used block) + and `max_blocks`, a bound on the blocks any routing needs. + """ + num_tokens, top_k = topk_ids.shape + device = topk_ids.device + flat = topk_ids.reshape(-1).long() - local_expert_offset + is_local = (flat >= 0) & (flat < num_local) + expert = torch.where( + is_local, flat, torch.full_like(flat, num_local) + ) # non-local routes sort last + counts = torch.zeros(num_local + 1, dtype=torch.long, device=device) + counts.scatter_add_(0, expert, torch.ones_like(expert)) + counts = counts[:num_local] + blocks_per_expert = (counts + block_m - 1) // block_m + block_end = torch.cumsum(blocks_per_expert, 0) + block_start = block_end - blocks_per_expert + max_blocks = num_tokens * top_k // block_m + num_local + order = torch.argsort(expert, stable=True) + sorted_expert = expert[order] + sorted_local = sorted_expert < num_local + e = sorted_expert.clamp(max=num_local - 1) + rank_in_expert = ( + torch.arange(order.numel(), device=device) - (torch.cumsum(counts, 0) - counts)[e] + ) + rows = max_blocks * block_m + # Non-local routes write to one spare slot past the end. + row = torch.where( + sorted_local, block_start[e] * block_m + rank_in_expert, torch.full_like(e, rows) + ) + route_of_row = torch.full((rows + 1,), -1, dtype=torch.int32, device=device) + route_of_row.scatter_(0, row, order.to(torch.int32)) + owner = torch.searchsorted(block_end, torch.arange(max_blocks, device=device), right=True) + expert_of_block = torch.where(owner < num_local, owner, torch.full_like(owner, -1)).to( + torch.int32 + ) + return route_of_row[:rows], expert_of_block, max_blocks + + +@triton.jit +def _dequant_tile(w_ptrs, s_ptrs, mask): + """E4M3 weights and their UE8M0 exponents -> bf16 (exact).""" + w = tl.load(w_ptrs, mask=mask, other=0.0).to(tl.float32) + e = tl.load(s_ptrs, mask=mask, other=127).to(tl.float32) + return (w * tl.exp2(e - 127.0)).to(tl.bfloat16) + + +@triton.jit +def _round_e4m3(x): + """Round fp32 values in [-448, 448] to the nearest E4M3 value (ties to even), in fp32. + + The quantum is 2^(e - 3) for a value in [2^e, 2^(e + 1)), and 2^-9 below 2^-6 (subnormals); + dividing by a power of two is exact, so `rint` gives the E4M3 rounding. + """ + exponent = ((x.to(tl.int32, bitcast=True) >> 23) & 0xFF) - 127 + quantum = tl.exp2((tl.maximum(exponent, -6) - 3).to(tl.float32)) + return libdevice.rint(x / quantum) * quantum + + +@triton.jit +def _gemm1_kernel( + X, + W13, + S13, + ROUTE, + EXPERT, + H, + top_k, + hidden, + inter, + stride_x, + stride_we, + stride_wr, + stride_se, + stride_sr, + stride_h, + gate_softcap, + up_softcap, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + block = tl.program_id(0) + expert = tl.load(EXPERT + block) + if expert < 0: + return + rows = block * BLOCK_M + tl.arange(0, BLOCK_M) + route = tl.load(ROUTE + rows) + row_ok = route >= 0 + token = tl.where(row_ok, route // top_k, 0).to(tl.int64) + cols = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) # intermediate columns j + col_ok = cols < inter + offs_k = tl.arange(0, BLOCK_K) + w_base = W13 + expert.to(tl.int64) * stride_we + s_base = S13 + expert.to(tl.int64) * stride_se + acc_g = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + acc_u = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + for k0 in range(0, hidden, BLOCK_K): + k = k0 + offs_k + x = tl.load(X + token[:, None] * stride_x + k[None, :], mask=row_ok[:, None], other=0.0) + gate_rows = (2 * cols).to(tl.int64) + mask = col_ok[:, None] + g = _dequant_tile( + w_base + gate_rows[:, None] * stride_wr + k[None, :], + s_base + gate_rows[:, None] * stride_sr + (k // 32)[None, :], + mask, + ) + u = _dequant_tile( + w_base + (gate_rows + 1)[:, None] * stride_wr + k[None, :], + s_base + (gate_rows + 1)[:, None] * stride_sr + (k // 32)[None, :], + mask, + ) + acc_g += tl.dot(x, tl.trans(g)) + acc_u += tl.dot(x, tl.trans(u)) + h = ( + gate_softcap + * libdevice.tanh(acc_g / gate_softcap) + * tl.sigmoid(acc_g) + * up_softcap + * libdevice.tanh(acc_u / up_softcap) + ) + # MXFP8 rounding per 32 columns: scale 2^ceil(log2(amax / 448)), values to E4M3. + groups = tl.reshape(h, [BLOCK_M, BLOCK_N // 32, 32]) + raw = tl.max(tl.abs(groups), axis=2) / 448.0 + bits = raw.to(tl.int32, bitcast=True) + exponent = ((bits + 0x007FFFFF) & 0x7F800000) >> 23 + # 2^-127 (an fp32 subnormal) and 2^127, built from their bits: as Python literals Triton + # would type the subnormal as fp64. + min_scale = tl.full([1, 1], 1 << 22, tl.int32).to(tl.float32, bitcast=True) + max_inverse = tl.full([1, 1], 254 << 23, tl.int32).to(tl.float32, bitcast=True) + exponent = tl.where(raw <= min_scale, 0, tl.minimum(exponent, 254)) + scale = tl.maximum((exponent << 23).to(tl.float32, bitcast=True), min_scale) + inverse = tl.where(exponent == 0, max_inverse, 1.0 / scale) + q = _round_e4m3(tl.clamp(groups * inverse[:, :, None], -448.0, 448.0)) + h_hat = tl.reshape(q * scale[:, :, None], [BLOCK_M, BLOCK_N]) + tl.store( + H + rows.to(tl.int64)[:, None] * stride_h + cols[None, :], + h_hat.to(tl.bfloat16), + mask=row_ok[:, None] & col_ok[None, :], + ) + + +@triton.jit +def _gemm2_kernel( + H, + W2, + S2, + ROUTE, + EXPERT, + WEIGHTS, + OUT, + top_k, + hidden, + inter, + stride_h, + stride_we, + stride_wr, + stride_se, + stride_sr, + stride_out, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + block = tl.program_id(0) + expert = tl.load(EXPERT + block) + if expert < 0: + return + rows = block * BLOCK_M + tl.arange(0, BLOCK_M) + route = tl.load(ROUTE + rows) + row_ok = route >= 0 + cols = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) # hidden columns + col_ok = cols < hidden + offs_k = tl.arange(0, BLOCK_K) + w_base = W2 + expert.to(tl.int64) * stride_we + s_base = S2 + expert.to(tl.int64) * stride_se + acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + for k0 in range(0, inter, BLOCK_K): + k = k0 + offs_k + h = tl.load( + H + rows.to(tl.int64)[:, None] * stride_h + k[None, :], mask=row_ok[:, None], other=0.0 + ) + w = _dequant_tile( + w_base + cols.to(tl.int64)[:, None] * stride_wr + k[None, :], + s_base + cols.to(tl.int64)[:, None] * stride_sr + (k // 32)[None, :], + col_ok[:, None], + ) + acc += tl.dot(h, tl.trans(w)) + weight = tl.load(WEIGHTS + tl.where(row_ok, route, 0), mask=row_ok, other=0.0) + token = tl.where(row_ok, route // top_k, 0).to(tl.int64) + tl.atomic_add( + OUT + token[:, None] * stride_out + cols[None, :], + acc * weight[:, None], + mask=row_ok[:, None] & col_ok[None, :], + sem="relaxed", + ) + + +def grouped_mxfp8_moe( + x_hat: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, +) -> torch.Tensor: + """`x_hat` `[T, H]` bf16 (MXFP8-rounded) -> fp32 `[T, H]`, this rank's experts' weighted sum.""" + num_tokens, hidden = x_hat.shape + local, rows, _ = w13.shape + inter = rows // 2 + if hidden % K_TILE or inter % K_TILE: + # Neither GEMM masks a partial K tile. + raise ValueError( + f"grouped MXFP8 MoE needs hidden ({hidden}) and intermediate ({inter}) sizes that are " + f"multiples of {K_TILE}" + ) + top_k = topk_ids.shape[1] + out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=x_hat.device) + if num_tokens == 0: + return out + route_of_row, expert_of_block, max_blocks = align_routes(topk_ids, local_expert_offset, local) + h = torch.empty(max_blocks * BLOCK_M, inter, dtype=torch.bfloat16, device=x_hat.device) + block_n, block_k = 128, K_TILE + _gemm1_kernel[(max_blocks, triton.cdiv(inter, block_n))]( + x_hat, + w13, + w13_s, + route_of_row, + expert_of_block, + h, + top_k, + hidden, + inter, + x_hat.stride(0), + w13.stride(0), + w13.stride(1), + w13_s.stride(0), + w13_s.stride(1), + h.stride(0), + float(gate_softcap), + float(up_softcap), + BLOCK_M=BLOCK_M, + BLOCK_N=block_n, + BLOCK_K=block_k, + num_warps=8, + ) + _gemm2_kernel[(max_blocks, triton.cdiv(hidden, block_n))]( + h, + w2, + w2_s, + route_of_row, + expert_of_block, + topk_weights.reshape(-1).contiguous(), + out, + top_k, + hidden, + inter, + h.stride(0), + w2.stride(0), + w2.stride(1), + w2_s.stride(0), + w2_s.stride(1), + out.stride(0), + BLOCK_M=BLOCK_M, + BLOCK_N=block_n, + BLOCK_K=block_k, + num_warps=8, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py new file mode 100644 index 000000000000..7dc624b6c105 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py @@ -0,0 +1,321 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton kernels of Inkling 2's paged attention: K/V append, and attention with a relative bias, +an attention sink and an output gate. + +The KV pool is one layer's `KVCacheManagerV2.get_buffers(layer, "HND")` view, `[pages, 2, kv +heads, page size, head dim]`; `page_table[r, i]` is the pool page of request `r`'s tokens +`i * page_size ..`. A step holds the new tokens of several requests, request-major +(`query_start`); request `r`'s new tokens sit at positions `kv_len[r] - n_r .. kv_len[r] - 1`. + +`append_kv` writes the new tokens' keys and values to their pages. `attention` then computes, per +query token `t` at position `p` and query head `h` (KV head `h // group`): + + s_j = q . k_j * sm_scale + rel_logits[t, h, p - j] (0 <= p - j < rel_extent, else no bias) + over keys j <= p, and p - j < window on a sliding layer + o = softmax(s) . v, lse = logsumexp(s) + out = o * sigmoid(lse - sink[t, h]) * sigmoid(gate[t, h]) + +in fp32, rounded once. `sigmoid(lse - sink)` is the probability mass the keys keep against one +extra logit `sink` of zero value. A program covers `BLOCK_T` tokens of one request and the `group` +query heads of one KV head, as `BLOCK_T * GROUP_PAD` rows of one tile. + +The kernels rely on the KV cache keeping every page they touch: `append_kv` writes only the step's +new tokens, and `attention` reads only keys `p - window + 1 .. p` on a sliding layer (`0 .. p` on a +global one). `KVCacheManagerV2` keeps a sliding layer's pages from `history + 1 - window` on, which +covers that range, and the model checks each step's tables for it on the host. `attention` also +masks a key whose page index is negative (a recycled page), so a broken contract cannot read +before the pool. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# Masked scores. Finite, so a row whose first key block is fully masked keeps a finite running +# maximum; the first valid key then rescales what the masked keys added to zero. +_MASKED = tl.constexpr(-1.0e30) + + +@triton.jit +def _append_kv_kernel( + K, + V, + POOL, + PAGE_TABLE, + QSTART, + KV_LEN, + TOKEN_REQ, + stride_kt, + stride_kh, + stride_vt, + stride_vh, + stride_pool_page, + stride_pool_kv, + stride_pool_head, + stride_pool_slot, + stride_pt, + PAGE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, +): + t = tl.program_id(0) + head = tl.program_id(1) + req = tl.load(TOKEN_REQ + t) + start = tl.load(QSTART + req) + n = tl.load(QSTART + req + 1) - start + pos = tl.load(KV_LEN + req) - n + (t - start) + page = tl.load(PAGE_TABLE + req.to(tl.int64) * stride_pt + pos // PAGE_SIZE).to(tl.int64) + dst = ( + POOL + + page * stride_pool_page + + head * stride_pool_head + + (pos % PAGE_SIZE) * stride_pool_slot + ) + offs = tl.arange(0, HEAD_DIM) + t64 = t.to(tl.int64) + tl.store(dst + offs, tl.load(K + t64 * stride_kt + head * stride_kh + offs)) + tl.store(dst + stride_pool_kv + offs, tl.load(V + t64 * stride_vt + head * stride_vh + offs)) + + +@triton.jit +def _attention_kernel( + Q, + POOL, + PAGE_TABLE, + QSTART, + KV_LEN, + TILE_REQ, + TILE_START, + REL, + SINK, + GATE, + OUT, + stride_qt, + stride_qh, + stride_pool_page, + stride_pool_kv, + stride_pool_head, + stride_pool_slot, + stride_pt, + stride_rt, + stride_rh, + stride_st, + stride_gt, + stride_gh, + stride_ot, + stride_oh, + sm_scale, + window, + GROUP: tl.constexpr, + GROUP_PAD: tl.constexpr, + REL_EXTENT: tl.constexpr, + PAGE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_N: tl.constexpr, +): + tile = tl.program_id(0) + kv_head = tl.program_id(1) + req = tl.load(TILE_REQ + tile) + q_begin = tl.load(TILE_START + tile) + start = tl.load(QSTART + req) + end = tl.load(QSTART + req + 1) + past = tl.load(KV_LEN + req) - (end - start) + + BLOCK_M: tl.constexpr = BLOCK_T * GROUP_PAD + rows = tl.arange(0, BLOCK_M) + token = q_begin + rows // GROUP_PAD + head_in_group = rows % GROUP_PAD + head = kv_head * GROUP + head_in_group + row_ok = (head_in_group < GROUP) & (token < end) + q_pos = past + (token - start) + token64 = token.to(tl.int64) + offs_d = tl.arange(0, HEAD_DIM) + + q = tl.load( + Q + token64[:, None] * stride_qt + head[:, None] * stride_qh + offs_d[None, :], + mask=row_ok[:, None], + other=0.0, + ) + + first_pos = past + (q_begin - start) + last_pos = past + (tl.minimum(q_begin + BLOCK_T, end) - start) - 1 + # The oldest key any row of the tile reads: the first row's window start on a sliding layer. + lo = tl.where(window > 0, tl.maximum(first_pos - window + 1, 0), 0) + hi = last_pos + 1 + + m_i = tl.full([BLOCK_M], _MASKED, dtype=tl.float32) + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + qk_scale = sm_scale * 1.4426950408889634 + pt_row = PAGE_TABLE + req.to(tl.int64) * stride_pt + kv_base = POOL + kv_head * stride_pool_head + + for block in range(lo // BLOCK_N * BLOCK_N, hi, BLOCK_N): + k_pos = block + tl.arange(0, BLOCK_N) + key_ok = (k_pos >= lo) & (k_pos < hi) + page = tl.load(pt_row + k_pos // PAGE_SIZE, mask=key_ok, other=0).to(tl.int64) + key_ok = key_ok & (page >= 0) + slot_ptr = kv_base + page * stride_pool_page + (k_pos % PAGE_SIZE) * stride_pool_slot + k = tl.load(slot_ptr[:, None] + offs_d[None, :], mask=key_ok[:, None], other=0.0) + v = tl.load( + slot_ptr[:, None] + stride_pool_kv + offs_d[None, :], mask=key_ok[:, None], other=0.0 + ) + + # Scores in log2 units: (q . k * sm_scale + bias) * log2(e). + s = tl.dot(q, tl.trans(k)) * qk_scale + dist = q_pos[:, None] - k_pos[None, :] + allowed = ( + row_ok[:, None] & key_ok[None, :] & (dist >= 0) & ((window <= 0) | (dist < window)) + ) + biased = allowed & (dist < REL_EXTENT) + bias = tl.load( + REL + token64[:, None] * stride_rt + head[:, None] * stride_rh + dist, + mask=biased, + other=0.0, + ) + s = tl.where(allowed, s + bias * 1.4426950408889634, _MASKED) + + m_new = tl.maximum(m_i, tl.max(s, 1)) + alpha = tl.math.exp2(m_i - m_new) + p = tl.math.exp2(s - m_new[:, None]) + l_i = l_i * alpha + tl.sum(p, 1) + acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) + m_i = m_new + + # Natural-log logsumexp of the scores, then the sink and the output gate. + lse = (m_i + tl.math.log2(l_i)) / 1.4426950408889634 + sink = tl.load(SINK + token64 * stride_st + head, mask=row_ok, other=0.0).to(tl.float32) + keep = tl.sigmoid(lse - sink) + gate = tl.load( + GATE + token64[:, None] * stride_gt + head[:, None] * stride_gh + offs_d[None, :], + mask=row_ok[:, None], + other=0.0, + ).to(tl.float32) + out = acc / l_i[:, None] * keep[:, None] * tl.sigmoid(gate) + tl.store( + OUT + token64[:, None] * stride_ot + head[:, None] * stride_oh + offs_d[None, :], + out.to(OUT.dtype.element_ty), + mask=row_ok[:, None], + ) + + +def append_kv( + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, +) -> None: + """Write `k`, `v` `[T, kv heads, head dim]` to their pages of `kv_pool`.""" + num_tokens, num_kv_heads, head_dim = k.shape + if num_tokens == 0: + return + _append_kv_kernel[(num_tokens, num_kv_heads)]( + k, + v, + kv_pool, + page_table, + query_start, + kv_len, + token_request, + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + kv_pool.stride(0), + kv_pool.stride(1), + kv_pool.stride(2), + kv_pool.stride(3), + page_table.stride(0), + PAGE_SIZE=kv_pool.shape[3], + HEAD_DIM=head_dim, + ) + + +def tile_plan( + query_lengths: list[int], block_t: int, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor]: + """Per program: its request and first token, from the host-side query lengths of one step.""" + tile_request, tile_start = [], [] + start = 0 + for r, n in enumerate(query_lengths): + for first in range(start, start + n, block_t): + tile_request.append(r) + tile_start.append(first) + start += n + as_tensor = lambda values: torch.tensor(values, dtype=torch.int32).to(device, non_blocking=True) # noqa: E731 + return as_tensor(tile_request), as_tensor(tile_start) + + +def block_tokens(max_query_length: int) -> int: + """Query tokens per program: 2 for a decode-only step (16 MMA rows with a group of up to 8 + heads), 16 otherwise.""" + return 2 if max_query_length <= 1 else 16 + + +def attention( + q: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, +) -> torch.Tensor: + """Launch the attention kernel; see the module docstring. Returns `[T, heads, head dim]`.""" + num_tokens, num_heads, head_dim = q.shape + num_kv_heads = kv_pool.shape[2] + group = num_heads // num_kv_heads + out = torch.empty_like(q) + num_tiles = tile_request.shape[0] + if num_tokens == 0 or num_tiles == 0: + return out + _attention_kernel[(num_tiles, num_kv_heads)]( + q, + kv_pool, + page_table, + query_start, + kv_len, + tile_request, + tile_start, + rel_logits, + sink, + gate, + out, + q.stride(0), + q.stride(1), + kv_pool.stride(0), + kv_pool.stride(1), + kv_pool.stride(2), + kv_pool.stride(3), + page_table.stride(0), + rel_logits.stride(0), + rel_logits.stride(1), + sink.stride(0), + gate.stride(0), + gate.stride(1), + out.stride(0), + out.stride(1), + float(sm_scale), + int(window), + GROUP=group, + GROUP_PAD=triton.next_power_of_2(group), + REL_EXTENT=rel_logits.shape[2], + PAGE_SIZE=kv_pool.shape[3], + HEAD_DIM=head_dim, + BLOCK_T=block_t, + BLOCK_N=64, + num_warps=8 if block_t * triton.next_power_of_2(group) >= 64 else 4, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py new file mode 100644 index 000000000000..c7568e4be259 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton kernels of Inkling 2's width-4 causal short convolutions, with a per-request window state. + +One step holds the new tokens of several requests, request-major (`query_start`). Token `t` of a +request reads its own input and the 3 inputs before it: inputs of the same step from `x`, earlier +ones from the request's window state. A request that starts at position 0 has no earlier inputs +(`has_state` 0), which reads as zeros. + +`conv_out` computes, in fp32, per channel `c`: + + out[t, c] = (sum over lag 3..0 of tap(t, c, lag) * x[t - lag, c] + x[t, c]) * out_scale + +with `tap = taps[lag, c]` (static), or `taps[lag, c] + dynamic_taps[t, c // channels_per_head * 4 ++ lag]` (dynamic), and rounds once to the output dtype. `update_state` then stores the last 3 +inputs of each request, oldest first, in its slot of `state` `[slots, 3, C]`. The state is read by +the first kernel and written by the second, so the two must run in this order on one stream. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +#: Kernel width of every Inkling 2 short convolution. +KERNEL_SIZE = 4 + + +@triton.jit +def _conv_out_kernel( + X, + TAPS, + DYN, + STATE, + SLOTS, + QSTART, + HAS_STATE, + TOKEN_REQ, + OUT, + num_tokens, + channels, + stride_x, + stride_dyn, + stride_state_slot, + stride_state_win, + stride_out, + out_scale, + CHANNELS_PER_HEAD: tl.constexpr, + DYNAMIC: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_C: tl.constexpr, +): + offs_t = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T) + offs_c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C) + mask_t = offs_t < num_tokens + mask_c = offs_c < channels + mask = mask_t[:, None] & mask_c[None, :] + + req = tl.load(TOKEN_REQ + offs_t, mask=mask_t, other=0) + local = offs_t - tl.load(QSTART + req, mask=mask_t, other=0) + has_state = tl.load(HAS_STATE + req, mask=mask_t, other=0) != 0 + slot = tl.load(SLOTS + req, mask=mask_t, other=0).to(tl.int64) + rows = offs_t.to(tl.int64) + + x0 = tl.load(X + rows[:, None] * stride_x + offs_c[None, :], mask=mask, other=0.0).to( + tl.float32 + ) + acc = tl.zeros([BLOCK_T, BLOCK_C], dtype=tl.float32) + # Oldest input first, the order of a convolution over the window. + for lag in tl.static_range(3, -1, -1): + if lag == 0: + xv = x0 + else: + src = local - lag + in_step = src >= 0 + xv_step = tl.load( + X + (rows - lag)[:, None] * stride_x + offs_c[None, :], + mask=mask & in_step[:, None], + other=0.0, + ) + # Window state index of an input `src` (< 0) steps before the request's first new token. + win = 3 + src + from_state = (~in_step) & has_state & (win >= 0) + xv_state = tl.load( + STATE + + slot[:, None] * stride_state_slot + + win[:, None] * stride_state_win + + offs_c[None, :], + mask=mask & from_state[:, None], + other=0.0, + ) + xv = tl.where(in_step[:, None], xv_step.to(tl.float32), xv_state.to(tl.float32)) + tap = tl.load(TAPS + lag * channels + offs_c, mask=mask_c, other=0.0)[None, :] + if DYNAMIC: + head = offs_c // CHANNELS_PER_HEAD + dyn = tl.load( + DYN + rows[:, None] * stride_dyn + head[None, :] * 4 + lag, mask=mask, other=0.0 + ).to(tl.float32) + tap = tap + dyn + acc += tap * xv + out = (acc + x0) * out_scale + tl.store( + OUT + rows[:, None] * stride_out + offs_c[None, :], out.to(OUT.dtype.element_ty), mask=mask + ) + + +@triton.jit +def _update_state_kernel( + X, + STATE, + SLOTS, + QSTART, + HAS_STATE, + channels, + stride_x, + stride_state_slot, + stride_state_win, + BLOCK_C: tl.constexpr, +): + req = tl.program_id(0) + offs_c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C) + mask_c = offs_c < channels + start = tl.load(QSTART + req) + n = tl.load(QSTART + req + 1) - start + has_state = tl.load(HAS_STATE + req) != 0 + base = STATE + tl.load(SLOTS + req).to(tl.int64) * stride_state_slot + + # New window entry j is the input at local index n - 3 + j: from this step, or from the old + # window (index n + j) when the request appended fewer than 3 tokens. All three are read + # before any is written. + v0 = _window_entry(X, base, start, n, 0, has_state, offs_c, mask_c, stride_x, stride_state_win) + v1 = _window_entry(X, base, start, n, 1, has_state, offs_c, mask_c, stride_x, stride_state_win) + v2 = _window_entry(X, base, start, n, 2, has_state, offs_c, mask_c, stride_x, stride_state_win) + tl.store(base + offs_c, v0, mask=mask_c) + tl.store(base + stride_state_win + offs_c, v1, mask=mask_c) + tl.store(base + 2 * stride_state_win + offs_c, v2, mask=mask_c) + + +@triton.jit +def _window_entry( + X, base, start, n, j: tl.constexpr, has_state, offs_c, mask_c, stride_x, stride_state_win +): + src = n - 3 + j + from_step = src >= 0 + v_step = tl.load( + X + (start + src).to(tl.int64) * stride_x + offs_c, mask=mask_c & from_step, other=0.0 + ) + v_state = tl.load( + base + (n + j) * stride_state_win + offs_c, + mask=mask_c & (~from_step) & has_state, + other=0.0, + ) + return tl.where(from_step, v_step.to(base.dtype.element_ty), v_state) + + +def short_conv( + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor | None, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + out_scale: float, + channels_per_head: int = 1, +) -> torch.Tensor: + """Launch both kernels; see the module docstring. Returns `out` `[T, C]` in `x`'s dtype.""" + num_tokens, channels = x.shape + out = torch.empty_like(x) + if num_tokens == 0: + return out + block_t, block_c = 16, 128 + dynamic = dynamic_taps is not None + _conv_out_kernel[(triton.cdiv(num_tokens, block_t), triton.cdiv(channels, block_c))]( + x, + taps, + dynamic_taps if dynamic else x, + state, + slots, + query_start, + has_state, + token_request, + out, + num_tokens, + channels, + x.stride(0), + dynamic_taps.stride(0) if dynamic else 0, + state.stride(0), + state.stride(1), + out.stride(0), + float(out_scale), + CHANNELS_PER_HEAD=channels_per_head, + DYNAMIC=dynamic, + BLOCK_T=block_t, + BLOCK_C=block_c, + ) + num_requests = slots.shape[0] + _update_state_kernel[(num_requests, triton.cdiv(channels, block_c))]( + x, + state, + slots, + query_start, + has_state, + channels, + x.stride(0), + state.stride(0), + state.stride(1), + BLOCK_C=block_c, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py new file mode 100644 index 000000000000..ae2216d8359b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""What Inkling 2's attention does to its projections before the attention core. + +Per token and head, each step in fp32 and rounded to bf16 where the source model rounds: + + q = l2clip(q * q_scale, 22.627) then on a global layer q = q * tau + k = l2clip(k, 22.627) (k after its short convolution) + v = v / rms(v) (eps 1e-6, no weight; v after its convolution) + r = l2clip(r * r_scale, 8) then on a global layer r = r * tau + +with `l2clip(x, c) = x * min(c / (|x| + 1e-6), 1)` over the head dimension, and the global +layers' log scaling `tau = 1 + alpha * ln(max((position + 1) / n_floor, 1))` per token. A +sliding layer has no tau. +""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +_TP4 = dict(heads=12, kv_heads=2, head_dim=128, d_rel=16) + + +def log_scaling(positions: torch.Tensor, alpha: float, n_floor: float) -> torch.Tensor: + """The global layers' per-token tau, fp32 `[T]`.""" + return 1.0 + alpha * torch.log(((positions.float() + 1.0) / n_floor).clamp(min=1.0)) + + +def _clip(x: torch.Tensor, threshold: float) -> torch.Tensor: + norm = torch.linalg.vector_norm(x, dim=-1, keepdim=True) + return x * (threshold / (norm + 1e-6)).clamp(max=1.0) + + +class Inkling2AttentionPrologue(OpWrapper): + """`q` `[T, H, 128]`, `k`, `v` `[T, KV, 128]`, `r` `[T, H, 16]` bf16, `q_scale` `[H, 128]`, + `r_scale` `[H, 16]`, `tau` `[T]` fp32 or None -> (q, k, v, r), new bf16 tensors.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="sliding layer at TP4, 33 tokens", spec=dict(_TP4, tokens=33, tau=False)), + Cell(why="global layer at TP4, tokens past n_floor", spec=dict(_TP4, tokens=33, tau=True)), + ) + + note = """ + The clip thresholds are 22.627 for q and k and 8 for r; tau multiplies q and r after their + clips, so a clipped q can exceed 22.627 on a global layer. tau is 1 below position 8191: + only positions past n_floor show it. + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + dtype = q.dtype + q = _clip(q.float() * q_scale.float(), qk_threshold).to(dtype) + k = _clip(k.float(), qk_threshold).to(dtype) + vf = v.float() + v = (vf * torch.rsqrt(vf.square().mean(-1, keepdim=True) + eps)).to(dtype) + r = _clip(r.float() * r_scale.float(), rel_threshold).to(dtype) + if tau is not None: + t = tau.view(-1, 1, 1) + q = (q.float() * t).to(dtype) + r = (r.float() * t).to(dtype) + return q, k, v, r + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """The same steps in fp64, rounded to bf16 at the same points.""" + dtype = q.dtype + q = _clip(q.double() * q_scale.double(), qk_threshold).to(dtype) + k = _clip(k.double(), qk_threshold).to(dtype) + vd = v.double() + v = (vd * torch.rsqrt(vd.square().mean(-1, keepdim=True) + eps)).to(dtype) + r = _clip(r.double() * r_scale.double(), rel_threshold).to(dtype) + if tau is not None: + t = tau.double().view(-1, 1, 1) + q = (q.double() * t).to(dtype) + r = (r.double() * t).to(dtype) + return q, k, v, r + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> None: + assert q.dim() == k.dim() == v.dim() == r.dim() == 3, "q, k, v, r are [T, heads, dim]" + assert q_scale.shape == q.shape[1:] and r_scale.shape == r.shape[1:], "per-head scales" + assert tau is None or (tau.dtype == torch.float32 and tau.shape == (q.shape[0],)), ( + "tau is fp32 [T]" + ) + + +inkling2_attention_prologue = Inkling2AttentionPrologue() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py new file mode 100644 index 000000000000..fb8972262ed4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2 attention on a paged KV cache: append the step's keys and values, then attend with +the relative position bias, the attention sink and the output gate. + +Inkling 2 has no rotary embedding. Position enters the logits only through a bias that each query +token mixes per head from a bank of 16 bias-vs-distance profiles, `rel_logits[t, h, d] = r[t, h] +. proj[:, d]` for backward distances `0 <= d < rel_extent` (1024), and zero beyond. A sliding +layer attends the last 1024 positions (`p - j < window`); a global layer the whole prefix. Each +query token and head also has a sink logit, from its own projection, that absorbs probability +mass without contributing a value. The attention output is then gated per element by +`sigmoid(gate)`. See `kernels/paged_attention.py` for the formula. + +The KV cache holds the keys and values after their short convolution, clipping and normalization +(bf16). The page table maps each request's token positions to pages of the layer's pool view. +""" + +from __future__ import annotations + +import math + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ...kernels.paged_attention import append_kv, attention + +#: `1 / sqrt(head_dim)` for Inkling 2's 128-wide heads. +SM_SCALE = 1.0 / math.sqrt(128) + +#: Requests of the cells, as (new tokens, cached tokens before the step). +_PREFILL = ((100, 0),) +_DECODE_PAST_WINDOW = ((1, 1500), (1, 7), (1, 1024), (1, 1023)) +_MIXED = ((70, 1000), (1, 1500), (37, 0), (1, 60)) + + +class Inkling2PagedAttention(OpWrapper): + """Append `k`, `v` `[T, kv heads, 128]` to `kv_pool` and attend with `q` `[T, heads, 128]`. + + `kv_pool` is the layer's HND pool view `[pages, 2, kv heads, page size, 128]`; `page_table` + `[R, W]` int32 maps request `r`'s token blocks to its pages. `query_start` `[R + 1]` and + `kv_len` `[R]` (cached plus new tokens) give each request's new tokens and positions; + `token_request` `[T]` maps tokens to requests. `tile_request`, `tile_start` (int32, one entry + per program) and `block_t` are the step's tile plan (`kernels.paged_attention.tile_plan`). + `rel_logits` `[T, heads, rel_extent]` fp32, `sink` `[T, heads]` and `gate` `[T, heads, 128]` + are per query token; `window` is 1024 on a sliding layer and 0 on a global one. Returns + `[T, heads, 128]` in `q`'s dtype. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="sliding layer at TP4 (12 q / 2 kv heads), one 100-token prefill", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_PREFILL), + ), + Cell( + why="sliding layer at TP4, decodes before, at and past the window edge", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_DECODE_PAST_WINDOW), + ), + Cell( + why="sliding layer at TP4, a mixed step with a chunk that crosses the window", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_MIXED), + ), + Cell( + why="global layer at TP4, decodes past the bias extent", + spec=dict(heads=12, kv_heads=2, window=0, requests=_DECODE_PAST_WINDOW), + ), + Cell( + why="global layer at TP1 (48 q / 8 kv heads), a mixed step", + spec=dict(heads=48, kv_heads=8, window=0, requests=_MIXED), + ), + ) + + note = """ + The bias is indexed by backward distance and is zero from rel_extent on, so a global + layer's keys past 1023 tokens carry no bias while a sliding layer's window ends exactly + where its bias does. The sink is a logit, compared with the logsumexp of the scaled and + biased scores; it is not scaled by sm_scale. Only decodes and chunks at positions past + 1023 tell the window, the bias extent and an off-by-one in either apart, which is what the + decode cells' positions 1023, 1024 and 1500 are for. + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> torch.Tensor: + append_kv(k, v, kv_pool, page_table, query_start, kv_len, token_request) + return attention( + q, + kv_pool, + page_table, + query_start, + kv_len, + tile_request, + tile_start, + block_t, + rel_logits, + sink, + gate, + window, + sm_scale, + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> torch.Tensor: + """Per request and head, dense fp32 attention over the pages it reads; writes `k`, `v` + to `kv_pool` as the op does. The tile plan is not needed.""" + page_size = kv_pool.shape[3] + num_heads = q.shape[1] + group = num_heads // kv_pool.shape[2] + rel_extent = rel_logits.shape[2] + bounds = query_start.tolist() + lengths = kv_len.tolist() + out = torch.empty_like(q) + for r in range(len(lengths)): + start, end = bounds[r], bounds[r + 1] + n, length = end - start, lengths[r] + positions = torch.arange(length - n, length, device=q.device) + pages = page_table[r, positions // page_size].long() + kv_pool[pages, 0, :, positions % page_size] = k[start:end] + kv_pool[pages, 1, :, positions % page_size] = v[start:end] + lo = max(0, length - n - window + 1) if window > 0 else 0 + keys_pos = torch.arange(lo, length, device=q.device) + key_pages = page_table[r, keys_pos // page_size].long() + keys = kv_pool[key_pages, 0, :, keys_pos % page_size].float() # [L, kv heads, D] + values = kv_pool[key_pages, 1, :, keys_pos % page_size].float() + dist = positions[:, None] - keys_pos[None, :] + allowed = dist >= 0 + if window > 0: + allowed &= dist < window + in_extent = (dist >= 0) & (dist < rel_extent) + for h in range(num_heads): + with true_fp32_matmul(): + logits = q[start:end, h].float() @ keys[:, h // group].t() * sm_scale + bias = rel_logits[start:end, h].gather(1, dist.clamp(0, rel_extent - 1)) + logits = logits + torch.where(in_extent, bias, torch.zeros_like(bias)) + logits = logits.masked_fill(~allowed, float("-inf")) + lse = torch.logsumexp(logits, dim=1) + with true_fp32_matmul(): + o = torch.softmax(logits, dim=1) @ values[:, h // group] + keep = torch.sigmoid(lse - sink[start:end, h].float()) + o = o * keep[:, None] * torch.sigmoid(gate[start:end, h].float()) + out[start:end, h] = o.to(q.dtype) + return out + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> None: + num_tokens, num_heads, head_dim = q.shape + assert kv_pool.dim() == 5 and kv_pool.shape[1] == 2 and kv_pool.shape[4] == head_dim, ( + "kv_pool must be the HND view [pages, 2, kv heads, page size, head dim]" + ) + assert kv_pool.stride(4) == 1 and q.stride(2) == 1 and gate.stride(2) == 1, ( + "head dims must be contiguous" + ) + num_kv_heads = kv_pool.shape[2] + assert num_heads % num_kv_heads == 0, "query heads must group evenly over kv heads" + assert k.shape == v.shape == (num_tokens, num_kv_heads, head_dim), ( + "k and v must be [T, kv heads, head dim]" + ) + assert k.stride(2) == 1 and v.stride(2) == 1, "k and v must have contiguous head dims" + assert k.dtype == v.dtype == q.dtype == kv_pool.dtype, "q, k, v and the pool share a dtype" + assert rel_logits.dtype == torch.float32 and rel_logits.shape[:2] == (num_tokens, num_heads) + assert rel_logits.stride(2) == 1, "rel_logits must be contiguous over distances" + assert sink.shape == (num_tokens, num_heads) and sink.stride(1) == 1, ( + "sink must be [T, heads]" + ) + assert gate.shape == q.shape, "gate must be [T, heads, head dim]" + assert window >= 0, "window is the sliding window length, or 0 for full attention" + if window > 0: + # A key past the bias extent inside the window would have no bias; Inkling 2's + # sliding layers have window == rel_extent. + assert window <= rel_logits.shape[2], "a sliding window may not exceed rel_extent" + bounds = query_start.tolist() + lengths = kv_len.tolist() + new = [b - a for a, b in zip(bounds, bounds[1:])] + assert bounds[0] == 0 and bounds[-1] == num_tokens and all(n > 0 for n in new) + assert all(length >= n for length, n in zip(lengths, new)), "kv_len counts the new tokens" + page_size = kv_pool.shape[3] + assert all(-(-length // page_size) <= page_table.shape[1] for length in lengths), ( + "page_table must cover every request's tokens" + ) + plan_request, plan_start = [], [] + for r, n in enumerate(new): + for first in range(bounds[r], bounds[r] + n, block_t): + plan_request.append(r) + plan_start.append(first) + assert tile_request.tolist() == plan_request and tile_start.tolist() == plan_start, ( + "the tile plan must cover each request's tokens in blocks of block_t" + ) + assert torch.equal( + token_request.cpu(), + torch.repeat_interleave(torch.arange(len(new), dtype=torch.int32), torch.tensor(new)), + ), "token_request must map each token to its request" + + +inkling2_paged_attention = Inkling2PagedAttention() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py new file mode 100644 index 000000000000..3ae1e0250a9e --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's width-4 causal short convolutions over a step of several requests. + +Every decoder layer runs two static convolutions on its full-width hidden states: one after the +attention output projection and one after the MoE. A sliding-window layer also convolves its key +and value projections (static); a global layer does so with dynamic taps, the static taps plus +per-token taps projected from the layer's key or value input, shared by the channels of one KV +head. Both forms add the input back and scale the sum:: + + static: out = (sum over lags of taps[lag] * x[t - lag] + x[t]) / sqrt(5) + dynamic: out = (sum over lags of (taps[lag] + dyn[t, head, lag]) * x[t - lag] + x[t]) / 3 + +in fp32, rounded once. `taps` is lag-major, `[4, C]` fp32: `taps[0]` weights the current token. +A checkpoint's `nn.Conv1d` weight `[C, 1, 4]` holds the same taps oldest first +(`taps[lag] = weight[:, 0, 3 - lag]`); a dynamic convolution's `static_w` `[4, C]` is already +lag-major. + +The 3 inputs before a request's first new token come from its window state `[slots, 3, C]` +(oldest first) when `has_state` is set, and are zeros otherwise. The call stores each request's +last 3 inputs back into its slot. +""" + +from __future__ import annotations + +import math + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from ...kernels.short_conv import KERNEL_SIZE, short_conv + +#: The static convolution's output scale, 1 / sqrt(kernel size + 1). +STATIC_SCALE = 1.0 / math.sqrt(KERNEL_SIZE + 1) +#: The dynamic convolution's output scale, 1 / sqrt(2 * kernel size + 1). +DYNAMIC_SCALE = 1.0 / math.sqrt(2 * KERNEL_SIZE + 1) + +#: Request mixes the cells run. Each entry is (new tokens, cached tokens before the step). +_FRESH_PREFILL = ((17, 0),) +_DECODE = ((1, 40), (1, 3), (1, 1), (1, 1000)) +_MIXED = ((5, 0), (2, 9), (1, 1), (1, 2), (33, 64)) + + +def _reference( + x, taps, dynamic_taps, state, slots, query_start, has_state, out_scale, channels_per_head +): + num_channels = x.shape[1] + out = torch.empty_like(x) + bounds = query_start.tolist() + updates = [] + for r, slot in enumerate(slots.tolist()): + start, end = bounds[r], bounds[r + 1] + n = end - start + if int(has_state[r]): + history = state[slot].float() + else: + history = torch.zeros(KERNEL_SIZE - 1, num_channels, device=x.device) + seq = torch.cat([history, x[start:end].float()]) + acc = torch.zeros(n, num_channels, device=x.device) + for lag in range(KERNEL_SIZE - 1, -1, -1): + tap = taps[lag].float().expand(n, num_channels) + if dynamic_taps is not None: + per_head = dynamic_taps[start:end].float().view(n, -1, KERNEL_SIZE)[:, :, lag] + tap = tap + per_head.repeat_interleave(channels_per_head, dim=1) + first = KERNEL_SIZE - 1 - lag + acc = acc + tap * seq[first : first + n] + out[start:end] = ((acc + seq[KERNEL_SIZE - 1 :]) * out_scale).to(x.dtype) + updates.append((slot, seq[-(KERNEL_SIZE - 1) :].to(state.dtype))) + for slot, window in updates: + state[slot] = window + return out + + +def _check( + x, taps, dynamic_taps, state, slots, query_start, has_state, token_request, channels_per_head +): + num_tokens, num_channels = x.shape + assert x.dim() == 2 and x.stride(1) == 1, "x must be [T, C] with contiguous channels" + assert taps.shape == (KERNEL_SIZE, num_channels) and taps.dtype == torch.float32, ( + f"taps must be lag-major fp32 [{KERNEL_SIZE}, {num_channels}]" + ) + assert taps.is_contiguous(), "taps must be contiguous" + assert state.dim() == 3 and state.shape[1:] == (KERNEL_SIZE - 1, num_channels), ( + f"state must be [slots, {KERNEL_SIZE - 1}, {num_channels}]" + ) + assert state.stride(2) == 1 and state.dtype == x.dtype, ( + "state must hold contiguous channels in x's dtype" + ) + num_requests = slots.shape[0] + metadata = ( + ("slots", slots), + ("query_start", query_start), + ("has_state", has_state), + ("token_request", token_request), + ) + for name, t in metadata: + assert t.dtype == torch.int32 and t.dim() == 1, f"{name} must be a 1-D int32 tensor" + # The kernels index request metadata at unit stride. + assert t.is_contiguous(), f"{name} must be contiguous" + assert query_start.shape[0] == num_requests + 1 and has_state.shape[0] == num_requests, ( + "query_start must be [R + 1] and has_state [R] for R = len(slots)" + ) + bounds = query_start.tolist() + assert bounds[0] == 0 and bounds[-1] == num_tokens, "query_start must run from 0 to T" + assert all(b > a for a, b in zip(bounds, bounds[1:])), "every request must append tokens" + slot_list = slots.tolist() + # Two requests in one slot race: each reads the other's window as its own. + assert len(set(slot_list)) == len(slot_list), "each request needs its own slot" + assert all(0 <= s < state.shape[0] for s in slot_list), "slots must index state" + expected = torch.repeat_interleave( + torch.arange(num_requests, dtype=torch.int32), torch.tensor(bounds).diff() + ) + assert token_request.dtype == torch.int32 and torch.equal(token_request.cpu(), expected), ( + "token_request must map each token to its request index" + ) + if dynamic_taps is not None: + assert num_channels % channels_per_head == 0, "channels must split into whole heads" + heads = num_channels // channels_per_head + assert dynamic_taps.shape == (num_tokens, heads * KERNEL_SIZE), ( + f"dynamic_taps must be [T, {heads} heads * {KERNEL_SIZE}]" + ) + assert dynamic_taps.stride(1) == 1, "dynamic_taps must be row-contiguous" + + +class Inkling2ShortConv(OpWrapper): + """Static short convolution: `x` `[T, C]` with lag-major fp32 `taps` `[4, C]`; returns `[T, C]`. + + `state` `[slots, 3, C]` (x's dtype) holds each request's last 3 inputs and is updated in + place. `slots` `[R]`, `query_start` `[R + 1]`, `has_state` `[R]` and `token_request` `[T]` + are int32 on the device; `token_request` maps each token to its request index. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="sliding-layer k/v at TP4 (256 channels), one fresh prefill", + spec=dict(channels=256, requests=_FRESH_PREFILL), + ), + Cell( + why="sliding-layer k/v at TP4, four decodes, one at position 1", + spec=dict(channels=256, requests=_DECODE), + ), + Cell( + why="hidden-width convolution (4608), a mixed step with 1- and 2-token appends", + spec=dict(channels=4608, requests=_MIXED), + ), + ) + + note = """ + The residual and the scale are part of the convolution, and the window holds the + convolution's input, not its output. A request at position 1 or 2 reads a window that is + partly the zeros before its first token, which only a decode right after a short prefill + exercises; the decode cell has one. + """ + + def __call__( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> torch.Tensor: + return short_conv( + x, taps, None, state, slots, query_start, has_state, token_request, STATIC_SCALE + ) + + def reference( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> torch.Tensor: + """Per request, the convolution over `[window or zeros, new inputs]` in fp32; updates + `state` in place as the op does. `token_request` is not needed.""" + return _reference(x, taps, None, state, slots, query_start, has_state, STATIC_SCALE, 1) + + def is_valid( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> None: + _check(x, taps, None, state, slots, query_start, has_state, token_request, 1) + + +class Inkling2DynamicShortConv(OpWrapper): + """Dynamic short convolution of a global layer's keys or values. + + As `Inkling2ShortConv`, with per-token taps `dynamic_taps` `[T, heads * 4]` (head-major, lag + minor; the layer's `weight_proj` output) added to `taps` for the `channels_per_head` channels + of each KV head, and the output scale 1/3. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="global-layer k/v at TP4 (2 heads of 128), one fresh prefill", + spec=dict(channels=256, heads=2, requests=_FRESH_PREFILL), + ), + Cell( + why="global-layer k/v at TP4, four decodes", + spec=dict(channels=256, heads=2, requests=_DECODE), + ), + Cell( + why="global-layer k/v at TP1 (8 heads), a mixed step", + spec=dict(channels=1024, heads=8, requests=_MIXED), + ), + ) + + note = """ + The per-token taps belong to the token being computed, not to the inputs it reads, and + one head's taps cover exactly that head's channels. Taps shared by every head, or applied + per input token, still give a plausible convolution; the cells use per-head random taps so + either mistake shows. + """ + + def __call__( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> torch.Tensor: + return short_conv( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + DYNAMIC_SCALE, + channels_per_head, + ) + + def reference( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> torch.Tensor: + """As `Inkling2ShortConv.reference`, with each token's head taps added to `taps`.""" + return _reference( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + DYNAMIC_SCALE, + channels_per_head, + ) + + def is_valid( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> None: + _check( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + channels_per_head, + ) + + +inkling2_short_conv = Inkling2ShortConv() +inkling2_dynamic_short_conv = Inkling2DynamicShortConv() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py new file mode 100644 index 000000000000..5d05b466a751 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's MXFP8 routed experts as a grouped MoE. + +One rank's experts `[offset, offset + E)` over every token of the step; the output is this +rank's partial sum. Per token and chosen expert `e` with weight `w`: + + x^ = mxfp8(x) E4M3 values, one power-of-two scale per 32 along K + h = situ(x^ @ gate_e^T, x^ @ up_e^T) + out += w * mxfp8(h) @ down_e^T + +`mxfp8` rounds each block's scale up to a power of two, `2 ** ceil(log2(amax / 448))`, and the +values to the nearest E4M3. Weights are the checkpoint's: `w13` `[E, 2I, H]` E4M3 with gate `j` +at row `2j` and up `j` at row `2j + 1`, `w2` `[E, H, I]`, and UE8M0 scale exponents `[E, rows, +K / 32]` (scale `2 ** (s - 127)`). + +The routes are grouped by expert on the device and both GEMMs run as one Triton launch each +(`kernels/mxfp8_moe.py`): the kernels expand the E4M3 weights to bf16 in registers, which is +exact, and accumulate in fp32, so only the summation order differs from an MXFP8 MMA. Experts' +outputs meet in fp32 atomic adds. Nothing synchronizes with the host. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ...kernels.mxfp8_moe import K_TILE, grouped_mxfp8_moe + +#: Values per MXFP8 scale. +BLOCK = 32 +_E4M3_MAX = 448.0 +_MIN_SCALE = 2.0**-127 + +_EP4 = dict(experts=384, local=96, hidden=4608, inter=2304, top_k=8) + + +def mxfp8_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """`[N, K]` -> (E4M3 values `[N, K]`, uint8 scale exponents `[N, K / 32]`).""" + n, k = x.shape + groups = x.float().view(n, k // BLOCK, BLOCK) + raw = groups.abs().amax(dim=-1) / _E4M3_MAX + # Round the scale up to a power of two: add the largest mantissa and keep the exponent. + exponent = ((raw.view(torch.int32) + 0x007FFFFF) & 0x7F800000) >> 23 + exponent = torch.where(raw <= _MIN_SCALE, torch.zeros_like(exponent), exponent.clamp(max=254)) + scale = torch.clamp((exponent << 23).view(torch.float32), min=_MIN_SCALE) + inverse = torch.where(exponent == 0, torch.full_like(scale, 2.0**127), 1.0 / scale) + values = (groups * inverse.unsqueeze(-1)).clamp(-_E4M3_MAX, _E4M3_MAX).to(torch.float8_e4m3fn) + return values.view(n, k), exponent.to(torch.uint8) + + +def mxfp8_dequantize( + values: torch.Tensor, exponents: torch.Tensor, dtype=torch.float32 +) -> torch.Tensor: + """E4M3 values `[..., K]` and uint8 exponents `[..., K / 32]` -> `[..., K]` in `dtype`.""" + scale = torch.exp2(exponents.view(torch.uint8).float() - 127.0).clamp(min=_MIN_SCALE) + out = values.float().unflatten(-1, (-1, BLOCK)) * scale.unsqueeze(-1) + return out.flatten(-2).to(dtype) + + +def _situ(gate, up, gate_softcap, up_softcap): + return ( + gate_softcap + * torch.tanh(gate / gate_softcap) + * torch.sigmoid(gate) + * up_softcap + * torch.tanh(up / up_softcap) + ) + + +class Inkling2Mxfp8Experts(OpWrapper): + """This rank's MXFP8 experts: `x` `[T, H]` bf16, `topk_ids` `[T, k]` int32 (global ids), + `topk_weights` `[T, k]` fp32 -> `[T, H]` bf16, the sum over the chosen experts this rank holds.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="EP4 rank 2, one decode token", spec=dict(_EP4, tokens=1, rank=2)), + Cell(why="EP4 rank 0, 64 tokens", spec=dict(_EP4, tokens=64, rank=0)), + Cell( + why="EP4 rank 3, 300 tokens: several row blocks per expert", + spec=dict(_EP4, tokens=300, rank=3), + ), + ) + + note = """ + Both GEMM inputs are MXFP8-quantized, the SiTU output from fp32. The gate and up rows + interleave per row in the checkpoint, unlike the NVFP4 layers' kernel layout; reading + them as two halves still gives a finite, wrong result. The fp32 atomics make the last + bits of repeated calls differ. + """ + + def __call__( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + x_hat = mxfp8_dequantize(*mxfp8_quantize(x), dtype=torch.bfloat16) + out = grouped_mxfp8_moe( + x_hat, + topk_ids, + topk_weights, + w13, + w13_s, + w2, + w2_s, + local_expert_offset, + gate_softcap, + up_softcap, + ) + return out.to(x.dtype) + + def reference( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + """True fp32 GEMMs on the fp32-dequantized weights and activations, gate and up taken from + the even and odd rows.""" + local = w13.shape[0] + x_hat = mxfp8_dequantize(*mxfp8_quantize(x)) + out = torch.zeros(x.shape, dtype=torch.float32, device=x.device) + for e in range(local): + tokens, slots = torch.where(topk_ids == local_expert_offset + e) + if tokens.numel() == 0: + continue + fc1 = mxfp8_dequantize(w13[e], w13_s[e]) + down = mxfp8_dequantize(w2[e], w2_s[e]) + with true_fp32_matmul(): + h = _situ( + x_hat[tokens] @ fc1[0::2].t(), + x_hat[tokens] @ fc1[1::2].t(), + gate_softcap, + up_softcap, + ) + y = mxfp8_dequantize(*mxfp8_quantize(h)) @ down.t() + out.index_add_(0, tokens, y * topk_weights[tokens, slots].float().unsqueeze(1)) + return out.to(x.dtype) + + def is_valid( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> None: + local, rows, hidden = w13.shape + assert w13.dtype == w2.dtype == torch.float8_e4m3fn, "MXFP8 weights are E4M3" + assert w13_s.shape == (local, rows, hidden // BLOCK) and w13_s.dtype == torch.uint8 + assert w2.shape == (local, hidden, rows // 2), "w2 must be [E, H, I]" + assert w2_s.shape == (local, hidden, rows // 2 // BLOCK) and w2_s.dtype == torch.uint8 + assert x.dtype == torch.bfloat16 and x.shape[1] == hidden + assert topk_ids.dtype == torch.int32 and topk_weights.dtype == torch.float32 + assert hidden % K_TILE == 0 and (rows // 2) % K_TILE == 0, ( + f"hidden and intermediate sizes must be multiples of the {K_TILE}-wide K tile" + ) + + +inkling2_mxfp8_experts = Inkling2Mxfp8Experts() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py new file mode 100644 index 000000000000..779eb3c7d5ce --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's NVFP4 routed experts on FlashInfer's CuTe DSL fused MoE (W4A4, SiTU). + +One rank's experts `[offset, offset + E)` over every token of the step; the output is this +rank's partial sum, which an all-reduce over the EP group completes. Per token and chosen expert +`e` with weight `w`: + + x^ = nvfp4(x) per token, 4/6 block scales (E4M3 max 256) + h = situ(x^ @ gate_e^T, x^ @ up_e^T) + = 4 tanh(g / 4) sigmoid(g) * 25 tanh(u / 25) + out += w * nvfp4(h) @ down_e^T + +Weights are the checkpoint's NVFP4 tensors: E2M1 pairs (low nibble first), one E4M3 block scale +per 16 values along K, and a per-expert global scale `alpha = amax / outer_scale_denominator` +applied after the block-scaled product. In the layout the kernel reads: + +* `w13` `[E, 2I, H / 2]`: rows in FlashInfer's gated FC1 order (`fc1_row_order`): in each group + of 128 rows, up rows `64g .. 64g + 63` then gate rows `64g ..`. The checkpoint holds gate `j` + at row `2j` and up `j` at row `2j + 1`. +* `w13_sf` `[E, 2I, H / 16]`, `w2_sf` `[E, H, I / 16]`: those rows' block scales, swizzled into + 128x4 atoms (`swizzle_block_scales`), as uint8. +* `w2` `[E, H, I / 2]` as stored. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +#: Per-token activation global scale of the 4/6 recipe with an E4M3 max of 256: 1 / (6 * 256). +ACT_GLOBAL_SCALE = 1.0 / (6.0 * 256.0) +#: FlashInfer's FC1 interleave: groups of 64 up rows, then 64 gate rows. +FC1_GROUP = 64 + +_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0) + +#: Inkling-2-Small at EP4: 96 local experts of 384, hidden 4608, intermediate 2304. +_EP4 = dict(experts=384, local=96, hidden=4608, inter=2304, top_k=8) + + +def recipe(): + """FlashInfer's 4/6 recipe the checkpoint was quantized for.""" + from flashinfer.quantization.nvfp4_quantization_utils import NVFP44Over6Config + + return NVFP44Over6Config(e4m3_max=256, err_mode="MAE", err_use_fast_math=False) + + +def fc1_row_order(intermediate: int, device: torch.device) -> torch.Tensor: + """Checkpoint row of each kernel FC1 row, for one expert's `[2I, ...]` w13 (see module doc).""" + j = torch.arange(intermediate, device=device) + halves = torch.stack([2 * j + 1, 2 * j]) # [up; gate] + return halves.view(2, intermediate // FC1_GROUP, FC1_GROUP).transpose(0, 1).reshape(-1) + + +def swizzle_block_scales(scales: torch.Tensor) -> torch.Tensor: + """`[E, M, K / 16]` uint8 linear block scales -> 128x4 atoms, the layout of FlashInfer's + `block_scale_interleave`: row `m`, column `k` at `(m // 128, k // 4, m % 32, (m % 128) // 32, + k % 4)`. Needs `M % 128 == 0` and `(K / 16) % 4 == 0`.""" + e, m, ks = scales.shape + assert m % 128 == 0 and ks % 4 == 0, (m, ks) + return ( + scales.reshape(e, m // 128, 4, 32, ks // 4, 4) + .permute(0, 1, 4, 3, 2, 5) + .contiguous() + .view(e, m, ks) + ) + + +def unswizzle_block_scales(scales: torch.Tensor) -> torch.Tensor: + """Inverse of `swizzle_block_scales`.""" + e, m, ks = scales.shape + return ( + scales.reshape(e, m // 128, ks // 4, 32, 4, 4) + .permute(0, 1, 4, 3, 2, 5) + .contiguous() + .view(e, m, ks) + ) + + +def dequantize_nvfp4(values: torch.Tensor, block_scales: torch.Tensor) -> torch.Tensor: + """`[..., K / 2]` packed E2M1 and `[..., K / 16]` E4M3 block scales (uint8 bits) -> fp32 + `[..., K]`, without any global scale.""" + lut = torch.tensor(_E2M1, device=values.device) + v = values.view(torch.uint8) + pairs = torch.stack([lut[(v & 0xF).long()], lut[(v >> 4).long()]], dim=-1).flatten(-2) + scales = block_scales.view(torch.uint8).view(torch.float8_e4m3fn).float() + return (pairs.unflatten(-1, (-1, 16)) * scales.unsqueeze(-1)).flatten(-2) + + +def quantize_activations(x: torch.Tensor): + """Per-token 4/6 NVFP4 activations: (packed values, linear E4M3 scales as uint8, per-token + decode scales), with `x ~= e2m1 * block_scale * per_token`.""" + from flashinfer import SfLayout, nvfp4_quantize + + return nvfp4_quantize( + x, + ACT_GLOBAL_SCALE, + sfLayout=SfLayout.layout_linear, + per_token_activation=True, + backend="cute-dsl", + nvfp4_4over6=recipe(), + ) + + +def fc2_input_scale(device: torch.device) -> torch.Tensor: + """The GEMM2-input global scale the kernel takes, `[1]` fp32 `1 / (6 * 256)`. + + FlashInfer treats it as a weight-pack constant: it reads it to the host once per tensor and + remembers it by storage and version, so a caller builds it once, outside inference mode, + and passes the same tensor on every call. + """ + with torch.inference_mode(False): + return torch.full((1,), ACT_GLOBAL_SCALE, device=device, dtype=torch.float32) + + +def _situ( + gate: torch.Tensor, up: torch.Tensor, gate_softcap: float, up_softcap: float +) -> torch.Tensor: + return ( + gate_softcap + * torch.tanh(gate / gate_softcap) + * torch.sigmoid(gate) + * up_softcap + * torch.tanh(up / up_softcap) + ) + + +class Inkling2Nvfp4Experts(OpWrapper): + """This rank's NVFP4 experts: `x` `[T, H]` bf16, `topk_ids` `[T, k]` int32 (global ids), + `topk_weights` `[T, k]` fp32 -> `[T, H]` bf16, the sum over the chosen experts this rank holds. + Weight layouts as in the module docstring; `w13_alpha`, `w2_alpha` `[E]` fp32; `fc2_scale` the + persistent GEMM2-input scale from `fc2_input_scale`.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="EP4 rank 1, one decode token", spec=dict(_EP4, tokens=1, rank=1)), + Cell(why="EP4 rank 0, 64 tokens", spec=dict(_EP4, tokens=64, rank=0)), + Cell(why="EP4 rank 3, 300 tokens", spec=dict(_EP4, tokens=300, rank=3)), + ) + + note = """ + The SiTU caps are the model's mlp_gate_softcap (4) and mlp_up_softcap (25); the sigmoid + sees the uncapped gate. FlashInfer requantizes the SiTU output per token with the same 4/6 + recipe before the down projection, from fp32; the reference requantizes the bf16-rounded + intermediate with the same quantizer, which moves a few block choices. The kernel adds the + experts' outputs with atomics, so repeated calls differ in the last bf16 bit. Swapping gate + and up, or the 64-row interleave halves, gives a finite, plausibly scaled and wrong output. + """ + + def __call__( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + from flashinfer import cute_dsl_fused_moe + from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout + + local, rows, packed_k = w13.shape + hidden, inter = packed_k * 2, rows // 2 + values, scales, per_token = quantize_activations(x) + return cute_dsl_fused_moe( + values, + scales, + topk_ids, + topk_weights, + w13, + convert_sf_to_mma_layout(w13_sf, m=rows, k=hidden, num_groups=local, sf_vec_size=16), + w13_alpha, + fc2_scale, + w2, + convert_sf_to_mma_layout(w2_sf, m=hidden, k=inter, num_groups=local, sf_vec_size=16), + w2_alpha, + num_experts, + topk_ids.shape[1], + num_local_experts=local, + local_expert_offset=local_expert_offset, + situ_beta=gate_softcap, + situ_linear_beta=up_softcap, + quant_mode="w4a4", + per_token_scale=per_token, + nvfp4_4over6=recipe(), + ) + + def reference( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + """Dense fp32 per expert from the dequantized weights; the activations and the SiTU output + (rounded to bf16) go through the same per-token 4/6 quantizer. `num_experts` and `fc2_scale` + (a constant) are unused.""" + local, rows, _ = w13.shape + inter = rows // 2 + values, scales, per_token = quantize_activations(x) + x_hat = dequantize_nvfp4(values, scales) * per_token.float().view(-1, 1) + out = torch.zeros(x.shape, dtype=torch.float32, device=x.device) + for e in range(local): + tokens, slots = torch.where(topk_ids == local_expert_offset + e) + if tokens.numel() == 0: + continue + fc1 = ( + dequantize_nvfp4(w13[e], unswizzle_block_scales(w13_sf[e : e + 1])[0]) + * w13_alpha[e] + ) + groups = fc1.view(inter // FC1_GROUP, 2, FC1_GROUP, -1) + up, gate = groups[:, 0].reshape(inter, -1), groups[:, 1].reshape(inter, -1) + down = ( + dequantize_nvfp4(w2[e], unswizzle_block_scales(w2_sf[e : e + 1])[0]) * w2_alpha[e] + ) + with true_fp32_matmul(): + h = _situ( + x_hat[tokens] @ gate.t(), x_hat[tokens] @ up.t(), gate_softcap, up_softcap + ) + h_values, h_scales, h_per_token = quantize_activations(h.to(torch.bfloat16)) + h_hat = dequantize_nvfp4(h_values, h_scales) * h_per_token.float().view(-1, 1) + out.index_add_( + 0, tokens, (h_hat @ down.t()) * topk_weights[tokens, slots].float().unsqueeze(1) + ) + return out.to(x.dtype) + + def is_valid( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> None: + local, rows, packed_k = w13.shape + assert x.dtype == torch.bfloat16 and x.shape[1] == 2 * packed_k, "x must be bf16 [T, H]" + assert w13.dtype == w2.dtype == torch.uint8, "packed NVFP4 weights are uint8" + assert w13_sf.shape == (local, rows, 2 * packed_k // 16), "w13_sf must be [E, 2I, H / 16]" + assert w2.shape == (local, 2 * packed_k, rows // 4), "w2 must be [E, H, I / 2]" + assert w2_sf.shape == (local, 2 * packed_k, rows // 32), "w2_sf must be [E, H, I / 16]" + assert w13_alpha.dtype == w2_alpha.dtype == torch.float32 and w13_alpha.shape == (local,) + assert topk_ids.dtype == torch.int32 and topk_weights.dtype == torch.float32 + assert 0 <= local_expert_offset and local_expert_offset + local <= num_experts + assert fc2_scale.shape == (1,) and abs(float(fc2_scale[0]) / ACT_GLOBAL_SCALE - 1) < 1e-6, ( + "fc2_scale is 1 / (6 * 256)" + ) + + +inkling2_nvfp4_experts = Inkling2Nvfp4Experts() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py new file mode 100644 index 000000000000..8e41caf2c5a6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's MoE router: fp32 sigmoid scores, bias-corrected top-k selection, renormalized +weights. + + logits = x @ W^T fp32 (bf16 operands, fp32 accumulation) + scores = sigmoid(logits) + experts = top_k(scores + bias) the bias steers selection only + weights = scores[experts] / (sum(scores[experts]) + 1e-20) * route_scale +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm + +_INKLING2 = dict(hidden=4608, experts=384, top_k=8, route_scale=8.0) + + +class Inkling2Router(OpWrapper): + """`x` `[T, H]` bf16, `weight` `[E, H]` bf16, `bias` `[E]` fp32 -> (`topk_ids` `[T, k]` int32, + `topk_weights` `[T, k]` fp32). The ids are global expert ids, in no particular order.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="Inkling-2-Small, one decode token", spec=dict(_INKLING2, tokens=1)), + Cell(why="Inkling-2-Small, 257 tokens", spec=dict(_INKLING2, tokens=257)), + ) + + note = """ + The selection adds the correction bias to the sigmoid scores, but the weights come from + the uncorrected scores, renormalized over the chosen experts and multiplied by + route_scale (8). A weight taken from the biased scores, or a softmax router, still sums + to route_scale; only the per-expert weights show the difference. + """ + + def __call__( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + logits = cublas_mm(x, weight.t(), None, torch.float32) + scores = torch.sigmoid(logits) + ids = torch.topk(scores + bias, top_k, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) * route_scale + return ids.to(torch.int32), weights + + def reference( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + """fp32 logits from a true fp32 product; selection and weights as the op.""" + with true_fp32_matmul(): + logits = x.float() @ weight.float().t() + scores = torch.sigmoid(logits) + ids = torch.topk(scores + bias.float(), top_k, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) * route_scale + return ids.to(torch.int32), weights + + def is_valid( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> None: + # A bf16 bias rounds the correction and changes which experts win; an fp32 weight + # would need a different GEMM. Neither raises. + assert bias.dtype == torch.float32, "the correction bias is fp32" + assert x.dtype == weight.dtype == torch.bfloat16, "x and the router weight are bf16" + assert weight.shape == (bias.shape[0], x.shape[1]), "weight must be [experts, hidden]" + + +inkling2_router = Inkling2Router() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py new file mode 100644 index 000000000000..e4a8c1ff4f34 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's block attention residual: a layer's input is a learned mix of the completed +residual blocks and the current partial block, followed by that input's RMSNorm. + +Every 6 layers the residual stream closes a block: the input of layers 0, 6, 12, ... (the +embedding for layer 0) becomes a block state, and the stream restarts from the next sublayer's +output. A sublayer reads, instead of the stream, a softmax mix over the unit-RMS-normalized +block states and partial block, with a learned fp32 query per input (3 for a global layer's q, k +and v inputs, 1 otherwise; the final norm has its own): + + n_s = s / rms(s) (eps 1e-6), for each block s and the partial block + p_q = softmax_s(q . n_s / sqrt(H)) + mix_q = sum_s p_q[s] * n_s + out_q = weight_q * mix_q / rms(mix_q) (eps), rounded to bf16 + +all in fp32. `blocks` holds the completed blocks already normalized, `[S, T, H]` fp32; a block's +normalization is the same in every later mix, so the caller computes it once +(`normalize_block`). +""" + +from __future__ import annotations + +import math +from typing import Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +#: The unweighted normalization of block states (Inkling2UnweightedRMSNorm's default). +BLOCK_EPS = 1e-6 + + +def normalize_block(state: torch.Tensor) -> torch.Tensor: + """A block state `[T, H]` divided by its RMS (eps 1e-6), fp32.""" + x = state.float() + return x * torch.rsqrt(x.square().mean(-1, keepdim=True) + BLOCK_EPS) + + +def rmsnorm_to_bf16(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + """`weight * x / rms(x)` in fp32, rounded once to bf16 (Inkling2RMSNorm on an fp32 input).""" + x = x.float() + return (weight.float() * (x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps))).to( + torch.bfloat16 + ) + + +class Inkling2ResidualMix(OpWrapper): + """`blocks` `[S, T, H]` fp32 (normalized), `partial` `[T, H]` (bf16 or fp32), `queries` `[Q, H]` + fp32, `norm_weights` Q weights `[H]` -> `[Q, T, H]` bf16, each mix through its RMSNorm.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="layer 1's attention input: the embedding block, one query", + spec=dict(blocks=1, queries=1, tokens=17), + ), + Cell( + why="a global layer's q/k/v inputs: 3 queries over 5 blocks", + spec=dict(blocks=5, queries=3, tokens=33), + ), + Cell( + why="the final norm's input: 8 blocks, one decode token", + spec=dict(blocks=8, queries=1, tokens=1), + ), + ) + + note = """ + Values are the normalized states, not the raw ones: a mix of raw states is plausible but + scaled by the states' RMS. The partial block is normalized here, the completed blocks by + the caller, once. + """ + + def __call__( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> torch.Tensor: + hidden = partial.shape[-1] + n_p = normalize_block(partial) + with true_fp32_matmul(): + logits_b = torch.einsum("sth,qh->tqs", blocks, queries) + logit_p = (n_p @ queries.t()).unsqueeze(-1) + probs = torch.softmax(torch.cat([logits_b, logit_p], dim=-1) / math.sqrt(hidden), dim=-1) + with true_fp32_matmul(): + mix = torch.einsum("tqs,sth->qth", probs[..., :-1].contiguous(), blocks) + mix = mix + probs[..., -1].t().unsqueeze(-1) * n_p.unsqueeze(0) + return torch.stack([rmsnorm_to_bf16(mix[i], w, eps) for i, w in enumerate(norm_weights)]) + + def reference( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> torch.Tensor: + """fp64 throughout, the partial block appended to the normalized blocks as one more state.""" + hidden = partial.shape[-1] + x = partial.double() + n_p = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + BLOCK_EPS) + stacked = torch.cat([blocks.double(), n_p.unsqueeze(0)]) # [S + 1, T, H] + probs = torch.softmax( + torch.einsum("sth,qh->tqs", stacked, queries.double()) / math.sqrt(hidden), dim=-1 + ) + mix = torch.einsum("tqs,sth->qth", probs, stacked) + out = [] + for i, w in enumerate(norm_weights): + m = mix[i] + out.append( + (w.double() * (m * torch.rsqrt(m.square().mean(-1, keepdim=True) + eps))).to( + torch.bfloat16 + ) + ) + return torch.stack(out) + + def is_valid( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> None: + assert blocks.dtype == torch.float32 and blocks.dim() == 3, ( + "blocks are normalized fp32 [S, T, H]" + ) + assert blocks.shape[1:] == partial.shape, "blocks and the partial block share [T, H]" + assert queries.dtype == torch.float32 and queries.shape[1] == partial.shape[1], ( + "queries are fp32 [Q, H]" + ) + assert len(norm_weights) == queries.shape[0], "one norm weight per query" + + +inkling2_residual_mix = Inkling2ResidualMix() diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 81c0bb1c2e5b..b7bee3368f09 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -177,6 +177,7 @@ l0_b300: - unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4 TIMEOUT (30) - unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency TIMEOUT (30) - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default TIMEOUT (30) + - unittest/_torch/modeling_v2/models/inkling2/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 9e9e052798f8..7e61f1e76d8e 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -157,6 +157,10 @@ l0_cpu: - unittest/_torch/modeling_v2/models/deepseek_v3/r1_0528_nvfp4__sm_10x__default/test_modeling_v2_deepseek_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_weight_table.py + - unittest/_torch/modeling_v2/models/inkling2/test_config.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py # The Kimi K3 SM107 targets' cpu_only cases: CuTe DSL hint tables, the FP4 MLA # decode plan geometry, each target's constants and model defaults, the routing # of the recipes' mappings, the context MLA split-K dispatch gates, the diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py new file mode 100644 index 000000000000..26be3f516865 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling 2 KV cache manager: block reuse refused, convolution slot ownership.""" + +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + cache, +) +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import ( + KVCacheManagerV2, + _max_admissible_sequences, +) +from tensorrt_llm.mapping import Mapping + +pytestmark = pytest.mark.cpu_only + + +def test_block_reuse_is_refused_before_any_allocation(monkeypatch): + def base_init(self, *args, **kwargs): + self.enable_block_reuse = True + self.gen_preemption_headroom = 0 + + monkeypatch.setattr(KVCacheManagerV2, "__init__", base_init) + # Anything past the refusal would read the config or allocate on the GPU. + with pytest.raises(ValueError, match="enable_block_reuse"): + cache.Inkling2KVCacheManager( + pretrained_config=SimpleNamespace(), mapping=None, max_batch_size=4 + ) + + +def _small_text_config() -> SimpleNamespace: + """Two layers (the second global) of the Inkling 2 text config, small enough for a CPU.""" + return SimpleNamespace( + num_hidden_layers=2, + local_layer_ids=[0], + hidden_size=64, + vocab_size=128, + num_attention_heads=4, + num_key_value_heads=4, + head_dim=16, + d_rel=16, + rel_extent=64, + sliding_window_size=64, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=8, + num_experts_per_tok=2, + intermediate_size=128, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + + +@pytest.mark.parametrize( + "disable_overlap_scheduler, admissible", + [(True, 23 + 9), (False, 2 * 23 + 9)], + ids=["overlap_off", "overlap_on"], +) +def test_conv_slots_cover_every_sequence_the_manager_admits( + monkeypatch, disable_overlap_scheduler, admissible +): + """Paused requests (GEN preemption headroom 9) and, under the overlap scheduler, retiring ones + hold their windows too: the pool is the base manager's admission count plus the spares.""" + mapping = Mapping(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4) + + def base_init(self, *args, **kwargs): + self.enable_block_reuse = False + self.max_admissible_sequences = _max_admissible_sequences( + 23, + mapping, + is_disagg=False, + disable_overlap_scheduler=disable_overlap_scheduler, + gen_preemption_headroom=9, + ) + + monkeypatch.setattr(KVCacheManagerV2, "__init__", base_init) + # The windows are allocated on the current CUDA device; a CPU stands in for it here. + zeros = torch.zeros + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + monkeypatch.setattr( + torch, "zeros", lambda *shape, dtype=None, device=None: zeros(*shape, dtype=dtype) + ) + manager = cache.Inkling2KVCacheManager( + pretrained_config=SimpleNamespace(text_config=_small_text_config()), + mapping=mapping, + max_batch_size=23, + ) + slots_total = admissible + cache.SPARE_SLOTS + assert manager.num_conv_slots == slots_total + assert all(t.shape[0] == slots_total for t in manager.conv_state(1).values()) + + resident = [f"request-{i}" for i in range(admissible)] + slots = manager.conv_slots(resident).tolist() + assert len(set(slots)) == admissible + # The earliest owners keep their slots while the rest are resident. + assert manager.conv_slots(resident[:3]).tolist() == slots[:3] + # The spare slots (CUDA-graph padding, warmup) are still there. + spares = manager.conv_slots([f"spare-{i}" for i in range(cache.SPARE_SLOTS)]).tolist() + assert len(set(spares)) == cache.SPARE_SLOTS + with pytest.raises(RuntimeError, match=f"all {slots_total} short-convolution slots are in use"): + manager.conv_slots(["one-too-many"]) + + +def test_a_released_slot_goes_to_the_next_request_and_others_keep_theirs(): + slots = cache.ConvSlots(3) + a, b = slots.acquire(["a", "b"]) + assert a != b + # A live request, paused or not, keeps its slot from step to step. + assert slots.acquire(["b", "a"]) == [b, a] + slots.release("a") + (c,) = slots.acquire(["c"]) + assert c == a + assert slots.acquire(["b", "c"]) == [b, c] + # Releasing a request that holds no slot changes nothing. + slots.release("never-seen") + slots.release("a") + assert slots.acquire(["b", "c"]) == [b, c] + + +def test_slots_run_out_only_when_every_one_is_held(): + slots = cache.ConvSlots(2) + slots.acquire(["a", "b"]) + with pytest.raises(RuntimeError, match="all 2 short-convolution slots are in use"): + slots.acquire(["c"]) + slots.release("b") + assert slots.acquire(["c"]) == [1] diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py new file mode 100644 index 000000000000..22eb7af27dbc --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling 2 target's engine glue: the page tables against the KV cache's +retention, and the config errors raised before any weight is declared.""" + +import re +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + modeling, +) + +pytestmark = pytest.mark.cpu_only + +#: Layer 0 slides over 8 tokens and layer 1 is global, with 4 tokens per KV cache block. +_WINDOW = 8 +_TOKENS_PER_BLOCK = 4 + + +class _Manager: + """The rows `KVCacheManagerV2.get_batch_cache_indices` returns: per request, a sliding layer's + pages (-1 for a block it has recycled) and a global layer's.""" + + tokens_per_block = _TOKENS_PER_BLOCK + + def __init__(self, sliding, full): + self.rows = {False: sliding, True: full} + + def get_batch_cache_indices(self, request_ids, layer_idx, num_blocks_per_seq): + rows = self.rows[layer_idx == 1] + return [rows[r][:n] for r, n in zip(request_ids, num_blocks_per_seq)] + + +def _page_tables(manager, cached, new): + core = SimpleNamespace( + dims=SimpleNamespace( + num_layers=2, sliding_window=_WINDOW, is_global=lambda layer: layer == 1 + ), + _shared_tables=None, + _check_live_pages=modeling.Inkling2SmallCore._check_live_pages, + ) + blocks = [-(-(c + n) // _TOKENS_PER_BLOCK) for c, n in zip(cached, new)] + request_ids = list(range(len(cached))) + return modeling.Inkling2SmallCore._page_tables( + core, manager, request_ids, blocks, cached, torch.device("cpu") + ) + + +@pytest.mark.parametrize("history", [20, 23], ids=["window_starts_in_block_3", "in_block_4"]) +def test_pages_recycled_before_the_window_are_not_read(history): + """A decode step at `history` reads tokens `history - 7 .. history`: from block 3 at 20 and + from block 4 at 23, where block 3 may be gone too.""" + first = (history + 1 - _WINDOW) // _TOKENS_PER_BLOCK + sliding = [-1] * first + list(range(first, 6)) + manager = _Manager(sliding=[sliding, [10, 11]], full=[list(range(6)), [10, 11]]) + tables = _page_tables(manager, cached=[history, 0], new=[1, 6]) + # A table is as wide as the longest request's blocks; a shorter request's row ends in -1. + assert tables[0].tolist() == [sliding, [10, 11, -1, -1, -1, -1]] + assert tables[1].tolist() == [list(range(6)), [10, 11, -1, -1, -1, -1]] + + +@pytest.mark.parametrize( + "sliding, full, hole", + [ + ([-1, -1, -1, -1, 4, 5], list(range(6)), "layer 0: request 0 reads KV cache block 3,"), + ([-1, -1, -1, 3, 4, 5], [-1, 1, 2, 3, 4, 5], "layer 1: request 0 reads KV cache block 0,"), + ([-1, -1, -1, 3, 4], list(range(6)), "layer 0: request 0 reads KV cache block 5,"), + ], + ids=["sliding_page_inside_the_window", "global_page", "row_shorter_than_the_read"], +) +def test_a_missing_page_the_step_reads_is_an_error(sliding, full, hole): + manager = _Manager(sliding=[sliding], full=[full]) + with pytest.raises(RuntimeError, match=re.escape(hole)): + _page_tables(manager, cached=[20], new=[1]) + + +def test_the_model_names_the_quant_config_it_could_not_find(tmp_path): + config = SimpleNamespace( + text_config=SimpleNamespace(), _name_or_path=str(tmp_path), torch_dtype=torch.bfloat16 + ) + with pytest.raises(ValueError, match=re.escape(f"no hf_quant_config.json in '{tmp_path}'")): + modeling.Inkling2SmallCore(SimpleNamespace(pretrained_config=config)) + + +def test_the_model_refuses_a_config_without_a_dtype(): + config = SimpleNamespace(text_config=SimpleNamespace(), torch_dtype=None) + with pytest.raises(ValueError, match="declares no dtype"): + modeling.ModelingV2Inkling2SmallNvfp4Sm103Tep4(SimpleNamespace(pretrained_config=config)) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py new file mode 100644 index 000000000000..3f2876d111c0 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py @@ -0,0 +1,419 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling-2-Small TP4/EP4 weight table against the checkpoint's tensor layout. + +The layout below restates the released checkpoint's text-decoder tensors (names, shapes, dtypes), +as its safetensors headers declare them. Every rank's transforms run on meta tensors of those +shapes, so the table's declared shares, its sources and the checkpoint cannot drift apart +unnoticed. +""" + +import os +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.config import ExpertFormat, InklingDims +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + weights as w, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_nvfp4_experts import ( + unswizzle_block_scales, +) + +pytestmark = pytest.mark.cpu_only + +U8, F8, BF16, F32 = torch.uint8, torch.float8_e4m3fn, torch.bfloat16, torch.float32 +GLOBAL = tuple(range(5, 48, 6)) + + +def _dims() -> InklingDims: + text = SimpleNamespace( + num_hidden_layers=48, + local_layer_ids=[i for i in range(48) if i not in GLOBAL], + hidden_size=4608, + vocab_size=201024, + unpadded_vocab_size=200058, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + d_rel=16, + rel_extent=1024, + sliding_window_size=1024, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=384, + num_experts_per_tok=8, + intermediate_size=2304, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + groups = [ + { + "compute_dtype": "nvfp4", + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in range(40)], + }, + { + "compute_dtype": "mxfp8", + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in range(40, 48)], + }, + ] + return InklingDims.from_config(text, {"model": {"parameter_groups": groups}}) + + +def checkpoint_layout(d: InklingDims) -> dict: + """name -> (shape, dtype) of the checkpoint's text-decoder tensors for geometry `d`. + + For `_dims()` these are the released checkpoint's shapes. + """ + h, e, i = d.hidden_size, d.num_experts, d.expert_intermediate_size + q, kv, r = d.num_heads * d.head_dim, d.num_kv_heads * d.head_dim, d.num_heads * d.d_rel + k = d.conv_kernel_size + out = { + "model.llm.embed.weight": ((d.vocab_size, h), BF16), + "model.llm.embed_norm.weight": ((h,), BF16), + "model.llm.norm.weight": ((h,), BF16), + "model.llm.unembed.weight": ((d.vocab_size, h), BF16), + "model.llm.attn_res.output_query": ((h,), F32), + } + for layer in range(d.num_layers): + p, a = f"model.llm.layers.{layer}", f"model.llm.layers.{layer}.attn" + out[f"model.llm.attn_res.mlp_queries.{layer}"] = ((h,), F32) + if layer > 0: + out[f"model.llm.attn_res.attn_queries.{layer - 1}"] = ( + (3, h) if d.is_global(layer) else (h,), + F32, + ) + out.update({ + f"{a}.wq_du.weight": ((q, h), BF16), + f"{a}.wk_dv.weight": ((kv, h), BF16), + f"{a}.wv_dv.weight": ((kv, h), BF16), + f"{a}.wr_du.weight": ((r, h), BF16), + f"{a}.wo_ud.weight": ((h, q), BF16), + f"{a}.gate_attn.weight": ((q, h), BF16), + f"{a}.attn_sink_proj.weight": ((d.num_heads, h), BF16), + f"{a}.q_l2_clip.scale": ((d.num_heads, d.head_dim), BF16), + f"{a}.rel_l2_clip.scale": ((d.num_heads, d.d_rel), BF16), + f"{a}.rel_logits_proj.proj": ((d.d_rel, d.rel_extent_of(layer)), BF16), + f"{p}.attn_sconv.weight": ((h, 1, k), BF16), + f"{p}.mlp_sconv.weight": ((h, 1, k), BF16), + f"{p}.mlp_norm.weight": ((h,), BF16), + f"{p}.mlp.gate.weight": ((e, h), BF16), + f"{p}.mlp.gate.bias": ((e,), F32), + }) # fmt: skip + if d.is_global(layer): + for s in ("k", "v"): + out[f"{a}.{s}_sconv.static_w"] = ((k, kv), BF16) + out[f"{a}.{s}_sconv.weight_proj"] = ((d.num_kv_heads * k, h), BF16) + for n in ("q", "k", "v"): + out[f"{p}.{n}_input_norm.weight"] = ((h,), BF16) + else: + out[f"{a}.k_sconv.weight"] = ((kv, 1, k), BF16) + out[f"{a}.v_sconv.weight"] = ((kv, 1, k), BF16) + out[f"{p}.attn_norm.weight"] = ((h,), BF16) + x = f"{p}.mlp.experts" + if d.expert_formats[layer] is ExpertFormat.NVFP4: + out.update({ + f"{x}.w13_weight": ((e, 2 * i, h // 2), U8), + f"{x}.w13_weight_scale": ((e, 2 * i, h // 16), F8), + f"{x}.w13_weight_amax": ((e,), F32), + f"{x}.w13_weight_outer_scale_denominator": ((1,), F32), + f"{x}.w2_weight": ((e, h, i // 2), U8), + f"{x}.w2_weight_scale": ((e, h, i // 16), F8), + f"{x}.w2_weight_amax": ((e,), F32), + f"{x}.w2_weight_outer_scale_denominator": ((1,), F32), + }) # fmt: skip + else: + out.update({ + f"{x}.w13_weight": ((e, 2 * i, h), F8), + f"{x}.w13_weight_scale": ((e, 2 * i, h // 32), U8), + f"{x}.w2_weight": ((e, h, i), F8), + f"{x}.w2_weight_scale": ((e, h, i // 32), U8), + }) # fmt: skip + return out + + +def _mapping(rank: int): + return SimpleNamespace( + pp_size=1, + cp_size=1, + enable_attention_dp=False, + moe_tp_size=1, + moe_ep_size=4, + tp_size=4, + tp_rank=rank, + moe_ep_rank=rank, + ) + + +@pytest.mark.parametrize("rank", range(4)) +def test_every_entry_transforms_to_its_declared_share(rank): + d = _dims() + s = w.shares_of(d, _mapping(rank)) + layout = checkpoint_layout(d) + used = set() + for layer, entry in w.all_entries(s): + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = [] + for name, rows in zip(entry.sources, flags): + assert name in layout, f"layer {layer} {entry.name}: {name} is not a checkpoint tensor" + shape, dtype = layout[name] + if rows: + shape = (s.local_experts, *shape[1:]) + srcs.append(torch.empty(shape, dtype=dtype, device="meta")) + assert name not in used, f"{name} is read twice" + used.add(name) + out = entry.transform(tuple(srcs), s) + if out.dtype != entry.dtype and out.dtype == U8 and entry.dtype.itemsize == 1: + out = out.view(entry.dtype) + assert tuple(out.shape) == tuple(entry.shape), ( + layer, + entry.name, + tuple(out.shape), + entry.shape, + ) + assert out.dtype == entry.dtype, (layer, entry.name, out.dtype, entry.dtype) + used.add("model.llm.unembed.weight") + assert used == set(layout), sorted(set(layout) ^ used)[:8] + + +def test_shares_split_heads_and_experts_evenly(): + s = w.shares_of(_dims(), _mapping(2)) + assert (s.heads, s.kv_heads, s.q_width, s.kv_width, s.r_width) == (12, 2, 1536, 256, 192) + assert (s.local_experts, s.expert_offset, s.experts()) == (96, 192, slice(192, 288)) + assert s.rows(6144) == slice(3072, 4608) and s.rows(32) == slice(16, 24) + + +def test_transforms_put_values_where_the_kernels_read_them(): + """Real tensors whose values name their source row, against expectations built by hand.""" + s = w.shares_of(_dims(), _mapping(2)) + # A conv weight stores the oldest tap first; the kernels read lag 0, the newest input, first. + taps = torch.arange(8, dtype=F32).reshape(2, 1, 4) + assert w._conv_taps((taps,), s).tolist() == [[3.0, 7.0], [2.0, 6.0], [1.0, 5.0], [0.0, 4.0]] + # A sliding layer's k/v conv: this rank's 256 of the 1024 channels, reversed the same way. + kv_taps = torch.arange(1024 * 4, dtype=F32).reshape(1024, 1, 4) + out = w._kv_conv_taps((kv_taps,), s) + assert out.shape == (4, 256) + assert all( + out[lag, c] == kv_taps[512 + c, 0, 3 - lag] for lag in range(4) for c in (0, 77, 255) + ) + # A global layer's static taps are lag-major already: this rank's channels. + static = torch.arange(4 * 1024, dtype=F32).reshape(4, 1024) + assert torch.equal(w._static_taps((static,), s), static[:, 512:768]) + # Stacked projections: each source's rows of this rank, in the order the entry names them. + sizes = (6144, 768, 6144, 48) # wq, wr, gate, sink + srcs = tuple(torch.arange(n, dtype=F32)[:, None] + 100_000 * k for k, n in enumerate(sizes)) + stacked = w._cat_rows(srcs, s)[:, 0].long().tolist() + rank_rows = [(k, range(n // 2, 3 * n // 4)) for k, n in enumerate(sizes)] # rank 2 of 4 + assert stacked == [100_000 * k + r for k, rows in rank_rows for r in rows] + # A global layer's k or v rows, then its dynamic conv's tap-projection rows. + proj = torch.arange(1024, dtype=F32)[:, None] + tap_proj = torch.arange(32, dtype=F32)[:, None] + 100_000 + out = w._kv_with_taps((proj, tap_proj), s)[:, 0].long().tolist() + assert out == list(range(512, 768)) + [100_000 + r for r in range(16, 24)] + # The output projection keeps this rank's input columns. + wo = torch.arange(6144, dtype=F32)[None, :].expand(2, 6144) + assert torch.equal(w._o_cols((wo,), s), wo[:, 3072:4608]) + # The NVFP4 global scale: this rank's experts' amax over the outer denominator. + amax = torch.arange(384, dtype=F32) + alpha = w._nvfp4_alpha((amax, torch.tensor([2688.0])), s) + assert torch.equal(alpha, amax[192:288] / 2688.0) + + +def test_nvfp4_fc1_weight_and_scale_rows_move_together(): + """Kernel row k of the FC1 weight and of its block scales is the same checkpoint row.""" + i, h = 2304, 4608 + ids = torch.arange(2 * i) + # Each checkpoint row carries its own index in its first two bytes. + tag = torch.stack([ids % 256, ids // 256], dim=-1).to(U8) + w13 = torch.cat([tag, torch.zeros(2 * i, h // 2 - 2, dtype=U8)], dim=-1)[None] + w13_sf = torch.cat([tag, torch.zeros(2 * i, h // 16 - 2, dtype=U8)], dim=-1)[None].view(F8) + s = w.shares_of(_dims(), _mapping(0)) + + def rows_of(t): + return (t[0, :, 0].long() + 256 * t[0, :, 1].long()).tolist() + + # The checkpoint interleaves gate (even) and up (odd) rows; the kernel wants, per 64 rows of + # each, the up rows first. + expected = [] + for group in range(i // 64): + expected += [2 * (64 * group + j) + 1 for j in range(64)] + expected += [2 * (64 * group + j) for j in range(64)] + assert rows_of(w._nvfp4_w13((w13,), s)) == expected + assert rows_of(unswizzle_block_scales(w._nvfp4_w13_scale((w13_sf,), s))) == expected + + +def _small_dims() -> InklingDims: + """Six layers (layer 5 global; NVFP4 experts in 0-3, MXFP8 in 4-5), small enough to load.""" + text = SimpleNamespace( + num_hidden_layers=6, + local_layer_ids=[0, 1, 2, 3, 4], + hidden_size=256, + vocab_size=1000, + unpadded_vocab_size=990, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=128, + d_rel=16, + rel_extent=64, + sliding_window_size=64, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=8, + num_experts_per_tok=2, + intermediate_size=128, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + groups = [ + { + "compute_dtype": fmt, + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in layers], + } + for fmt, layers in (("nvfp4", range(4)), ("mxfp8", (4, 5))) + ] + return InklingDims.from_config(text, {"model": {"parameter_groups": groups}}) + + +class _RecordingCheckpoint: + """The engine's lazy checkpoint view (`view[name][rows]` reads only those rows), recording.""" + + def __init__(self, tensors): + self.tensors = tensors + self.reads = [] + + def __contains__(self, name): + return name in self.tensors + + def keys(self): + return self.tensors.keys() + + def __getitem__(self, name): + return _RecordedTensor(self, name) + + +class _RecordedTensor: + def __init__(self, checkpoint: _RecordingCheckpoint, name: str): + self.checkpoint, self.name = checkpoint, name + + def __getitem__(self, rows): + self.checkpoint.reads.append((self.name, rows)) + return self.checkpoint.tensors[self.name][rows] + + +def _random_checkpoint(layout: dict, seed: int) -> dict: + g = torch.Generator().manual_seed(seed) + out = {} + for name, (shape, dtype) in layout.items(): + if dtype.itemsize == 1: + out[name] = torch.randint(0, 256, shape, generator=g, dtype=U8).view(dtype) + else: + out[name] = (torch.rand(shape, generator=g) + 0.5).to(dtype) + return out + + +def _bits(t: torch.Tensor) -> torch.Tensor: + return t.view(U8) if t.dtype.itemsize == 1 else t + + +def _model(s: "w.Shares", lm_head_rows: int): + """The parts of the engine model `load` fills, every value poisoned first.""" + model = SimpleNamespace( + model=SimpleNamespace(w=w.declare(s)), + lm_head=SimpleNamespace( + weight=torch.nn.Parameter( + torch.empty(lm_head_rows, s.model.hidden_size, dtype=BF16), requires_grad=False + ) + ), + ) + for p in [*model.model.w.values(), model.lm_head.weight]: + if p.dtype.itemsize == 1: + _bits(p.data).fill_(0x7F) + else: + p.data.fill_(float("nan")) + return model + + +@pytest.mark.parametrize("rank", [1, 3]) +def test_load_reads_this_ranks_rows_and_fills_every_weight(rank, monkeypatch): + monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *args, **kwargs: None) + d = _small_dims() + s = w.shares_of(d, _mapping(rank)) + tensors = _random_checkpoint(checkpoint_layout(d), seed=rank) + tensors["model.mtp.fc.weight"] = torch.zeros(2, 2) # the drafter's: not the table's to read + ckpt = _RecordingCheckpoint(tensors) + rows = 256 # this rank's share of the vocab-parallel lm_head (1000 rows over 4 ranks) + model = _model(s, rows) + w.load(model, ckpt, s) + + reads = dict(ckpt.reads) + assert len(reads) == len(ckpt.reads), "a checkpoint tensor was read twice" + assert set(reads) == set(checkpoint_layout(d)) + for layer, entry in w.all_entries(s): + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = [] + for name, by_expert in zip(entry.sources, flags): + # Expert tensors are read at this rank's experts only. + want = s.experts() if by_expert else slice(None) + assert reads[name] == want, (name, reads[name], want) + srcs.append(tensors[name][want]) + expected = entry.transform(tuple(srcs), s) + loaded = model.model.w[w._key(layer, entry.name)].data + assert torch.equal(_bits(loaded), _bits(expected)), (layer, entry.name) + + start, end = rank * rows, min(rank * rows + rows, d.vocab_size) + assert reads["model.llm.unembed.weight"] == slice(start, end) + head = model.lm_head.weight.data + assert torch.equal(head[: end - start], tensors["model.llm.unembed.weight"][start:end]) + assert not head[end - start :].any(), "rows past the vocabulary must be zero" + + +def test_load_refuses_a_text_decoder_tensor_the_table_does_not_name(monkeypatch): + monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *args, **kwargs: None) + d = _small_dims() + s = w.shares_of(d, _mapping(0)) + tensors = _random_checkpoint(checkpoint_layout(d), seed=0) + tensors["model.llm.layers.0.attn.extra.weight"] = torch.zeros(2, 2) + with pytest.raises(AssertionError, match="does not name"): + w.load(_model(s, 256), _RecordingCheckpoint(tensors), s) + + +@pytest.mark.skipif( + not os.path.isdir(os.path.join(os.environ.get("LLM_MODELS_ROOT", ""), "Inkling-2-Small")), + reason="needs the Inkling-2-Small checkpoint under LLM_MODELS_ROOT", +) +def test_layout_matches_the_checkpoint_headers(): + import json + import struct + + root = os.path.join(os.environ["LLM_MODELS_ROOT"], "Inkling-2-Small") + index = json.load(open(os.path.join(root, "model.safetensors.index.json")))["weight_map"] + headers = {} + for f in sorted(set(index.values())): + with open(os.path.join(root, f), "rb") as fh: + (n,) = struct.unpack(" None: # noqa: F811 + certifying(op) + args = _build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31) + for out, ref in zip(op(*args), op.reference(*args)): + assert_within_ulp( + out.reshape(-1, out.shape[-1]), + ref.reshape(-1, ref.shape[-1]), + element_ulp=1.0, + rms_ulp=0.1, + ) + + +def test_clip_bounds_and_tau() -> None: + args = _build(dict(tokens=9, heads=12, kv_heads=2, head_dim=128, d_rel=16, tau=False), 1) + q, k, v, r = op(*args) + assert ( + q.float().norm(dim=-1).max() <= QK_CLIP * 1.01 + and k.float().norm(dim=-1).max() <= QK_CLIP * 1.01 + ) + assert r.float().norm(dim=-1).max() <= REL_CLIP * 1.01 + torch.testing.assert_close( + v.float().pow(2).mean(-1), torch.ones(9, 2, device="cuda"), rtol=0.02, atol=0.02 + ) + tau = log_scaling(torch.tensor([0, 8190, 8191, 8192 * 4 - 1], device="cuda"), 0.1, 8192.0) + torch.testing.assert_close( + tau, torch.tensor([1.0, 1.0, 1.0, 1.0 + 0.1 * 1.3862944], device="cuda") + ) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py new file mode 100644 index 000000000000..19cae9df2fb9 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_paged_attention entry.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.paged_attention import ( + block_tokens, + tile_plan, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.attention.inkling2_paged_attention import ( + SM_SCALE, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.attention.inkling2_paged_attention import ( + inkling2_paged_attention as op, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "inkling2_paged_attention requires a CUDA device" + +PAGE_SIZE = 32 +HEAD_DIM = 128 +REL_EXTENT = 1024 + + +def _build(spec, seed): + """A step of `spec["requests"]` ((new, cached) pairs) over a pool of shuffled pages whose + cached positions already hold keys and values.""" + g = torch.Generator(device="cuda").manual_seed(seed) + cpu = torch.Generator().manual_seed(seed) + heads, kv_heads, window = spec["heads"], spec["kv_heads"], spec["window"] + requests = spec["requests"] + lengths = [n for n, _ in requests] + kv_lens = [n + c for n, c in requests] + widths = [-(-length // PAGE_SIZE) for length in kv_lens] + num_pages = sum(widths) + 3 + pool = torch.randn(num_pages, 2, kv_heads, PAGE_SIZE, HEAD_DIM, generator=g, device="cuda") + # Keys after the clip have norm <= 22.6; values after the RMS norm have unit RMS. + pool[:, 0] *= 1.5 + pool = pool.to(torch.bfloat16) + order = torch.randperm(num_pages, generator=cpu) + page_table = torch.full((len(requests), max(widths)), -1, dtype=torch.int32) + used = 0 + for r, width in enumerate(widths): + page_table[r, :width] = order[used : used + width].to(torch.int32) + used += width + num_tokens = sum(lengths) + # Clipped queries reach norm 22.6 and the global layers' tau scales them further. + q = (1.5 * torch.randn(num_tokens, heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + k = (1.5 * torch.randn(num_tokens, kv_heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + v = torch.randn(num_tokens, kv_heads, HEAD_DIM, generator=g, device="cuda").to(torch.bfloat16) + rel_logits = 2.0 * torch.randn(num_tokens, heads, REL_EXTENT, generator=g, device="cuda") + sink = (2.0 * torch.randn(num_tokens, heads, generator=g, device="cuda")).to(torch.bfloat16) + gate = (2.0 * torch.randn(num_tokens, heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + block_t = block_tokens(max(lengths)) + tile_request, tile_start = tile_plan(lengths, block_t, torch.device("cuda")) + return [ + q, + k, + v, + pool, + page_table.cuda(), + torch.tensor(bounds, dtype=torch.int32, device="cuda"), + torch.tensor(kv_lens, dtype=torch.int32, device="cuda"), + torch.repeat_interleave( + torch.arange(len(lengths), dtype=torch.int32), torch.tensor(lengths) + ).cuda(), + tile_request, + tile_start, + block_t, + rel_logits, + sink, + gate, + window, + SM_SCALE, + ] + + +def _run_both(args): + op_args, ref_args = list(args), list(args) + op_args[3] = args[3].clone() + ref_args[3] = args[3].clone() + out = op(*op_args) + ref = op.reference(*ref_args) + return out, ref, op_args[3], ref_args[3] + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:48] for c in op.CELLS]) +def test_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(op) + out, ref, pool, ref_pool = _run_both(_build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31)) + torch.testing.assert_close(pool, ref_pool, rtol=0, atol=0) + # Flash attention rounds the unnormalized probabilities to bf16 for the value product and + # sums in another order than the dense fp32 reference: a few bf16 ulps of each row's scale. + assert_within_ulp( + out.reshape(-1, out.shape[-1]), ref.reshape(-1, ref.shape[-1]), element_ulp=4.0, rms_ulp=0.5 + ) + + +@pytest.mark.parametrize("what", ["bias_extent", "window", "sink"]) +def test_negative_controls_are_visible(what) -> None: + """Each mechanism the cells certify changes the output when it is broken.""" + spec = dict( + heads=12, + kv_heads=2, + window=1024 if what == "window" else 0, + requests=((1, 1500), (40, 990)), + ) + args = _build(spec, 21) + if what == "sink": + # Sinks near the logsumexp of 1500 keys, where they take a visible share of the mass. + args[12] = (args[12].float() + 10.0).to(torch.bfloat16) + out, *_ = _run_both(args) + broken = list(args) + if what == "bias_extent": + # Bias leaking past the extent: shift it by one distance. + broken[11] = torch.roll(args[11], shifts=1, dims=2) + elif what == "window": + broken[14] = 1023 + else: + broken[12] = torch.zeros_like(args[12]) + broken[3] = args[3].clone() + assert not torch.allclose(op(*broken), out, rtol=0.01, atol=0.01) + + +@pytest.mark.parametrize( + "case,match", + [ + ("bf16_rel_logits", None), + ("short_page_table", "page_table must cover"), + ("wrong_tiles", "tile plan"), + ("strided_kv", "contiguous head dims"), + ], +) +def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: + args = _build(dict(heads=12, kv_heads=2, window=1024, requests=((5, 3), (1, 40))), 4) + if case == "bf16_rel_logits": + args[11] = args[11].to(torch.bfloat16) + elif case == "short_page_table": + args[4] = args[4][:, :1] + elif case == "strided_kv": + # The same values every other element of a wider buffer: the kernels read unit stride. + for i in (1, 2): + t = args[i] + wide = torch.zeros(*t.shape[:2], 2 * t.shape[2], dtype=t.dtype, device=t.device) + wide[..., ::2] = t + args[i] = wide[..., ::2] + else: + args[9] = torch.zeros_like(args[9]) + with pytest.raises(AssertionError, match=match): + with validating(op): + op(*args) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py new file mode 100644 index 000000000000..052d5b06a0a4 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_short_conv and inkling2_dynamic_short_conv entries.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.conv.inkling2_short_conv import ( + inkling2_dynamic_short_conv, + inkling2_short_conv, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "the Inkling 2 short convolutions require a CUDA device" + +NUM_SLOTS = 12 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) % 2**31 + + +def _build(spec, seed, dynamic): + """The step of `spec["requests"]` ((new, cached) pairs) on random inputs and windows.""" + g = torch.Generator(device="cuda").manual_seed(seed) + channels = spec["channels"] + lengths = [n for n, _ in spec["requests"]] + num_tokens = sum(lengths) + x = torch.randn(num_tokens, channels, generator=g, device="cuda").to(torch.bfloat16) + taps = 0.5 * torch.randn(4, channels, generator=g, device="cuda") + # Every slot holds a window, including the ones of fresh requests: has_state 0 must ignore it. + state = torch.randn(NUM_SLOTS, 3, channels, generator=g, device="cuda").to(torch.bfloat16) + perm = torch.randperm(NUM_SLOTS, generator=torch.Generator().manual_seed(seed)) + slots = perm[: len(lengths)].to(torch.int32).cuda() + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + query_start = torch.tensor(bounds, dtype=torch.int32, device="cuda") + has_state = torch.tensor( + [int(c > 0) for _, c in spec["requests"]], dtype=torch.int32, device="cuda" + ) + token_request = torch.repeat_interleave( + torch.arange(len(lengths), dtype=torch.int32), torch.tensor(lengths) + ).cuda() + if not dynamic: + return x, taps, state, slots, query_start, has_state, token_request + heads = spec["heads"] + dynamic_taps = (0.3 * torch.randn(num_tokens, heads * 4, generator=g, device="cuda")).to( + torch.bfloat16 + ) + return ( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + channels // heads, + ) + + +def _check_cell(op, args, state_index): + op_args, ref_args = list(args), list(args) + op_args[state_index] = args[state_index].clone() + ref_args[state_index] = args[state_index].clone() + out = op(*op_args) + ref = op.reference(*ref_args) + assert out.shape == ref.shape and out.dtype == ref.dtype + # The op and the reference sum the same four fp32 products in the same order; FMA + # contraction moves the fp32 sum by an ulp, which a bf16 rounding boundary can turn into one + # bf16 ulp of the output. + assert_within_ulp(out, ref, element_ulp=1.0, rms_ulp=0.05) + # The window is a copy of inputs: exact. + torch.testing.assert_close(op_args[state_index], ref_args[state_index], rtol=0, atol=0) + + +@pytest.mark.parametrize( + "cell", inkling2_short_conv.CELLS, ids=[c.why[:48] for c in inkling2_short_conv.CELLS] +) +def test_static_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_short_conv) + _check_cell( + inkling2_short_conv, _build(cell.spec, _seed(cell.why), dynamic=False), state_index=2 + ) + + +@pytest.mark.parametrize( + "cell", + inkling2_dynamic_short_conv.CELLS, + ids=[c.why[:48] for c in inkling2_dynamic_short_conv.CELLS], +) +def test_dynamic_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_dynamic_short_conv) + _check_cell( + inkling2_dynamic_short_conv, _build(cell.spec, _seed(cell.why), dynamic=True), state_index=3 + ) + + +@pytest.mark.parametrize("first", [1, 2, 21]) +@pytest.mark.parametrize("dynamic", [False, True], ids=["static", "dynamic"]) +def test_chunked_steps_equal_one_prefill(dynamic, first) -> None: + """A fresh prefill of `first` tokens over a stale window, then chunks and decodes, equals one + 29-token prefill: the window carries exactly the inputs a step needs from the steps before it, + and a fresh request shorter than the window starts it with zeros, not the stale inputs.""" + channels, heads = 256, 2 + g = torch.Generator(device="cuda").manual_seed(5) + total = 29 + x = torch.randn(total, channels, generator=g, device="cuda").to(torch.bfloat16) + taps = 0.5 * torch.randn(4, channels, generator=g, device="cuda") + dyn = (0.3 * torch.randn(total, heads * 4, generator=g, device="cuda")).to(torch.bfloat16) + + def run(start, end, state, has_state): + n = end - start + args = [x[start:end], taps] + if dynamic: + args.append(dyn[start:end]) + args += [ + state, + torch.tensor([1], dtype=torch.int32, device="cuda"), + torch.tensor([0, n], dtype=torch.int32, device="cuda"), + torch.tensor([has_state], dtype=torch.int32, device="cuda"), + torch.zeros(n, dtype=torch.int32, device="cuda"), + ] + if dynamic: + args.append(channels // heads) + op = inkling2_dynamic_short_conv if dynamic else inkling2_short_conv + return op(*args) + + whole = run(0, total, torch.zeros(3, 3, channels, dtype=torch.bfloat16, device="cuda"), 0) + state = torch.randn(3, 3, channels, device="cuda").to(torch.bfloat16) + pieces, start = [], 0 + for end in sorted({first, *(e for e in (21, 23, 24, 25, 26, 27, 28, 29) if e > first)}): + pieces.append(run(start, end, state, int(start > 0))) + if start == 0 and end < 3: + zeros = torch.zeros(3 - end, channels, dtype=torch.bfloat16, device="cuda") + torch.testing.assert_close(state[1], torch.cat([zeros, x[:end]]), rtol=0, atol=0) + start = end + torch.testing.assert_close(torch.cat(pieces), whole, rtol=0, atol=0) + torch.testing.assert_close(state[1], x[-3:], rtol=0, atol=0) + + +def test_dynamic_taps_are_per_head() -> None: + """Swapping the two heads' dynamic taps changes the output: one head's taps never reach the + other head's channels.""" + spec = dict(channels=256, heads=2, requests=((7, 0),)) + args = list(_build(spec, 3, dynamic=True)) + out = inkling2_dynamic_short_conv(*args[:3], args[3].clone(), *args[4:]) + swapped = args[2].view(-1, 2, 4).flip(1).reshape(-1, 8).contiguous() + out_swapped = inkling2_dynamic_short_conv(args[0], args[1], swapped, args[3].clone(), *args[4:]) + assert not torch.equal(out[:, :128], out_swapped[:, :128]) + + +@pytest.mark.parametrize( + "case,match", + [ + ("duplicate_slot", "own slot"), + ("bf16_taps", "lag-major fp32"), + ("bad_token_request", "token_request"), + ("strided_slots", "slots must be contiguous"), + ], +) +def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: + spec = dict(channels=256, requests=((3, 0), (1, 4))) + x, taps, state, slots, query_start, has_state, token_request = _build(spec, 9, dynamic=False) + if case == "duplicate_slot": + slots = slots[:1].repeat(2) + elif case == "bf16_taps": + taps = taps.to(torch.bfloat16) + elif case == "strided_slots": + # The same slot values, as a view the kernels would index at unit stride. + slots = torch.stack([slots, slots + NUM_SLOTS], dim=1)[:, 0] + else: + token_request = torch.zeros_like(token_request) + with pytest.raises(AssertionError, match=match): + with validating(inkling2_short_conv): + inkling2_short_conv(x, taps, state, slots, query_start, has_state, token_request) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py new file mode 100644 index 000000000000..0ed0fda6af1f --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Inkling 2 MoE entries: router, NVFP4 experts, MXFP8 experts.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_mxfp8_experts import ( + inkling2_mxfp8_experts, + mxfp8_dequantize, + mxfp8_quantize, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_nvfp4_experts import ( + fc1_row_order, + fc2_input_scale, + inkling2_nvfp4_experts, + swizzle_block_scales, + unswizzle_block_scales, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_router import inkling2_router + +__extra_import_path__ = ["../../../.."] +from _validating import certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "the Inkling 2 MoE entries require a CUDA device" + +GATE_SOFTCAP, UP_SOFTCAP = 4.0, 25.0 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) % 2**31 + + +def _routing(g, tokens, experts, top_k, local, offset): + """Distinct experts per token, about half of them on this rank; positive weights summing to 8.""" + pool = torch.cat( + [ + torch.arange(offset, offset + local), + torch.arange(0, offset), + torch.arange(offset + local, experts), + ] + ) + pick = torch.stack([torch.randperm(2 * local, generator=g)[:top_k] for _ in range(tokens)]) + ids = pool[pick.clamp(max=experts - 1)] + weights = torch.rand(tokens, top_k, generator=g) + 0.1 + weights = weights / weights.sum(-1, keepdim=True) * 8.0 + return ids.to(torch.int32).cuda(), weights.float().cuda() + + +def _rel_rms(out, ref): + return float(((out.float() - ref.float()).pow(2).mean() / ref.float().pow(2).mean()).sqrt()) + + +def _e4m3_scales(g, shape, lo=-4, hi=3): + exps = torch.randint(lo, hi, shape, generator=g, device="cuda") + return torch.exp2(exps.float()).to(torch.float8_e4m3fn).view(torch.uint8) + + +def _nvfp4_args(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + e, h, i = spec["local"], spec["hidden"], spec["inter"] + dev = "cuda" + w13 = torch.randint(0, 256, (e, 2 * i, h // 2), generator=g, dtype=torch.uint8, device=dev) + w13_sf = _e4m3_scales(g, (e, 2 * i, h // 16)) + w2 = torch.randint(0, 256, (e, h, i // 2), generator=g, dtype=torch.uint8, device=dev) + w2_sf = _e4m3_scales(g, (e, h, i // 16)) + # Global scales that keep the experts' outputs O(1) for unit-RMS inputs. + w13_alpha = (0.02 + 0.01 * torch.rand(e, generator=g, device=dev)).float() + w2_alpha = (0.02 + 0.01 * torch.rand(e, generator=g, device=dev)).float() + offset = spec["rank"] * e + x = torch.randn(spec["tokens"], h, generator=g, device=dev).to(torch.bfloat16) + ids, weights = _routing( + torch.Generator().manual_seed(seed), + spec["tokens"], + spec["experts"], + spec["top_k"], + e, + offset, + ) + return [ + x, + ids, + weights, + w13, + swizzle_block_scales(w13_sf), + w13_alpha, + w2, + swizzle_block_scales(w2_sf), + w2_alpha, + fc2_input_scale(torch.device("cuda")), + spec["experts"], + offset, + GATE_SOFTCAP, + UP_SOFTCAP, + ] + + +def _mxfp8_args(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + e, h, i = spec["local"], spec["hidden"], spec["inter"] + dev = "cuda" + + def e4m3(shape): + return ( + (64 * torch.randn(shape, generator=g, device=dev)) + .clamp(-448, 448) + .to(torch.float8_e4m3fn) + ) + + def exponents(shape): + return torch.randint(127 - 14, 127 - 10, shape, generator=g, dtype=torch.uint8, device=dev) + + w13, w13_s = e4m3((e, 2 * i, h)), exponents((e, 2 * i, h // 32)) + w2, w2_s = e4m3((e, h, i)), exponents((e, h, i // 32)) + offset = spec["rank"] * e + x = torch.randn(spec["tokens"], h, generator=g, device=dev).to(torch.bfloat16) + ids, weights = _routing( + torch.Generator().manual_seed(seed), + spec["tokens"], + spec["experts"], + spec["top_k"], + e, + offset, + ) + return [x, ids, weights, w13, w13_s, w2, w2_s, offset, GATE_SOFTCAP, UP_SOFTCAP] + + +# --------------------------------------------------------------------------------------- router + + +@pytest.mark.parametrize("cell", inkling2_router.CELLS, ids=[c.why for c in inkling2_router.CELLS]) +def test_router_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_router) + spec = cell.spec + g = torch.Generator(device="cuda").manual_seed(_seed(cell.why)) + x = torch.randn(spec["tokens"], spec["hidden"], generator=g, device="cuda").to(torch.bfloat16) + weight = (0.015 * torch.randn(spec["experts"], spec["hidden"], generator=g, device="cuda")).to( + torch.bfloat16 + ) + bias = 0.05 * torch.randn(spec["experts"], generator=g, device="cuda") + ids, weights = inkling2_router(x, weight, bias, spec["top_k"], spec["route_scale"]) + ref_ids, ref_weights = inkling2_router.reference( + x, weight, bias, spec["top_k"], spec["route_scale"] + ) + # The selection is a set; order the two by expert id before comparing. + order, ref_order = ids.sort(-1), ref_ids.sort(-1) + assert torch.equal(order.values, ref_order.values) + torch.testing.assert_close( + weights.gather(1, order.indices.long()), + ref_weights.gather(1, ref_order.indices.long()), + rtol=1e-5, + atol=1e-5, + ) + torch.testing.assert_close( + weights.sum(-1), torch.full_like(weights[:, 0], 8.0), rtol=1e-5, atol=1e-5 + ) + + +# ----------------------------------------------------------------------------------------- NVFP4 + + +def test_fc1_row_order_and_swizzle_round_trip() -> None: + order = fc1_row_order(2304, torch.device("cpu")) + # The checkpoint interleaves gate (even) and up (odd) rows. Kernel group g (128 rows) holds the + # up rows of gate/up pairs 64 g .. 64 g + 63, then their gate rows: every group, in order. + expected = [] + for group in range(2304 // 64): + pairs = range(64 * group, 64 * group + 64) + expected += [2 * p + 1 for p in pairs] + [2 * p for p in pairs] + assert order.tolist() == expected + scales = torch.randint(0, 256, (3, 256, 288), dtype=torch.uint8) + assert torch.equal(unswizzle_block_scales(swizzle_block_scales(scales)), scales) + + +def test_swizzle_matches_flashinfer_block_scale_interleave() -> None: + from flashinfer.quantization.fp4_quantization import block_scale_interleave + + scales = torch.randint(0, 256, (2, 256, 144), dtype=torch.uint8, device="cuda") + assert torch.equal( + swizzle_block_scales(scales).reshape(-1), block_scale_interleave(scales).reshape(-1) + ) + + +@pytest.mark.parametrize( + "cell", inkling2_nvfp4_experts.CELLS, ids=[c.why for c in inkling2_nvfp4_experts.CELLS] +) +def test_nvfp4_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_nvfp4_experts) + args = _nvfp4_args(cell.spec, _seed(cell.why)) + out = inkling2_nvfp4_experts(*args) + ref = inkling2_nvfp4_experts.reference(*args) + assert out.shape == ref.shape and out.dtype == torch.bfloat16 + # The kernel requantizes the fp32 SiTU output, the reference its bf16 rounding: a few block + # scale choices and E2M1 roundings differ, each by a quantization step. + assert _rel_rms(out, ref) < 0.02, _rel_rms(out, ref) + + +def test_nvfp4_runs_under_inference_mode() -> None: + """The engine runs every forward under inference mode, a layer's first call included.""" + args = _nvfp4_args(dict(inkling2_nvfp4_experts.CELLS[1].spec), 23) + ref = inkling2_nvfp4_experts.reference(*args) + with torch.inference_mode(): + out = inkling2_nvfp4_experts(*args) + assert _rel_rms(out, ref) < 0.02, _rel_rms(out, ref) + + +def test_nvfp4_swapped_gate_and_up_is_visible() -> None: + spec = dict(inkling2_nvfp4_experts.CELLS[1].spec) + args = _nvfp4_args(spec, 17) + ref = inkling2_nvfp4_experts.reference(*args) + swapped = list(args) + i = args[3].shape[1] // 2 + groups = args[3].view(-1, i // 64, 2, 64, args[3].shape[-1]).flip(2).reshape(args[3].shape) + swapped[3] = groups.contiguous() + sf = unswizzle_block_scales(args[4]) + swapped[4] = swizzle_block_scales( + sf.view(-1, i // 64, 2, 64, sf.shape[-1]).flip(2).reshape(sf.shape).contiguous() + ) + assert _rel_rms(inkling2_nvfp4_experts(*swapped), ref) > 0.3 + + +# ----------------------------------------------------------------------------------------- MXFP8 + + +def test_align_routes_puts_each_local_route_in_one_row_of_its_expert() -> None: + from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.mxfp8_moe import ( + BLOCK_M, + align_routes, + ) + + ids, _ = _routing(torch.Generator().manual_seed(4), 100, 384, 8, 96, 96) + route_of_row, expert_of_block, max_blocks = align_routes(ids, 96, 96) + rows = route_of_row.view(max_blocks, BLOCK_M).cpu() + experts = expert_of_block.cpu() + flat = ids.reshape(-1).cpu().long() - 96 + local = set(torch.nonzero((flat >= 0) & (flat < 96)).flatten().tolist()) + placed = [] + for b in range(max_blocks): + routes = [r for r in rows[b].tolist() if r >= 0] + if experts[b] < 0: + assert not routes + assert all(flat[r] == experts[b] for r in routes) + placed += routes + assert sorted(placed) == sorted(local) + + +def test_mxfp8_expert_with_more_routes_than_a_row_block() -> None: + """33 routes to one expert fill a 32-row block and spill into a second: both count.""" + from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.mxfp8_moe import ( + BLOCK_M, + align_routes, + ) + + tokens = BLOCK_M + 1 + spec = dict(local=4, hidden=256, inter=256, rank=1, tokens=tokens, experts=16, top_k=2) + args = _mxfp8_args(spec, 31) + offset = args[7] + # Every token's first choice is local expert 2; its second is expert 0, on another rank. + args[1] = torch.tensor([[offset + 2, 0]] * tokens, dtype=torch.int32, device="cuda") + args[2] = torch.full((tokens, 2), 4.0, device="cuda") + route_of_row, expert_of_block, max_blocks = align_routes(args[1], offset, spec["local"]) + rows = route_of_row.view(max_blocks, BLOCK_M).cpu() + blocks = [b for b in range(max_blocks) if int(expert_of_block[b]) == 2] + assert len(blocks) == 2 + placed = sorted(r for b in blocks for r in rows[b].tolist() if r >= 0) + assert placed == [2 * t for t in range(tokens)], "each route exactly once" + out = inkling2_mxfp8_experts(*args) + ref = inkling2_mxfp8_experts.reference(*args) + assert _rel_rms(out, ref) < 2e-3, _rel_rms(out, ref) + + +def test_mxfp8_refuses_a_partial_k_tile() -> None: + """Neither GEMM masks a partial 128-wide K tile, so such a size is refused before launch.""" + spec = dict(local=2, hidden=256, inter=160, rank=0, tokens=4, experts=8, top_k=2) + args = _mxfp8_args(spec, 7) + with pytest.raises(ValueError, match="multiples of 128"): + inkling2_mxfp8_experts(*args) + with pytest.raises(AssertionError, match="128-wide K tile"): + with validating(inkling2_mxfp8_experts): + inkling2_mxfp8_experts(*args) + + +def test_mxfp8_quantize_round_trip() -> None: + x = torch.randn(64, 256, device="cuda") * torch.logspace(-3, 3, 256, device="cuda") + values, exps = mxfp8_quantize(x) + back = mxfp8_dequantize(values, exps) + # E4M3 keeps 3 mantissa bits: half an ulp is 1/16 of the block's power-of-two range. + blocks = x.view(64, 8, 32).abs().amax(-1, keepdim=True) + assert ((back - x).view(64, 8, 32).abs() <= blocks / 16 + 1e-30).all() + assert ((back.view(64, 8, 32).abs().amax(-1, keepdim=True) / blocks) > 0.5).all() + + +@pytest.mark.parametrize( + "cell", inkling2_mxfp8_experts.CELLS, ids=[c.why for c in inkling2_mxfp8_experts.CELLS] +) +def test_mxfp8_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_mxfp8_experts) + args = _mxfp8_args(cell.spec, _seed(cell.why)) + out = inkling2_mxfp8_experts(*args) + ref = inkling2_mxfp8_experts.reference(*args) + # Same products; the bf16 GEMMs accumulate in another order, which moves a few SiTU outputs + # across an E4M3 rounding boundary before the down projection. + assert _rel_rms(out, ref) < 2e-3, _rel_rms(out, ref) + + +@pytest.mark.parametrize("case", ["bf16_bias", "fp16_x"]) +def test_router_guard(case) -> None: + x = torch.randn(4, 64, device="cuda").to(torch.bfloat16) + weight = torch.randn(16, 64, device="cuda").to(torch.bfloat16) + bias = torch.zeros(16, device="cuda") + if case == "bf16_bias": + bias = bias.to(torch.bfloat16) + else: + x = x.to(torch.float16) + with pytest.raises(AssertionError): + with validating(inkling2_router): + inkling2_router(x, weight, bias, 4, 8.0) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py new file mode 100644 index 000000000000..936ac7018fd6 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_residual_mix entry.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.residual.inkling2_residual_mix import ( + inkling2_residual_mix as op, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.residual.inkling2_residual_mix import ( + normalize_block, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "inkling2_residual_mix requires a CUDA device" + +HIDDEN = 4608 +EPS = 1e-6 + + +def _build(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + t, s, q = spec["tokens"], spec["blocks"], spec["queries"] + # Block states at different scales: the normalization, not the raw scale, sets their weight. + raw = torch.randn(s, t, HIDDEN, generator=g, device="cuda") * torch.logspace( + -1, 1, s, device="cuda" + ).view(s, 1, 1) + blocks = torch.stack([normalize_block(raw[i].to(torch.bfloat16)) for i in range(s)]) + partial = (3.0 * torch.randn(t, HIDDEN, generator=g, device="cuda")).to(torch.bfloat16) + # Queries with norms like the checkpoint's, so the softmax is neither flat nor one-hot. + queries = 0.05 * torch.randn(q, HIDDEN, generator=g, device="cuda") + weights = tuple( + (1.0 + 0.2 * torch.randn(HIDDEN, generator=g, device="cuda")).to(torch.bfloat16) + for _ in range(q) + ) + return blocks, partial, queries, weights, EPS + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:48] for c in op.CELLS]) +def test_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(op) + args = _build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31) + out = op(*args) + ref = op.reference(*args) + assert out.shape == (cell.spec["queries"], cell.spec["tokens"], HIDDEN) + # fp32 against fp64, one bf16 rounding at the end: at most one bf16 ulp per element. + assert_within_ulp( + out.reshape(-1, HIDDEN), ref.reshape(-1, HIDDEN), element_ulp=1.0, rms_ulp=0.1 + ) + + +def test_values_are_the_normalized_states() -> None: + """Scaling the partial block does not change the mix: it enters normalized.""" + args = list(_build(dict(blocks=3, queries=1, tokens=5), 7)) + out = op(*args) + args[1] = (args[1].float() * 8.0).to(torch.bfloat16) + torch.testing.assert_close(op(*args), out, rtol=0, atol=0) + + +def test_fp32_partial_block_keeps_its_low_bits() -> None: + """The decoder hands the partial block over in fp32; the mix must not round it to bf16.""" + blocks, partial, queries, weights, eps = _build(dict(blocks=3, queries=2, tokens=5), 11) + g = torch.Generator(device="cuda").manual_seed(12) + partial = partial.float() + 1e-3 * torch.randn(partial.shape, generator=g, device="cuda") + rounded = partial.to(torch.bfloat16) + assert not torch.equal(partial, rounded.float()), "values bf16 cannot hold" + out = op(blocks, partial, queries, weights, eps) + ref = op.reference(blocks, partial, queries, weights, eps) + assert_within_ulp( + out.reshape(-1, HIDDEN), ref.reshape(-1, HIDDEN), element_ulp=1.0, rms_ulp=0.1 + ) + # Had the op rounded the partial block first, both inputs would give the same mix. + assert not torch.equal(out, op(blocks, rounded, queries, weights, eps)) + + +def test_guard_refuses_bf16_queries() -> None: + blocks, partial, queries, weights, eps = _build(dict(blocks=2, queries=1, tokens=3), 3) + with pytest.raises(AssertionError, match="fp32"): + with validating(op): + op(blocks, partial, queries.to(torch.bfloat16), weights, eps) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py b/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py new file mode 100644 index 000000000000..20af2a9b2a7f --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of `InklingDims`: both config spellings give one geometry, and bad inputs fail.""" + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch.modeling_v2.models.inkling2.config import ExpertFormat, InklingDims + +pytestmark = pytest.mark.cpu_only + +GLOBAL = tuple(range(5, 48, 6)) +_COMMON = dict( + num_hidden_layers=48, + hidden_size=4608, + vocab_size=201024, + unpadded_vocab_size=200058, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + d_rel=16, + rel_extent=1024, + n_layers_per_attn_res_block=6, + n_routed_experts=384, + num_experts_per_tok=8, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, +) + + +def _trtllm_style(**overrides) -> SimpleNamespace: + """The `Inkling2TextConfig` spelling: `local_layer_ids` and the checkpoint's field names.""" + fields = dict( + _COMMON, + local_layer_ids=[i for i in range(48) if i not in GLOBAL], + sliding_window_size=1024, + sconv_kernel_size=4, + intermediate_size=2304, + route_scale=8.0, + ) + return SimpleNamespace(**{**fields, **overrides}) + + +def _native_style() -> SimpleNamespace: + """The Transformers spelling: `layer_types` (`hybrid` / `hybrid_sliding`) and its aliases.""" + return SimpleNamespace( + **_COMMON, + layer_types=["hybrid" if i in GLOBAL else "hybrid_sliding" for i in range(48)], + sliding_window=1024, + conv_kernel_size=4, + moe_intermediate_size=2304, + routed_scaling_factor=8.0, + ) + + +def _groups(**layers_by_format): + return { + "model": { + "parameter_groups": [ + { + "compute_dtype": fmt, + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in layers], + } + for fmt, layers in layers_by_format.items() + ] + } + } + + +def test_both_config_spellings_give_one_geometry(): + quant = _groups(nvfp4=range(40), mxfp8=range(40, 48)) + native = InklingDims.from_config(_native_style(), quant) + assert native == InklingDims.from_config(_trtllm_style(), quant) + assert native.global_layers == GLOBAL + assert (native.conv_kernel_size, native.expert_intermediate_size, native.route_scale) == ( + 4, + 2304, + 8.0, + ) + assert native.expert_formats == (ExpertFormat.NVFP4,) * 40 + (ExpertFormat.MXFP8,) * 8 + + +def test_layers_no_parameter_group_names_keep_bf16_experts(): + dims = InklingDims.from_config(_trtllm_style(), _groups(nvfp4=[0])) + assert dims.expert_formats[:2] == (ExpertFormat.NVFP4, ExpertFormat.BF16) + + +@pytest.mark.parametrize( + "quant, match", + [ + (_groups(nvfp4=[3], mxfp8=[3]), "layer 3 two expert formats"), + (_groups(nvfp4=[48]), r"layers \[48\], outside the 48 decoder layers"), + ], + ids=["two_formats_on_one_layer", "layer_out_of_range"], +) +def test_malformed_quant_config_is_refused(quant, match): + with pytest.raises(ValueError, match=match): + InklingDims.from_config(_trtllm_style(), quant) + + +def test_sliding_layers_with_another_attention_shape_are_refused(): + with pytest.raises(ValueError, match="different attention shape"): + InklingDims.from_config(_trtllm_style(swa_num_key_value_heads=4)) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py index ba6e79f4ab8d..5a80210522cf 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py @@ -242,7 +242,7 @@ def test_accuracy_directories_are_targets(repo: Path = _REPO): # Families that ship directly constructed components without a registered model. # An exemption is invalid as soon as routing or registry imports are introduced. -_COMPONENT_ONLY_FAMILIES: set[str] = {"beam"} +_COMPONENT_ONLY_FAMILIES: set[str] = {"beam", "inkling2"} def test_every_routed_target_has_accuracy_tests(repo: Path = _REPO):