From 9b4c6401b6019c660ba690149c3713d48eca5203 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Wed, 18 Nov 2026 21:10:25 -0800 Subject: [PATCH 1/4] [None][feat] Add the Inkling 2 Modeling V2 family: operators, kernels and the Inkling-2-Small target The inkling2 family for Inkling2ForConditionalGeneration (thinkingmachines/Inkling-2-Small), not routed yet: - Triton kernels: the paged attention with the relative position bias, the attention sink and the output gate; the width-4 short convolutions with per-request window state; the grouped MXFP8 MoE. - OpWrappers with torch references: the attention prologue (clips, value norm, log scaling), the block-residual mix, the router, and the NVFP4 (FlashInfer CuTe DSL, SiTU, per-token 4/6 activations) and MXFP8 experts. - The inkling_2_small_nvfp4__sm_103__tep4 target for one GB300 tray (attention TP4, MoE EP4): weight table, 48-layer forward and the KVCacheManagerV2 subclass that keeps the convolution windows. Signed-off-by: Michal Guzek --- .../modeling_v2/models/inkling2/README.md | 51 +++ .../modeling_v2/models/inkling2/__init__.py | 3 + .../modeling_v2/models/inkling2/config.py | 203 +++++++++ .../__init__.py | 2 + .../cache.py | 92 ++++ .../modeling.py | 398 ++++++++++++++++ .../weights.py | 430 ++++++++++++++++++ .../models/inkling2/kernels/__init__.py | 2 + .../models/inkling2/kernels/mxfp8_moe.py | 301 ++++++++++++ .../inkling2/kernels/paged_attention.py | 313 +++++++++++++ .../models/inkling2/kernels/short_conv.py | 212 +++++++++ .../models/inkling2/ops/__init__.py | 2 + .../models/inkling2/ops/attention/__init__.py | 2 + .../attention/inkling2_attention_prologue.py | 126 +++++ .../ops/attention/inkling2_paged_attention.py | 245 ++++++++++ .../models/inkling2/ops/conv/__init__.py | 2 + .../inkling2/ops/conv/inkling2_short_conv.py | 299 ++++++++++++ .../models/inkling2/ops/moe/__init__.py | 2 + .../ops/moe/inkling2_mxfp8_experts.py | 179 ++++++++ .../ops/moe/inkling2_nvfp4_experts.py | 279 ++++++++++++ .../inkling2/ops/moe/inkling2_router.py | 87 ++++ .../models/inkling2/ops/residual/__init__.py | 2 + .../ops/residual/inkling2_residual_mix.py | 140 ++++++ .../test_lists/test-db/l0_b300.yml | 1 + .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../test_weights.py | 208 +++++++++ ...modeling_v2_inkling2_attention_prologue.py | 83 ++++ ...st_modeling_v2_inkling2_paged_attention.py | 158 +++++++ .../test_modeling_v2_inkling2_short_conv.py | 173 +++++++ .../ops/moe/test_modeling_v2_inkling2_moe.py | 280 ++++++++++++ .../test_modeling_v2_inkling2_residual_mix.py | 69 +++ .../test_modeling_v2_ci_selection.py | 2 +- 32 files changed, 4346 insertions(+), 1 deletion(-) create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py create mode 100644 tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_attention_prologue.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md b/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md new file mode 100644 index 000000000000..2eef809c8ce4 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/README.md @@ -0,0 +1,51 @@ + + + +# Inkling 2 + +`Inkling2ForConditionalGeneration` (`thinkingmachines/Inkling-2-Small`): a 48-layer MoE text decoder +(hidden 4608, 384 routed experts, top 8) with a vision tower. The family implements the text model. + +## What the architecture needs + +* **Attention without rotary embeddings.** Position enters through a relative bias each token mixes + per head from 16 bias-vs-distance profiles (`rel_logits_proj`), over distances below 1024. 40 + sliding-window layers attend the last 1024 tokens; 8 global layers (5, 11, ..., 47) attend the + whole prefix and scale q and the relative states by `tau = 1 + 0.1 ln(max((p + 1) / 8192, 1))`. + Each token and head has a sink logit, and the output is gated by `sigmoid(gate)`. q, k and the + relative states are L2-clipped; v is RMS-normalized without a weight. +* **Short convolutions.** Width-4 causal convolutions with a residual run on k and v (dynamic + per-token taps on the global layers) and on each sublayer's output. Their windows are + per-request state. +* **Block attention residuals.** Every 6 layers the residual stream closes a block; each + sublayer reads a learned softmax mix of the normalized blocks and the current partial block. +* **Experts.** fp32 sigmoid router with a selection bias; SiTU activation + (`4 tanh(g / 4) sigmoid(g) * 25 tanh(u / 25)`). The checkpoint stores layers 0-39 in NVFP4 (with + per-token 4/6 NVFP4 activations) and layers 40-47 in MXFP8. + +## Layout + +```text +inkling2/ +├── config.py # InklingDims: the decoder's shape and per-layer expert formats +├── kernels/ # Triton: short convolution, paged attention with the relative bias +├── ops/ # OpWrappers: conv, attention prologue and core, residual mix, +│ # router, NVFP4 experts (FlashInfer CuTe DSL), MXFP8 experts +└── inkling_2_small_nvfp4__sm_103__tep4/ + ├── weights.py # weight table: TP4 heads, EP4 experts, kernel layouts + ├── modeling.py # the 48-layer forward and the engine shell + └── cache.py # KVCacheManagerV2 with the convolution windows +``` + +## Target `inkling_2_small_nvfp4__sm_103__tep4` + +The released checkpoint on one GB300 tray: attention TP4 (12 query and 2 KV heads per rank), MoE +EP4 (96 experts per rank), BF16 KV cache. The baseline runs eagerly, without CUDA graphs, the +overlap scheduler, chunked prefill, KV cache block reuse or speculative decoding. + +The family is not routed yet: these are the operators and the target's components; the route that +serves the target through `LLM()` and `trtllm-serve` follows separately. + +The NVFP4 experts run FlashInfer's CuTe DSL fused MoE with SiTU and per-token 4/6 activations. The +MXFP8 experts run a grouped Triton MoE that expands the E4M3 weights to bf16 in registers (exactly) +and accumulates in fp32. diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py new file mode 100644 index 000000000000..49fdf3f909aa --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2 (`Inkling2ForConditionalGeneration`): ops, kernels and targets of the family.""" diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py new file mode 100644 index 000000000000..f4f97bc41f58 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/config.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The Inkling 2 text decoder's shape, as the family's ops and targets read it. + +`InklingDims` is built once from the checkpoint's `config.json` text config and its +`hf_quant_config.json`, and is the only place a target reads model geometry from. It accepts both +`tensorrt_llm._torch.configs.Inkling2TextConfig` and a native Transformers `Inkling2TextConfig`, +which name a few fields differently (`conv_kernel_size` for `sconv_kernel_size`, +`moe_intermediate_size` for `intermediate_size`) and mark local layers by `layer_types` value +`hybrid_sliding` instead of listing `local_layer_ids`. + +Expert precision is per layer and comes from `hf_quant_config.json`, whose `parameter_groups` name +every expert tensor with the format it is stored in. The released checkpoint stores layers 0-39 +in NVFP4 and layers 40-47 in MXFP8; nothing here assumes that split. +""" + +from __future__ import annotations + +import enum +import json +import os +import re +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +#: The expert weight tensor `hf_quant_config.json` names for each MoE layer. +_EXPERT_W13 = re.compile(r"^model\.llm\.layers\.(\d+)\.mlp\.experts\.w13_weight$") + + +class ExpertFormat(str, enum.Enum): + """How a layer's routed experts are stored and how their activations are quantized.""" + + #: E2M1 values in packed uint8, an FP8 E4M3 scale per 16 values along K, and a per-expert FP32 + #: global scale `amax / outer_scale_denominator`. Activations are quantized per token with the + #: 4/6 block-scale rule. + NVFP4 = "nvfp4" + #: FP8 E4M3 values and a UE8M0 scale per 32 values along K; activations are MXFP8 too. + MXFP8 = "mxfp8" + #: BF16 experts, as in a dequantized or randomly initialized test checkpoint. + BF16 = "bfloat16" + + +def _first(config: Any, *names: str, default: Any = None) -> Any: + for name in names: + value = getattr(config, name, None) + if value is not None: + return value + return default + + +def expert_formats_from_quant_config( + hf_quant_config: Optional[Mapping[str, Any]], num_layers: int +) -> tuple[ExpertFormat, ...]: + """Each layer's expert format, from an Inkling `hf_quant_config.json` payload. + + A layer that no parameter group names keeps BF16 experts; a layer that two groups name is a + malformed file. + """ + formats: dict[int, ExpertFormat] = {} + groups = ((hf_quant_config or {}).get("model") or {}).get("parameter_groups") or [] + for group in groups: + compute = str(group.get("compute_dtype", "")).lower() + fmt = {"nvfp4": ExpertFormat.NVFP4, "mxfp8": ExpertFormat.MXFP8}.get(compute) + if fmt is None: + continue + for tensor in group.get("tensors", []): + match = _EXPERT_W13.match(tensor) + if match is None: + continue + layer = int(match.group(1)) + if layer in formats and formats[layer] is not fmt: + raise ValueError(f"hf_quant_config.json gives layer {layer} two expert formats") + formats[layer] = fmt + out_of_range = sorted(i for i in formats if not 0 <= i < num_layers) + if out_of_range: + raise ValueError( + f"hf_quant_config.json names experts of layers {out_of_range}, " + f"outside the {num_layers} decoder layers" + ) + return tuple(formats.get(i, ExpertFormat.BF16) for i in range(num_layers)) + + +def read_hf_quant_config(checkpoint_dir: Optional[str]) -> Optional[dict]: + """The checkpoint's `hf_quant_config.json`, or None when it has none.""" + if not checkpoint_dir: + return None + path = os.path.join(checkpoint_dir, "hf_quant_config.json") + if not os.path.isfile(path): + return None + with open(path) as f: + return json.load(f) + + +@dataclass(frozen=True) +class InklingDims: + """Geometry of the Inkling 2 text decoder, whole-model (not per rank).""" + + hidden_size: int + num_layers: int + vocab_size: int + unpadded_vocab_size: int + num_heads: int + num_kv_heads: int + head_dim: int + d_rel: int + rel_extent: int + sliding_window: int + global_layers: tuple[int, ...] + conv_kernel_size: int + attn_res_block_size: int + num_experts: int + top_k: int + expert_intermediate_size: int + route_scale: float + gate_softcap: float + up_softcap: float + logit_softcap: float + qk_clip_threshold: float + rel_clip_threshold: float + log_scaling_alpha: float + log_scaling_n_floor: float + rms_norm_eps: float + expert_formats: tuple[ExpertFormat, ...] + + @classmethod + def from_config( + cls, text_config: Any, hf_quant_config: Optional[Mapping[str, Any]] = None + ) -> "InklingDims": + """Read the text config (either config class) and the quant config payload.""" + num_layers = int(text_config.num_hidden_layers) + local = getattr(text_config, "local_layer_ids", None) + if local is not None: + local = set(int(i) for i in local) + global_layers = tuple(i for i in range(num_layers) if i not in local) + else: + layer_types = list(text_config.layer_types) + global_layers = tuple( + i for i, kind in enumerate(layer_types) if "sliding" not in str(kind).lower() + ) + swa = ( + _first(text_config, "swa_num_attention_heads"), + _first(text_config, "swa_num_key_value_heads"), + _first(text_config, "swa_head_dim"), + ) + full = ( + text_config.num_attention_heads, + text_config.num_key_value_heads, + text_config.head_dim, + ) + if any(s is not None and s != f for s, f in zip(swa, full)): + raise ValueError( + f"sliding-window layers with a different attention shape {swa} from " + f"the global layers {full} are not supported" + ) + return cls( + hidden_size=int(text_config.hidden_size), + num_layers=num_layers, + vocab_size=int(text_config.vocab_size), + unpadded_vocab_size=int( + _first(text_config, "unpadded_vocab_size", default=text_config.vocab_size) + ), + num_heads=int(text_config.num_attention_heads), + num_kv_heads=int(text_config.num_key_value_heads), + head_dim=int(text_config.head_dim), + d_rel=int(text_config.d_rel), + rel_extent=int(text_config.rel_extent), + sliding_window=int(_first(text_config, "sliding_window_size", "sliding_window")), + global_layers=global_layers, + conv_kernel_size=int(_first(text_config, "sconv_kernel_size", "conv_kernel_size")), + attn_res_block_size=int(text_config.n_layers_per_attn_res_block), + num_experts=int(text_config.n_routed_experts), + top_k=int(text_config.num_experts_per_tok), + expert_intermediate_size=int( + _first(text_config, "moe_intermediate_size", "intermediate_size") + ), + route_scale=float(_first(text_config, "route_scale", "routed_scaling_factor")), + gate_softcap=float(text_config.mlp_gate_softcap), + up_softcap=float(text_config.mlp_up_softcap), + logit_softcap=float(text_config.final_logit_softcapping), + qk_clip_threshold=float(text_config.qk_l2_clip_threshold), + rel_clip_threshold=float(text_config.rel_l2_clip_threshold), + log_scaling_alpha=float(text_config.log_scaling_alpha), + log_scaling_n_floor=float(text_config.log_scaling_n_floor), + rms_norm_eps=float(text_config.rms_norm_eps), + expert_formats=expert_formats_from_quant_config(hf_quant_config, num_layers), + ) + + def is_global(self, layer: int) -> bool: + """Whether `layer` attends over the whole context rather than the sliding window.""" + return layer in self.global_layers + + def rel_extent_of(self, layer: int) -> int: + """Relative distances the layer's position bias covers: the window in a sliding layer.""" + return self.rel_extent if self.is_global(layer) else self.sliding_window + + def is_block_boundary(self, layer: int) -> bool: + """Whether `layer` starts a new attention-residual block.""" + return layer % self.attn_res_block_size == 0 + + @property + def num_block_states(self) -> int: + """Completed blocks the final output mix reads: the embedding plus one per later boundary.""" + return len([i for i in range(self.num_layers) if self.is_block_boundary(i)]) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py new file mode 100644 index 000000000000..14f41159c340 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""`KVCacheManagerV2` with Inkling 2's short-convolution windows. + +Every decoder layer convolves four streams over the last 4 tokens: its keys and values (this +rank's KV heads) and its attention and MoE outputs (the full hidden width). A request's step +needs the 3 inputs of each stream before its first new token, so the manager keeps them, per +layer and request, in a slot of four tensors `[slots, 3, C]` (bf16): `k`, `v`, `attn`, `mlp`. + +A request takes a slot the first time a step names it and keeps it until the engine frees its +KV cache. The slot pool holds `max_batch_size` requests plus the engine's padding and warmup +requests. The windows are not part of the KV cache's prefix reuse, so block reuse must be off. +""" + +from __future__ import annotations + +from typing import Iterable + +import torch + +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 + +from ..config import InklingDims + +#: Slots beyond max_batch_size: CUDA-graph padding and warmup requests. +SPARE_SLOTS = 8 +#: Inputs a width-4 window keeps. +WINDOW = 3 + + +class Inkling2KVCacheManager(KVCacheManagerV2): + """The KV cache plus each request's short-convolution windows, for every rank-local layer.""" + + def __init__( + self, *args, pretrained_config=None, mapping=None, max_batch_size: int = 1, **kwargs + ): + super().__init__( + *args, + pretrained_config=pretrained_config, + mapping=mapping, + max_batch_size=max_batch_size, + **kwargs, + ) + text = getattr(pretrained_config, "text_config", None) or pretrained_config + dims = InklingDims.from_config(text) + tp = mapping.tp_size if mapping is not None else 1 + kv_width = dims.num_kv_heads * dims.head_dim // tp + self.num_conv_slots = max_batch_size + SPARE_SLOTS + widths = {"k": kv_width, "v": kv_width, "attn": dims.hidden_size, "mlp": dims.hidden_size} + device = torch.device("cuda", torch.cuda.current_device()) + self._conv_state = [ + { + name: torch.zeros( + self.num_conv_slots, WINDOW, width, dtype=torch.bfloat16, device=device + ) + for name, width in widths.items() + } + for _ in range(dims.num_layers) + ] + self._conv_slot_of: dict[int, int] = {} + self._free_conv_slots = list(range(self.num_conv_slots - 1, -1, -1)) + + def conv_state(self, layer: int) -> dict: + """Layer `layer`'s windows: `k`, `v`, `attn`, `mlp`, each `[slots, 3, C]` bf16.""" + return self._conv_state[layer] + + def conv_slots(self, request_ids: Iterable[int]) -> torch.Tensor: + """Each request's slot, as int32 on the device; a request seen for the first time gets one.""" + slots = [] + for request_id in request_ids: + slot = self._conv_slot_of.get(request_id) + if slot is None: + if not self._free_conv_slots: + raise RuntimeError( + f"all {self.num_conv_slots} short-convolution slots are in use; " + "more requests are live than max_batch_size allows" + ) + slot = self._free_conv_slots.pop() + self._conv_slot_of[request_id] = slot + slots.append(slot) + return torch.tensor(slots, dtype=torch.int32).to( + self._conv_state[0]["k"].device, non_blocking=True + ) + + def _release_conv_slot(self, request_id: int) -> None: + slot = self._conv_slot_of.pop(request_id, None) + if slot is not None: + self._free_conv_slots.append(slot) + + def free_resources(self, request, pin_on_release: bool = False): + self._release_conv_slot(request.py_request_id) + return super().free_resources(request, pin_on_release) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py new file mode 100644 index 000000000000..d1c5d62580fc --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling-2-Small, NVFP4/MXFP8 experts / sm_103 / attention TP4 + MoE EP4: token ids to logits. + +One engine step through the 48 decoder layers of the text model. Per layer, with `P` the +partial residual block (bf16 between layers) and `blocks` the normalized completed blocks: + + attention inputs mix(blocks, P) through the input norm(s): one input on a sliding layer, + q, k and v inputs on a global layer (layer 0 reads the embedding block) + projections q, r (16 per head), gate, sink, k and v; a global layer's k and v also + give their convolution's per-token taps + k, v short convolution (dynamic on a global layer), then the prologue: q and r + scaled and clipped (times tau on a global layer), k clipped, v normalized + attention paged, relative bias from r, sink and output gate (window 1024 on a + sliding layer); output projection; all-reduce over TP; short convolution + residual a block boundary (layer % 6 == 0) closes P into blocks and restarts the + partial block from the attention output; otherwise P' = P + a + MoE mix(blocks, P') through the MLP norm; fp32 sigmoid router, top-8 of 384; + this rank's 96 experts (NVFP4 or MXFP8); all-reduce over EP; short conv + output P'' = P' + m, rounded to bf16 + +then the final mix over the 8 blocks and the last partial block through the final norm. The +shell's vocab-parallel lm_head gives fp32 logits, which are soft-capped (30 tanh(x / 30)) and +cut to the tokenizer's 200058 entries (the padded rows are set to -inf). + +The KV cache is the engine's `KVCacheManagerV2` (BF16, 2 KV heads of 128 per rank, a 1024-token +window on the 40 sliding layers) extended with each request's short-convolution windows +(`Inkling2KVCacheManager`, `cache.py`). The forward runs eagerly: no CUDA graphs, no chunked +prefill, no KV cache reuse, no speculative decoding; routing admits only that. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modeling_v2.catalog._op import true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import allreduce, allreduce_group_args +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM +from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks +from tensorrt_llm.functional import AllReduceFusionOp +from tensorrt_llm.logger import logger + +from ..config import ExpertFormat, InklingDims, read_hf_quant_config +from ..kernels.paged_attention import block_tokens, tile_plan +from ..ops.attention.inkling2_attention_prologue import inkling2_attention_prologue, log_scaling +from ..ops.attention.inkling2_paged_attention import SM_SCALE, inkling2_paged_attention +from ..ops.conv.inkling2_short_conv import inkling2_dynamic_short_conv, inkling2_short_conv +from ..ops.moe.inkling2_mxfp8_experts import inkling2_mxfp8_experts +from ..ops.moe.inkling2_nvfp4_experts import fc2_input_scale, inkling2_nvfp4_experts +from ..ops.moe.inkling2_router import inkling2_router +from ..ops.residual.inkling2_residual_mix import ( + inkling2_residual_mix, + normalize_block, + rmsnorm_to_bf16, +) +from . import weights as _weights +from .cache import Inkling2KVCacheManager + +_NO_FUSION = int(AllReduceFusionOp.NONE) + + +def text_config_of(pretrained_config: PretrainedConfig) -> PretrainedConfig: + return getattr(pretrained_config, "text_config", None) or pretrained_config + + +@dataclass +class _Step: + """Per-step views of the attention metadata, shared by every layer.""" + + num_tokens: int + query_start: torch.Tensor # [R + 1] int32 + kv_len: torch.Tensor # [R] int32, cached + new + token_request: torch.Tensor # [T] int32 + has_state: torch.Tensor # [R] int32: the request has convolution history + conv_slots: torch.Tensor # [R] int32 + tile_request: torch.Tensor + tile_start: torch.Tensor + block_t: int + page_tables: dict # layer -> [R, W] int32 + tau: torch.Tensor # [T] fp32 + + +class Inkling2SmallCore(DecoderModel): + """The 48-layer text decoder on one rank of attention TP4 + MoE EP4.""" + + def __init__(self, model_config: ModelConfig): + super().__init__(model_config) + text = text_config_of(model_config.pretrained_config) + checkpoint_dir = getattr(model_config.pretrained_config, "_name_or_path", None) + self.dims = InklingDims.from_config(text, read_hf_quant_config(checkpoint_dir)) + self.shares = _weights.shares_of(self.dims, model_config.mapping) + self.eps = self.dims.rms_norm_eps + self.w = _weights.declare(self.shares) + self._layers: Optional[list] = None + self._allreduce: Optional[dict] = None + self._fc2_scale: Optional[torch.Tensor] = None + self._shared_tables: Optional[bool] = None + + def post_load_weights(self) -> None: + """Per-layer weight views and the TP communicator, once the weights are real.""" + self._layers = [_weights.layer_view(self.w, layer) for layer in range(self.dims.num_layers)] + self._allreduce = allreduce_group_args(self.model_config.mapping) + self._fc2_scale = fc2_input_scale(self.w["embed"].device) + + # ------------------------------------------------------------------------------- step plan + + def _plan(self, md: AttentionMetadata, position_ids: torch.Tensor, num_tokens: int) -> _Step: + manager: Inkling2KVCacheManager = md.kv_cache_manager + num_seqs = md.num_seqs + lengths = [int(n) for n in md.seq_lens[:num_seqs].tolist()] + cached = [int(c) for c in md.kv_cache_params.num_cached_tokens_per_seq[:num_seqs]] + request_ids = list(md.request_ids[:num_seqs]) + assert sum(lengths) == num_tokens, (lengths, num_tokens) + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + device = position_ids.device + + def i32(values): + return torch.tensor(values, dtype=torch.int32).to(device, non_blocking=True) + + block_t = block_tokens(max(lengths)) + tile_request, tile_start = tile_plan(lengths, block_t, device) + kv_lens = [c + n for c, n in zip(cached, lengths)] + blocks = [-(-length // manager.tokens_per_block) for length in kv_lens] + page_tables = self._page_tables(manager, request_ids, blocks, device) + return _Step( + num_tokens=num_tokens, + query_start=i32(bounds), + kv_len=i32(kv_lens), + token_request=i32([r for r, n in enumerate(lengths) for _ in range(n)]), + has_state=i32([int(c > 0) for c in cached]), + conv_slots=manager.conv_slots(request_ids), + tile_request=tile_request, + tile_start=tile_start, + block_t=block_t, + page_tables=page_tables, + tau=log_scaling( + position_ids.reshape(-1), self.dims.log_scaling_alpha, self.dims.log_scaling_n_floor + ), + ) + + def _page_tables(self, manager, request_ids, blocks, device) -> dict: + """Each layer's page table `[R, W]` int32 (-1 past a request's pages). + + A layer's indices address its own `get_buffers` view. The manager gives the layers of one + window the same indices; the first step checks that once, after which one table per window + is built and shared. + """ + + def build(layer): + rows = manager.get_batch_cache_indices( + request_ids, layer_idx=layer, num_blocks_per_seq=blocks + ) + table = torch.full((len(rows), max(len(r) for r in rows)), -1, dtype=torch.int32) + for i, r in enumerate(rows): + table[i, : len(r)] = torch.tensor(r, dtype=torch.int32) + return table + + layers = range(self.dims.num_layers) + window = {layer: self.dims.is_global(layer) for layer in layers} + if self._shared_tables is None: + tables = {layer: build(layer) for layer in layers} + first = {} + for layer in layers: + first.setdefault(window[layer], layer) + self._shared_tables = all( + torch.equal(tables[layer], tables[first[window[layer]]]) for layer in layers + ) + if not self._shared_tables: + logger.warning( + "Inkling2: layers of one window have different page tables; building one per layer" + ) + elif self._shared_tables: + per_window = {} + for layer in layers: + if window[layer] not in per_window: + per_window[window[layer]] = build(layer) + tables = {layer: per_window[window[layer]] for layer in layers} + else: + tables = {layer: build(layer) for layer in layers} + uploaded = {} + return { + layer: uploaded.setdefault(id(t), t.to(device, non_blocking=True)) + for layer, t in tables.items() + } + + # ------------------------------------------------------------------------------- forward + + def _all_reduce(self, x: torch.Tensor) -> torch.Tensor: + if self.model_config.mapping.tp_size == 1: + return x + return allreduce( + x, + None, + None, + None, + None, + op=_NO_FUSION, + eps=self.eps, + trigger_completion_at_end=True, + **self._allreduce, + ) + + def _conv(self, x, taps, state, step: _Step): + return inkling2_short_conv( + x, taps, state, step.conv_slots, step.query_start, step.has_state, step.token_request + ) + + def _attention( + self, layer: int, inputs: torch.Tensor, step: _Step, md: AttentionMetadata + ) -> torch.Tensor: + d, s, w = self.dims, self.shares, self._layers[layer] + manager: Inkling2KVCacheManager = md.kv_cache_manager + state = manager.conv_state(layer) + t = step.num_tokens + if d.is_global(layer): + yq = cublas_mm(inputs[0], w.w_q.t()) + q, r, gate, sink = yq.split([s.q_width, s.r_width, s.q_width, s.heads], dim=1) + yk = cublas_mm(inputs[1], w.w_k.t()) + yv = cublas_mm(inputs[2], w.w_v.t()) + k_lin, k_taps = yk.split([s.kv_width, s.kv_heads * 4], dim=1) + v_lin, v_taps = yv.split([s.kv_width, s.kv_heads * 4], dim=1) + conv_args = ( + step.conv_slots, + step.query_start, + step.has_state, + step.token_request, + d.head_dim, + ) + k = inkling2_dynamic_short_conv( + k_lin.contiguous(), w.k_conv, k_taps.contiguous(), state["k"], *conv_args + ) + v = inkling2_dynamic_short_conv( + v_lin.contiguous(), w.v_conv, v_taps.contiguous(), state["v"], *conv_args + ) + tau, window = step.tau, 0 + else: + y = cublas_mm(inputs[0], w.w_in.t()) + q, r, gate, sink, k_lin, v_lin = y.split( + [s.q_width, s.r_width, s.q_width, s.heads, s.kv_width, s.kv_width], dim=1 + ) + k = self._conv(k_lin.contiguous(), w.k_conv, state["k"], step) + v = self._conv(v_lin.contiguous(), w.v_conv, state["v"], step) + tau, window = None, d.sliding_window + q, k, v, r = inkling2_attention_prologue( + q.reshape(t, s.heads, d.head_dim), + k.view(t, s.kv_heads, d.head_dim), + v.view(t, s.kv_heads, d.head_dim), + r.reshape(t, s.heads, d.d_rel), + w.q_scale, + w.r_scale, + tau, + d.qk_clip_threshold, + d.rel_clip_threshold, + d.rms_norm_eps, + ) + # The bias profiles mix 16 values per head; TF32 would round them, so a true fp32 product. + with true_fp32_matmul(): + rel_logits = torch.matmul(r.float(), w.rel_proj.float()) # [T, heads, rel_extent] + o = inkling2_paged_attention( + q, + k, + v, + manager.get_buffers(layer, kv_layout="HND"), + step.page_tables[layer], + step.query_start, + step.kv_len, + step.token_request, + step.tile_request, + step.tile_start, + step.block_t, + rel_logits, + sink.contiguous(), + gate.reshape(t, s.heads, d.head_dim), + window, + SM_SCALE, + ) + a = self._all_reduce(cublas_mm(o.view(t, s.q_width), w.wo.t())) + return self._conv(a, w.attn_conv, state["attn"], step) + + def _moe(self, layer: int, x: torch.Tensor, step: _Step, md: AttentionMetadata) -> torch.Tensor: + d, s, w = self.dims, self.shares, self._layers[layer] + ids, weights = inkling2_router(x, w.router, w.router_bias, d.top_k, d.route_scale) + if d.expert_formats[layer] is ExpertFormat.NVFP4: + m = inkling2_nvfp4_experts( + x, ids, weights, w.w13, w.w13_sf, w.w13_alpha, w.w2, w.w2_sf, w.w2_alpha, self._fc2_scale, + d.num_experts, s.expert_offset, d.gate_softcap, d.up_softcap, + ) # fmt: skip + else: + m = inkling2_mxfp8_experts( + x, + ids, + weights, + w.w13, + w.w13_s, + w.w2, + w.w2_s, + s.expert_offset, + d.gate_softcap, + d.up_softcap, + ) + m = self._all_reduce(m) + return self._conv(m, w.mlp_conv, md.kv_cache_manager.conv_state(layer)["mlp"], step) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + assert inputs_embeds is None, "the text-only target embeds token ids itself" + d, w = self.dims, self.w + ids = input_ids.reshape(-1) + num_tokens = ids.shape[0] + step = self._plan(attn_metadata, position_ids, num_tokens) + + partial = rmsnorm_to_bf16( + torch.nn.functional.embedding(ids, w.embed), w.embed_norm, self.eps + ) + blocks = torch.empty( + d.num_block_states, num_tokens, d.hidden_size, dtype=torch.float32, device=ids.device + ) + num_blocks = 0 + for layer in range(d.num_layers): + lw = self._layers[layer] + norms = (lw.q_norm, lw.k_norm, lw.v_norm) if d.is_global(layer) else (lw.attn_norm,) + if layer == 0: + inputs = [rmsnorm_to_bf16(partial, n, self.eps) for n in norms] + else: + inputs = inkling2_residual_mix( + blocks[:num_blocks], partial, lw.attn_query, norms, self.eps + ) + a = self._attention(layer, inputs, step, attn_metadata) + if d.is_block_boundary(layer): + blocks[num_blocks] = normalize_block(partial) + num_blocks += 1 + residual = a.float() + else: + residual = partial.float() + a.float() + x = inkling2_residual_mix( + blocks[:num_blocks], residual, lw.mlp_query, (lw.mlp_norm,), self.eps + )[0] + m = self._moe(layer, x, step, attn_metadata) + partial = (residual + m.float()).to(torch.bfloat16) + assert num_blocks == d.num_block_states + return inkling2_residual_mix(blocks, partial, w.final_query, (w.final_norm,), self.eps)[0] + + +class ModelingV2Inkling2SmallNvfp4Sm103Tep4( + DecoderModelForCausalLM[Inkling2SmallCore, PretrainedConfig] +): + """Inkling-2-Small on GB300 with attention TP4 and MoE EP4: the engine-facing shell.""" + + def __init__(self, model_config: ModelConfig): + cfg = model_config.pretrained_config + text = text_config_of(cfg) + # config.json declares bf16 only in its text config; lm_head is built in this dtype. + if getattr(cfg, "torch_dtype", None) is None: + cfg.torch_dtype = model_config.torch_dtype or torch.bfloat16 + super().__init__( + Inkling2SmallCore(model_config), + config=model_config, + hidden_size=text.hidden_size, + vocab_size=text.vocab_size, + ) + dims = self.model.dims + self.logit_softcap = dims.logit_softcap + self.unpadded_vocab_size = dims.unpadded_vocab_size + + #: The engine builds the KV cache manager with the short-convolution windows. + runtime_hooks = RuntimeHooks(kv_cache_manager_cls=Inkling2KVCacheManager) + + def load_weights(self, weights, *args, **kwargs): + _weights.load(self, weights, self.model.shares) + + def post_load_weights(self): + super().post_load_weights() + self.model.post_load_weights() + + def forward(self, attn_metadata: AttentionMetadata, *args, **kwargs) -> torch.Tensor: + logits = super().forward(attn_metadata, *args, **kwargs) + logits = torch.tanh(logits / self.logit_softcap) * self.logit_softcap + if logits.shape[-1] > self.unpadded_vocab_size: + logits[..., self.unpadded_vocab_size :] = float("-inf") + return logits + + +__all__ = ["Inkling2SmallCore", "ModelingV2Inkling2SmallNvfp4Sm103Tep4"] diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py new file mode 100644 index 000000000000..b9194d7d7b32 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/weights.py @@ -0,0 +1,430 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Weight table: Inkling-2-Small / sm_103 / attention TP4 + MoE EP4. + +Every weight the target holds, at this rank's share, with the checkpoint tensors it comes from +and the transform on the way. `declare` allocates them (the engine materializes the storage), +`load` fills them from the engine's lazy checkpoint view. + +* **Attention heads** are split over TP: a rank holds 12 of the 48 query heads and 2 of the 8 KV + heads, the matching rows of the q, k, v, relative-state (16 per head), gate and sink + projections, of the q and relative clip scales and of the k/v short convolutions (their + channels are the KV heads' 128-wide slices, and a global layer's dynamic taps are 4 per KV + head), and the matching input columns of the output projection, whose partial sums meet in an + all-reduce. A sliding layer's six projections read one input and are stacked into one weight; + a global layer reads three inputs (q, k and v each have their own residual mix and norm), so it + stacks q, r, gate and sink, and appends each dynamic convolution's tap projection to its k or v + projection. + +* **Experts** are split over EP: a rank holds the 96 experts `[96 * rank, 96 * rank + 96)`, and + reads only those rows of the checkpoint. NVFP4 layers (0-39 in the release) are stored for + FlashInfer's CuTe DSL MoE: `w13` rows reordered from the checkpoint's per-row gate/up + interleave (gate `j` at row `2j`, up `j` at `2j + 1`) into the kernel's order, block scales + reordered the same way and swizzled into 128x4 atoms, and the per-expert global scale + `amax / outer_scale_denominator` as the GEMM alpha. MXFP8 layers keep the checkpoint's bytes. + +* **Short convolutions** are stored as lag-major fp32 taps `[4, C]` (`taps[0]` weights the + current token): a checkpoint `nn.Conv1d` weight `[C, 1, 4]` is reversed along its last axis; a + dynamic convolution's `static_w` is lag-major already. + +* **Everything else** is replicated: the embedding (whole, so no all-reduce follows the lookup), + the norms, the block-residual queries (fp32), the relative-logit profiles and the router (bf16 + weight, fp32 logits; fp32 bias). `lm_head` is the shell's vocab-parallel module; `load` fills + its rows of `unembed`. + +Only the text decoder's tensors (`model.llm.*`) are loaded: the target is text only and drafts nothing. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Callable, Optional + +import torch +import torch.nn as nn + +from ..config import ExpertFormat, InklingDims +from ..ops.moe.inkling2_nvfp4_experts import fc1_row_order, swizzle_block_scales + +#: Prefix of the text decoder's tensors in the checkpoint. The target loads every one of them and +#: nothing else: not the vision tower (`model.visual.*`) nor the DSpark drafter (`mtp.safetensors`). +LLM = "model.llm" + + +@dataclass(frozen=True) +class Shares: + """This rank's share of every split dimension, from the config and the engine's Mapping.""" + + model: InklingDims + tp_size: int + tp_rank: int + ep_size: int + ep_rank: int + + @property + def heads(self) -> int: + return self.model.num_heads // self.tp_size + + @property + def kv_heads(self) -> int: + return self.model.num_kv_heads // self.tp_size + + @property + def q_width(self) -> int: + return self.heads * self.model.head_dim + + @property + def kv_width(self) -> int: + return self.kv_heads * self.model.head_dim + + @property + def r_width(self) -> int: + return self.heads * self.model.d_rel + + @property + def local_experts(self) -> int: + return self.model.num_experts // self.ep_size + + @property + def expert_offset(self) -> int: + return self.ep_rank * self.local_experts + + def rows(self, width_total: int) -> slice: + """This rank's rows of a projection whose output is split evenly over TP.""" + width = width_total // self.tp_size + return slice(self.tp_rank * width, (self.tp_rank + 1) * width) + + def experts(self) -> slice: + return slice(self.expert_offset, self.expert_offset + self.local_experts) + + +def shares_of(model: InklingDims, mapping) -> Shares: + assert mapping.pp_size == 1 and mapping.cp_size == 1, "no pipeline or context parallelism" + assert not mapping.enable_attention_dp, "attention DP is not implemented" + assert mapping.moe_tp_size == 1, "the experts are split by EP only" + assert mapping.moe_ep_size == mapping.tp_size, "EP must span the TP group" + s = Shares(model, mapping.tp_size, mapping.tp_rank, mapping.moe_ep_size, mapping.moe_ep_rank) + assert model.num_heads % s.tp_size == 0 and model.num_kv_heads % s.tp_size == 0 + assert model.num_experts % s.ep_size == 0 + return s + + +# --------------------------------------------------------------------------------------------- +# Transforms: (sources, shares) -> the tensor to copy in. Sources are the checkpoint tensors in +# the order the entry names them, already on the device. +# --------------------------------------------------------------------------------------------- + + +def _cat_rows(srcs, s: Shares) -> torch.Tensor: + """Each source's TP rows, stacked in source order (projections that read one input).""" + return torch.cat([t[s.rows(t.shape[0])] for t in srcs], dim=0).contiguous() + + +def _rows(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[s.rows(t.shape[0])].contiguous() + + +def _o_cols(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[:, s.rows(t.shape[1])].contiguous() + + +def _conv_taps(srcs, s: Shares) -> torch.Tensor: + """`nn.Conv1d` weight `[C, 1, 4]` (oldest tap first) -> lag-major fp32 `[4, C]`.""" + (t,) = srcs + return t[:, 0, :].flip(-1).t().float().contiguous() + + +def _kv_conv_taps(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return _conv_taps((t[s.rows(t.shape[0])],), s) + + +def _static_taps(srcs, s: Shares) -> torch.Tensor: + """A dynamic convolution's `static_w` `[4, C]` (lag-major): this rank's channels, fp32.""" + (t,) = srcs + return t[:, s.rows(t.shape[1])].float().contiguous() + + +def _kv_with_taps(srcs, s: Shares) -> torch.Tensor: + """k or v projection rows, then the dynamic convolution's tap projection rows (4 per head).""" + proj, taps = srcs + return torch.cat([proj[s.rows(proj.shape[0])], taps[s.rows(taps.shape[0])]], dim=0).contiguous() + + +def _fp32(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.float().reshape(-1, t.shape[-1]).contiguous() + + +def _as_is(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.contiguous() + + +def _nvfp4_w13(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t[:, fc1_row_order(t.shape[1] // 2, t.device)].contiguous() + + +def _nvfp4_w13_scale(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + rows = t.view(torch.uint8)[:, fc1_row_order(t.shape[1] // 2, t.device)] + return swizzle_block_scales(rows) + + +def _nvfp4_scale(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return swizzle_block_scales(t.view(torch.uint8)) + + +def _nvfp4_alpha(srcs, s: Shares) -> torch.Tensor: + """Per local expert: `amax / outer_scale_denominator`, fp32.""" + amax, denominator = srcs + return (amax[s.experts()].float() / denominator.float().reshape(())).contiguous() + + +def _raw_u8(srcs, s: Shares) -> torch.Tensor: + (t,) = srcs + return t.view(torch.uint8).contiguous() + + +@dataclass(frozen=True) +class Entry: + """One weight of one layer (or of the model): shape, dtype, checkpoint sources, transform. + + `sources` are checkpoint names; `expert_rows` marks sources read only at this rank's expert + rows (the lazy view reads just those bytes). + """ + + name: str + shape: tuple + dtype: torch.dtype + sources: tuple[str, ...] + transform: Callable[[Any, Shares], torch.Tensor] + expert_rows: tuple[bool, ...] = () + + +def layer_entries(s: Shares, layer: int) -> list[Entry]: + d = s.model + p = f"{LLM}.layers.{layer}" + a = f"{p}.attn" + h, bf16, f32, u8 = d.hidden_size, torch.bfloat16, torch.float32, torch.uint8 + entries = [ + Entry("mlp_norm", (h,), bf16, (f"{p}.mlp_norm.weight",), _as_is), + Entry("wo", (h, s.q_width), bf16, (f"{a}.wo_ud.weight",), _o_cols), + Entry("q_scale", (s.heads, d.head_dim), bf16, (f"{a}.q_l2_clip.scale",), _rows), + Entry("r_scale", (s.heads, d.d_rel), bf16, (f"{a}.rel_l2_clip.scale",), _rows), + Entry( + "rel_proj", + (d.d_rel, d.rel_extent_of(layer)), + bf16, + (f"{a}.rel_logits_proj.proj",), + _as_is, + ), + Entry("attn_conv", (4, h), f32, (f"{p}.attn_sconv.weight",), _conv_taps), + Entry("mlp_conv", (4, h), f32, (f"{p}.mlp_sconv.weight",), _conv_taps), + Entry("router", (d.num_experts, h), bf16, (f"{p}.mlp.gate.weight",), _as_is), + Entry("router_bias", (d.num_experts,), f32, (f"{p}.mlp.gate.bias",), _as_is), + Entry("mlp_query", (1, h), f32, (f"{LLM}.attn_res.mlp_queries.{layer}",), _fp32), + ] + stacked_q = s.q_width + s.r_width + s.q_width + s.heads + if d.is_global(layer): + dyn = s.kv_heads * 4 + entries += [ + Entry("q_norm", (h,), bf16, (f"{p}.q_input_norm.weight",), _as_is), + Entry("k_norm", (h,), bf16, (f"{p}.k_input_norm.weight",), _as_is), + Entry("v_norm", (h,), bf16, (f"{p}.v_input_norm.weight",), _as_is), + Entry( + "w_q", + (stacked_q, h), + bf16, + ( + f"{a}.wq_du.weight", + f"{a}.wr_du.weight", + f"{a}.gate_attn.weight", + f"{a}.attn_sink_proj.weight", + ), + _cat_rows, + ), + Entry( + "w_k", + (s.kv_width + dyn, h), + bf16, + (f"{a}.wk_dv.weight", f"{a}.k_sconv.weight_proj"), + _kv_with_taps, + ), + Entry( + "w_v", + (s.kv_width + dyn, h), + bf16, + (f"{a}.wv_dv.weight", f"{a}.v_sconv.weight_proj"), + _kv_with_taps, + ), + Entry("k_conv", (4, s.kv_width), f32, (f"{a}.k_sconv.static_w",), _static_taps), + Entry("v_conv", (4, s.kv_width), f32, (f"{a}.v_sconv.static_w",), _static_taps), + ] + else: + entries += [ + Entry("attn_norm", (h,), bf16, (f"{p}.attn_norm.weight",), _as_is), + Entry( + "w_in", + (stacked_q + 2 * s.kv_width, h), + bf16, + ( + f"{a}.wq_du.weight", + f"{a}.wr_du.weight", + f"{a}.gate_attn.weight", + f"{a}.attn_sink_proj.weight", + f"{a}.wk_dv.weight", + f"{a}.wv_dv.weight", + ), + _cat_rows, + ), + Entry("k_conv", (4, s.kv_width), f32, (f"{a}.k_sconv.weight",), _kv_conv_taps), + Entry("v_conv", (4, s.kv_width), f32, (f"{a}.v_sconv.weight",), _kv_conv_taps), + ] + if layer > 0: + queries = 3 if d.is_global(layer) else 1 + entries.append( + Entry( + "attn_query", + (queries, h), + f32, + (f"{LLM}.attn_res.attn_queries.{layer - 1}",), + _fp32, + ) + ) + + e, i = s.local_experts, d.expert_intermediate_size + x = f"{p}.mlp.experts" + fmt = d.expert_formats[layer] + if fmt is ExpertFormat.NVFP4: + entries += [ + Entry("w13", (e, 2 * i, h // 2), u8, (f"{x}.w13_weight",), _nvfp4_w13, (True,)), + Entry( + "w13_sf", + (e, 2 * i, h // 16), + u8, + (f"{x}.w13_weight_scale",), + _nvfp4_w13_scale, + (True,), + ), + Entry( + "w13_alpha", + (e,), + f32, + (f"{x}.w13_weight_amax", f"{x}.w13_weight_outer_scale_denominator"), + _nvfp4_alpha, + ), + Entry("w2", (e, h, i // 2), u8, (f"{x}.w2_weight",), _as_is, (True,)), + Entry("w2_sf", (e, h, i // 16), u8, (f"{x}.w2_weight_scale",), _nvfp4_scale, (True,)), + Entry( + "w2_alpha", + (e,), + f32, + (f"{x}.w2_weight_amax", f"{x}.w2_weight_outer_scale_denominator"), + _nvfp4_alpha, + ), + ] + elif fmt is ExpertFormat.MXFP8: + e4m3 = torch.float8_e4m3fn + entries += [ + Entry("w13", (e, 2 * i, h), e4m3, (f"{x}.w13_weight",), _as_is, (True,)), + Entry("w13_s", (e, 2 * i, h // 32), u8, (f"{x}.w13_weight_scale",), _raw_u8, (True,)), + Entry("w2", (e, h, i), e4m3, (f"{x}.w2_weight",), _as_is, (True,)), + Entry("w2_s", (e, h, i // 32), u8, (f"{x}.w2_weight_scale",), _raw_u8, (True,)), + ] + else: + raise NotImplementedError( + f"layer {layer}: {fmt.value} experts are not supported by this target" + ) + return entries + + +def model_entries(s: Shares) -> list[Entry]: + d = s.model + h, bf16, f32 = d.hidden_size, torch.bfloat16, torch.float32 + return [ + Entry("embed", (d.vocab_size, h), bf16, (f"{LLM}.embed.weight",), _as_is), + Entry("embed_norm", (h,), bf16, (f"{LLM}.embed_norm.weight",), _as_is), + Entry("final_query", (1, h), f32, (f"{LLM}.attn_res.output_query",), _fp32), + Entry("final_norm", (h,), bf16, (f"{LLM}.norm.weight",), _as_is), + ] + + +def _key(layer: Optional[int], name: str) -> str: + return name if layer is None else f"l{layer}_{name}" + + +def all_entries(s: Shares) -> list[tuple[Optional[int], Entry]]: + rows = [(None, entry) for entry in model_entries(s)] + for layer in range(s.model.num_layers): + rows += [(layer, entry) for entry in layer_entries(s, layer)] + return rows + + +def declare(s: Shares) -> nn.ParameterDict: + """Every weight at this rank's share; the engine's meta init gives them storage later.""" + w = nn.ParameterDict() + for layer, entry in all_entries(s): + w[_key(layer, entry.name)] = nn.Parameter( + torch.empty(entry.shape, dtype=entry.dtype), requires_grad=False + ) + return w + + +def layer_view(w: nn.ParameterDict, layer: int) -> SimpleNamespace: + """Layer `layer`'s weights by name, `w.`.""" + prefix = f"l{layer}_" + return SimpleNamespace(**{k[len(prefix) :]: v for k, v in w.items() if k.startswith(prefix)}) + + +def load(model, weights, s: Shares) -> None: + """Fill every declared weight and the shell's lm_head rows from the checkpoint. + + `weights` is the engine's lazy view (`weights[name][a:b]` reads only those rows). Every + text-decoder tensor of the checkpoint must be consumed; other tensors are not read. + """ + core = model.model + consumed: set[str] = set() + expert_rows = s.experts() + + def read(name: str, rows: Optional[slice], device: torch.device) -> torch.Tensor: + assert name in weights, f"checkpoint tensor missing: {name}" + consumed.add(name) + t = weights[name][rows] if rows is not None else weights[name][:] + return t.to(device, non_blocking=True) + + for layer, entry in all_entries(s): + dst = core.w[_key(layer, entry.name)].data + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = tuple( + read(n, expert_rows if f else None, dst.device) for n, f in zip(entry.sources, flags) + ) + src = entry.transform(srcs, s) + if src.dtype != dst.dtype and src.dtype == torch.uint8 and dst.dtype.itemsize == 1: + src = src.view(dst.dtype) + assert src.shape == dst.shape, (layer, entry.name, tuple(src.shape), tuple(dst.shape)) + assert src.dtype == dst.dtype, (layer, entry.name, src.dtype, dst.dtype) + dst.copy_(src, non_blocking=True) + if entry.expert_rows: + del srcs, src + torch.cuda.empty_cache() + + lm_head = model.lm_head.weight + vocab = s.model.vocab_size + rows = lm_head.shape[0] + start = 0 if rows == vocab else s.tp_rank * rows + unembed = read(f"{LLM}.unembed.weight", slice(start, min(start + rows, vocab)), lm_head.device) + if unembed.shape[0] < rows: + lm_head.data.zero_() + lm_head.data[: unembed.shape[0]].copy_(unembed) + torch.cuda.synchronize() + + leftover = [ + name for name in weights.keys() if name.startswith(f"{LLM}.") and name not in consumed + ] + assert not leftover, f"text-decoder tensors the table does not name: {sorted(leftover)[:8]}" diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py new file mode 100644 index 000000000000..77b54d94f579 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton grouped MoE for Inkling 2's MXFP8 experts: one launch per GEMM for all local experts. + +The routes `(token, slot)` that land on this rank's experts are grouped by expert into blocks of +`BLOCK_M` rows (`align_routes`, on the device, without a host synchronization; the block count is +bounded by `T * top_k / BLOCK_M + E`). Then: + +* `gemm1`: per block and 128 intermediate columns, `x^ @ gate^T` and `x^ @ up^T` (gate rows `2j`, + up rows `2j + 1` of `w13`), with the E4M3 weights and their UE8M0 scales expanded to bf16 in the + kernel (exactly: an E4M3 value times a power of two is a bf16 value), fp32 accumulation, then + SiTU and the MXFP8 rounding of the result per 32 columns; writes `h^` `[routes, I]` bf16. +* `gemm2`: per block and 128 hidden columns, `h^ @ down^T` in fp32, times the route's weight, + added atomically into the fp32 output row of its token. + +`x^`, the MXFP8-rounded input, is computed once per call by the caller. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + +BLOCK_M = 32 + + +def align_routes( + topk_ids: torch.Tensor, local_expert_offset: int, num_local: int, block_m: int = BLOCK_M +): + """Group this rank's routes by expert into blocks of `block_m` rows, on the device and without + a host synchronization. + + Returns `route_of_row` `[max_blocks * block_m]` int32 (the flat route `token * top_k + slot` of + each row, -1 for padding), `expert_of_block` `[max_blocks]` int32 (-1 past the last used block) + and `max_blocks`, a bound on the blocks any routing needs. + """ + num_tokens, top_k = topk_ids.shape + device = topk_ids.device + flat = topk_ids.reshape(-1).long() - local_expert_offset + is_local = (flat >= 0) & (flat < num_local) + expert = torch.where( + is_local, flat, torch.full_like(flat, num_local) + ) # non-local routes sort last + counts = torch.zeros(num_local + 1, dtype=torch.long, device=device) + counts.scatter_add_(0, expert, torch.ones_like(expert)) + counts = counts[:num_local] + blocks_per_expert = (counts + block_m - 1) // block_m + block_end = torch.cumsum(blocks_per_expert, 0) + block_start = block_end - blocks_per_expert + max_blocks = num_tokens * top_k // block_m + num_local + order = torch.argsort(expert, stable=True) + sorted_expert = expert[order] + sorted_local = sorted_expert < num_local + e = sorted_expert.clamp(max=num_local - 1) + rank_in_expert = ( + torch.arange(order.numel(), device=device) - (torch.cumsum(counts, 0) - counts)[e] + ) + rows = max_blocks * block_m + # Non-local routes write to one spare slot past the end. + row = torch.where( + sorted_local, block_start[e] * block_m + rank_in_expert, torch.full_like(e, rows) + ) + route_of_row = torch.full((rows + 1,), -1, dtype=torch.int32, device=device) + route_of_row.scatter_(0, row, order.to(torch.int32)) + owner = torch.searchsorted(block_end, torch.arange(max_blocks, device=device), right=True) + expert_of_block = torch.where(owner < num_local, owner, torch.full_like(owner, -1)).to( + torch.int32 + ) + return route_of_row[:rows], expert_of_block, max_blocks + + +@triton.jit +def _dequant_tile(w_ptrs, s_ptrs, mask): + """E4M3 weights and their UE8M0 exponents -> bf16 (exact).""" + w = tl.load(w_ptrs, mask=mask, other=0.0).to(tl.float32) + e = tl.load(s_ptrs, mask=mask, other=127).to(tl.float32) + return (w * tl.exp2(e - 127.0)).to(tl.bfloat16) + + +@triton.jit +def _round_e4m3(x): + """Round fp32 values in [-448, 448] to the nearest E4M3 value (ties to even), in fp32. + + The quantum is 2^(e - 3) for a value in [2^e, 2^(e + 1)), and 2^-9 below 2^-6 (subnormals); + dividing by a power of two is exact, so `rint` gives the E4M3 rounding. + """ + exponent = ((x.to(tl.int32, bitcast=True) >> 23) & 0xFF) - 127 + quantum = tl.exp2((tl.maximum(exponent, -6) - 3).to(tl.float32)) + return libdevice.rint(x / quantum) * quantum + + +@triton.jit +def _gemm1_kernel( + X, + W13, + S13, + ROUTE, + EXPERT, + H, + top_k, + hidden, + inter, + stride_x, + stride_we, + stride_wr, + stride_se, + stride_sr, + stride_h, + gate_softcap, + up_softcap, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + block = tl.program_id(0) + expert = tl.load(EXPERT + block) + if expert < 0: + return + rows = block * BLOCK_M + tl.arange(0, BLOCK_M) + route = tl.load(ROUTE + rows) + row_ok = route >= 0 + token = tl.where(row_ok, route // top_k, 0).to(tl.int64) + cols = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) # intermediate columns j + col_ok = cols < inter + offs_k = tl.arange(0, BLOCK_K) + w_base = W13 + expert.to(tl.int64) * stride_we + s_base = S13 + expert.to(tl.int64) * stride_se + acc_g = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + acc_u = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + for k0 in range(0, hidden, BLOCK_K): + k = k0 + offs_k + x = tl.load(X + token[:, None] * stride_x + k[None, :], mask=row_ok[:, None], other=0.0) + gate_rows = (2 * cols).to(tl.int64) + mask = col_ok[:, None] + g = _dequant_tile( + w_base + gate_rows[:, None] * stride_wr + k[None, :], + s_base + gate_rows[:, None] * stride_sr + (k // 32)[None, :], + mask, + ) + u = _dequant_tile( + w_base + (gate_rows + 1)[:, None] * stride_wr + k[None, :], + s_base + (gate_rows + 1)[:, None] * stride_sr + (k // 32)[None, :], + mask, + ) + acc_g += tl.dot(x, tl.trans(g)) + acc_u += tl.dot(x, tl.trans(u)) + h = ( + gate_softcap + * libdevice.tanh(acc_g / gate_softcap) + * tl.sigmoid(acc_g) + * up_softcap + * libdevice.tanh(acc_u / up_softcap) + ) + # MXFP8 rounding per 32 columns: scale 2^ceil(log2(amax / 448)), values to E4M3. + groups = tl.reshape(h, [BLOCK_M, BLOCK_N // 32, 32]) + raw = tl.max(tl.abs(groups), axis=2) / 448.0 + bits = raw.to(tl.int32, bitcast=True) + exponent = ((bits + 0x007FFFFF) & 0x7F800000) >> 23 + # 2^-127 (an fp32 subnormal) and 2^127, built from their bits: as Python literals Triton + # would type the subnormal as fp64. + min_scale = tl.full([1, 1], 1 << 22, tl.int32).to(tl.float32, bitcast=True) + max_inverse = tl.full([1, 1], 254 << 23, tl.int32).to(tl.float32, bitcast=True) + exponent = tl.where(raw <= min_scale, 0, tl.minimum(exponent, 254)) + scale = tl.maximum((exponent << 23).to(tl.float32, bitcast=True), min_scale) + inverse = tl.where(exponent == 0, max_inverse, 1.0 / scale) + q = _round_e4m3(tl.clamp(groups * inverse[:, :, None], -448.0, 448.0)) + h_hat = tl.reshape(q * scale[:, :, None], [BLOCK_M, BLOCK_N]) + tl.store( + H + rows.to(tl.int64)[:, None] * stride_h + cols[None, :], + h_hat.to(tl.bfloat16), + mask=row_ok[:, None] & col_ok[None, :], + ) + + +@triton.jit +def _gemm2_kernel( + H, + W2, + S2, + ROUTE, + EXPERT, + WEIGHTS, + OUT, + top_k, + hidden, + inter, + stride_h, + stride_we, + stride_wr, + stride_se, + stride_sr, + stride_out, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + block = tl.program_id(0) + expert = tl.load(EXPERT + block) + if expert < 0: + return + rows = block * BLOCK_M + tl.arange(0, BLOCK_M) + route = tl.load(ROUTE + rows) + row_ok = route >= 0 + cols = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) # hidden columns + col_ok = cols < hidden + offs_k = tl.arange(0, BLOCK_K) + w_base = W2 + expert.to(tl.int64) * stride_we + s_base = S2 + expert.to(tl.int64) * stride_se + acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + for k0 in range(0, inter, BLOCK_K): + k = k0 + offs_k + h = tl.load( + H + rows.to(tl.int64)[:, None] * stride_h + k[None, :], mask=row_ok[:, None], other=0.0 + ) + w = _dequant_tile( + w_base + cols.to(tl.int64)[:, None] * stride_wr + k[None, :], + s_base + cols.to(tl.int64)[:, None] * stride_sr + (k // 32)[None, :], + col_ok[:, None], + ) + acc += tl.dot(h, tl.trans(w)) + weight = tl.load(WEIGHTS + tl.where(row_ok, route, 0), mask=row_ok, other=0.0) + token = tl.where(row_ok, route // top_k, 0).to(tl.int64) + tl.atomic_add( + OUT + token[:, None] * stride_out + cols[None, :], + acc * weight[:, None], + mask=row_ok[:, None] & col_ok[None, :], + sem="relaxed", + ) + + +def grouped_mxfp8_moe( + x_hat: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, +) -> torch.Tensor: + """`x_hat` `[T, H]` bf16 (MXFP8-rounded) -> fp32 `[T, H]`, this rank's experts' weighted sum.""" + num_tokens, hidden = x_hat.shape + local, rows, _ = w13.shape + inter = rows // 2 + top_k = topk_ids.shape[1] + out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=x_hat.device) + if num_tokens == 0: + return out + route_of_row, expert_of_block, max_blocks = align_routes(topk_ids, local_expert_offset, local) + h = torch.empty(max_blocks * BLOCK_M, inter, dtype=torch.bfloat16, device=x_hat.device) + block_n, block_k = 128, 128 + _gemm1_kernel[(max_blocks, triton.cdiv(inter, block_n))]( + x_hat, + w13, + w13_s, + route_of_row, + expert_of_block, + h, + top_k, + hidden, + inter, + x_hat.stride(0), + w13.stride(0), + w13.stride(1), + w13_s.stride(0), + w13_s.stride(1), + h.stride(0), + float(gate_softcap), + float(up_softcap), + BLOCK_M=BLOCK_M, + BLOCK_N=block_n, + BLOCK_K=block_k, + num_warps=8, + ) + _gemm2_kernel[(max_blocks, triton.cdiv(hidden, block_n))]( + h, + w2, + w2_s, + route_of_row, + expert_of_block, + topk_weights.reshape(-1).contiguous(), + out, + top_k, + hidden, + inter, + h.stride(0), + w2.stride(0), + w2.stride(1), + w2_s.stride(0), + w2_s.stride(1), + out.stride(0), + BLOCK_M=BLOCK_M, + BLOCK_N=block_n, + BLOCK_K=block_k, + num_warps=8, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py new file mode 100644 index 000000000000..9fa0ff77cf8d --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton kernels of Inkling 2's paged attention: K/V append, and attention with a relative bias, +an attention sink and an output gate. + +The KV pool is one layer's `KVCacheManagerV2.get_buffers(layer, "HND")` view, `[pages, 2, kv +heads, page size, head dim]`; `page_table[r, i]` is the pool page of request `r`'s tokens +`i * page_size ..`. A step holds the new tokens of several requests, request-major +(`query_start`); request `r`'s new tokens sit at positions `kv_len[r] - n_r .. kv_len[r] - 1`. + +`append_kv` writes the new tokens' keys and values to their pages. `attention` then computes, per +query token `t` at position `p` and query head `h` (KV head `h // group`): + + s_j = q . k_j * sm_scale + rel_logits[t, h, p - j] (0 <= p - j < rel_extent, else no bias) + over keys j <= p, and p - j < window on a sliding layer + o = softmax(s) . v, lse = logsumexp(s) + out = o * sigmoid(lse - sink[t, h]) * sigmoid(gate[t, h]) + +in fp32, rounded once. `sigmoid(lse - sink)` is the probability mass the keys keep against one +extra logit `sink` of zero value. A program covers `BLOCK_T` tokens of one request and the `group` +query heads of one KV head, as `BLOCK_T * GROUP_PAD` rows of one tile. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# Masked scores. Finite, so a row whose first key block is fully masked keeps a finite running +# maximum; the first valid key then rescales what the masked keys added to zero. +_MASKED = tl.constexpr(-1.0e30) + + +@triton.jit +def _append_kv_kernel( + K, + V, + POOL, + PAGE_TABLE, + QSTART, + KV_LEN, + TOKEN_REQ, + stride_kt, + stride_kh, + stride_vt, + stride_vh, + stride_pool_page, + stride_pool_kv, + stride_pool_head, + stride_pool_slot, + stride_pt, + PAGE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, +): + t = tl.program_id(0) + head = tl.program_id(1) + req = tl.load(TOKEN_REQ + t) + start = tl.load(QSTART + req) + n = tl.load(QSTART + req + 1) - start + pos = tl.load(KV_LEN + req) - n + (t - start) + page = tl.load(PAGE_TABLE + req.to(tl.int64) * stride_pt + pos // PAGE_SIZE).to(tl.int64) + dst = ( + POOL + + page * stride_pool_page + + head * stride_pool_head + + (pos % PAGE_SIZE) * stride_pool_slot + ) + offs = tl.arange(0, HEAD_DIM) + t64 = t.to(tl.int64) + tl.store(dst + offs, tl.load(K + t64 * stride_kt + head * stride_kh + offs)) + tl.store(dst + stride_pool_kv + offs, tl.load(V + t64 * stride_vt + head * stride_vh + offs)) + + +@triton.jit +def _attention_kernel( + Q, + POOL, + PAGE_TABLE, + QSTART, + KV_LEN, + TILE_REQ, + TILE_START, + REL, + SINK, + GATE, + OUT, + stride_qt, + stride_qh, + stride_pool_page, + stride_pool_kv, + stride_pool_head, + stride_pool_slot, + stride_pt, + stride_rt, + stride_rh, + stride_st, + stride_gt, + stride_gh, + stride_ot, + stride_oh, + sm_scale, + window, + GROUP: tl.constexpr, + GROUP_PAD: tl.constexpr, + REL_EXTENT: tl.constexpr, + PAGE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_N: tl.constexpr, +): + tile = tl.program_id(0) + kv_head = tl.program_id(1) + req = tl.load(TILE_REQ + tile) + q_begin = tl.load(TILE_START + tile) + start = tl.load(QSTART + req) + end = tl.load(QSTART + req + 1) + past = tl.load(KV_LEN + req) - (end - start) + + BLOCK_M: tl.constexpr = BLOCK_T * GROUP_PAD + rows = tl.arange(0, BLOCK_M) + token = q_begin + rows // GROUP_PAD + head_in_group = rows % GROUP_PAD + head = kv_head * GROUP + head_in_group + row_ok = (head_in_group < GROUP) & (token < end) + q_pos = past + (token - start) + token64 = token.to(tl.int64) + offs_d = tl.arange(0, HEAD_DIM) + + q = tl.load( + Q + token64[:, None] * stride_qt + head[:, None] * stride_qh + offs_d[None, :], + mask=row_ok[:, None], + other=0.0, + ) + + first_pos = past + (q_begin - start) + last_pos = past + (tl.minimum(q_begin + BLOCK_T, end) - start) - 1 + # The oldest key any row of the tile reads: the first row's window start on a sliding layer. + lo = tl.where(window > 0, tl.maximum(first_pos - window + 1, 0), 0) + hi = last_pos + 1 + + m_i = tl.full([BLOCK_M], _MASKED, dtype=tl.float32) + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + qk_scale = sm_scale * 1.4426950408889634 + pt_row = PAGE_TABLE + req.to(tl.int64) * stride_pt + kv_base = POOL + kv_head * stride_pool_head + + for block in range(lo // BLOCK_N * BLOCK_N, hi, BLOCK_N): + k_pos = block + tl.arange(0, BLOCK_N) + key_ok = (k_pos >= lo) & (k_pos < hi) + page = tl.load(pt_row + k_pos // PAGE_SIZE, mask=key_ok, other=0).to(tl.int64) + slot_ptr = kv_base + page * stride_pool_page + (k_pos % PAGE_SIZE) * stride_pool_slot + k = tl.load(slot_ptr[:, None] + offs_d[None, :], mask=key_ok[:, None], other=0.0) + v = tl.load( + slot_ptr[:, None] + stride_pool_kv + offs_d[None, :], mask=key_ok[:, None], other=0.0 + ) + + # Scores in log2 units: (q . k * sm_scale + bias) * log2(e). + s = tl.dot(q, tl.trans(k)) * qk_scale + dist = q_pos[:, None] - k_pos[None, :] + allowed = ( + row_ok[:, None] & key_ok[None, :] & (dist >= 0) & ((window <= 0) | (dist < window)) + ) + biased = allowed & (dist < REL_EXTENT) + bias = tl.load( + REL + token64[:, None] * stride_rt + head[:, None] * stride_rh + dist, + mask=biased, + other=0.0, + ) + s = tl.where(allowed, s + bias * 1.4426950408889634, _MASKED) + + m_new = tl.maximum(m_i, tl.max(s, 1)) + alpha = tl.math.exp2(m_i - m_new) + p = tl.math.exp2(s - m_new[:, None]) + l_i = l_i * alpha + tl.sum(p, 1) + acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) + m_i = m_new + + # Natural-log logsumexp of the scores, then the sink and the output gate. + lse = (m_i + tl.math.log2(l_i)) / 1.4426950408889634 + sink = tl.load(SINK + token64 * stride_st + head, mask=row_ok, other=0.0).to(tl.float32) + keep = tl.sigmoid(lse - sink) + gate = tl.load( + GATE + token64[:, None] * stride_gt + head[:, None] * stride_gh + offs_d[None, :], + mask=row_ok[:, None], + other=0.0, + ).to(tl.float32) + out = acc / l_i[:, None] * keep[:, None] * tl.sigmoid(gate) + tl.store( + OUT + token64[:, None] * stride_ot + head[:, None] * stride_oh + offs_d[None, :], + out.to(OUT.dtype.element_ty), + mask=row_ok[:, None], + ) + + +def append_kv( + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, +) -> None: + """Write `k`, `v` `[T, kv heads, head dim]` to their pages of `kv_pool`.""" + num_tokens, num_kv_heads, head_dim = k.shape + if num_tokens == 0: + return + _append_kv_kernel[(num_tokens, num_kv_heads)]( + k, + v, + kv_pool, + page_table, + query_start, + kv_len, + token_request, + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + kv_pool.stride(0), + kv_pool.stride(1), + kv_pool.stride(2), + kv_pool.stride(3), + page_table.stride(0), + PAGE_SIZE=kv_pool.shape[3], + HEAD_DIM=head_dim, + ) + + +def tile_plan( + query_lengths: list[int], block_t: int, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor]: + """Per program: its request and first token, from the host-side query lengths of one step.""" + tile_request, tile_start = [], [] + start = 0 + for r, n in enumerate(query_lengths): + for first in range(start, start + n, block_t): + tile_request.append(r) + tile_start.append(first) + start += n + as_tensor = lambda values: torch.tensor(values, dtype=torch.int32).to(device, non_blocking=True) # noqa: E731 + return as_tensor(tile_request), as_tensor(tile_start) + + +def block_tokens(max_query_length: int) -> int: + """Query tokens per program: 2 for a decode-only step (16 MMA rows with a group of up to 8 + heads), 16 otherwise.""" + return 2 if max_query_length <= 1 else 16 + + +def attention( + q: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, +) -> torch.Tensor: + """Launch the attention kernel; see the module docstring. Returns `[T, heads, head dim]`.""" + num_tokens, num_heads, head_dim = q.shape + num_kv_heads = kv_pool.shape[2] + group = num_heads // num_kv_heads + out = torch.empty_like(q) + num_tiles = tile_request.shape[0] + if num_tokens == 0 or num_tiles == 0: + return out + _attention_kernel[(num_tiles, num_kv_heads)]( + q, + kv_pool, + page_table, + query_start, + kv_len, + tile_request, + tile_start, + rel_logits, + sink, + gate, + out, + q.stride(0), + q.stride(1), + kv_pool.stride(0), + kv_pool.stride(1), + kv_pool.stride(2), + kv_pool.stride(3), + page_table.stride(0), + rel_logits.stride(0), + rel_logits.stride(1), + sink.stride(0), + gate.stride(0), + gate.stride(1), + out.stride(0), + out.stride(1), + float(sm_scale), + int(window), + GROUP=group, + GROUP_PAD=triton.next_power_of_2(group), + REL_EXTENT=rel_logits.shape[2], + PAGE_SIZE=kv_pool.shape[3], + HEAD_DIM=head_dim, + BLOCK_T=block_t, + BLOCK_N=64, + num_warps=8 if block_t * triton.next_power_of_2(group) >= 64 else 4, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py new file mode 100644 index 000000000000..c7568e4be259 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/short_conv.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Triton kernels of Inkling 2's width-4 causal short convolutions, with a per-request window state. + +One step holds the new tokens of several requests, request-major (`query_start`). Token `t` of a +request reads its own input and the 3 inputs before it: inputs of the same step from `x`, earlier +ones from the request's window state. A request that starts at position 0 has no earlier inputs +(`has_state` 0), which reads as zeros. + +`conv_out` computes, in fp32, per channel `c`: + + out[t, c] = (sum over lag 3..0 of tap(t, c, lag) * x[t - lag, c] + x[t, c]) * out_scale + +with `tap = taps[lag, c]` (static), or `taps[lag, c] + dynamic_taps[t, c // channels_per_head * 4 ++ lag]` (dynamic), and rounds once to the output dtype. `update_state` then stores the last 3 +inputs of each request, oldest first, in its slot of `state` `[slots, 3, C]`. The state is read by +the first kernel and written by the second, so the two must run in this order on one stream. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +#: Kernel width of every Inkling 2 short convolution. +KERNEL_SIZE = 4 + + +@triton.jit +def _conv_out_kernel( + X, + TAPS, + DYN, + STATE, + SLOTS, + QSTART, + HAS_STATE, + TOKEN_REQ, + OUT, + num_tokens, + channels, + stride_x, + stride_dyn, + stride_state_slot, + stride_state_win, + stride_out, + out_scale, + CHANNELS_PER_HEAD: tl.constexpr, + DYNAMIC: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_C: tl.constexpr, +): + offs_t = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T) + offs_c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C) + mask_t = offs_t < num_tokens + mask_c = offs_c < channels + mask = mask_t[:, None] & mask_c[None, :] + + req = tl.load(TOKEN_REQ + offs_t, mask=mask_t, other=0) + local = offs_t - tl.load(QSTART + req, mask=mask_t, other=0) + has_state = tl.load(HAS_STATE + req, mask=mask_t, other=0) != 0 + slot = tl.load(SLOTS + req, mask=mask_t, other=0).to(tl.int64) + rows = offs_t.to(tl.int64) + + x0 = tl.load(X + rows[:, None] * stride_x + offs_c[None, :], mask=mask, other=0.0).to( + tl.float32 + ) + acc = tl.zeros([BLOCK_T, BLOCK_C], dtype=tl.float32) + # Oldest input first, the order of a convolution over the window. + for lag in tl.static_range(3, -1, -1): + if lag == 0: + xv = x0 + else: + src = local - lag + in_step = src >= 0 + xv_step = tl.load( + X + (rows - lag)[:, None] * stride_x + offs_c[None, :], + mask=mask & in_step[:, None], + other=0.0, + ) + # Window state index of an input `src` (< 0) steps before the request's first new token. + win = 3 + src + from_state = (~in_step) & has_state & (win >= 0) + xv_state = tl.load( + STATE + + slot[:, None] * stride_state_slot + + win[:, None] * stride_state_win + + offs_c[None, :], + mask=mask & from_state[:, None], + other=0.0, + ) + xv = tl.where(in_step[:, None], xv_step.to(tl.float32), xv_state.to(tl.float32)) + tap = tl.load(TAPS + lag * channels + offs_c, mask=mask_c, other=0.0)[None, :] + if DYNAMIC: + head = offs_c // CHANNELS_PER_HEAD + dyn = tl.load( + DYN + rows[:, None] * stride_dyn + head[None, :] * 4 + lag, mask=mask, other=0.0 + ).to(tl.float32) + tap = tap + dyn + acc += tap * xv + out = (acc + x0) * out_scale + tl.store( + OUT + rows[:, None] * stride_out + offs_c[None, :], out.to(OUT.dtype.element_ty), mask=mask + ) + + +@triton.jit +def _update_state_kernel( + X, + STATE, + SLOTS, + QSTART, + HAS_STATE, + channels, + stride_x, + stride_state_slot, + stride_state_win, + BLOCK_C: tl.constexpr, +): + req = tl.program_id(0) + offs_c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C) + mask_c = offs_c < channels + start = tl.load(QSTART + req) + n = tl.load(QSTART + req + 1) - start + has_state = tl.load(HAS_STATE + req) != 0 + base = STATE + tl.load(SLOTS + req).to(tl.int64) * stride_state_slot + + # New window entry j is the input at local index n - 3 + j: from this step, or from the old + # window (index n + j) when the request appended fewer than 3 tokens. All three are read + # before any is written. + v0 = _window_entry(X, base, start, n, 0, has_state, offs_c, mask_c, stride_x, stride_state_win) + v1 = _window_entry(X, base, start, n, 1, has_state, offs_c, mask_c, stride_x, stride_state_win) + v2 = _window_entry(X, base, start, n, 2, has_state, offs_c, mask_c, stride_x, stride_state_win) + tl.store(base + offs_c, v0, mask=mask_c) + tl.store(base + stride_state_win + offs_c, v1, mask=mask_c) + tl.store(base + 2 * stride_state_win + offs_c, v2, mask=mask_c) + + +@triton.jit +def _window_entry( + X, base, start, n, j: tl.constexpr, has_state, offs_c, mask_c, stride_x, stride_state_win +): + src = n - 3 + j + from_step = src >= 0 + v_step = tl.load( + X + (start + src).to(tl.int64) * stride_x + offs_c, mask=mask_c & from_step, other=0.0 + ) + v_state = tl.load( + base + (n + j) * stride_state_win + offs_c, + mask=mask_c & (~from_step) & has_state, + other=0.0, + ) + return tl.where(from_step, v_step.to(base.dtype.element_ty), v_state) + + +def short_conv( + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor | None, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + out_scale: float, + channels_per_head: int = 1, +) -> torch.Tensor: + """Launch both kernels; see the module docstring. Returns `out` `[T, C]` in `x`'s dtype.""" + num_tokens, channels = x.shape + out = torch.empty_like(x) + if num_tokens == 0: + return out + block_t, block_c = 16, 128 + dynamic = dynamic_taps is not None + _conv_out_kernel[(triton.cdiv(num_tokens, block_t), triton.cdiv(channels, block_c))]( + x, + taps, + dynamic_taps if dynamic else x, + state, + slots, + query_start, + has_state, + token_request, + out, + num_tokens, + channels, + x.stride(0), + dynamic_taps.stride(0) if dynamic else 0, + state.stride(0), + state.stride(1), + out.stride(0), + float(out_scale), + CHANNELS_PER_HEAD=channels_per_head, + DYNAMIC=dynamic, + BLOCK_T=block_t, + BLOCK_C=block_c, + ) + num_requests = slots.shape[0] + _update_state_kernel[(num_requests, triton.cdiv(channels, block_c))]( + x, + state, + slots, + query_start, + has_state, + channels, + x.stride(0), + state.stride(0), + state.stride(1), + BLOCK_C=block_c, + ) + return out diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py new file mode 100644 index 000000000000..ae2216d8359b --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_attention_prologue.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""What Inkling 2's attention does to its projections before the attention core. + +Per token and head, each step in fp32 and rounded to bf16 where the source model rounds: + + q = l2clip(q * q_scale, 22.627) then on a global layer q = q * tau + k = l2clip(k, 22.627) (k after its short convolution) + v = v / rms(v) (eps 1e-6, no weight; v after its convolution) + r = l2clip(r * r_scale, 8) then on a global layer r = r * tau + +with `l2clip(x, c) = x * min(c / (|x| + 1e-6), 1)` over the head dimension, and the global +layers' log scaling `tau = 1 + alpha * ln(max((position + 1) / n_floor, 1))` per token. A +sliding layer has no tau. +""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +_TP4 = dict(heads=12, kv_heads=2, head_dim=128, d_rel=16) + + +def log_scaling(positions: torch.Tensor, alpha: float, n_floor: float) -> torch.Tensor: + """The global layers' per-token tau, fp32 `[T]`.""" + return 1.0 + alpha * torch.log(((positions.float() + 1.0) / n_floor).clamp(min=1.0)) + + +def _clip(x: torch.Tensor, threshold: float) -> torch.Tensor: + norm = torch.linalg.vector_norm(x, dim=-1, keepdim=True) + return x * (threshold / (norm + 1e-6)).clamp(max=1.0) + + +class Inkling2AttentionPrologue(OpWrapper): + """`q` `[T, H, 128]`, `k`, `v` `[T, KV, 128]`, `r` `[T, H, 16]` bf16, `q_scale` `[H, 128]`, + `r_scale` `[H, 16]`, `tau` `[T]` fp32 or None -> (q, k, v, r), new bf16 tensors.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="sliding layer at TP4, 33 tokens", spec=dict(_TP4, tokens=33, tau=False)), + Cell(why="global layer at TP4, tokens past n_floor", spec=dict(_TP4, tokens=33, tau=True)), + ) + + note = """ + The clip thresholds are 22.627 for q and k and 8 for r; tau multiplies q and r after their + clips, so a clipped q can exceed 22.627 on a global layer. tau is 1 below position 8191: + only positions past n_floor show it. + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + dtype = q.dtype + q = _clip(q.float() * q_scale.float(), qk_threshold).to(dtype) + k = _clip(k.float(), qk_threshold).to(dtype) + vf = v.float() + v = (vf * torch.rsqrt(vf.square().mean(-1, keepdim=True) + eps)).to(dtype) + r = _clip(r.float() * r_scale.float(), rel_threshold).to(dtype) + if tau is not None: + t = tau.view(-1, 1, 1) + q = (q.float() * t).to(dtype) + r = (r.float() * t).to(dtype) + return q, k, v, r + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """The same steps in fp64, rounded to bf16 at the same points.""" + dtype = q.dtype + q = _clip(q.double() * q_scale.double(), qk_threshold).to(dtype) + k = _clip(k.double(), qk_threshold).to(dtype) + vd = v.double() + v = (vd * torch.rsqrt(vd.square().mean(-1, keepdim=True) + eps)).to(dtype) + r = _clip(r.double() * r_scale.double(), rel_threshold).to(dtype) + if tau is not None: + t = tau.double().view(-1, 1, 1) + q = (q.double() * t).to(dtype) + r = (r.double() * t).to(dtype) + return q, k, v, r + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + r: torch.Tensor, + q_scale: torch.Tensor, + r_scale: torch.Tensor, + tau: Optional[torch.Tensor], + qk_threshold: float, + rel_threshold: float, + eps: float, + ) -> None: + assert q.dim() == k.dim() == v.dim() == r.dim() == 3, "q, k, v, r are [T, heads, dim]" + assert q_scale.shape == q.shape[1:] and r_scale.shape == r.shape[1:], "per-head scales" + assert tau is None or (tau.dtype == torch.float32 and tau.shape == (q.shape[0],)), ( + "tau is fp32 [T]" + ) + + +inkling2_attention_prologue = Inkling2AttentionPrologue() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py new file mode 100644 index 000000000000..85f31179950a --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2 attention on a paged KV cache: append the step's keys and values, then attend with +the relative position bias, the attention sink and the output gate. + +Inkling 2 has no rotary embedding. Position enters the logits only through a bias that each query +token mixes per head from a bank of 16 bias-vs-distance profiles, `rel_logits[t, h, d] = r[t, h] +. proj[:, d]` for backward distances `0 <= d < rel_extent` (1024), and zero beyond. A sliding +layer attends the last 1024 positions (`p - j < window`); a global layer the whole prefix. Each +query token and head also has a sink logit, from its own projection, that absorbs probability +mass without contributing a value. The attention output is then gated per element by +`sigmoid(gate)`. See `kernels/paged_attention.py` for the formula. + +The KV cache holds the keys and values after their short convolution, clipping and normalization +(bf16). The page table maps each request's token positions to pages of the layer's pool view. +""" + +from __future__ import annotations + +import math + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ...kernels.paged_attention import append_kv, attention + +#: `1 / sqrt(head_dim)` for Inkling 2's 128-wide heads. +SM_SCALE = 1.0 / math.sqrt(128) + +#: Requests of the cells, as (new tokens, cached tokens before the step). +_PREFILL = ((100, 0),) +_DECODE_PAST_WINDOW = ((1, 1500), (1, 7), (1, 1024), (1, 1023)) +_MIXED = ((70, 1000), (1, 1500), (37, 0), (1, 60)) + + +class Inkling2PagedAttention(OpWrapper): + """Append `k`, `v` `[T, kv heads, 128]` to `kv_pool` and attend with `q` `[T, heads, 128]`. + + `kv_pool` is the layer's HND pool view `[pages, 2, kv heads, page size, 128]`; `page_table` + `[R, W]` int32 maps request `r`'s token blocks to its pages. `query_start` `[R + 1]` and + `kv_len` `[R]` (cached plus new tokens) give each request's new tokens and positions; + `token_request` `[T]` maps tokens to requests. `tile_request`, `tile_start` (int32, one entry + per program) and `block_t` are the step's tile plan (`kernels.paged_attention.tile_plan`). + `rel_logits` `[T, heads, rel_extent]` fp32, `sink` `[T, heads]` and `gate` `[T, heads, 128]` + are per query token; `window` is 1024 on a sliding layer and 0 on a global one. Returns + `[T, heads, 128]` in `q`'s dtype. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="sliding layer at TP4 (12 q / 2 kv heads), one 100-token prefill", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_PREFILL), + ), + Cell( + why="sliding layer at TP4, decodes before, at and past the window edge", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_DECODE_PAST_WINDOW), + ), + Cell( + why="sliding layer at TP4, a mixed step with a chunk that crosses the window", + spec=dict(heads=12, kv_heads=2, window=1024, requests=_MIXED), + ), + Cell( + why="global layer at TP4, decodes past the bias extent", + spec=dict(heads=12, kv_heads=2, window=0, requests=_DECODE_PAST_WINDOW), + ), + Cell( + why="global layer at TP1 (48 q / 8 kv heads), a mixed step", + spec=dict(heads=48, kv_heads=8, window=0, requests=_MIXED), + ), + ) + + note = """ + The bias is indexed by backward distance and is zero from rel_extent on, so a global + layer's keys past 1023 tokens carry no bias while a sliding layer's window ends exactly + where its bias does. The sink is a logit, compared with the logsumexp of the scaled and + biased scores; it is not scaled by sm_scale. Only decodes and chunks at positions past + 1023 tell the window, the bias extent and an off-by-one in either apart, which is what the + decode cells' positions 1023, 1024 and 1500 are for. + """ + + def __call__( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> torch.Tensor: + append_kv(k, v, kv_pool, page_table, query_start, kv_len, token_request) + return attention( + q, + kv_pool, + page_table, + query_start, + kv_len, + tile_request, + tile_start, + block_t, + rel_logits, + sink, + gate, + window, + sm_scale, + ) + + def reference( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> torch.Tensor: + """Per request and head, dense fp32 attention over the pages it reads; writes `k`, `v` + to `kv_pool` as the op does. The tile plan is not needed.""" + page_size = kv_pool.shape[3] + num_heads = q.shape[1] + group = num_heads // kv_pool.shape[2] + rel_extent = rel_logits.shape[2] + bounds = query_start.tolist() + lengths = kv_len.tolist() + out = torch.empty_like(q) + for r in range(len(lengths)): + start, end = bounds[r], bounds[r + 1] + n, length = end - start, lengths[r] + positions = torch.arange(length - n, length, device=q.device) + pages = page_table[r, positions // page_size].long() + kv_pool[pages, 0, :, positions % page_size] = k[start:end] + kv_pool[pages, 1, :, positions % page_size] = v[start:end] + lo = max(0, length - n - window + 1) if window > 0 else 0 + keys_pos = torch.arange(lo, length, device=q.device) + key_pages = page_table[r, keys_pos // page_size].long() + keys = kv_pool[key_pages, 0, :, keys_pos % page_size].float() # [L, kv heads, D] + values = kv_pool[key_pages, 1, :, keys_pos % page_size].float() + dist = positions[:, None] - keys_pos[None, :] + allowed = dist >= 0 + if window > 0: + allowed &= dist < window + in_extent = (dist >= 0) & (dist < rel_extent) + for h in range(num_heads): + with true_fp32_matmul(): + logits = q[start:end, h].float() @ keys[:, h // group].t() * sm_scale + bias = rel_logits[start:end, h].gather(1, dist.clamp(0, rel_extent - 1)) + logits = logits + torch.where(in_extent, bias, torch.zeros_like(bias)) + logits = logits.masked_fill(~allowed, float("-inf")) + lse = torch.logsumexp(logits, dim=1) + with true_fp32_matmul(): + o = torch.softmax(logits, dim=1) @ values[:, h // group] + keep = torch.sigmoid(lse - sink[start:end, h].float()) + o = o * keep[:, None] * torch.sigmoid(gate[start:end, h].float()) + out[start:end, h] = o.to(q.dtype) + return out + + def is_valid( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_pool: torch.Tensor, + page_table: torch.Tensor, + query_start: torch.Tensor, + kv_len: torch.Tensor, + token_request: torch.Tensor, + tile_request: torch.Tensor, + tile_start: torch.Tensor, + block_t: int, + rel_logits: torch.Tensor, + sink: torch.Tensor, + gate: torch.Tensor, + window: int, + sm_scale: float, + ) -> None: + num_tokens, num_heads, head_dim = q.shape + assert kv_pool.dim() == 5 and kv_pool.shape[1] == 2 and kv_pool.shape[4] == head_dim, ( + "kv_pool must be the HND view [pages, 2, kv heads, page size, head dim]" + ) + assert kv_pool.stride(4) == 1 and q.stride(2) == 1 and gate.stride(2) == 1, ( + "head dims must be contiguous" + ) + num_kv_heads = kv_pool.shape[2] + assert num_heads % num_kv_heads == 0, "query heads must group evenly over kv heads" + assert k.shape == v.shape == (num_tokens, num_kv_heads, head_dim), ( + "k and v must be [T, kv heads, head dim]" + ) + assert k.dtype == v.dtype == q.dtype == kv_pool.dtype, "q, k, v and the pool share a dtype" + assert rel_logits.dtype == torch.float32 and rel_logits.shape[:2] == (num_tokens, num_heads) + assert rel_logits.stride(2) == 1, "rel_logits must be contiguous over distances" + assert sink.shape == (num_tokens, num_heads) and sink.stride(1) == 1, ( + "sink must be [T, heads]" + ) + assert gate.shape == q.shape, "gate must be [T, heads, head dim]" + assert window >= 0, "window is the sliding window length, or 0 for full attention" + if window > 0: + # A key past the bias extent inside the window would have no bias; Inkling 2's + # sliding layers have window == rel_extent. + assert window <= rel_logits.shape[2], "a sliding window may not exceed rel_extent" + bounds = query_start.tolist() + lengths = kv_len.tolist() + new = [b - a for a, b in zip(bounds, bounds[1:])] + assert bounds[0] == 0 and bounds[-1] == num_tokens and all(n > 0 for n in new) + assert all(length >= n for length, n in zip(lengths, new)), "kv_len counts the new tokens" + page_size = kv_pool.shape[3] + assert all(-(-length // page_size) <= page_table.shape[1] for length in lengths), ( + "page_table must cover every request's tokens" + ) + plan_request, plan_start = [], [] + for r, n in enumerate(new): + for first in range(bounds[r], bounds[r] + n, block_t): + plan_request.append(r) + plan_start.append(first) + assert tile_request.tolist() == plan_request and tile_start.tolist() == plan_start, ( + "the tile plan must cover each request's tokens in blocks of block_t" + ) + assert torch.equal( + token_request.cpu(), + torch.repeat_interleave(torch.arange(len(new), dtype=torch.int32), torch.tensor(new)), + ), "token_request must map each token to its request" + + +inkling2_paged_attention = Inkling2PagedAttention() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py new file mode 100644 index 000000000000..de531e39aa13 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py @@ -0,0 +1,299 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's width-4 causal short convolutions over a step of several requests. + +Every decoder layer runs two static convolutions on its full-width hidden states: one after the +attention output projection and one after the MoE. A sliding-window layer also convolves its key +and value projections (static); a global layer does so with dynamic taps, the static taps plus +per-token taps projected from the layer's key or value input, shared by the channels of one KV +head. Both forms add the input back and scale the sum:: + + static: out = (sum over lags of taps[lag] * x[t - lag] + x[t]) / sqrt(5) + dynamic: out = (sum over lags of (taps[lag] + dyn[t, head, lag]) * x[t - lag] + x[t]) / 3 + +in fp32, rounded once. `taps` is lag-major, `[4, C]` fp32: `taps[0]` weights the current token. +A checkpoint's `nn.Conv1d` weight `[C, 1, 4]` holds the same taps oldest first +(`taps[lag] = weight[:, 0, 3 - lag]`); a dynamic convolution's `static_w` `[4, C]` is already +lag-major. + +The 3 inputs before a request's first new token come from its window state `[slots, 3, C]` +(oldest first) when `has_state` is set, and are zeros otherwise. The call stores each request's +last 3 inputs back into its slot. +""" + +from __future__ import annotations + +import math + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from ...kernels.short_conv import KERNEL_SIZE, short_conv + +#: The static convolution's output scale, 1 / sqrt(kernel size + 1). +STATIC_SCALE = 1.0 / math.sqrt(KERNEL_SIZE + 1) +#: The dynamic convolution's output scale, 1 / sqrt(2 * kernel size + 1). +DYNAMIC_SCALE = 1.0 / math.sqrt(2 * KERNEL_SIZE + 1) + +#: Request mixes the cells run. Each entry is (new tokens, cached tokens before the step). +_FRESH_PREFILL = ((17, 0),) +_DECODE = ((1, 40), (1, 3), (1, 1), (1, 1000)) +_MIXED = ((5, 0), (2, 9), (1, 1), (1, 2), (33, 64)) + + +def _reference( + x, taps, dynamic_taps, state, slots, query_start, has_state, out_scale, channels_per_head +): + num_channels = x.shape[1] + out = torch.empty_like(x) + bounds = query_start.tolist() + updates = [] + for r, slot in enumerate(slots.tolist()): + start, end = bounds[r], bounds[r + 1] + n = end - start + if int(has_state[r]): + history = state[slot].float() + else: + history = torch.zeros(KERNEL_SIZE - 1, num_channels, device=x.device) + seq = torch.cat([history, x[start:end].float()]) + acc = torch.zeros(n, num_channels, device=x.device) + for lag in range(KERNEL_SIZE - 1, -1, -1): + tap = taps[lag].float().expand(n, num_channels) + if dynamic_taps is not None: + per_head = dynamic_taps[start:end].float().view(n, -1, KERNEL_SIZE)[:, :, lag] + tap = tap + per_head.repeat_interleave(channels_per_head, dim=1) + first = KERNEL_SIZE - 1 - lag + acc = acc + tap * seq[first : first + n] + out[start:end] = ((acc + seq[KERNEL_SIZE - 1 :]) * out_scale).to(x.dtype) + updates.append((slot, seq[-(KERNEL_SIZE - 1) :].to(state.dtype))) + for slot, window in updates: + state[slot] = window + return out + + +def _check( + x, taps, dynamic_taps, state, slots, query_start, has_state, token_request, channels_per_head +): + num_tokens, num_channels = x.shape + assert x.dim() == 2 and x.stride(1) == 1, "x must be [T, C] with contiguous channels" + assert taps.shape == (KERNEL_SIZE, num_channels) and taps.dtype == torch.float32, ( + f"taps must be lag-major fp32 [{KERNEL_SIZE}, {num_channels}]" + ) + assert taps.is_contiguous(), "taps must be contiguous" + assert state.dim() == 3 and state.shape[1:] == (KERNEL_SIZE - 1, num_channels), ( + f"state must be [slots, {KERNEL_SIZE - 1}, {num_channels}]" + ) + assert state.stride(2) == 1 and state.dtype == x.dtype, ( + "state must hold contiguous channels in x's dtype" + ) + num_requests = slots.shape[0] + for name, t in (("slots", slots), ("query_start", query_start), ("has_state", has_state)): + assert t.dtype == torch.int32 and t.dim() == 1, f"{name} must be a 1-D int32 tensor" + assert query_start.shape[0] == num_requests + 1 and has_state.shape[0] == num_requests, ( + "query_start must be [R + 1] and has_state [R] for R = len(slots)" + ) + bounds = query_start.tolist() + assert bounds[0] == 0 and bounds[-1] == num_tokens, "query_start must run from 0 to T" + assert all(b > a for a, b in zip(bounds, bounds[1:])), "every request must append tokens" + slot_list = slots.tolist() + # Two requests in one slot race: each reads the other's window as its own. + assert len(set(slot_list)) == len(slot_list), "each request needs its own slot" + assert all(0 <= s < state.shape[0] for s in slot_list), "slots must index state" + expected = torch.repeat_interleave( + torch.arange(num_requests, dtype=torch.int32), torch.tensor(bounds).diff() + ) + assert token_request.dtype == torch.int32 and torch.equal(token_request.cpu(), expected), ( + "token_request must map each token to its request index" + ) + if dynamic_taps is not None: + assert num_channels % channels_per_head == 0, "channels must split into whole heads" + heads = num_channels // channels_per_head + assert dynamic_taps.shape == (num_tokens, heads * KERNEL_SIZE), ( + f"dynamic_taps must be [T, {heads} heads * {KERNEL_SIZE}]" + ) + assert dynamic_taps.stride(1) == 1, "dynamic_taps must be row-contiguous" + + +class Inkling2ShortConv(OpWrapper): + """Static short convolution: `x` `[T, C]` with lag-major fp32 `taps` `[4, C]`; returns `[T, C]`. + + `state` `[slots, 3, C]` (x's dtype) holds each request's last 3 inputs and is updated in + place. `slots` `[R]`, `query_start` `[R + 1]`, `has_state` `[R]` and `token_request` `[T]` + are int32 on the device; `token_request` maps each token to its request index. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="sliding-layer k/v at TP4 (256 channels), one fresh prefill", + spec=dict(channels=256, requests=_FRESH_PREFILL), + ), + Cell( + why="sliding-layer k/v at TP4, four decodes, one at position 1", + spec=dict(channels=256, requests=_DECODE), + ), + Cell( + why="hidden-width convolution (4608), a mixed step with 1- and 2-token appends", + spec=dict(channels=4608, requests=_MIXED), + ), + ) + + note = """ + The residual and the scale are part of the convolution, and the window holds the + convolution's input, not its output. A request at position 1 or 2 reads a window that is + partly the zeros before its first token, which only a decode right after a short prefill + exercises; the decode cell has one. + """ + + def __call__( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> torch.Tensor: + return short_conv( + x, taps, None, state, slots, query_start, has_state, token_request, STATIC_SCALE + ) + + def reference( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> torch.Tensor: + """Per request, the convolution over `[window or zeros, new inputs]` in fp32; updates + `state` in place as the op does. `token_request` is not needed.""" + return _reference(x, taps, None, state, slots, query_start, has_state, STATIC_SCALE, 1) + + def is_valid( + self, + x: torch.Tensor, + taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + ) -> None: + _check(x, taps, None, state, slots, query_start, has_state, token_request, 1) + + +class Inkling2DynamicShortConv(OpWrapper): + """Dynamic short convolution of a global layer's keys or values. + + As `Inkling2ShortConv`, with per-token taps `dynamic_taps` `[T, heads * 4]` (head-major, lag + minor; the layer's `weight_proj` output) added to `taps` for the `channels_per_head` channels + of each KV head, and the output scale 1/3. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="global-layer k/v at TP4 (2 heads of 128), one fresh prefill", + spec=dict(channels=256, heads=2, requests=_FRESH_PREFILL), + ), + Cell( + why="global-layer k/v at TP4, four decodes", + spec=dict(channels=256, heads=2, requests=_DECODE), + ), + Cell( + why="global-layer k/v at TP1 (8 heads), a mixed step", + spec=dict(channels=1024, heads=8, requests=_MIXED), + ), + ) + + note = """ + The per-token taps belong to the token being computed, not to the inputs it reads, and + one head's taps cover exactly that head's channels. Taps shared by every head, or applied + per input token, still give a plausible convolution; the cells use per-head random taps so + either mistake shows. + """ + + def __call__( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> torch.Tensor: + return short_conv( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + DYNAMIC_SCALE, + channels_per_head, + ) + + def reference( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> torch.Tensor: + """As `Inkling2ShortConv.reference`, with each token's head taps added to `taps`.""" + return _reference( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + DYNAMIC_SCALE, + channels_per_head, + ) + + def is_valid( + self, + x: torch.Tensor, + taps: torch.Tensor, + dynamic_taps: torch.Tensor, + state: torch.Tensor, + slots: torch.Tensor, + query_start: torch.Tensor, + has_state: torch.Tensor, + token_request: torch.Tensor, + channels_per_head: int, + ) -> None: + _check( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + channels_per_head, + ) + + +inkling2_short_conv = Inkling2ShortConv() +inkling2_dynamic_short_conv = Inkling2DynamicShortConv() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py new file mode 100644 index 000000000000..eb9da96d2d50 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's MXFP8 routed experts as a grouped MoE. + +One rank's experts `[offset, offset + E)` over every token of the step; the output is this +rank's partial sum. Per token and chosen expert `e` with weight `w`: + + x^ = mxfp8(x) E4M3 values, one power-of-two scale per 32 along K + h = situ(x^ @ gate_e^T, x^ @ up_e^T) + out += w * mxfp8(h) @ down_e^T + +`mxfp8` rounds each block's scale up to a power of two, `2 ** ceil(log2(amax / 448))`, and the +values to the nearest E4M3. Weights are the checkpoint's: `w13` `[E, 2I, H]` E4M3 with gate `j` +at row `2j` and up `j` at row `2j + 1`, `w2` `[E, H, I]`, and UE8M0 scale exponents `[E, rows, +K / 32]` (scale `2 ** (s - 127)`). + +The routes are grouped by expert on the device and both GEMMs run as one Triton launch each +(`kernels/mxfp8_moe.py`): the kernels expand the E4M3 weights to bf16 in registers, which is +exact, and accumulate in fp32, so only the summation order differs from an MXFP8 MMA. Experts' +outputs meet in fp32 atomic adds. Nothing synchronizes with the host. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +from ...kernels.mxfp8_moe import grouped_mxfp8_moe + +#: Values per MXFP8 scale. +BLOCK = 32 +_E4M3_MAX = 448.0 +_MIN_SCALE = 2.0**-127 + +_EP4 = dict(experts=384, local=96, hidden=4608, inter=2304, top_k=8) + + +def mxfp8_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """`[N, K]` -> (E4M3 values `[N, K]`, uint8 scale exponents `[N, K / 32]`).""" + n, k = x.shape + groups = x.float().view(n, k // BLOCK, BLOCK) + raw = groups.abs().amax(dim=-1) / _E4M3_MAX + # Round the scale up to a power of two: add the largest mantissa and keep the exponent. + exponent = ((raw.view(torch.int32) + 0x007FFFFF) & 0x7F800000) >> 23 + exponent = torch.where(raw <= _MIN_SCALE, torch.zeros_like(exponent), exponent.clamp(max=254)) + scale = torch.clamp((exponent << 23).view(torch.float32), min=_MIN_SCALE) + inverse = torch.where(exponent == 0, torch.full_like(scale, 2.0**127), 1.0 / scale) + values = (groups * inverse.unsqueeze(-1)).clamp(-_E4M3_MAX, _E4M3_MAX).to(torch.float8_e4m3fn) + return values.view(n, k), exponent.to(torch.uint8) + + +def mxfp8_dequantize( + values: torch.Tensor, exponents: torch.Tensor, dtype=torch.float32 +) -> torch.Tensor: + """E4M3 values `[..., K]` and uint8 exponents `[..., K / 32]` -> `[..., K]` in `dtype`.""" + scale = torch.exp2(exponents.view(torch.uint8).float() - 127.0).clamp(min=_MIN_SCALE) + out = values.float().unflatten(-1, (-1, BLOCK)) * scale.unsqueeze(-1) + return out.flatten(-2).to(dtype) + + +def _situ(gate, up, gate_softcap, up_softcap): + return ( + gate_softcap + * torch.tanh(gate / gate_softcap) + * torch.sigmoid(gate) + * up_softcap + * torch.tanh(up / up_softcap) + ) + + +class Inkling2Mxfp8Experts(OpWrapper): + """This rank's MXFP8 experts: `x` `[T, H]` bf16, `topk_ids` `[T, k]` int32 (global ids), + `topk_weights` `[T, k]` fp32 -> `[T, H]` bf16, the sum over the chosen experts this rank holds.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="EP4 rank 2, one decode token", spec=dict(_EP4, tokens=1, rank=2)), + Cell(why="EP4 rank 0, 64 tokens", spec=dict(_EP4, tokens=64, rank=0)), + Cell( + why="EP4 rank 3, 300 tokens: several row blocks per expert", + spec=dict(_EP4, tokens=300, rank=3), + ), + ) + + note = """ + Both GEMM inputs are MXFP8-quantized, the SiTU output from fp32. The gate and up rows + interleave per row in the checkpoint, unlike the NVFP4 layers' kernel layout; reading + them as two halves still gives a finite, wrong result. The fp32 atomics make the last + bits of repeated calls differ. + """ + + def __call__( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + x_hat = mxfp8_dequantize(*mxfp8_quantize(x), dtype=torch.bfloat16) + out = grouped_mxfp8_moe( + x_hat, + topk_ids, + topk_weights, + w13, + w13_s, + w2, + w2_s, + local_expert_offset, + gate_softcap, + up_softcap, + ) + return out.to(x.dtype) + + def reference( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + """True fp32 GEMMs on the fp32-dequantized weights and activations, gate and up taken from + the even and odd rows.""" + local = w13.shape[0] + x_hat = mxfp8_dequantize(*mxfp8_quantize(x)) + out = torch.zeros(x.shape, dtype=torch.float32, device=x.device) + for e in range(local): + tokens, slots = torch.where(topk_ids == local_expert_offset + e) + if tokens.numel() == 0: + continue + fc1 = mxfp8_dequantize(w13[e], w13_s[e]) + down = mxfp8_dequantize(w2[e], w2_s[e]) + with true_fp32_matmul(): + h = _situ( + x_hat[tokens] @ fc1[0::2].t(), + x_hat[tokens] @ fc1[1::2].t(), + gate_softcap, + up_softcap, + ) + y = mxfp8_dequantize(*mxfp8_quantize(h)) @ down.t() + out.index_add_(0, tokens, y * topk_weights[tokens, slots].float().unsqueeze(1)) + return out.to(x.dtype) + + def is_valid( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_s: torch.Tensor, + w2: torch.Tensor, + w2_s: torch.Tensor, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> None: + local, rows, hidden = w13.shape + assert w13.dtype == w2.dtype == torch.float8_e4m3fn, "MXFP8 weights are E4M3" + assert w13_s.shape == (local, rows, hidden // BLOCK) and w13_s.dtype == torch.uint8 + assert w2.shape == (local, hidden, rows // 2), "w2 must be [E, H, I]" + assert w2_s.shape == (local, hidden, rows // 2 // BLOCK) and w2_s.dtype == torch.uint8 + assert x.dtype == torch.bfloat16 and x.shape[1] == hidden + assert topk_ids.dtype == torch.int32 and topk_weights.dtype == torch.float32 + + +inkling2_mxfp8_experts = Inkling2Mxfp8Experts() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py new file mode 100644 index 000000000000..779eb3c7d5ce --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_nvfp4_experts.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's NVFP4 routed experts on FlashInfer's CuTe DSL fused MoE (W4A4, SiTU). + +One rank's experts `[offset, offset + E)` over every token of the step; the output is this +rank's partial sum, which an all-reduce over the EP group completes. Per token and chosen expert +`e` with weight `w`: + + x^ = nvfp4(x) per token, 4/6 block scales (E4M3 max 256) + h = situ(x^ @ gate_e^T, x^ @ up_e^T) + = 4 tanh(g / 4) sigmoid(g) * 25 tanh(u / 25) + out += w * nvfp4(h) @ down_e^T + +Weights are the checkpoint's NVFP4 tensors: E2M1 pairs (low nibble first), one E4M3 block scale +per 16 values along K, and a per-expert global scale `alpha = amax / outer_scale_denominator` +applied after the block-scaled product. In the layout the kernel reads: + +* `w13` `[E, 2I, H / 2]`: rows in FlashInfer's gated FC1 order (`fc1_row_order`): in each group + of 128 rows, up rows `64g .. 64g + 63` then gate rows `64g ..`. The checkpoint holds gate `j` + at row `2j` and up `j` at row `2j + 1`. +* `w13_sf` `[E, 2I, H / 16]`, `w2_sf` `[E, H, I / 16]`: those rows' block scales, swizzled into + 128x4 atoms (`swizzle_block_scales`), as uint8. +* `w2` `[E, H, I / 2]` as stored. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +#: Per-token activation global scale of the 4/6 recipe with an E4M3 max of 256: 1 / (6 * 256). +ACT_GLOBAL_SCALE = 1.0 / (6.0 * 256.0) +#: FlashInfer's FC1 interleave: groups of 64 up rows, then 64 gate rows. +FC1_GROUP = 64 + +_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0) + +#: Inkling-2-Small at EP4: 96 local experts of 384, hidden 4608, intermediate 2304. +_EP4 = dict(experts=384, local=96, hidden=4608, inter=2304, top_k=8) + + +def recipe(): + """FlashInfer's 4/6 recipe the checkpoint was quantized for.""" + from flashinfer.quantization.nvfp4_quantization_utils import NVFP44Over6Config + + return NVFP44Over6Config(e4m3_max=256, err_mode="MAE", err_use_fast_math=False) + + +def fc1_row_order(intermediate: int, device: torch.device) -> torch.Tensor: + """Checkpoint row of each kernel FC1 row, for one expert's `[2I, ...]` w13 (see module doc).""" + j = torch.arange(intermediate, device=device) + halves = torch.stack([2 * j + 1, 2 * j]) # [up; gate] + return halves.view(2, intermediate // FC1_GROUP, FC1_GROUP).transpose(0, 1).reshape(-1) + + +def swizzle_block_scales(scales: torch.Tensor) -> torch.Tensor: + """`[E, M, K / 16]` uint8 linear block scales -> 128x4 atoms, the layout of FlashInfer's + `block_scale_interleave`: row `m`, column `k` at `(m // 128, k // 4, m % 32, (m % 128) // 32, + k % 4)`. Needs `M % 128 == 0` and `(K / 16) % 4 == 0`.""" + e, m, ks = scales.shape + assert m % 128 == 0 and ks % 4 == 0, (m, ks) + return ( + scales.reshape(e, m // 128, 4, 32, ks // 4, 4) + .permute(0, 1, 4, 3, 2, 5) + .contiguous() + .view(e, m, ks) + ) + + +def unswizzle_block_scales(scales: torch.Tensor) -> torch.Tensor: + """Inverse of `swizzle_block_scales`.""" + e, m, ks = scales.shape + return ( + scales.reshape(e, m // 128, ks // 4, 32, 4, 4) + .permute(0, 1, 4, 3, 2, 5) + .contiguous() + .view(e, m, ks) + ) + + +def dequantize_nvfp4(values: torch.Tensor, block_scales: torch.Tensor) -> torch.Tensor: + """`[..., K / 2]` packed E2M1 and `[..., K / 16]` E4M3 block scales (uint8 bits) -> fp32 + `[..., K]`, without any global scale.""" + lut = torch.tensor(_E2M1, device=values.device) + v = values.view(torch.uint8) + pairs = torch.stack([lut[(v & 0xF).long()], lut[(v >> 4).long()]], dim=-1).flatten(-2) + scales = block_scales.view(torch.uint8).view(torch.float8_e4m3fn).float() + return (pairs.unflatten(-1, (-1, 16)) * scales.unsqueeze(-1)).flatten(-2) + + +def quantize_activations(x: torch.Tensor): + """Per-token 4/6 NVFP4 activations: (packed values, linear E4M3 scales as uint8, per-token + decode scales), with `x ~= e2m1 * block_scale * per_token`.""" + from flashinfer import SfLayout, nvfp4_quantize + + return nvfp4_quantize( + x, + ACT_GLOBAL_SCALE, + sfLayout=SfLayout.layout_linear, + per_token_activation=True, + backend="cute-dsl", + nvfp4_4over6=recipe(), + ) + + +def fc2_input_scale(device: torch.device) -> torch.Tensor: + """The GEMM2-input global scale the kernel takes, `[1]` fp32 `1 / (6 * 256)`. + + FlashInfer treats it as a weight-pack constant: it reads it to the host once per tensor and + remembers it by storage and version, so a caller builds it once, outside inference mode, + and passes the same tensor on every call. + """ + with torch.inference_mode(False): + return torch.full((1,), ACT_GLOBAL_SCALE, device=device, dtype=torch.float32) + + +def _situ( + gate: torch.Tensor, up: torch.Tensor, gate_softcap: float, up_softcap: float +) -> torch.Tensor: + return ( + gate_softcap + * torch.tanh(gate / gate_softcap) + * torch.sigmoid(gate) + * up_softcap + * torch.tanh(up / up_softcap) + ) + + +class Inkling2Nvfp4Experts(OpWrapper): + """This rank's NVFP4 experts: `x` `[T, H]` bf16, `topk_ids` `[T, k]` int32 (global ids), + `topk_weights` `[T, k]` fp32 -> `[T, H]` bf16, the sum over the chosen experts this rank holds. + Weight layouts as in the module docstring; `w13_alpha`, `w2_alpha` `[E]` fp32; `fc2_scale` the + persistent GEMM2-input scale from `fc2_input_scale`.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="EP4 rank 1, one decode token", spec=dict(_EP4, tokens=1, rank=1)), + Cell(why="EP4 rank 0, 64 tokens", spec=dict(_EP4, tokens=64, rank=0)), + Cell(why="EP4 rank 3, 300 tokens", spec=dict(_EP4, tokens=300, rank=3)), + ) + + note = """ + The SiTU caps are the model's mlp_gate_softcap (4) and mlp_up_softcap (25); the sigmoid + sees the uncapped gate. FlashInfer requantizes the SiTU output per token with the same 4/6 + recipe before the down projection, from fp32; the reference requantizes the bf16-rounded + intermediate with the same quantizer, which moves a few block choices. The kernel adds the + experts' outputs with atomics, so repeated calls differ in the last bf16 bit. Swapping gate + and up, or the 64-row interleave halves, gives a finite, plausibly scaled and wrong output. + """ + + def __call__( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + from flashinfer import cute_dsl_fused_moe + from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout + + local, rows, packed_k = w13.shape + hidden, inter = packed_k * 2, rows // 2 + values, scales, per_token = quantize_activations(x) + return cute_dsl_fused_moe( + values, + scales, + topk_ids, + topk_weights, + w13, + convert_sf_to_mma_layout(w13_sf, m=rows, k=hidden, num_groups=local, sf_vec_size=16), + w13_alpha, + fc2_scale, + w2, + convert_sf_to_mma_layout(w2_sf, m=hidden, k=inter, num_groups=local, sf_vec_size=16), + w2_alpha, + num_experts, + topk_ids.shape[1], + num_local_experts=local, + local_expert_offset=local_expert_offset, + situ_beta=gate_softcap, + situ_linear_beta=up_softcap, + quant_mode="w4a4", + per_token_scale=per_token, + nvfp4_4over6=recipe(), + ) + + def reference( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> torch.Tensor: + """Dense fp32 per expert from the dequantized weights; the activations and the SiTU output + (rounded to bf16) go through the same per-token 4/6 quantizer. `num_experts` and `fc2_scale` + (a constant) are unused.""" + local, rows, _ = w13.shape + inter = rows // 2 + values, scales, per_token = quantize_activations(x) + x_hat = dequantize_nvfp4(values, scales) * per_token.float().view(-1, 1) + out = torch.zeros(x.shape, dtype=torch.float32, device=x.device) + for e in range(local): + tokens, slots = torch.where(topk_ids == local_expert_offset + e) + if tokens.numel() == 0: + continue + fc1 = ( + dequantize_nvfp4(w13[e], unswizzle_block_scales(w13_sf[e : e + 1])[0]) + * w13_alpha[e] + ) + groups = fc1.view(inter // FC1_GROUP, 2, FC1_GROUP, -1) + up, gate = groups[:, 0].reshape(inter, -1), groups[:, 1].reshape(inter, -1) + down = ( + dequantize_nvfp4(w2[e], unswizzle_block_scales(w2_sf[e : e + 1])[0]) * w2_alpha[e] + ) + with true_fp32_matmul(): + h = _situ( + x_hat[tokens] @ gate.t(), x_hat[tokens] @ up.t(), gate_softcap, up_softcap + ) + h_values, h_scales, h_per_token = quantize_activations(h.to(torch.bfloat16)) + h_hat = dequantize_nvfp4(h_values, h_scales) * h_per_token.float().view(-1, 1) + out.index_add_( + 0, tokens, (h_hat @ down.t()) * topk_weights[tokens, slots].float().unsqueeze(1) + ) + return out.to(x.dtype) + + def is_valid( + self, + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13: torch.Tensor, + w13_sf: torch.Tensor, + w13_alpha: torch.Tensor, + w2: torch.Tensor, + w2_sf: torch.Tensor, + w2_alpha: torch.Tensor, + fc2_scale: torch.Tensor, + num_experts: int, + local_expert_offset: int, + gate_softcap: float, + up_softcap: float, + ) -> None: + local, rows, packed_k = w13.shape + assert x.dtype == torch.bfloat16 and x.shape[1] == 2 * packed_k, "x must be bf16 [T, H]" + assert w13.dtype == w2.dtype == torch.uint8, "packed NVFP4 weights are uint8" + assert w13_sf.shape == (local, rows, 2 * packed_k // 16), "w13_sf must be [E, 2I, H / 16]" + assert w2.shape == (local, 2 * packed_k, rows // 4), "w2 must be [E, H, I / 2]" + assert w2_sf.shape == (local, 2 * packed_k, rows // 32), "w2_sf must be [E, H, I / 16]" + assert w13_alpha.dtype == w2_alpha.dtype == torch.float32 and w13_alpha.shape == (local,) + assert topk_ids.dtype == torch.int32 and topk_weights.dtype == torch.float32 + assert 0 <= local_expert_offset and local_expert_offset + local <= num_experts + assert fc2_scale.shape == (1,) and abs(float(fc2_scale[0]) / ACT_GLOBAL_SCALE - 1) < 1e-6, ( + "fc2_scale is 1 / (6 * 256)" + ) + + +inkling2_nvfp4_experts = Inkling2Nvfp4Experts() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py new file mode 100644 index 000000000000..8e41caf2c5a6 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_router.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's MoE router: fp32 sigmoid scores, bias-corrected top-k selection, renormalized +weights. + + logits = x @ W^T fp32 (bf16 operands, fp32 accumulation) + scores = sigmoid(logits) + experts = top_k(scores + bias) the bias steers selection only + weights = scores[experts] / (sum(scores[experts]) + 1e-20) * route_scale +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm + +_INKLING2 = dict(hidden=4608, experts=384, top_k=8, route_scale=8.0) + + +class Inkling2Router(OpWrapper): + """`x` `[T, H]` bf16, `weight` `[E, H]` bf16, `bias` `[E]` fp32 -> (`topk_ids` `[T, k]` int32, + `topk_weights` `[T, k]` fp32). The ids are global expert ids, in no particular order.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="Inkling-2-Small, one decode token", spec=dict(_INKLING2, tokens=1)), + Cell(why="Inkling-2-Small, 257 tokens", spec=dict(_INKLING2, tokens=257)), + ) + + note = """ + The selection adds the correction bias to the sigmoid scores, but the weights come from + the uncorrected scores, renormalized over the chosen experts and multiplied by + route_scale (8). A weight taken from the biased scores, or a softmax router, still sums + to route_scale; only the per-expert weights show the difference. + """ + + def __call__( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + logits = cublas_mm(x, weight.t(), None, torch.float32) + scores = torch.sigmoid(logits) + ids = torch.topk(scores + bias, top_k, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) * route_scale + return ids.to(torch.int32), weights + + def reference( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> tuple[torch.Tensor, torch.Tensor]: + """fp32 logits from a true fp32 product; selection and weights as the op.""" + with true_fp32_matmul(): + logits = x.float() @ weight.float().t() + scores = torch.sigmoid(logits) + ids = torch.topk(scores + bias.float(), top_k, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) * route_scale + return ids.to(torch.int32), weights + + def is_valid( + self, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + top_k: int, + route_scale: float, + ) -> None: + # A bf16 bias rounds the correction and changes which experts win; an fp32 weight + # would need a different GEMM. Neither raises. + assert bias.dtype == torch.float32, "the correction bias is fp32" + assert x.dtype == weight.dtype == torch.bfloat16, "x and the router weight are bf16" + assert weight.shape == (bias.shape[0], x.shape[1]), "weight must be [experts, hidden]" + + +inkling2_router = Inkling2Router() diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py new file mode 100644 index 000000000000..e4a8c1ff4f34 --- /dev/null +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/residual/inkling2_residual_mix.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inkling 2's block attention residual: a layer's input is a learned mix of the completed +residual blocks and the current partial block, followed by that input's RMSNorm. + +Every 6 layers the residual stream closes a block: the input of layers 0, 6, 12, ... (the +embedding for layer 0) becomes a block state, and the stream restarts from the next sublayer's +output. A sublayer reads, instead of the stream, a softmax mix over the unit-RMS-normalized +block states and partial block, with a learned fp32 query per input (3 for a global layer's q, k +and v inputs, 1 otherwise; the final norm has its own): + + n_s = s / rms(s) (eps 1e-6), for each block s and the partial block + p_q = softmax_s(q . n_s / sqrt(H)) + mix_q = sum_s p_q[s] * n_s + out_q = weight_q * mix_q / rms(mix_q) (eps), rounded to bf16 + +all in fp32. `blocks` holds the completed blocks already normalized, `[S, T, H]` fp32; a block's +normalization is the same in every later mix, so the caller computes it once +(`normalize_block`). +""" + +from __future__ import annotations + +import math +from typing import Sequence + +import torch + +from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul + +#: The unweighted normalization of block states (Inkling2UnweightedRMSNorm's default). +BLOCK_EPS = 1e-6 + + +def normalize_block(state: torch.Tensor) -> torch.Tensor: + """A block state `[T, H]` divided by its RMS (eps 1e-6), fp32.""" + x = state.float() + return x * torch.rsqrt(x.square().mean(-1, keepdim=True) + BLOCK_EPS) + + +def rmsnorm_to_bf16(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + """`weight * x / rms(x)` in fp32, rounded once to bf16 (Inkling2RMSNorm on an fp32 input).""" + x = x.float() + return (weight.float() * (x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps))).to( + torch.bfloat16 + ) + + +class Inkling2ResidualMix(OpWrapper): + """`blocks` `[S, T, H]` fp32 (normalized), `partial` `[T, H]` (bf16 or fp32), `queries` `[Q, H]` + fp32, `norm_weights` Q weights `[H]` -> `[Q, T, H]` bf16, each mix through its RMSNorm.""" + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell( + why="layer 1's attention input: the embedding block, one query", + spec=dict(blocks=1, queries=1, tokens=17), + ), + Cell( + why="a global layer's q/k/v inputs: 3 queries over 5 blocks", + spec=dict(blocks=5, queries=3, tokens=33), + ), + Cell( + why="the final norm's input: 8 blocks, one decode token", + spec=dict(blocks=8, queries=1, tokens=1), + ), + ) + + note = """ + Values are the normalized states, not the raw ones: a mix of raw states is plausible but + scaled by the states' RMS. The partial block is normalized here, the completed blocks by + the caller, once. + """ + + def __call__( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> torch.Tensor: + hidden = partial.shape[-1] + n_p = normalize_block(partial) + with true_fp32_matmul(): + logits_b = torch.einsum("sth,qh->tqs", blocks, queries) + logit_p = (n_p @ queries.t()).unsqueeze(-1) + probs = torch.softmax(torch.cat([logits_b, logit_p], dim=-1) / math.sqrt(hidden), dim=-1) + with true_fp32_matmul(): + mix = torch.einsum("tqs,sth->qth", probs[..., :-1].contiguous(), blocks) + mix = mix + probs[..., -1].t().unsqueeze(-1) * n_p.unsqueeze(0) + return torch.stack([rmsnorm_to_bf16(mix[i], w, eps) for i, w in enumerate(norm_weights)]) + + def reference( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> torch.Tensor: + """fp64 throughout, the partial block appended to the normalized blocks as one more state.""" + hidden = partial.shape[-1] + x = partial.double() + n_p = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + BLOCK_EPS) + stacked = torch.cat([blocks.double(), n_p.unsqueeze(0)]) # [S + 1, T, H] + probs = torch.softmax( + torch.einsum("sth,qh->tqs", stacked, queries.double()) / math.sqrt(hidden), dim=-1 + ) + mix = torch.einsum("tqs,sth->qth", probs, stacked) + out = [] + for i, w in enumerate(norm_weights): + m = mix[i] + out.append( + (w.double() * (m * torch.rsqrt(m.square().mean(-1, keepdim=True) + eps))).to( + torch.bfloat16 + ) + ) + return torch.stack(out) + + def is_valid( + self, + blocks: torch.Tensor, + partial: torch.Tensor, + queries: torch.Tensor, + norm_weights: Sequence[torch.Tensor], + eps: float, + ) -> None: + assert blocks.dtype == torch.float32 and blocks.dim() == 3, ( + "blocks are normalized fp32 [S, T, H]" + ) + assert blocks.shape[1:] == partial.shape, "blocks and the partial block share [T, H]" + assert queries.dtype == torch.float32 and queries.shape[1] == partial.shape[1], ( + "queries are fp32 [Q, H]" + ) + assert len(norm_weights) == queries.shape[0], "one norm weight per query" + + +inkling2_residual_mix = Inkling2ResidualMix() diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 81c0bb1c2e5b..b7bee3368f09 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -177,6 +177,7 @@ l0_b300: - unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4 TIMEOUT (30) - unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency TIMEOUT (30) - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default TIMEOUT (30) + - unittest/_torch/modeling_v2/models/inkling2/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_fp8__sm_103__unified TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 9e9e052798f8..a006667b5dd7 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -157,6 +157,7 @@ l0_cpu: - unittest/_torch/modeling_v2/models/deepseek_v3/r1_0528_nvfp4__sm_10x__default/test_modeling_v2_deepseek_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_weight_table.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py # The Kimi K3 SM107 targets' cpu_only cases: CuTe DSL hint tables, the FP4 MLA # decode plan geometry, each target's constants and model defaults, the routing # of the recipes' mappings, the context MLA split-K dispatch gates, the diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py new file mode 100644 index 000000000000..799216d402a2 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling-2-Small TP4/EP4 weight table against the checkpoint's tensor layout. + +The layout below restates the released checkpoint's text-decoder tensors (names, shapes, dtypes), +as its safetensors headers declare them. Every rank's transforms run on meta tensors of those +shapes, so the table's declared shares, its sources and the checkpoint cannot drift apart +unnoticed. +""" + +import os +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.config import ExpertFormat, InklingDims +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + weights as w, +) + +pytestmark = pytest.mark.cpu_only + +U8, F8, BF16, F32 = torch.uint8, torch.float8_e4m3fn, torch.bfloat16, torch.float32 +GLOBAL = tuple(range(5, 48, 6)) + + +def _dims() -> InklingDims: + text = SimpleNamespace( + num_hidden_layers=48, + local_layer_ids=[i for i in range(48) if i not in GLOBAL], + hidden_size=4608, + vocab_size=201024, + unpadded_vocab_size=200058, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + d_rel=16, + rel_extent=1024, + sliding_window_size=1024, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=384, + num_experts_per_tok=8, + intermediate_size=2304, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + groups = [ + { + "compute_dtype": "nvfp4", + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in range(40)], + }, + { + "compute_dtype": "mxfp8", + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in range(40, 48)], + }, + ] + return InklingDims.from_config(text, {"model": {"parameter_groups": groups}}) + + +def checkpoint_layout(d: InklingDims) -> dict: + """name -> (shape, dtype) of the released checkpoint's text-decoder tensors.""" + h, e, i = d.hidden_size, d.num_experts, d.expert_intermediate_size + out = { + "model.llm.embed.weight": ((d.vocab_size, h), BF16), + "model.llm.embed_norm.weight": ((h,), BF16), + "model.llm.norm.weight": ((h,), BF16), + "model.llm.unembed.weight": ((d.vocab_size, h), BF16), + "model.llm.attn_res.output_query": ((h,), F32), + } + for layer in range(d.num_layers): + p, a = f"model.llm.layers.{layer}", f"model.llm.layers.{layer}.attn" + out[f"model.llm.attn_res.mlp_queries.{layer}"] = ((h,), F32) + if layer > 0: + out[f"model.llm.attn_res.attn_queries.{layer - 1}"] = ( + (3, h) if d.is_global(layer) else (h,), + F32, + ) + out.update({ + f"{a}.wq_du.weight": ((6144, h), BF16), + f"{a}.wk_dv.weight": ((1024, h), BF16), + f"{a}.wv_dv.weight": ((1024, h), BF16), + f"{a}.wr_du.weight": ((768, h), BF16), + f"{a}.wo_ud.weight": ((h, 6144), BF16), + f"{a}.gate_attn.weight": ((6144, h), BF16), + f"{a}.attn_sink_proj.weight": ((48, h), BF16), + f"{a}.q_l2_clip.scale": ((48, 128), BF16), + f"{a}.rel_l2_clip.scale": ((48, 16), BF16), + f"{a}.rel_logits_proj.proj": ((16, 1024), BF16), + f"{p}.attn_sconv.weight": ((h, 1, 4), BF16), + f"{p}.mlp_sconv.weight": ((h, 1, 4), BF16), + f"{p}.mlp_norm.weight": ((h,), BF16), + f"{p}.mlp.gate.weight": ((e, h), BF16), + f"{p}.mlp.gate.bias": ((e,), F32), + }) # fmt: skip + if d.is_global(layer): + for s in ("k", "v"): + out[f"{a}.{s}_sconv.static_w"] = ((4, 1024), BF16) + out[f"{a}.{s}_sconv.weight_proj"] = ((32, h), BF16) + for n in ("q", "k", "v"): + out[f"{p}.{n}_input_norm.weight"] = ((h,), BF16) + else: + out[f"{a}.k_sconv.weight"] = ((1024, 1, 4), BF16) + out[f"{a}.v_sconv.weight"] = ((1024, 1, 4), BF16) + out[f"{p}.attn_norm.weight"] = ((h,), BF16) + x = f"{p}.mlp.experts" + if d.expert_formats[layer] is ExpertFormat.NVFP4: + out.update({ + f"{x}.w13_weight": ((e, 2 * i, h // 2), U8), + f"{x}.w13_weight_scale": ((e, 2 * i, h // 16), F8), + f"{x}.w13_weight_amax": ((e,), F32), + f"{x}.w13_weight_outer_scale_denominator": ((1,), F32), + f"{x}.w2_weight": ((e, h, i // 2), U8), + f"{x}.w2_weight_scale": ((e, h, i // 16), F8), + f"{x}.w2_weight_amax": ((e,), F32), + f"{x}.w2_weight_outer_scale_denominator": ((1,), F32), + }) # fmt: skip + else: + out.update({ + f"{x}.w13_weight": ((e, 2 * i, h), F8), + f"{x}.w13_weight_scale": ((e, 2 * i, h // 32), U8), + f"{x}.w2_weight": ((e, h, i), F8), + f"{x}.w2_weight_scale": ((e, h, i // 32), U8), + }) # fmt: skip + return out + + +def _mapping(rank: int): + return SimpleNamespace( + pp_size=1, + cp_size=1, + enable_attention_dp=False, + moe_tp_size=1, + moe_ep_size=4, + tp_size=4, + tp_rank=rank, + moe_ep_rank=rank, + ) + + +@pytest.mark.parametrize("rank", range(4)) +def test_every_entry_transforms_to_its_declared_share(rank): + d = _dims() + s = w.shares_of(d, _mapping(rank)) + layout = checkpoint_layout(d) + used = set() + for layer, entry in w.all_entries(s): + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = [] + for name, rows in zip(entry.sources, flags): + assert name in layout, f"layer {layer} {entry.name}: {name} is not a checkpoint tensor" + shape, dtype = layout[name] + if rows: + shape = (s.local_experts, *shape[1:]) + srcs.append(torch.empty(shape, dtype=dtype, device="meta")) + assert name not in used, f"{name} is read twice" + used.add(name) + out = entry.transform(tuple(srcs), s) + if out.dtype != entry.dtype and out.dtype == U8 and entry.dtype.itemsize == 1: + out = out.view(entry.dtype) + assert tuple(out.shape) == tuple(entry.shape), ( + layer, + entry.name, + tuple(out.shape), + entry.shape, + ) + assert out.dtype == entry.dtype, (layer, entry.name, out.dtype, entry.dtype) + used.add("model.llm.unembed.weight") + assert used == set(layout), sorted(set(layout) ^ used)[:8] + + +def test_shares_split_heads_and_experts_evenly(): + s = w.shares_of(_dims(), _mapping(2)) + assert (s.heads, s.kv_heads, s.q_width, s.kv_width, s.r_width) == (12, 2, 1536, 256, 192) + assert (s.local_experts, s.expert_offset, s.experts()) == (96, 192, slice(192, 288)) + assert s.rows(6144) == slice(3072, 4608) and s.rows(32) == slice(16, 24) + + +@pytest.mark.skipif( + not os.path.isdir(os.path.join(os.environ.get("LLM_MODELS_ROOT", ""), "Inkling-2-Small")), + reason="needs the Inkling-2-Small checkpoint under LLM_MODELS_ROOT", +) +def test_layout_matches_the_checkpoint_headers(): + import json + import struct + + root = os.path.join(os.environ["LLM_MODELS_ROOT"], "Inkling-2-Small") + index = json.load(open(os.path.join(root, "model.safetensors.index.json")))["weight_map"] + headers = {} + for f in sorted(set(index.values())): + with open(os.path.join(root, f), "rb") as fh: + (n,) = struct.unpack(" None: # noqa: F811 + certifying(op) + args = _build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31) + for out, ref in zip(op(*args), op.reference(*args)): + assert_within_ulp( + out.reshape(-1, out.shape[-1]), + ref.reshape(-1, ref.shape[-1]), + element_ulp=1.0, + rms_ulp=0.1, + ) + + +def test_clip_bounds_and_tau() -> None: + args = _build(dict(tokens=9, heads=12, kv_heads=2, head_dim=128, d_rel=16, tau=False), 1) + q, k, v, r = op(*args) + assert ( + q.float().norm(dim=-1).max() <= QK_CLIP * 1.01 + and k.float().norm(dim=-1).max() <= QK_CLIP * 1.01 + ) + assert r.float().norm(dim=-1).max() <= REL_CLIP * 1.01 + torch.testing.assert_close( + v.float().pow(2).mean(-1), torch.ones(9, 2, device="cuda"), rtol=0.02, atol=0.02 + ) + tau = log_scaling(torch.tensor([0, 8190, 8191, 8192 * 4 - 1], device="cuda"), 0.1, 8192.0) + torch.testing.assert_close( + tau, torch.tensor([1.0, 1.0, 1.0, 1.0 + 0.1 * 1.3862944], device="cuda") + ) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py new file mode 100644 index 000000000000..419f22b57059 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_paged_attention entry.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.paged_attention import ( + block_tokens, + tile_plan, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.attention.inkling2_paged_attention import ( + SM_SCALE, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.attention.inkling2_paged_attention import ( + inkling2_paged_attention as op, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "inkling2_paged_attention requires a CUDA device" + +PAGE_SIZE = 32 +HEAD_DIM = 128 +REL_EXTENT = 1024 + + +def _build(spec, seed): + """A step of `spec["requests"]` ((new, cached) pairs) over a pool of shuffled pages whose + cached positions already hold keys and values.""" + g = torch.Generator(device="cuda").manual_seed(seed) + cpu = torch.Generator().manual_seed(seed) + heads, kv_heads, window = spec["heads"], spec["kv_heads"], spec["window"] + requests = spec["requests"] + lengths = [n for n, _ in requests] + kv_lens = [n + c for n, c in requests] + widths = [-(-length // PAGE_SIZE) for length in kv_lens] + num_pages = sum(widths) + 3 + pool = torch.randn(num_pages, 2, kv_heads, PAGE_SIZE, HEAD_DIM, generator=g, device="cuda") + # Keys after the clip have norm <= 22.6; values after the RMS norm have unit RMS. + pool[:, 0] *= 1.5 + pool = pool.to(torch.bfloat16) + order = torch.randperm(num_pages, generator=cpu) + page_table = torch.full((len(requests), max(widths)), -1, dtype=torch.int32) + used = 0 + for r, width in enumerate(widths): + page_table[r, :width] = order[used : used + width].to(torch.int32) + used += width + num_tokens = sum(lengths) + # Clipped queries reach norm 22.6 and the global layers' tau scales them further. + q = (1.5 * torch.randn(num_tokens, heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + k = (1.5 * torch.randn(num_tokens, kv_heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + v = torch.randn(num_tokens, kv_heads, HEAD_DIM, generator=g, device="cuda").to(torch.bfloat16) + rel_logits = 2.0 * torch.randn(num_tokens, heads, REL_EXTENT, generator=g, device="cuda") + sink = (2.0 * torch.randn(num_tokens, heads, generator=g, device="cuda")).to(torch.bfloat16) + gate = (2.0 * torch.randn(num_tokens, heads, HEAD_DIM, generator=g, device="cuda")).to( + torch.bfloat16 + ) + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + block_t = block_tokens(max(lengths)) + tile_request, tile_start = tile_plan(lengths, block_t, torch.device("cuda")) + return [ + q, + k, + v, + pool, + page_table.cuda(), + torch.tensor(bounds, dtype=torch.int32, device="cuda"), + torch.tensor(kv_lens, dtype=torch.int32, device="cuda"), + torch.repeat_interleave( + torch.arange(len(lengths), dtype=torch.int32), torch.tensor(lengths) + ).cuda(), + tile_request, + tile_start, + block_t, + rel_logits, + sink, + gate, + window, + SM_SCALE, + ] + + +def _run_both(args): + op_args, ref_args = list(args), list(args) + op_args[3] = args[3].clone() + ref_args[3] = args[3].clone() + out = op(*op_args) + ref = op.reference(*ref_args) + return out, ref, op_args[3], ref_args[3] + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:48] for c in op.CELLS]) +def test_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(op) + out, ref, pool, ref_pool = _run_both(_build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31)) + torch.testing.assert_close(pool, ref_pool, rtol=0, atol=0) + # Flash attention rounds the unnormalized probabilities to bf16 for the value product and + # sums in another order than the dense fp32 reference: a few bf16 ulps of each row's scale. + assert_within_ulp( + out.reshape(-1, out.shape[-1]), ref.reshape(-1, ref.shape[-1]), element_ulp=4.0, rms_ulp=0.5 + ) + + +@pytest.mark.parametrize("what", ["bias_extent", "window", "sink"]) +def test_negative_controls_are_visible(what) -> None: + """Each mechanism the cells certify changes the output when it is broken.""" + spec = dict( + heads=12, + kv_heads=2, + window=1024 if what == "window" else 0, + requests=((1, 1500), (40, 990)), + ) + args = _build(spec, 21) + if what == "sink": + # Sinks near the logsumexp of 1500 keys, where they take a visible share of the mass. + args[12] = (args[12].float() + 10.0).to(torch.bfloat16) + out, *_ = _run_both(args) + broken = list(args) + if what == "bias_extent": + # Bias leaking past the extent: shift it by one distance. + broken[11] = torch.roll(args[11], shifts=1, dims=2) + elif what == "window": + broken[14] = 1023 + else: + broken[12] = torch.zeros_like(args[12]) + broken[3] = args[3].clone() + assert not torch.allclose(op(*broken), out, rtol=0.01, atol=0.01) + + +@pytest.mark.parametrize( + "case,match", + [ + ("bf16_rel_logits", None), + ("short_page_table", "page_table must cover"), + ("wrong_tiles", "tile plan"), + ], +) +def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: + args = _build(dict(heads=12, kv_heads=2, window=1024, requests=((5, 3), (1, 40))), 4) + if case == "bf16_rel_logits": + args[11] = args[11].to(torch.bfloat16) + elif case == "short_page_table": + args[4] = args[4][:, :1] + else: + args[9] = torch.zeros_like(args[9]) + with pytest.raises(AssertionError, match=match): + with validating(op): + op(*args) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py new file mode 100644 index 000000000000..2775c15ed730 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_short_conv and inkling2_dynamic_short_conv entries.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.conv.inkling2_short_conv import ( + inkling2_dynamic_short_conv, + inkling2_short_conv, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "the Inkling 2 short convolutions require a CUDA device" + +NUM_SLOTS = 12 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) % 2**31 + + +def _build(spec, seed, dynamic): + """The step of `spec["requests"]` ((new, cached) pairs) on random inputs and windows.""" + g = torch.Generator(device="cuda").manual_seed(seed) + channels = spec["channels"] + lengths = [n for n, _ in spec["requests"]] + num_tokens = sum(lengths) + x = torch.randn(num_tokens, channels, generator=g, device="cuda").to(torch.bfloat16) + taps = 0.5 * torch.randn(4, channels, generator=g, device="cuda") + # Every slot holds a window, including the ones of fresh requests: has_state 0 must ignore it. + state = torch.randn(NUM_SLOTS, 3, channels, generator=g, device="cuda").to(torch.bfloat16) + perm = torch.randperm(NUM_SLOTS, generator=torch.Generator().manual_seed(seed)) + slots = perm[: len(lengths)].to(torch.int32).cuda() + bounds = [0] + for n in lengths: + bounds.append(bounds[-1] + n) + query_start = torch.tensor(bounds, dtype=torch.int32, device="cuda") + has_state = torch.tensor( + [int(c > 0) for _, c in spec["requests"]], dtype=torch.int32, device="cuda" + ) + token_request = torch.repeat_interleave( + torch.arange(len(lengths), dtype=torch.int32), torch.tensor(lengths) + ).cuda() + if not dynamic: + return x, taps, state, slots, query_start, has_state, token_request + heads = spec["heads"] + dynamic_taps = (0.3 * torch.randn(num_tokens, heads * 4, generator=g, device="cuda")).to( + torch.bfloat16 + ) + return ( + x, + taps, + dynamic_taps, + state, + slots, + query_start, + has_state, + token_request, + channels // heads, + ) + + +def _check_cell(op, args, state_index): + op_args, ref_args = list(args), list(args) + op_args[state_index] = args[state_index].clone() + ref_args[state_index] = args[state_index].clone() + out = op(*op_args) + ref = op.reference(*ref_args) + assert out.shape == ref.shape and out.dtype == ref.dtype + # The op and the reference sum the same four fp32 products in the same order; FMA + # contraction moves the fp32 sum by an ulp, which a bf16 rounding boundary can turn into one + # bf16 ulp of the output. + assert_within_ulp(out, ref, element_ulp=1.0, rms_ulp=0.05) + # The window is a copy of inputs: exact. + torch.testing.assert_close(op_args[state_index], ref_args[state_index], rtol=0, atol=0) + + +@pytest.mark.parametrize( + "cell", inkling2_short_conv.CELLS, ids=[c.why[:48] for c in inkling2_short_conv.CELLS] +) +def test_static_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_short_conv) + _check_cell( + inkling2_short_conv, _build(cell.spec, _seed(cell.why), dynamic=False), state_index=2 + ) + + +@pytest.mark.parametrize( + "cell", + inkling2_dynamic_short_conv.CELLS, + ids=[c.why[:48] for c in inkling2_dynamic_short_conv.CELLS], +) +def test_dynamic_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_dynamic_short_conv) + _check_cell( + inkling2_dynamic_short_conv, _build(cell.spec, _seed(cell.why), dynamic=True), state_index=3 + ) + + +@pytest.mark.parametrize("dynamic", [False, True], ids=["static", "dynamic"]) +def test_chunked_steps_equal_one_prefill(dynamic) -> None: + """A prefill of 21 tokens, then chunks of 2 and 1 and decodes, equals one 29-token prefill: + the window carries exactly the inputs a step needs from the steps before it.""" + channels, heads = 256, 2 + g = torch.Generator(device="cuda").manual_seed(5) + total = 29 + x = torch.randn(total, channels, generator=g, device="cuda").to(torch.bfloat16) + taps = 0.5 * torch.randn(4, channels, generator=g, device="cuda") + dyn = (0.3 * torch.randn(total, heads * 4, generator=g, device="cuda")).to(torch.bfloat16) + + def run(start, end, state, has_state): + n = end - start + args = [x[start:end], taps] + if dynamic: + args.append(dyn[start:end]) + args += [ + state, + torch.tensor([1], dtype=torch.int32, device="cuda"), + torch.tensor([0, n], dtype=torch.int32, device="cuda"), + torch.tensor([has_state], dtype=torch.int32, device="cuda"), + torch.zeros(n, dtype=torch.int32, device="cuda"), + ] + if dynamic: + args.append(channels // heads) + op = inkling2_dynamic_short_conv if dynamic else inkling2_short_conv + return op(*args) + + whole = run(0, total, torch.zeros(3, 3, channels, dtype=torch.bfloat16, device="cuda"), 0) + state = torch.randn(3, 3, channels, device="cuda").to(torch.bfloat16) + pieces, start = [], 0 + for end in (21, 23, 24, 25, 26, 27, 28, 29): + pieces.append(run(start, end, state, int(start > 0))) + start = end + torch.testing.assert_close(torch.cat(pieces), whole, rtol=0, atol=0) + torch.testing.assert_close(state[1], x[-3:], rtol=0, atol=0) + + +def test_dynamic_taps_are_per_head() -> None: + """Swapping the two heads' dynamic taps changes the output: one head's taps never reach the + other head's channels.""" + spec = dict(channels=256, heads=2, requests=((7, 0),)) + args = list(_build(spec, 3, dynamic=True)) + out = inkling2_dynamic_short_conv(*args[:3], args[3].clone(), *args[4:]) + swapped = args[2].view(-1, 2, 4).flip(1).reshape(-1, 8).contiguous() + out_swapped = inkling2_dynamic_short_conv(args[0], args[1], swapped, args[3].clone(), *args[4:]) + assert not torch.equal(out[:, :128], out_swapped[:, :128]) + + +@pytest.mark.parametrize( + "case,match", + [ + ("duplicate_slot", "own slot"), + ("bf16_taps", "lag-major fp32"), + ("bad_token_request", "token_request"), + ], +) +def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: + spec = dict(channels=256, requests=((3, 0), (1, 4))) + x, taps, state, slots, query_start, has_state, token_request = _build(spec, 9, dynamic=False) + if case == "duplicate_slot": + slots = slots[:1].repeat(2) + elif case == "bf16_taps": + taps = taps.to(torch.bfloat16) + else: + token_request = torch.zeros_like(token_request) + with pytest.raises(AssertionError, match=match): + with validating(inkling2_short_conv): + inkling2_short_conv(x, taps, state, slots, query_start, has_state, token_request) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py new file mode 100644 index 000000000000..9eceb4b1da70 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Inkling 2 MoE entries: router, NVFP4 experts, MXFP8 experts.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_mxfp8_experts import ( + inkling2_mxfp8_experts, + mxfp8_dequantize, + mxfp8_quantize, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_nvfp4_experts import ( + fc1_row_order, + fc2_input_scale, + inkling2_nvfp4_experts, + swizzle_block_scales, + unswizzle_block_scales, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_router import inkling2_router + +__extra_import_path__ = ["../../../.."] +from _validating import certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "the Inkling 2 MoE entries require a CUDA device" + +GATE_SOFTCAP, UP_SOFTCAP = 4.0, 25.0 + + +def _seed(text: str) -> int: + return zlib.crc32(text.encode()) % 2**31 + + +def _routing(g, tokens, experts, top_k, local, offset): + """Distinct experts per token, about half of them on this rank; positive weights summing to 8.""" + pool = torch.cat( + [ + torch.arange(offset, offset + local), + torch.arange(0, offset), + torch.arange(offset + local, experts), + ] + ) + pick = torch.stack([torch.randperm(2 * local, generator=g)[:top_k] for _ in range(tokens)]) + ids = pool[pick.clamp(max=experts - 1)] + weights = torch.rand(tokens, top_k, generator=g) + 0.1 + weights = weights / weights.sum(-1, keepdim=True) * 8.0 + return ids.to(torch.int32).cuda(), weights.float().cuda() + + +def _rel_rms(out, ref): + return float(((out.float() - ref.float()).pow(2).mean() / ref.float().pow(2).mean()).sqrt()) + + +def _e4m3_scales(g, shape, lo=-4, hi=3): + exps = torch.randint(lo, hi, shape, generator=g, device="cuda") + return torch.exp2(exps.float()).to(torch.float8_e4m3fn).view(torch.uint8) + + +def _nvfp4_args(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + e, h, i = spec["local"], spec["hidden"], spec["inter"] + dev = "cuda" + w13 = torch.randint(0, 256, (e, 2 * i, h // 2), generator=g, dtype=torch.uint8, device=dev) + w13_sf = _e4m3_scales(g, (e, 2 * i, h // 16)) + w2 = torch.randint(0, 256, (e, h, i // 2), generator=g, dtype=torch.uint8, device=dev) + w2_sf = _e4m3_scales(g, (e, h, i // 16)) + # Global scales that keep the experts' outputs O(1) for unit-RMS inputs. + w13_alpha = (0.02 + 0.01 * torch.rand(e, generator=g, device=dev)).float() + w2_alpha = (0.02 + 0.01 * torch.rand(e, generator=g, device=dev)).float() + offset = spec["rank"] * e + x = torch.randn(spec["tokens"], h, generator=g, device=dev).to(torch.bfloat16) + ids, weights = _routing( + torch.Generator().manual_seed(seed), + spec["tokens"], + spec["experts"], + spec["top_k"], + e, + offset, + ) + return [ + x, + ids, + weights, + w13, + swizzle_block_scales(w13_sf), + w13_alpha, + w2, + swizzle_block_scales(w2_sf), + w2_alpha, + fc2_input_scale(torch.device("cuda")), + spec["experts"], + offset, + GATE_SOFTCAP, + UP_SOFTCAP, + ] + + +def _mxfp8_args(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + e, h, i = spec["local"], spec["hidden"], spec["inter"] + dev = "cuda" + + def e4m3(shape): + return ( + (64 * torch.randn(shape, generator=g, device=dev)) + .clamp(-448, 448) + .to(torch.float8_e4m3fn) + ) + + def exponents(shape): + return torch.randint(127 - 14, 127 - 10, shape, generator=g, dtype=torch.uint8, device=dev) + + w13, w13_s = e4m3((e, 2 * i, h)), exponents((e, 2 * i, h // 32)) + w2, w2_s = e4m3((e, h, i)), exponents((e, h, i // 32)) + offset = spec["rank"] * e + x = torch.randn(spec["tokens"], h, generator=g, device=dev).to(torch.bfloat16) + ids, weights = _routing( + torch.Generator().manual_seed(seed), + spec["tokens"], + spec["experts"], + spec["top_k"], + e, + offset, + ) + return [x, ids, weights, w13, w13_s, w2, w2_s, offset, GATE_SOFTCAP, UP_SOFTCAP] + + +# --------------------------------------------------------------------------------------- router + + +@pytest.mark.parametrize("cell", inkling2_router.CELLS, ids=[c.why for c in inkling2_router.CELLS]) +def test_router_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_router) + spec = cell.spec + g = torch.Generator(device="cuda").manual_seed(_seed(cell.why)) + x = torch.randn(spec["tokens"], spec["hidden"], generator=g, device="cuda").to(torch.bfloat16) + weight = (0.015 * torch.randn(spec["experts"], spec["hidden"], generator=g, device="cuda")).to( + torch.bfloat16 + ) + bias = 0.05 * torch.randn(spec["experts"], generator=g, device="cuda") + ids, weights = inkling2_router(x, weight, bias, spec["top_k"], spec["route_scale"]) + ref_ids, ref_weights = inkling2_router.reference( + x, weight, bias, spec["top_k"], spec["route_scale"] + ) + # The selection is a set; order the two by expert id before comparing. + order, ref_order = ids.sort(-1), ref_ids.sort(-1) + assert torch.equal(order.values, ref_order.values) + torch.testing.assert_close( + weights.gather(1, order.indices.long()), + ref_weights.gather(1, ref_order.indices.long()), + rtol=1e-5, + atol=1e-5, + ) + torch.testing.assert_close( + weights.sum(-1), torch.full_like(weights[:, 0], 8.0), rtol=1e-5, atol=1e-5 + ) + + +# ----------------------------------------------------------------------------------------- NVFP4 + + +def test_fc1_row_order_and_swizzle_round_trip() -> None: + order = fc1_row_order(2304, torch.device("cpu")) + assert sorted(order.tolist()) == list(range(4608)) + # Rows 0..63 of the kernel are up rows 1, 3, ..; rows 64..127 gate rows 0, 2, .. + assert order[:3].tolist() == [1, 3, 5] and order[64:67].tolist() == [0, 2, 4] + scales = torch.randint(0, 256, (3, 256, 288), dtype=torch.uint8) + assert torch.equal(unswizzle_block_scales(swizzle_block_scales(scales)), scales) + + +def test_swizzle_matches_flashinfer_block_scale_interleave() -> None: + from flashinfer.quantization.fp4_quantization import block_scale_interleave + + scales = torch.randint(0, 256, (2, 256, 144), dtype=torch.uint8, device="cuda") + assert torch.equal( + swizzle_block_scales(scales).reshape(-1), block_scale_interleave(scales).reshape(-1) + ) + + +@pytest.mark.parametrize( + "cell", inkling2_nvfp4_experts.CELLS, ids=[c.why for c in inkling2_nvfp4_experts.CELLS] +) +def test_nvfp4_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_nvfp4_experts) + args = _nvfp4_args(cell.spec, _seed(cell.why)) + out = inkling2_nvfp4_experts(*args) + ref = inkling2_nvfp4_experts.reference(*args) + assert out.shape == ref.shape and out.dtype == torch.bfloat16 + # The kernel requantizes the fp32 SiTU output, the reference its bf16 rounding: a few block + # scale choices and E2M1 roundings differ, each by a quantization step. + assert _rel_rms(out, ref) < 0.02, _rel_rms(out, ref) + + +def test_nvfp4_runs_under_inference_mode() -> None: + """The engine runs every forward under inference mode; the op gives the same answer there.""" + args = _nvfp4_args(dict(inkling2_nvfp4_experts.CELLS[1].spec), 23) + expected = inkling2_nvfp4_experts(*args) + with torch.inference_mode(): + out = inkling2_nvfp4_experts(*args) + # The experts' outputs meet in atomic adds, whose order changes the last bf16 bit. + assert _rel_rms(out, expected) < 1e-2, _rel_rms(out, expected) + + +def test_nvfp4_swapped_gate_and_up_is_visible() -> None: + spec = dict(inkling2_nvfp4_experts.CELLS[1].spec) + args = _nvfp4_args(spec, 17) + ref = inkling2_nvfp4_experts.reference(*args) + swapped = list(args) + i = args[3].shape[1] // 2 + groups = args[3].view(-1, i // 64, 2, 64, args[3].shape[-1]).flip(2).reshape(args[3].shape) + swapped[3] = groups.contiguous() + sf = unswizzle_block_scales(args[4]) + swapped[4] = swizzle_block_scales( + sf.view(-1, i // 64, 2, 64, sf.shape[-1]).flip(2).reshape(sf.shape).contiguous() + ) + assert _rel_rms(inkling2_nvfp4_experts(*swapped), ref) > 0.3 + + +# ----------------------------------------------------------------------------------------- MXFP8 + + +def test_align_routes_puts_each_local_route_in_one_row_of_its_expert() -> None: + from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.mxfp8_moe import ( + BLOCK_M, + align_routes, + ) + + ids, _ = _routing(torch.Generator().manual_seed(4), 100, 384, 8, 96, 96) + route_of_row, expert_of_block, max_blocks = align_routes(ids, 96, 96) + rows = route_of_row.view(max_blocks, BLOCK_M).cpu() + experts = expert_of_block.cpu() + flat = ids.reshape(-1).cpu().long() - 96 + local = set(torch.nonzero((flat >= 0) & (flat < 96)).flatten().tolist()) + placed = [] + for b in range(max_blocks): + routes = [r for r in rows[b].tolist() if r >= 0] + if experts[b] < 0: + assert not routes + assert all(flat[r] == experts[b] for r in routes) + placed += routes + assert sorted(placed) == sorted(local) + + +def test_mxfp8_quantize_round_trip() -> None: + x = torch.randn(64, 256, device="cuda") * torch.logspace(-3, 3, 256, device="cuda") + values, exps = mxfp8_quantize(x) + back = mxfp8_dequantize(values, exps) + # E4M3 keeps 3 mantissa bits: half an ulp is 1/16 of the block's power-of-two range. + blocks = x.view(64, 8, 32).abs().amax(-1, keepdim=True) + assert ((back - x).view(64, 8, 32).abs() <= blocks / 16 + 1e-30).all() + assert ((back.view(64, 8, 32).abs().amax(-1, keepdim=True) / blocks) > 0.5).all() + + +@pytest.mark.parametrize( + "cell", inkling2_mxfp8_experts.CELLS, ids=[c.why for c in inkling2_mxfp8_experts.CELLS] +) +def test_mxfp8_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(inkling2_mxfp8_experts) + args = _mxfp8_args(cell.spec, _seed(cell.why)) + out = inkling2_mxfp8_experts(*args) + ref = inkling2_mxfp8_experts.reference(*args) + # Same products; the bf16 GEMMs accumulate in another order, which moves a few SiTU outputs + # across an E4M3 rounding boundary before the down projection. + assert _rel_rms(out, ref) < 2e-3, _rel_rms(out, ref) + + +@pytest.mark.parametrize("case", ["bf16_bias", "fp16_x"]) +def test_router_guard(case) -> None: + x = torch.randn(4, 64, device="cuda").to(torch.bfloat16) + weight = torch.randn(16, 64, device="cuda").to(torch.bfloat16) + bias = torch.zeros(16, device="cuda") + if case == "bf16_bias": + bias = bias.to(torch.bfloat16) + else: + x = x.to(torch.float16) + with pytest.raises(AssertionError): + with validating(inkling2_router): + inkling2_router(x, weight, bias, 4, 8.0) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py new file mode 100644 index 000000000000..ed7e4c136a06 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the inkling2_residual_mix entry.""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.residual.inkling2_residual_mix import ( + inkling2_residual_mix as op, +) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.residual.inkling2_residual_mix import ( + normalize_block, +) + +__extra_import_path__ = ["../../../.."] +from _validating import assert_within_ulp, certifying, validating # noqa: E402,F401 + +assert torch.cuda.is_available(), "inkling2_residual_mix requires a CUDA device" + +HIDDEN = 4608 +EPS = 1e-6 + + +def _build(spec, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + t, s, q = spec["tokens"], spec["blocks"], spec["queries"] + # Block states at different scales: the normalization, not the raw scale, sets their weight. + raw = torch.randn(s, t, HIDDEN, generator=g, device="cuda") * torch.logspace( + -1, 1, s, device="cuda" + ).view(s, 1, 1) + blocks = torch.stack([normalize_block(raw[i].to(torch.bfloat16)) for i in range(s)]) + partial = (3.0 * torch.randn(t, HIDDEN, generator=g, device="cuda")).to(torch.bfloat16) + # Queries with norms like the checkpoint's, so the softmax is neither flat nor one-hot. + queries = 0.05 * torch.randn(q, HIDDEN, generator=g, device="cuda") + weights = tuple( + (1.0 + 0.2 * torch.randn(HIDDEN, generator=g, device="cuda")).to(torch.bfloat16) + for _ in range(q) + ) + return blocks, partial, queries, weights, EPS + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why[:48] for c in op.CELLS]) +def test_certified_cells(cell, certifying) -> None: # noqa: F811 + certifying(op) + args = _build(cell.spec, zlib.crc32(cell.why.encode()) % 2**31) + out = op(*args) + ref = op.reference(*args) + assert out.shape == (cell.spec["queries"], cell.spec["tokens"], HIDDEN) + # fp32 against fp64, one bf16 rounding at the end: at most one bf16 ulp per element. + assert_within_ulp( + out.reshape(-1, HIDDEN), ref.reshape(-1, HIDDEN), element_ulp=1.0, rms_ulp=0.1 + ) + + +def test_values_are_the_normalized_states() -> None: + """Scaling the partial block does not change the mix: it enters normalized.""" + args = list(_build(dict(blocks=3, queries=1, tokens=5), 7)) + out = op(*args) + args[1] = (args[1].float() * 8.0).to(torch.bfloat16) + torch.testing.assert_close(op(*args), out, rtol=0, atol=0) + + +def test_guard_refuses_bf16_queries() -> None: + blocks, partial, queries, weights, eps = _build(dict(blocks=2, queries=1, tokens=3), 3) + with pytest.raises(AssertionError, match="fp32"): + with validating(op): + op(blocks, partial, queries.to(torch.bfloat16), weights, eps) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py index ba6e79f4ab8d..5a80210522cf 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py @@ -242,7 +242,7 @@ def test_accuracy_directories_are_targets(repo: Path = _REPO): # Families that ship directly constructed components without a registered model. # An exemption is invalid as soon as routing or registry imports are introduced. -_COMPONENT_ONLY_FAMILIES: set[str] = {"beam"} +_COMPONENT_ONLY_FAMILIES: set[str] = {"beam", "inkling2"} def test_every_routed_target_has_accuracy_tests(repo: Path = _REPO): From e2baee7e78d6799ab1a1e4890609733bc510af3f Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 19 Nov 2026 02:21:12 -0800 Subject: [PATCH 2/4] [None][fix] Inkling 2 family: input guards, conv slots and tests - Inkling2KVCacheManager refuses KV cache block reuse and sizes the convolution slots for requests paused by GEN preemption. - Each distinct page table is uploaded once per step. - The MXFP8 MoE refuses sizes that are not multiples of its 128-wide K tile; paged attention refuses strided K/V head dims; the short convolutions refuse non-contiguous request metadata. - The module docstring states the logits' bf16 GEMM output and fp32 soft-cap. - Tests: InklingDims spellings and errors, convolution slot ownership, the lazy loader at a nonzero rank, transform values and FC1 row order, an expert spanning two row blocks, short fresh prefills over stale windows, fp32 partial residuals and the first NVFP4 call under inference mode. Signed-off-by: Michal Guzek --- .../cache.py | 76 ++++-- .../modeling.py | 14 +- .../models/inkling2/kernels/mxfp8_moe.py | 10 +- .../ops/attention/inkling2_paged_attention.py | 1 + .../inkling2/ops/conv/inkling2_short_conv.py | 10 +- .../ops/moe/inkling2_mxfp8_experts.py | 5 +- .../integration/test_lists/test-db/l0_cpu.yml | 2 + .../test_cache.py | 52 ++++ .../test_weights.py | 245 ++++++++++++++++-- ...st_modeling_v2_inkling2_paged_attention.py | 8 + .../test_modeling_v2_inkling2_short_conv.py | 17 +- .../ops/moe/test_modeling_v2_inkling2_moe.py | 53 +++- .../test_modeling_v2_inkling2_residual_mix.py | 16 ++ .../models/inkling2/test_config.py | 110 ++++++++ 14 files changed, 558 insertions(+), 61 deletions(-) create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py index 14f41159c340..f94de21d0e2e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py @@ -22,14 +22,53 @@ from ..config import InklingDims -#: Slots beyond max_batch_size: CUDA-graph padding and warmup requests. +#: Slots beyond the resident requests: CUDA-graph padding and warmup requests. SPARE_SLOTS = 8 #: Inputs a width-4 window keeps. WINDOW = 3 +class ConvSlots: + """Which short-convolution slot each live request owns. + + A request keeps its slot, and so its windows, from its first step until it is released, also + while it is paused; a released slot goes to the next new request. + """ + + def __init__(self, num_slots: int): + self.num_slots = num_slots + self._slot_of: dict[int, int] = {} + self._free = list(range(num_slots - 1, -1, -1)) + + def acquire(self, request_ids: Iterable[int]) -> list[int]: + """Each request's slot; a request seen for the first time gets a free one.""" + slots = [] + for request_id in request_ids: + slot = self._slot_of.get(request_id) + if slot is None: + if not self._free: + raise RuntimeError( + f"all {self.num_slots} short-convolution slots are in use; " + "more requests are live than the KV cache manager admits" + ) + slot = self._free.pop() + self._slot_of[request_id] = slot + slots.append(slot) + return slots + + def release(self, request_id: int) -> None: + """Free the request's slot; a request that never got one is ignored.""" + slot = self._slot_of.pop(request_id, None) + if slot is not None: + self._free.append(slot) + + class Inkling2KVCacheManager(KVCacheManagerV2): - """The KV cache plus each request's short-convolution windows, for every rank-local layer.""" + """The KV cache plus each request's short-convolution windows, for every rank-local layer. + + Block reuse is refused: a reused prefix would come without the convolution windows of the + tokens it covers. + """ def __init__( self, *args, pretrained_config=None, mapping=None, max_batch_size: int = 1, **kwargs @@ -41,11 +80,18 @@ def __init__( max_batch_size=max_batch_size, **kwargs, ) + if self.enable_block_reuse: + raise ValueError( + "Inkling2KVCacheManager does not support KV cache block reuse: a reused prefix " + "comes without its short-convolution windows. Set " + "kv_cache_config.enable_block_reuse to false." + ) text = getattr(pretrained_config, "text_config", None) or pretrained_config dims = InklingDims.from_config(text) tp = mapping.tp_size if mapping is not None else 1 kv_width = dims.num_kv_heads * dims.head_dim // tp - self.num_conv_slots = max_batch_size + SPARE_SLOTS + # Requests paused by GEN preemption stay resident with their windows, beyond max_batch_size. + self.num_conv_slots = max_batch_size + self.gen_preemption_headroom + SPARE_SLOTS widths = {"k": kv_width, "v": kv_width, "attn": dims.hidden_size, "mlp": dims.hidden_size} device = torch.device("cuda", torch.cuda.current_device()) self._conv_state = [ @@ -57,8 +103,7 @@ def __init__( } for _ in range(dims.num_layers) ] - self._conv_slot_of: dict[int, int] = {} - self._free_conv_slots = list(range(self.num_conv_slots - 1, -1, -1)) + self._conv_slots = ConvSlots(self.num_conv_slots) def conv_state(self, layer: int) -> dict: """Layer `layer`'s windows: `k`, `v`, `attn`, `mlp`, each `[slots, 3, C]` bf16.""" @@ -66,27 +111,10 @@ def conv_state(self, layer: int) -> dict: def conv_slots(self, request_ids: Iterable[int]) -> torch.Tensor: """Each request's slot, as int32 on the device; a request seen for the first time gets one.""" - slots = [] - for request_id in request_ids: - slot = self._conv_slot_of.get(request_id) - if slot is None: - if not self._free_conv_slots: - raise RuntimeError( - f"all {self.num_conv_slots} short-convolution slots are in use; " - "more requests are live than max_batch_size allows" - ) - slot = self._free_conv_slots.pop() - self._conv_slot_of[request_id] = slot - slots.append(slot) - return torch.tensor(slots, dtype=torch.int32).to( + return torch.tensor(self._conv_slots.acquire(request_ids), dtype=torch.int32).to( self._conv_state[0]["k"].device, non_blocking=True ) - def _release_conv_slot(self, request_id: int) -> None: - slot = self._conv_slot_of.pop(request_id, None) - if slot is not None: - self._free_conv_slots.append(slot) - def free_resources(self, request, pin_on_release: bool = False): - self._release_conv_slot(request.py_request_id) + self._conv_slots.release(request.py_request_id) return super().free_resources(request, pin_on_release) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py index d1c5d62580fc..a9b482047386 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py @@ -20,8 +20,9 @@ output P'' = P' + m, rounded to bf16 then the final mix over the 8 blocks and the last partial block through the final norm. The -shell's vocab-parallel lm_head gives fp32 logits, which are soft-capped (30 tanh(x / 30)) and -cut to the tokenizer's 200058 entries (the padded rows are set to -inf). +shell's vocab-parallel lm_head GEMM writes bf16 logits, which the shell upcasts to fp32 (as the +Hugging Face implementation does); they are soft-capped in fp32 (30 tanh(x / 30)) and cut to +the tokenizer's 200058 entries (the padded rows are set to -inf). The KV cache is the engine's `KVCacheManagerV2` (BF16, 2 KV heads of 128 per rank, a 1024-token window on the 40 sliding layers) extended with each request's short-convolution windows @@ -186,11 +187,12 @@ def build(layer): tables = {layer: per_window[window[layer]] for layer in layers} else: tables = {layer: build(layer) for layer in layers} + # One upload per distinct table: the layers of a window share theirs. uploaded = {} - return { - layer: uploaded.setdefault(id(t), t.to(device, non_blocking=True)) - for layer, t in tables.items() - } + for t in tables.values(): + if id(t) not in uploaded: + uploaded[id(t)] = t.to(device, non_blocking=True) + return {layer: uploaded[id(t)] for layer, t in tables.items()} # ------------------------------------------------------------------------------- forward diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py index 77b54d94f579..ecc578fa33bd 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py @@ -24,6 +24,8 @@ from triton.language.extra import libdevice BLOCK_M = 32 +#: The K tile of both GEMMs: the hidden and intermediate sizes must be multiples of it. +K_TILE = 128 def align_routes( @@ -246,13 +248,19 @@ def grouped_mxfp8_moe( num_tokens, hidden = x_hat.shape local, rows, _ = w13.shape inter = rows // 2 + if hidden % K_TILE or inter % K_TILE: + # Neither GEMM masks a partial K tile. + raise ValueError( + f"grouped MXFP8 MoE needs hidden ({hidden}) and intermediate ({inter}) sizes that are " + f"multiples of {K_TILE}" + ) top_k = topk_ids.shape[1] out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=x_hat.device) if num_tokens == 0: return out route_of_row, expert_of_block, max_blocks = align_routes(topk_ids, local_expert_offset, local) h = torch.empty(max_blocks * BLOCK_M, inter, dtype=torch.bfloat16, device=x_hat.device) - block_n, block_k = 128, 128 + block_n, block_k = 128, K_TILE _gemm1_kernel[(max_blocks, triton.cdiv(inter, block_n))]( x_hat, w13, diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py index 85f31179950a..fb8972262ed4 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/attention/inkling2_paged_attention.py @@ -207,6 +207,7 @@ def is_valid( assert k.shape == v.shape == (num_tokens, num_kv_heads, head_dim), ( "k and v must be [T, kv heads, head dim]" ) + assert k.stride(2) == 1 and v.stride(2) == 1, "k and v must have contiguous head dims" assert k.dtype == v.dtype == q.dtype == kv_pool.dtype, "q, k, v and the pool share a dtype" assert rel_logits.dtype == torch.float32 and rel_logits.shape[:2] == (num_tokens, num_heads) assert rel_logits.stride(2) == 1, "rel_logits must be contiguous over distances" diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py index de531e39aa13..3ae1e0250a9e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/conv/inkling2_short_conv.py @@ -88,8 +88,16 @@ def _check( "state must hold contiguous channels in x's dtype" ) num_requests = slots.shape[0] - for name, t in (("slots", slots), ("query_start", query_start), ("has_state", has_state)): + metadata = ( + ("slots", slots), + ("query_start", query_start), + ("has_state", has_state), + ("token_request", token_request), + ) + for name, t in metadata: assert t.dtype == torch.int32 and t.dim() == 1, f"{name} must be a 1-D int32 tensor" + # The kernels index request metadata at unit stride. + assert t.is_contiguous(), f"{name} must be contiguous" assert query_start.shape[0] == num_requests + 1 and has_state.shape[0] == num_requests, ( "query_start must be [R + 1] and has_state [R] for R = len(slots)" ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py index eb9da96d2d50..5d05b466a751 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/ops/moe/inkling2_mxfp8_experts.py @@ -26,7 +26,7 @@ from tensorrt_llm._torch.modeling_v2.catalog._op import Arch, Cell, OpWrapper, true_fp32_matmul -from ...kernels.mxfp8_moe import grouped_mxfp8_moe +from ...kernels.mxfp8_moe import K_TILE, grouped_mxfp8_moe #: Values per MXFP8 scale. BLOCK = 32 @@ -174,6 +174,9 @@ def is_valid( assert w2_s.shape == (local, hidden, rows // 2 // BLOCK) and w2_s.dtype == torch.uint8 assert x.dtype == torch.bfloat16 and x.shape[1] == hidden assert topk_ids.dtype == torch.int32 and topk_weights.dtype == torch.float32 + assert hidden % K_TILE == 0 and (rows // 2) % K_TILE == 0, ( + f"hidden and intermediate sizes must be multiples of the {K_TILE}-wide K tile" + ) inkling2_mxfp8_experts = Inkling2Mxfp8Experts() diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index a006667b5dd7..4db3f2ca9ac9 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -157,6 +157,8 @@ l0_cpu: - unittest/_torch/modeling_v2/models/deepseek_v3/r1_0528_nvfp4__sm_10x__default/test_modeling_v2_deepseek_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_default_shards.py - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_weight_table.py + - unittest/_torch/modeling_v2/models/inkling2/test_config.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py # The Kimi K3 SM107 targets' cpu_only cases: CuTe DSL hint tables, the FP4 MLA # decode plan geometry, each target's constants and model defaults, the routing diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py new file mode 100644 index 000000000000..c4e44421fe72 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling 2 KV cache manager: block reuse refused, convolution slot ownership.""" + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + cache, +) +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 + +pytestmark = pytest.mark.cpu_only + + +def test_block_reuse_is_refused_before_any_allocation(monkeypatch): + def base_init(self, *args, **kwargs): + self.enable_block_reuse = True + self.gen_preemption_headroom = 0 + + monkeypatch.setattr(KVCacheManagerV2, "__init__", base_init) + # Anything past the refusal would read the config or allocate on the GPU. + with pytest.raises(ValueError, match="enable_block_reuse"): + cache.Inkling2KVCacheManager( + pretrained_config=SimpleNamespace(), mapping=None, max_batch_size=4 + ) + + +def test_a_released_slot_goes_to_the_next_request_and_others_keep_theirs(): + slots = cache.ConvSlots(3) + a, b = slots.acquire(["a", "b"]) + assert a != b + # A live request, paused or not, keeps its slot from step to step. + assert slots.acquire(["b", "a"]) == [b, a] + slots.release("a") + (c,) = slots.acquire(["c"]) + assert c == a + assert slots.acquire(["b", "c"]) == [b, c] + # Releasing a request that holds no slot changes nothing. + slots.release("never-seen") + slots.release("a") + assert slots.acquire(["b", "c"]) == [b, c] + + +def test_slots_run_out_only_when_every_one_is_held(): + slots = cache.ConvSlots(2) + slots.acquire(["a", "b"]) + with pytest.raises(RuntimeError, match="all 2 short-convolution slots are in use"): + slots.acquire(["c"]) + slots.release("b") + assert slots.acquire(["c"]) == [1] diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py index 799216d402a2..3f2876d111c0 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py @@ -18,6 +18,9 @@ from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( weights as w, ) +from tensorrt_llm._torch.modeling_v2.models.inkling2.ops.moe.inkling2_nvfp4_experts import ( + unswizzle_block_scales, +) pytestmark = pytest.mark.cpu_only @@ -67,8 +70,13 @@ def _dims() -> InklingDims: def checkpoint_layout(d: InklingDims) -> dict: - """name -> (shape, dtype) of the released checkpoint's text-decoder tensors.""" + """name -> (shape, dtype) of the checkpoint's text-decoder tensors for geometry `d`. + + For `_dims()` these are the released checkpoint's shapes. + """ h, e, i = d.hidden_size, d.num_experts, d.expert_intermediate_size + q, kv, r = d.num_heads * d.head_dim, d.num_kv_heads * d.head_dim, d.num_heads * d.d_rel + k = d.conv_kernel_size out = { "model.llm.embed.weight": ((d.vocab_size, h), BF16), "model.llm.embed_norm.weight": ((h,), BF16), @@ -85,31 +93,31 @@ def checkpoint_layout(d: InklingDims) -> dict: F32, ) out.update({ - f"{a}.wq_du.weight": ((6144, h), BF16), - f"{a}.wk_dv.weight": ((1024, h), BF16), - f"{a}.wv_dv.weight": ((1024, h), BF16), - f"{a}.wr_du.weight": ((768, h), BF16), - f"{a}.wo_ud.weight": ((h, 6144), BF16), - f"{a}.gate_attn.weight": ((6144, h), BF16), - f"{a}.attn_sink_proj.weight": ((48, h), BF16), - f"{a}.q_l2_clip.scale": ((48, 128), BF16), - f"{a}.rel_l2_clip.scale": ((48, 16), BF16), - f"{a}.rel_logits_proj.proj": ((16, 1024), BF16), - f"{p}.attn_sconv.weight": ((h, 1, 4), BF16), - f"{p}.mlp_sconv.weight": ((h, 1, 4), BF16), + f"{a}.wq_du.weight": ((q, h), BF16), + f"{a}.wk_dv.weight": ((kv, h), BF16), + f"{a}.wv_dv.weight": ((kv, h), BF16), + f"{a}.wr_du.weight": ((r, h), BF16), + f"{a}.wo_ud.weight": ((h, q), BF16), + f"{a}.gate_attn.weight": ((q, h), BF16), + f"{a}.attn_sink_proj.weight": ((d.num_heads, h), BF16), + f"{a}.q_l2_clip.scale": ((d.num_heads, d.head_dim), BF16), + f"{a}.rel_l2_clip.scale": ((d.num_heads, d.d_rel), BF16), + f"{a}.rel_logits_proj.proj": ((d.d_rel, d.rel_extent_of(layer)), BF16), + f"{p}.attn_sconv.weight": ((h, 1, k), BF16), + f"{p}.mlp_sconv.weight": ((h, 1, k), BF16), f"{p}.mlp_norm.weight": ((h,), BF16), f"{p}.mlp.gate.weight": ((e, h), BF16), f"{p}.mlp.gate.bias": ((e,), F32), }) # fmt: skip if d.is_global(layer): for s in ("k", "v"): - out[f"{a}.{s}_sconv.static_w"] = ((4, 1024), BF16) - out[f"{a}.{s}_sconv.weight_proj"] = ((32, h), BF16) + out[f"{a}.{s}_sconv.static_w"] = ((k, kv), BF16) + out[f"{a}.{s}_sconv.weight_proj"] = ((d.num_kv_heads * k, h), BF16) for n in ("q", "k", "v"): out[f"{p}.{n}_input_norm.weight"] = ((h,), BF16) else: - out[f"{a}.k_sconv.weight"] = ((1024, 1, 4), BF16) - out[f"{a}.v_sconv.weight"] = ((1024, 1, 4), BF16) + out[f"{a}.k_sconv.weight"] = ((kv, 1, k), BF16) + out[f"{a}.v_sconv.weight"] = ((kv, 1, k), BF16) out[f"{p}.attn_norm.weight"] = ((h,), BF16) x = f"{p}.mlp.experts" if d.expert_formats[layer] is ExpertFormat.NVFP4: @@ -184,6 +192,209 @@ def test_shares_split_heads_and_experts_evenly(): assert s.rows(6144) == slice(3072, 4608) and s.rows(32) == slice(16, 24) +def test_transforms_put_values_where_the_kernels_read_them(): + """Real tensors whose values name their source row, against expectations built by hand.""" + s = w.shares_of(_dims(), _mapping(2)) + # A conv weight stores the oldest tap first; the kernels read lag 0, the newest input, first. + taps = torch.arange(8, dtype=F32).reshape(2, 1, 4) + assert w._conv_taps((taps,), s).tolist() == [[3.0, 7.0], [2.0, 6.0], [1.0, 5.0], [0.0, 4.0]] + # A sliding layer's k/v conv: this rank's 256 of the 1024 channels, reversed the same way. + kv_taps = torch.arange(1024 * 4, dtype=F32).reshape(1024, 1, 4) + out = w._kv_conv_taps((kv_taps,), s) + assert out.shape == (4, 256) + assert all( + out[lag, c] == kv_taps[512 + c, 0, 3 - lag] for lag in range(4) for c in (0, 77, 255) + ) + # A global layer's static taps are lag-major already: this rank's channels. + static = torch.arange(4 * 1024, dtype=F32).reshape(4, 1024) + assert torch.equal(w._static_taps((static,), s), static[:, 512:768]) + # Stacked projections: each source's rows of this rank, in the order the entry names them. + sizes = (6144, 768, 6144, 48) # wq, wr, gate, sink + srcs = tuple(torch.arange(n, dtype=F32)[:, None] + 100_000 * k for k, n in enumerate(sizes)) + stacked = w._cat_rows(srcs, s)[:, 0].long().tolist() + rank_rows = [(k, range(n // 2, 3 * n // 4)) for k, n in enumerate(sizes)] # rank 2 of 4 + assert stacked == [100_000 * k + r for k, rows in rank_rows for r in rows] + # A global layer's k or v rows, then its dynamic conv's tap-projection rows. + proj = torch.arange(1024, dtype=F32)[:, None] + tap_proj = torch.arange(32, dtype=F32)[:, None] + 100_000 + out = w._kv_with_taps((proj, tap_proj), s)[:, 0].long().tolist() + assert out == list(range(512, 768)) + [100_000 + r for r in range(16, 24)] + # The output projection keeps this rank's input columns. + wo = torch.arange(6144, dtype=F32)[None, :].expand(2, 6144) + assert torch.equal(w._o_cols((wo,), s), wo[:, 3072:4608]) + # The NVFP4 global scale: this rank's experts' amax over the outer denominator. + amax = torch.arange(384, dtype=F32) + alpha = w._nvfp4_alpha((amax, torch.tensor([2688.0])), s) + assert torch.equal(alpha, amax[192:288] / 2688.0) + + +def test_nvfp4_fc1_weight_and_scale_rows_move_together(): + """Kernel row k of the FC1 weight and of its block scales is the same checkpoint row.""" + i, h = 2304, 4608 + ids = torch.arange(2 * i) + # Each checkpoint row carries its own index in its first two bytes. + tag = torch.stack([ids % 256, ids // 256], dim=-1).to(U8) + w13 = torch.cat([tag, torch.zeros(2 * i, h // 2 - 2, dtype=U8)], dim=-1)[None] + w13_sf = torch.cat([tag, torch.zeros(2 * i, h // 16 - 2, dtype=U8)], dim=-1)[None].view(F8) + s = w.shares_of(_dims(), _mapping(0)) + + def rows_of(t): + return (t[0, :, 0].long() + 256 * t[0, :, 1].long()).tolist() + + # The checkpoint interleaves gate (even) and up (odd) rows; the kernel wants, per 64 rows of + # each, the up rows first. + expected = [] + for group in range(i // 64): + expected += [2 * (64 * group + j) + 1 for j in range(64)] + expected += [2 * (64 * group + j) for j in range(64)] + assert rows_of(w._nvfp4_w13((w13,), s)) == expected + assert rows_of(unswizzle_block_scales(w._nvfp4_w13_scale((w13_sf,), s))) == expected + + +def _small_dims() -> InklingDims: + """Six layers (layer 5 global; NVFP4 experts in 0-3, MXFP8 in 4-5), small enough to load.""" + text = SimpleNamespace( + num_hidden_layers=6, + local_layer_ids=[0, 1, 2, 3, 4], + hidden_size=256, + vocab_size=1000, + unpadded_vocab_size=990, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=128, + d_rel=16, + rel_extent=64, + sliding_window_size=64, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=8, + num_experts_per_tok=2, + intermediate_size=128, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + groups = [ + { + "compute_dtype": fmt, + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in layers], + } + for fmt, layers in (("nvfp4", range(4)), ("mxfp8", (4, 5))) + ] + return InklingDims.from_config(text, {"model": {"parameter_groups": groups}}) + + +class _RecordingCheckpoint: + """The engine's lazy checkpoint view (`view[name][rows]` reads only those rows), recording.""" + + def __init__(self, tensors): + self.tensors = tensors + self.reads = [] + + def __contains__(self, name): + return name in self.tensors + + def keys(self): + return self.tensors.keys() + + def __getitem__(self, name): + return _RecordedTensor(self, name) + + +class _RecordedTensor: + def __init__(self, checkpoint: _RecordingCheckpoint, name: str): + self.checkpoint, self.name = checkpoint, name + + def __getitem__(self, rows): + self.checkpoint.reads.append((self.name, rows)) + return self.checkpoint.tensors[self.name][rows] + + +def _random_checkpoint(layout: dict, seed: int) -> dict: + g = torch.Generator().manual_seed(seed) + out = {} + for name, (shape, dtype) in layout.items(): + if dtype.itemsize == 1: + out[name] = torch.randint(0, 256, shape, generator=g, dtype=U8).view(dtype) + else: + out[name] = (torch.rand(shape, generator=g) + 0.5).to(dtype) + return out + + +def _bits(t: torch.Tensor) -> torch.Tensor: + return t.view(U8) if t.dtype.itemsize == 1 else t + + +def _model(s: "w.Shares", lm_head_rows: int): + """The parts of the engine model `load` fills, every value poisoned first.""" + model = SimpleNamespace( + model=SimpleNamespace(w=w.declare(s)), + lm_head=SimpleNamespace( + weight=torch.nn.Parameter( + torch.empty(lm_head_rows, s.model.hidden_size, dtype=BF16), requires_grad=False + ) + ), + ) + for p in [*model.model.w.values(), model.lm_head.weight]: + if p.dtype.itemsize == 1: + _bits(p.data).fill_(0x7F) + else: + p.data.fill_(float("nan")) + return model + + +@pytest.mark.parametrize("rank", [1, 3]) +def test_load_reads_this_ranks_rows_and_fills_every_weight(rank, monkeypatch): + monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *args, **kwargs: None) + d = _small_dims() + s = w.shares_of(d, _mapping(rank)) + tensors = _random_checkpoint(checkpoint_layout(d), seed=rank) + tensors["model.mtp.fc.weight"] = torch.zeros(2, 2) # the drafter's: not the table's to read + ckpt = _RecordingCheckpoint(tensors) + rows = 256 # this rank's share of the vocab-parallel lm_head (1000 rows over 4 ranks) + model = _model(s, rows) + w.load(model, ckpt, s) + + reads = dict(ckpt.reads) + assert len(reads) == len(ckpt.reads), "a checkpoint tensor was read twice" + assert set(reads) == set(checkpoint_layout(d)) + for layer, entry in w.all_entries(s): + flags = entry.expert_rows or (False,) * len(entry.sources) + srcs = [] + for name, by_expert in zip(entry.sources, flags): + # Expert tensors are read at this rank's experts only. + want = s.experts() if by_expert else slice(None) + assert reads[name] == want, (name, reads[name], want) + srcs.append(tensors[name][want]) + expected = entry.transform(tuple(srcs), s) + loaded = model.model.w[w._key(layer, entry.name)].data + assert torch.equal(_bits(loaded), _bits(expected)), (layer, entry.name) + + start, end = rank * rows, min(rank * rows + rows, d.vocab_size) + assert reads["model.llm.unembed.weight"] == slice(start, end) + head = model.lm_head.weight.data + assert torch.equal(head[: end - start], tensors["model.llm.unembed.weight"][start:end]) + assert not head[end - start :].any(), "rows past the vocabulary must be zero" + + +def test_load_refuses_a_text_decoder_tensor_the_table_does_not_name(monkeypatch): + monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) + monkeypatch.setattr(torch.cuda, "synchronize", lambda *args, **kwargs: None) + d = _small_dims() + s = w.shares_of(d, _mapping(0)) + tensors = _random_checkpoint(checkpoint_layout(d), seed=0) + tensors["model.llm.layers.0.attn.extra.weight"] = torch.zeros(2, 2) + with pytest.raises(AssertionError, match="does not name"): + w.load(_model(s, 256), _RecordingCheckpoint(tensors), s) + + @pytest.mark.skipif( not os.path.isdir(os.path.join(os.environ.get("LLM_MODELS_ROOT", ""), "Inkling-2-Small")), reason="needs the Inkling-2-Small checkpoint under LLM_MODELS_ROOT", diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py index 419f22b57059..19cae9df2fb9 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/attention/test_modeling_v2_inkling2_paged_attention.py @@ -143,6 +143,7 @@ def test_negative_controls_are_visible(what) -> None: ("bf16_rel_logits", None), ("short_page_table", "page_table must cover"), ("wrong_tiles", "tile plan"), + ("strided_kv", "contiguous head dims"), ], ) def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: @@ -151,6 +152,13 @@ def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: args[11] = args[11].to(torch.bfloat16) elif case == "short_page_table": args[4] = args[4][:, :1] + elif case == "strided_kv": + # The same values every other element of a wider buffer: the kernels read unit stride. + for i in (1, 2): + t = args[i] + wide = torch.zeros(*t.shape[:2], 2 * t.shape[2], dtype=t.dtype, device=t.device) + wide[..., ::2] = t + args[i] = wide[..., ::2] else: args[9] = torch.zeros_like(args[9]) with pytest.raises(AssertionError, match=match): diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py index 2775c15ed730..052d5b06a0a4 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/conv/test_modeling_v2_inkling2_short_conv.py @@ -102,10 +102,12 @@ def test_dynamic_certified_cells(cell, certifying) -> None: # noqa: F811 ) +@pytest.mark.parametrize("first", [1, 2, 21]) @pytest.mark.parametrize("dynamic", [False, True], ids=["static", "dynamic"]) -def test_chunked_steps_equal_one_prefill(dynamic) -> None: - """A prefill of 21 tokens, then chunks of 2 and 1 and decodes, equals one 29-token prefill: - the window carries exactly the inputs a step needs from the steps before it.""" +def test_chunked_steps_equal_one_prefill(dynamic, first) -> None: + """A fresh prefill of `first` tokens over a stale window, then chunks and decodes, equals one + 29-token prefill: the window carries exactly the inputs a step needs from the steps before it, + and a fresh request shorter than the window starts it with zeros, not the stale inputs.""" channels, heads = 256, 2 g = torch.Generator(device="cuda").manual_seed(5) total = 29 @@ -133,8 +135,11 @@ def run(start, end, state, has_state): whole = run(0, total, torch.zeros(3, 3, channels, dtype=torch.bfloat16, device="cuda"), 0) state = torch.randn(3, 3, channels, device="cuda").to(torch.bfloat16) pieces, start = [], 0 - for end in (21, 23, 24, 25, 26, 27, 28, 29): + for end in sorted({first, *(e for e in (21, 23, 24, 25, 26, 27, 28, 29) if e > first)}): pieces.append(run(start, end, state, int(start > 0))) + if start == 0 and end < 3: + zeros = torch.zeros(3 - end, channels, dtype=torch.bfloat16, device="cuda") + torch.testing.assert_close(state[1], torch.cat([zeros, x[:end]]), rtol=0, atol=0) start = end torch.testing.assert_close(torch.cat(pieces), whole, rtol=0, atol=0) torch.testing.assert_close(state[1], x[-3:], rtol=0, atol=0) @@ -157,6 +162,7 @@ def test_dynamic_taps_are_per_head() -> None: ("duplicate_slot", "own slot"), ("bf16_taps", "lag-major fp32"), ("bad_token_request", "token_request"), + ("strided_slots", "slots must be contiguous"), ], ) def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: @@ -166,6 +172,9 @@ def test_guard_refuses_inputs_the_op_answers_wrongly(case, match) -> None: slots = slots[:1].repeat(2) elif case == "bf16_taps": taps = taps.to(torch.bfloat16) + elif case == "strided_slots": + # The same slot values, as a view the kernels would index at unit stride. + slots = torch.stack([slots, slots + NUM_SLOTS], dim=1)[:, 0] else: token_request = torch.zeros_like(token_request) with pytest.raises(AssertionError, match=match): diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py index 9eceb4b1da70..0ed0fda6af1f 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/moe/test_modeling_v2_inkling2_moe.py @@ -163,9 +163,13 @@ def test_router_certified_cells(cell, certifying) -> None: # noqa: F811 def test_fc1_row_order_and_swizzle_round_trip() -> None: order = fc1_row_order(2304, torch.device("cpu")) - assert sorted(order.tolist()) == list(range(4608)) - # Rows 0..63 of the kernel are up rows 1, 3, ..; rows 64..127 gate rows 0, 2, .. - assert order[:3].tolist() == [1, 3, 5] and order[64:67].tolist() == [0, 2, 4] + # The checkpoint interleaves gate (even) and up (odd) rows. Kernel group g (128 rows) holds the + # up rows of gate/up pairs 64 g .. 64 g + 63, then their gate rows: every group, in order. + expected = [] + for group in range(2304 // 64): + pairs = range(64 * group, 64 * group + 64) + expected += [2 * p + 1 for p in pairs] + [2 * p for p in pairs] + assert order.tolist() == expected scales = torch.randint(0, 256, (3, 256, 288), dtype=torch.uint8) assert torch.equal(unswizzle_block_scales(swizzle_block_scales(scales)), scales) @@ -194,13 +198,12 @@ def test_nvfp4_certified_cells(cell, certifying) -> None: # noqa: F811 def test_nvfp4_runs_under_inference_mode() -> None: - """The engine runs every forward under inference mode; the op gives the same answer there.""" + """The engine runs every forward under inference mode, a layer's first call included.""" args = _nvfp4_args(dict(inkling2_nvfp4_experts.CELLS[1].spec), 23) - expected = inkling2_nvfp4_experts(*args) + ref = inkling2_nvfp4_experts.reference(*args) with torch.inference_mode(): out = inkling2_nvfp4_experts(*args) - # The experts' outputs meet in atomic adds, whose order changes the last bf16 bit. - assert _rel_rms(out, expected) < 1e-2, _rel_rms(out, expected) + assert _rel_rms(out, ref) < 0.02, _rel_rms(out, ref) def test_nvfp4_swapped_gate_and_up_is_visible() -> None: @@ -243,6 +246,42 @@ def test_align_routes_puts_each_local_route_in_one_row_of_its_expert() -> None: assert sorted(placed) == sorted(local) +def test_mxfp8_expert_with_more_routes_than_a_row_block() -> None: + """33 routes to one expert fill a 32-row block and spill into a second: both count.""" + from tensorrt_llm._torch.modeling_v2.models.inkling2.kernels.mxfp8_moe import ( + BLOCK_M, + align_routes, + ) + + tokens = BLOCK_M + 1 + spec = dict(local=4, hidden=256, inter=256, rank=1, tokens=tokens, experts=16, top_k=2) + args = _mxfp8_args(spec, 31) + offset = args[7] + # Every token's first choice is local expert 2; its second is expert 0, on another rank. + args[1] = torch.tensor([[offset + 2, 0]] * tokens, dtype=torch.int32, device="cuda") + args[2] = torch.full((tokens, 2), 4.0, device="cuda") + route_of_row, expert_of_block, max_blocks = align_routes(args[1], offset, spec["local"]) + rows = route_of_row.view(max_blocks, BLOCK_M).cpu() + blocks = [b for b in range(max_blocks) if int(expert_of_block[b]) == 2] + assert len(blocks) == 2 + placed = sorted(r for b in blocks for r in rows[b].tolist() if r >= 0) + assert placed == [2 * t for t in range(tokens)], "each route exactly once" + out = inkling2_mxfp8_experts(*args) + ref = inkling2_mxfp8_experts.reference(*args) + assert _rel_rms(out, ref) < 2e-3, _rel_rms(out, ref) + + +def test_mxfp8_refuses_a_partial_k_tile() -> None: + """Neither GEMM masks a partial 128-wide K tile, so such a size is refused before launch.""" + spec = dict(local=2, hidden=256, inter=160, rank=0, tokens=4, experts=8, top_k=2) + args = _mxfp8_args(spec, 7) + with pytest.raises(ValueError, match="multiples of 128"): + inkling2_mxfp8_experts(*args) + with pytest.raises(AssertionError, match="128-wide K tile"): + with validating(inkling2_mxfp8_experts): + inkling2_mxfp8_experts(*args) + + def test_mxfp8_quantize_round_trip() -> None: x = torch.randn(64, 256, device="cuda") * torch.logspace(-3, 3, 256, device="cuda") values, exps = mxfp8_quantize(x) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py index ed7e4c136a06..936ac7018fd6 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/ops/residual/test_modeling_v2_inkling2_residual_mix.py @@ -62,6 +62,22 @@ def test_values_are_the_normalized_states() -> None: torch.testing.assert_close(op(*args), out, rtol=0, atol=0) +def test_fp32_partial_block_keeps_its_low_bits() -> None: + """The decoder hands the partial block over in fp32; the mix must not round it to bf16.""" + blocks, partial, queries, weights, eps = _build(dict(blocks=3, queries=2, tokens=5), 11) + g = torch.Generator(device="cuda").manual_seed(12) + partial = partial.float() + 1e-3 * torch.randn(partial.shape, generator=g, device="cuda") + rounded = partial.to(torch.bfloat16) + assert not torch.equal(partial, rounded.float()), "values bf16 cannot hold" + out = op(blocks, partial, queries, weights, eps) + ref = op.reference(blocks, partial, queries, weights, eps) + assert_within_ulp( + out.reshape(-1, HIDDEN), ref.reshape(-1, HIDDEN), element_ulp=1.0, rms_ulp=0.1 + ) + # Had the op rounded the partial block first, both inputs would give the same mix. + assert not torch.equal(out, op(blocks, rounded, queries, weights, eps)) + + def test_guard_refuses_bf16_queries() -> None: blocks, partial, queries, weights, eps = _build(dict(blocks=2, queries=1, tokens=3), 3) with pytest.raises(AssertionError, match="fp32"): diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py b/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py new file mode 100644 index 000000000000..20af2a9b2a7f --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/test_config.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of `InklingDims`: both config spellings give one geometry, and bad inputs fail.""" + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch.modeling_v2.models.inkling2.config import ExpertFormat, InklingDims + +pytestmark = pytest.mark.cpu_only + +GLOBAL = tuple(range(5, 48, 6)) +_COMMON = dict( + num_hidden_layers=48, + hidden_size=4608, + vocab_size=201024, + unpadded_vocab_size=200058, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + d_rel=16, + rel_extent=1024, + n_layers_per_attn_res_block=6, + n_routed_experts=384, + num_experts_per_tok=8, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, +) + + +def _trtllm_style(**overrides) -> SimpleNamespace: + """The `Inkling2TextConfig` spelling: `local_layer_ids` and the checkpoint's field names.""" + fields = dict( + _COMMON, + local_layer_ids=[i for i in range(48) if i not in GLOBAL], + sliding_window_size=1024, + sconv_kernel_size=4, + intermediate_size=2304, + route_scale=8.0, + ) + return SimpleNamespace(**{**fields, **overrides}) + + +def _native_style() -> SimpleNamespace: + """The Transformers spelling: `layer_types` (`hybrid` / `hybrid_sliding`) and its aliases.""" + return SimpleNamespace( + **_COMMON, + layer_types=["hybrid" if i in GLOBAL else "hybrid_sliding" for i in range(48)], + sliding_window=1024, + conv_kernel_size=4, + moe_intermediate_size=2304, + routed_scaling_factor=8.0, + ) + + +def _groups(**layers_by_format): + return { + "model": { + "parameter_groups": [ + { + "compute_dtype": fmt, + "tensors": [f"model.llm.layers.{i}.mlp.experts.w13_weight" for i in layers], + } + for fmt, layers in layers_by_format.items() + ] + } + } + + +def test_both_config_spellings_give_one_geometry(): + quant = _groups(nvfp4=range(40), mxfp8=range(40, 48)) + native = InklingDims.from_config(_native_style(), quant) + assert native == InklingDims.from_config(_trtllm_style(), quant) + assert native.global_layers == GLOBAL + assert (native.conv_kernel_size, native.expert_intermediate_size, native.route_scale) == ( + 4, + 2304, + 8.0, + ) + assert native.expert_formats == (ExpertFormat.NVFP4,) * 40 + (ExpertFormat.MXFP8,) * 8 + + +def test_layers_no_parameter_group_names_keep_bf16_experts(): + dims = InklingDims.from_config(_trtllm_style(), _groups(nvfp4=[0])) + assert dims.expert_formats[:2] == (ExpertFormat.NVFP4, ExpertFormat.BF16) + + +@pytest.mark.parametrize( + "quant, match", + [ + (_groups(nvfp4=[3], mxfp8=[3]), "layer 3 two expert formats"), + (_groups(nvfp4=[48]), r"layers \[48\], outside the 48 decoder layers"), + ], + ids=["two_formats_on_one_layer", "layer_out_of_range"], +) +def test_malformed_quant_config_is_refused(quant, match): + with pytest.raises(ValueError, match=match): + InklingDims.from_config(_trtllm_style(), quant) + + +def test_sliding_layers_with_another_attention_shape_are_refused(): + with pytest.raises(ValueError, match="different attention shape"): + InklingDims.from_config(_trtllm_style(swa_num_key_value_heads=4)) From 8f9ff78619aaa3947d5309702c6be39565b0f5a2 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 19 Nov 2026 10:02:27 -0800 Subject: [PATCH 3/4] [None][test] Test the convolution slots with GEN preemption headroom The cache manager with nonzero headroom holds 40 slots for a batch of 23, 9 paused requests and 8 spares, and resident requests keep theirs. Signed-off-by: Michal Guzek --- .../test_cache.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py index c4e44421fe72..b7c67f40c24f 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest +import torch from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( cache, @@ -27,6 +28,69 @@ def base_init(self, *args, **kwargs): ) +def _small_text_config() -> SimpleNamespace: + """Two layers (the second global) of the Inkling 2 text config, small enough for a CPU.""" + return SimpleNamespace( + num_hidden_layers=2, + local_layer_ids=[0], + hidden_size=64, + vocab_size=128, + num_attention_heads=4, + num_key_value_heads=4, + head_dim=16, + d_rel=16, + rel_extent=64, + sliding_window_size=64, + sconv_kernel_size=4, + n_layers_per_attn_res_block=6, + n_routed_experts=8, + num_experts_per_tok=2, + intermediate_size=128, + route_scale=8.0, + mlp_gate_softcap=4.0, + mlp_up_softcap=25.0, + final_logit_softcapping=30.0, + qk_l2_clip_threshold=22.627416997969522, + rel_l2_clip_threshold=8.0, + log_scaling_alpha=0.1, + log_scaling_n_floor=8192, + rms_norm_eps=1e-6, + ) + + +def test_conv_slots_cover_requests_paused_by_gen_preemption(monkeypatch): + """Paused requests stay resident with their windows: the pool holds them beyond the batch.""" + + def base_init(self, *args, **kwargs): + self.enable_block_reuse = False + self.gen_preemption_headroom = 9 + + monkeypatch.setattr(KVCacheManagerV2, "__init__", base_init) + # The windows are allocated on the current CUDA device; a CPU stands in for it here. + zeros = torch.zeros + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + monkeypatch.setattr( + torch, "zeros", lambda *shape, dtype=None, device=None: zeros(*shape, dtype=dtype) + ) + manager = cache.Inkling2KVCacheManager( + pretrained_config=SimpleNamespace(text_config=_small_text_config()), + mapping=SimpleNamespace(tp_size=4), + max_batch_size=23, + ) + assert manager.num_conv_slots == 23 + 9 + cache.SPARE_SLOTS == 40 + assert all(t.shape[0] == 40 for t in manager.conv_state(1).values()) + + resident = [f"request-{i}" for i in range(23 + 9)] + slots = manager.conv_slots(resident).tolist() + assert len(set(slots)) == 32 + # The earliest owners keep their slots while the rest are resident. + assert manager.conv_slots(resident[:3]).tolist() == slots[:3] + # The spare slots (CUDA-graph padding, warmup) are still there. + assert len(set(manager.conv_slots([f"spare-{i}" for i in range(8)]).tolist())) == 8 + with pytest.raises(RuntimeError, match="all 40 short-convolution slots are in use"): + manager.conv_slots(["one-too-many"]) + + def test_a_released_slot_goes_to_the_next_request_and_others_keep_theirs(): slots = cache.ConvSlots(3) a, b = slots.acquire(["a", "b"]) From f45d2c21aeff8f04b8d1576f40a3d15b25272815 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 19 Nov 2026 14:16:38 -0800 Subject: [PATCH 4/4] [None][fix] Inkling 2: slot pool from the manager's count; check page tables The short-convolution slot pool is the base manager's `max_admissible_sequences` plus the spares, which also covers the requests the overlap scheduler retires a step late. Each step checks on the host that every KV cache block a request reads has a page, and the attention kernel masks a key whose page index is negative; its docstring states the retention it relies on. The core names the hf_quant_config.json it could not find, the shell no longer writes `torch_dtype` onto the shared config, and the MXFP8 kernel records its scale rule, the same as SGLang's. Signed-off-by: Michal Guzek --- .../cache.py | 6 +- .../modeling.py | 59 ++++++++++-- .../models/inkling2/kernels/mxfp8_moe.py | 6 ++ .../inkling2/kernels/paged_attention.py | 8 ++ .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../test_cache.py | 43 ++++++--- .../test_modeling.py | 91 +++++++++++++++++++ 7 files changed, 192 insertions(+), 22 deletions(-) create mode 100644 tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py index f94de21d0e2e..5e384e7e3b11 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/cache.py @@ -90,8 +90,10 @@ def __init__( dims = InklingDims.from_config(text) tp = mapping.tp_size if mapping is not None else 1 kv_width = dims.num_kv_heads * dims.head_dim // tp - # Requests paused by GEN preemption stay resident with their windows, beyond max_batch_size. - self.num_conv_slots = max_batch_size + self.gen_preemption_headroom + SPARE_SLOTS + # Every sequence the manager admits holds its windows: a retiring request keeps its slot + # one more iteration under the overlap scheduler, and a request paused by GEN preemption + # keeps its slot, so the base manager's own count sizes the pool. + self.num_conv_slots = self.max_admissible_sequences + SPARE_SLOTS widths = {"k": kv_width, "v": kv_width, "attn": dims.hidden_size, "mlp": dims.hidden_size} device = torch.device("cuda", torch.cuda.current_device()) self._conv_state = [ diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py index a9b482047386..f71f660f60c6 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/modeling.py @@ -95,7 +95,15 @@ def __init__(self, model_config: ModelConfig): super().__init__(model_config) text = text_config_of(model_config.pretrained_config) checkpoint_dir = getattr(model_config.pretrained_config, "_name_or_path", None) - self.dims = InklingDims.from_config(text, read_hf_quant_config(checkpoint_dir)) + hf_quant_config = read_hf_quant_config(checkpoint_dir) + if hf_quant_config is None: + # Without it every layer would read as BF16 experts, which this target does not serve. + raise ValueError( + f"Inkling2: found no hf_quant_config.json in {checkpoint_dir!r}, the directory the " + "config was loaded from (`_name_or_path`); this target reads each layer's expert " + "format from that file, so load the config from the checkpoint directory" + ) + self.dims = InklingDims.from_config(text, hf_quant_config) self.shares = _weights.shares_of(self.dims, model_config.mapping) self.eps = self.dims.rms_norm_eps self.w = _weights.declare(self.shares) @@ -131,7 +139,7 @@ def i32(values): tile_request, tile_start = tile_plan(lengths, block_t, device) kv_lens = [c + n for c, n in zip(cached, lengths)] blocks = [-(-length // manager.tokens_per_block) for length in kv_lens] - page_tables = self._page_tables(manager, request_ids, blocks, device) + page_tables = self._page_tables(manager, request_ids, blocks, cached, device) return _Step( num_tokens=num_tokens, query_start=i32(bounds), @@ -148,19 +156,21 @@ def i32(values): ), ) - def _page_tables(self, manager, request_ids, blocks, device) -> dict: + def _page_tables(self, manager, request_ids, blocks, cached, device) -> dict: """Each layer's page table `[R, W]` int32 (-1 past a request's pages). A layer's indices address its own `get_buffers` view. The manager gives the layers of one window the same indices; the first step checks that once, after which one table per window - is built and shared. + is built and shared. Every table is checked to have a live page for each block a request + reads this step (see `_check_live_pages`). """ def build(layer): rows = manager.get_batch_cache_indices( request_ids, layer_idx=layer, num_blocks_per_seq=blocks ) - table = torch.full((len(rows), max(len(r) for r in rows)), -1, dtype=torch.int32) + # As wide as the longest read, so a row the manager returns short shows as -1. + table = torch.full((len(rows), max(blocks)), -1, dtype=torch.int32) for i, r in enumerate(rows): table[i, : len(r)] = torch.tensor(r, dtype=torch.int32) return table @@ -187,13 +197,40 @@ def build(layer): tables = {layer: per_window[window[layer]] for layer in layers} else: tables = {layer: build(layer) for layer in layers} - # One upload per distinct table: the layers of a window share theirs. + # One check and one upload per distinct table: the layers of a window share theirs. + tokens_per_block = manager.tokens_per_block + read_end = torch.tensor(blocks) + sliding_start = torch.tensor( + [max(0, c - self.dims.sliding_window + 1) // tokens_per_block for c in cached] + ) uploaded = {} - for t in tables.values(): + for layer, t in tables.items(): if id(t) not in uploaded: + start = torch.zeros_like(read_end) if window[layer] else sliding_start + self._check_live_pages(t, start, read_end, layer, request_ids) uploaded[id(t)] = t.to(device, non_blocking=True) return {layer: uploaded[id(t)] for layer, t in tables.items()} + @staticmethod + def _check_live_pages(table, start, end, layer, request_ids) -> None: + """Fail if a block a request reads this step has no page (-1) in `table`. + + Request `r` reads blocks `[start[r], end[r])`: from the window of its first new token on a + sliding layer (from block 0 on a global one) to its last token. `KVCacheManagerV2` keeps + a sliding layer's pages from `history + 1 - window` on, which covers that range, so a hole + there is a broken retention contract, reported here instead of read by the kernels. + """ + blocks = torch.arange(table.shape[1]) + read = (blocks[None, :] >= start[:, None]) & (blocks[None, :] < end[:, None]) + holes = (table < 0) & read + if bool(holes.any()): + row, block = (int(i) for i in holes.nonzero()[0]) + raise RuntimeError( + f"Inkling2: layer {layer}: request {request_ids[row]} reads KV cache block " + f"{block}, which has no page; the KV cache manager did not keep the window " + "the attention kernels read" + ) + # ------------------------------------------------------------------------------- forward def _all_reduce(self, x: torch.Tensor) -> torch.Tensor: @@ -366,9 +403,13 @@ class ModelingV2Inkling2SmallNvfp4Sm103Tep4( def __init__(self, model_config: ModelConfig): cfg = model_config.pretrained_config text = text_config_of(cfg) - # config.json declares bf16 only in its text config; lm_head is built in this dtype. + # lm_head is built in the config's dtype, which `ModelConfig.from_pretrained` takes from + # the text config when config.json declares none at the top level. if getattr(cfg, "torch_dtype", None) is None: - cfg.torch_dtype = model_config.torch_dtype or torch.bfloat16 + raise ValueError( + "Inkling2: the config declares no dtype (`torch_dtype`, or `dtype` in its text " + "config) to build lm_head in" + ) super().__init__( Inkling2SmallCore(model_config), config=model_config, diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py index ecc578fa33bd..899a8b324345 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/mxfp8_moe.py @@ -14,6 +14,12 @@ added atomically into the fp32 output row of its token. `x^`, the MXFP8-rounded input, is computed once per call by the caller. + +Both activation roundings, `x^` (the caller's `mxfp8_quantize`) and `h^`, take the block scale +`2^ceil(log2(amax / 448))`, as SGLang's Inkling 2 MoE does (`_mxfp8_block` in +`sglang/kernels/ops/moe/inkling2_moe.py`), so that no value of a block saturates. The OCP MX rule +`2^(floor(log2(amax)) - 8)` would instead clip a block whose `amax` lies in +`(448 * 2^k, 512 * 2^k)`. """ from __future__ import annotations diff --git a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py index 9fa0ff77cf8d..7dc624b6c105 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py +++ b/tensorrt_llm/_torch/modeling_v2/models/inkling2/kernels/paged_attention.py @@ -19,6 +19,13 @@ in fp32, rounded once. `sigmoid(lse - sink)` is the probability mass the keys keep against one extra logit `sink` of zero value. A program covers `BLOCK_T` tokens of one request and the `group` query heads of one KV head, as `BLOCK_T * GROUP_PAD` rows of one tile. + +The kernels rely on the KV cache keeping every page they touch: `append_kv` writes only the step's +new tokens, and `attention` reads only keys `p - window + 1 .. p` on a sliding layer (`0 .. p` on a +global one). `KVCacheManagerV2` keeps a sliding layer's pages from `history + 1 - window` on, which +covers that range, and the model checks each step's tables for it on the host. `attention` also +masks a key whose page index is negative (a recycled page), so a broken contract cannot read +before the pool. """ from __future__ import annotations @@ -150,6 +157,7 @@ def _attention_kernel( k_pos = block + tl.arange(0, BLOCK_N) key_ok = (k_pos >= lo) & (k_pos < hi) page = tl.load(pt_row + k_pos // PAGE_SIZE, mask=key_ok, other=0).to(tl.int64) + key_ok = key_ok & (page >= 0) slot_ptr = kv_base + page * stride_pool_page + (k_pos % PAGE_SIZE) * stride_pool_slot k = tl.load(slot_ptr[:, None] + offs_d[None, :], mask=key_ok[:, None], other=0.0) v = tl.load( diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 4db3f2ca9ac9..7e61f1e76d8e 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -159,6 +159,7 @@ l0_cpu: - unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_weight_table.py - unittest/_torch/modeling_v2/models/inkling2/test_config.py - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py + - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py - unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_weights.py # The Kimi K3 SM107 targets' cpu_only cases: CuTe DSL hint tables, the FP4 MLA # decode plan geometry, each target's constants and model defaults, the routing diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py index b7c67f40c24f..26be3f516865 100644 --- a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_cache.py @@ -10,7 +10,11 @@ from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( cache, ) -from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import ( + KVCacheManagerV2, + _max_admissible_sequences, +) +from tensorrt_llm.mapping import Mapping pytestmark = pytest.mark.cpu_only @@ -58,12 +62,27 @@ def _small_text_config() -> SimpleNamespace: ) -def test_conv_slots_cover_requests_paused_by_gen_preemption(monkeypatch): - """Paused requests stay resident with their windows: the pool holds them beyond the batch.""" +@pytest.mark.parametrize( + "disable_overlap_scheduler, admissible", + [(True, 23 + 9), (False, 2 * 23 + 9)], + ids=["overlap_off", "overlap_on"], +) +def test_conv_slots_cover_every_sequence_the_manager_admits( + monkeypatch, disable_overlap_scheduler, admissible +): + """Paused requests (GEN preemption headroom 9) and, under the overlap scheduler, retiring ones + hold their windows too: the pool is the base manager's admission count plus the spares.""" + mapping = Mapping(world_size=4, tp_size=4, moe_tp_size=1, moe_ep_size=4) def base_init(self, *args, **kwargs): self.enable_block_reuse = False - self.gen_preemption_headroom = 9 + self.max_admissible_sequences = _max_admissible_sequences( + 23, + mapping, + is_disagg=False, + disable_overlap_scheduler=disable_overlap_scheduler, + gen_preemption_headroom=9, + ) monkeypatch.setattr(KVCacheManagerV2, "__init__", base_init) # The windows are allocated on the current CUDA device; a CPU stands in for it here. @@ -74,20 +93,22 @@ def base_init(self, *args, **kwargs): ) manager = cache.Inkling2KVCacheManager( pretrained_config=SimpleNamespace(text_config=_small_text_config()), - mapping=SimpleNamespace(tp_size=4), + mapping=mapping, max_batch_size=23, ) - assert manager.num_conv_slots == 23 + 9 + cache.SPARE_SLOTS == 40 - assert all(t.shape[0] == 40 for t in manager.conv_state(1).values()) + slots_total = admissible + cache.SPARE_SLOTS + assert manager.num_conv_slots == slots_total + assert all(t.shape[0] == slots_total for t in manager.conv_state(1).values()) - resident = [f"request-{i}" for i in range(23 + 9)] + resident = [f"request-{i}" for i in range(admissible)] slots = manager.conv_slots(resident).tolist() - assert len(set(slots)) == 32 + assert len(set(slots)) == admissible # The earliest owners keep their slots while the rest are resident. assert manager.conv_slots(resident[:3]).tolist() == slots[:3] # The spare slots (CUDA-graph padding, warmup) are still there. - assert len(set(manager.conv_slots([f"spare-{i}" for i in range(8)]).tolist())) == 8 - with pytest.raises(RuntimeError, match="all 40 short-convolution slots are in use"): + spares = manager.conv_slots([f"spare-{i}" for i in range(cache.SPARE_SLOTS)]).tolist() + assert len(set(spares)) == cache.SPARE_SLOTS + with pytest.raises(RuntimeError, match=f"all {slots_total} short-convolution slots are in use"): manager.conv_slots(["one-too-many"]) diff --git a/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py new file mode 100644 index 000000000000..22eb7af27dbc --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/inkling2/inkling_2_small_nvfp4__sm_103__tep4/test_modeling.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU tests of the Inkling 2 target's engine glue: the page tables against the KV cache's +retention, and the config errors raised before any weight is declared.""" + +import re +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.inkling2.inkling_2_small_nvfp4__sm_103__tep4 import ( + modeling, +) + +pytestmark = pytest.mark.cpu_only + +#: Layer 0 slides over 8 tokens and layer 1 is global, with 4 tokens per KV cache block. +_WINDOW = 8 +_TOKENS_PER_BLOCK = 4 + + +class _Manager: + """The rows `KVCacheManagerV2.get_batch_cache_indices` returns: per request, a sliding layer's + pages (-1 for a block it has recycled) and a global layer's.""" + + tokens_per_block = _TOKENS_PER_BLOCK + + def __init__(self, sliding, full): + self.rows = {False: sliding, True: full} + + def get_batch_cache_indices(self, request_ids, layer_idx, num_blocks_per_seq): + rows = self.rows[layer_idx == 1] + return [rows[r][:n] for r, n in zip(request_ids, num_blocks_per_seq)] + + +def _page_tables(manager, cached, new): + core = SimpleNamespace( + dims=SimpleNamespace( + num_layers=2, sliding_window=_WINDOW, is_global=lambda layer: layer == 1 + ), + _shared_tables=None, + _check_live_pages=modeling.Inkling2SmallCore._check_live_pages, + ) + blocks = [-(-(c + n) // _TOKENS_PER_BLOCK) for c, n in zip(cached, new)] + request_ids = list(range(len(cached))) + return modeling.Inkling2SmallCore._page_tables( + core, manager, request_ids, blocks, cached, torch.device("cpu") + ) + + +@pytest.mark.parametrize("history", [20, 23], ids=["window_starts_in_block_3", "in_block_4"]) +def test_pages_recycled_before_the_window_are_not_read(history): + """A decode step at `history` reads tokens `history - 7 .. history`: from block 3 at 20 and + from block 4 at 23, where block 3 may be gone too.""" + first = (history + 1 - _WINDOW) // _TOKENS_PER_BLOCK + sliding = [-1] * first + list(range(first, 6)) + manager = _Manager(sliding=[sliding, [10, 11]], full=[list(range(6)), [10, 11]]) + tables = _page_tables(manager, cached=[history, 0], new=[1, 6]) + # A table is as wide as the longest request's blocks; a shorter request's row ends in -1. + assert tables[0].tolist() == [sliding, [10, 11, -1, -1, -1, -1]] + assert tables[1].tolist() == [list(range(6)), [10, 11, -1, -1, -1, -1]] + + +@pytest.mark.parametrize( + "sliding, full, hole", + [ + ([-1, -1, -1, -1, 4, 5], list(range(6)), "layer 0: request 0 reads KV cache block 3,"), + ([-1, -1, -1, 3, 4, 5], [-1, 1, 2, 3, 4, 5], "layer 1: request 0 reads KV cache block 0,"), + ([-1, -1, -1, 3, 4], list(range(6)), "layer 0: request 0 reads KV cache block 5,"), + ], + ids=["sliding_page_inside_the_window", "global_page", "row_shorter_than_the_read"], +) +def test_a_missing_page_the_step_reads_is_an_error(sliding, full, hole): + manager = _Manager(sliding=[sliding], full=[full]) + with pytest.raises(RuntimeError, match=re.escape(hole)): + _page_tables(manager, cached=[20], new=[1]) + + +def test_the_model_names_the_quant_config_it_could_not_find(tmp_path): + config = SimpleNamespace( + text_config=SimpleNamespace(), _name_or_path=str(tmp_path), torch_dtype=torch.bfloat16 + ) + with pytest.raises(ValueError, match=re.escape(f"no hf_quant_config.json in '{tmp_path}'")): + modeling.Inkling2SmallCore(SimpleNamespace(pretrained_config=config)) + + +def test_the_model_refuses_a_config_without_a_dtype(): + config = SimpleNamespace(text_config=SimpleNamespace(), torch_dtype=None) + with pytest.raises(ValueError, match="declares no dtype"): + modeling.ModelingV2Inkling2SmallNvfp4Sm103Tep4(SimpleNamespace(pretrained_config=config))