From c43d0c9500f60468338cc0f339691bfb42e2e874 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Thu, 12 Nov 2026 13:08:35 -0800 Subject: [PATCH 01/33] [TRTLLM-17361][test] Add Beam component coverage Rebase the remaining test-only changes onto the merged component implementation. Preserve independent single-GPU CI entries, manual large/distributed qualification, and the validated expectation fixes. Signed-off-by: Maxim Gluhovskoi --- .../test_lists/test-db/l0_b300.yml | 14 +- .../integration/test_lists/test-db/l0_cpu.yml | 5 + .../test-db/manual_b300_beam_components.yml | 21 + .../test-db/manual_b300_beam_distributed.yml | 19 + .../models/beam/_beam_component_support.py | 647 ++++++++++ .../dense_block/_dense_block_reference.py | 501 ++++++++ .../dense_block/test_dense_block.py | 943 +++++++++++++++ .../beam/_components/moe/_moe_reference.py | 287 +++++ .../models/beam/_components/moe/test_moe.py | 871 ++++++++++++++ .../_components/moe_expert/_expert_cases.py | 7 - .../_components/moe_expert/test_expert.py | 363 ++++++ .../_components/moe_router/test_router.py | 898 ++++++++++++++ .../six_layer_model/test_cached_generation.py | 759 ++++++++++++ .../test_small_model_prefill.py | 716 +++++++++++ .../sparse_block/_sparse_block_reference.py | 306 +++++ .../sparse_block/test_sparse_block.py | 1072 +++++++++++++++++ .../test_checkpoint_layout.py | 524 ++++++++ .../test_checkpoint_layout_nvfp4.py | 625 ++++++++++ .../models/beam/test_diagnostic_mode.py | 235 ++++ .../models/beam/test_dispatch_combine.py | 327 +++++ .../models/beam/test_distributed_loading.py | 108 ++ 21 files changed, 9239 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_lists/test-db/manual_b300_beam_components.yml create mode 100644 tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 3ecab24a2cf5..d8092951d32b 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -159,14 +159,24 @@ l0_b300: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True] # modeling_v2: one entry per GPU-run top-level file, catalog category and target, so CBTS # runs only what a change reaches. The cpu_only files run on the CPU stage; comm needs more - # GPUs. The whole set took 13m54s on one GB300 (3122 cases), so TIMEOUT (30) per entry is - # ample. + # GPUs. Each entry has its own timeout; no aggregate ModelingV2 timeout is shared. - unittest/_torch/modeling_v2/test_modeling_v2_no_stale_claims.py TIMEOUT (30) - unittest/_torch/modeling_v2/activation TIMEOUT (30) + # Beam: lightweight single-GPU entries. High-memory MoE, sparse-block and six-layer + # suites belong to manual_b300_beam_components.yml, not L0 or nightly/LR. + # Startup-inclusive measurements on one GB300 (ARM64, retained cache): operators + # 58s, GEMM/FP4 entries <=23s, expert 35s and router 26s. The B300 x86 operator + # entry measured 83s in CI. Each keeps an independent 30-minute ceiling. - unittest/_torch/modeling_v2/models/beam/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/beam/test_modeling_v2_beam_cublas_mm.py TIMEOUT (30) - unittest/_torch/modeling_v2/models/beam/test_modeling_v2_beam_nvfp4_gemm.py TIMEOUT (30) - unittest/_torch/modeling_v2/models/beam/test_modeling_v2_beam_fp4_quantize.py TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/_components/dense_block TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/_components/moe_expert TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/_components/moe_router TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py TIMEOUT (30) + - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py TIMEOUT (30) - unittest/_torch/modeling_v2/attention TIMEOUT (30) - unittest/_torch/modeling_v2/gemm TIMEOUT (30) - unittest/_torch/modeling_v2/graph TIMEOUT (30) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 0a09cc746034..d922ed058336 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -128,6 +128,8 @@ l0_cpu: # tensors and the CPU block_scale_interleave. The rest of unittest/_torch/modeling_v2 # needs a GPU. - unittest/_torch/modeling_v2/models/beam/test_component_contracts.py + - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py + - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py - unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py - unittest/_torch/modeling_v2/test_modeling_v2_contracts.py - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -454,3 +456,6 @@ l0_cpu: - unittest/utils/test_confidential_compute.py - unittest/utils/test_llm_data.py - unittest/visual_gen/test_output.py + + # Beam baseline diagnostic checks (no checkpoints or GPU). + - unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py diff --git a/tests/integration/test_lists/test-db/manual_b300_beam_components.yml b/tests/integration/test_lists/test-db/manual_b300_beam_components.yml new file mode 100644 index 000000000000..beb10e18d52b --- /dev/null +++ b/tests/integration/test_lists/test-db/manual_b300_beam_components.yml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +version: 0.0.1 +# High-memory generated models; explicitly selected manual qualification, not L0. +manual_b300_beam_components: +- condition: + ranges: + system_gpu_count: + gte: 1 + lte: 1 + wildcards: + gpu: + - '*b300*' + linux_distribution_name: ubuntu* + terms: + stage: manual + backend: pytorch + tests: + - unittest/_torch/modeling_v2/models/beam/_components/moe TIMEOUT (45) + - unittest/_torch/modeling_v2/models/beam/_components/sparse_block TIMEOUT (45) + - unittest/_torch/modeling_v2/models/beam/_components/six_layer_model TIMEOUT (45) diff --git a/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml b/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml new file mode 100644 index 000000000000..26a5d9c83753 --- /dev/null +++ b/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +version: 0.0.1 +manual_b300_beam_distributed: +# Tiny generated-data test; launch with mpirun -n 4 as documented in the module. +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*b300*' + linux_distribution_name: ubuntu* + terms: + stage: manual + backend: pytorch + tests: + - unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py TIMEOUT (2) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py new file mode 100644 index 000000000000..ff7bbbdeedec --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -0,0 +1,647 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test support shared by the Beam GPU tests under ``models/beam/``. + +Only harness code lives here: the IEEE fp32 session guard, the optional evidence writer, +result bookkeeping that judges with the rules each test file passes in, op-path +instrumentation, the one-request KV cache context of the block tests, and the generated +configuration and state of the six-block small model. No reference math lives here; each +reference stays in its own ``_*_reference.py`` or ``_*_cases.py`` module, and every rule +and tolerance stays in the test file that applies it. + +Imported through ``__extra_import_path__``. Deliberately not a conftest: the CPU tests in +this tree must not pick up the CUDA probe, and nothing here is autouse. The KV cache and +attention-metadata modules are imported where they are used, so a test that needs none of +them imports nothing more than before. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import math +import os +import platform +import re +import subprocess +import sys +import types +import zlib +from collections import OrderedDict +from pathlib import Path +from typing import TYPE_CHECKING + +import torch +from torch.overrides import TorchFunctionMode, resolve_name + +import tensorrt_llm + +if TYPE_CHECKING: + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + +#: CPU op names of PyTorch's SDPA paths, and CUDA kernel names of attention kernels. +SDPA_OPS = re.compile(r"scaled_dot_product|_flash_attention|_efficient_attention|_cudnn_attention") +ATTENTION_KERNEL = re.compile(r"(?i)fmha|attention") + +# -------------------------------------------------- precision and environment + + +def fp32_gemm_probe() -> float: + """Worst relative error of an fp32 GEMM against fp64; TF32 substitution shows as ~1e-3.""" + gen = torch.Generator().manual_seed(1234) + x = torch.randn(64, 5120, generator=gen).to(torch.bfloat16).float().cuda() + w = (torch.randn(512, 5120, generator=gen) * 5120**-0.5).float().cuda() + y = torch.nn.functional.linear(x, w) + ref = x.double() @ w.double().t() + return float((y.double() - ref).abs().max()) / float(ref.pow(2).mean().sqrt()) + + +def math_state() -> dict: + return { + "float32_matmul_precision": torch.get_float32_matmul_precision(), + "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, + "cuda_matmul_allow_tf32": bool(torch.backends.cuda.matmul.allow_tf32), + "cudnn_allow_tf32": bool(torch.backends.cudnn.allow_tf32), + } + + +def ieee_fp32_session(write_evidence): + """Enforce IEEE fp32 and verify it before and after the module's tests.""" + matmul = torch.backends.cuda.matmul + saved = ( + torch.get_float32_matmul_precision(), + matmul.allow_tf32, + torch.backends.cudnn.allow_tf32, + matmul.fp32_precision, + torch.backends.cudnn.fp32_precision, + ) + try: + torch.set_float32_matmul_precision("highest") + matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + matmul.fp32_precision = "ieee" + before = fp32_gemm_probe() + assert before <= 1e-5, f"fp32 GEMM is not IEEE fp32 (relative error {before:.3g})" + state = {"math": math_state(), "fp32_gemm_probe_before": before} + yield state + after = fp32_gemm_probe() + state.update(math_after=math_state(), fp32_gemm_probe_after=after) + write_evidence("fp32-session", {"fixture": "ieee_fp32", "math": state}) + assert after <= 1e-5, f"fp32 GEMM stopped being IEEE fp32 (relative error {after:.3g})" + finally: + # Legacy setters can also change the newer precision controls. + precision, allow_matmul, allow_cudnn, matmul_precision, cudnn_precision = saved + matmul.allow_tf32 = allow_matmul + torch.set_float32_matmul_precision(precision) + torch.backends.cudnn.allow_tf32 = allow_cudnn + matmul.fp32_precision = matmul_precision + torch.backends.cudnn.fp32_precision = cudnn_precision + + +def git_revision() -> str | None: + root = Path(tensorrt_llm.__file__).resolve().parents[1] + try: + out = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() or None + + +def environment(ieee: dict, execution, **extra) -> dict: + """The device and software a case ran on; ``extra`` entries follow ``execution``.""" + device = torch.cuda.current_device() + return { + "device_name": torch.cuda.get_device_name(device), + "device_capability": list(torch.cuda.get_device_capability(device)), + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "tensorrt_llm": tensorrt_llm.__version__, + "tensorrt_llm_file": tensorrt_llm.__file__, + "source_revision": git_revision(), + "python": sys.version.split()[0], + "machine": platform.machine(), + "argv": sys.argv, + "slurm_job_id": os.environ.get("SLURM_JOB_ID"), + "slurm_step_id": os.environ.get("SLURM_STEP_ID"), + "execution": dict(execution), + **extra, + "math": ieee, + } + + +def write_evidence(env_var: str, name: str, record: dict) -> None: + """Write ``record`` as ``.json`` into the directory named by ``env_var``, if set.""" + directory = os.environ.get(env_var) + if not directory: + return + path = Path(directory) + path.mkdir(parents=True, exist_ok=True) + (path / f"{name}.json").write_text(json.dumps(record, indent=1, default=str) + "\n") + + +def tensor_sha256(t: torch.Tensor) -> str: + flat = t.detach().contiguous().cpu().reshape(-1) + h = hashlib.sha256() + h.update(f"{t.dtype}|{tuple(t.shape)}|".encode()) + h.update(flat.view(torch.uint8).numpy().tobytes()) + return h.hexdigest() + + +# -------------------------------------------------- comparison bookkeeping + + +def compare( + name: str, + actual: torch.Tensor, + expected: torch.Tensor, + rule: str | None, + tolerance: tuple[float, float] | None, + dtype: torch.dtype, +) -> dict: + """Score one tensor against one rule; ``rule=None`` measures without judging. + + ``tolerance`` is the rule's ``(atol, rtol)`` from the calling test file and ``dtype`` + the dtype both tensors must have. An element passes when |actual - expected| <= atol + + rtol * |expected|, both widened to fp64; no NaN or Inf is accepted. + """ + result = { + "tensor": name, + "rule": rule, + "shape": list(expected.shape), + "dtype": str(actual.dtype), + } + problems = [] + if tuple(actual.shape) != tuple(expected.shape): + problems.append(f"shape {tuple(actual.shape)} != {tuple(expected.shape)}") + if actual.dtype != dtype or expected.dtype != dtype: + problems.append(f"dtypes {actual.dtype}/{expected.dtype}, need {dtype}") + if problems: + result.update(passed=False, problems=problems) + return result + a, r = actual.double(), expected.to(actual.device).double() + if not bool(torch.isfinite(a).all()) or not bool(torch.isfinite(r).all()): + result.update(passed=False, problems=["non-finite values"]) + return result + diff = (a - r).abs() + numel = diff.numel() + result.update(numel=numel, unequal=int((a != r).sum()), max_abs=0.0) + if numel: + worst = int(diff.argmax()) + result.update( + max_abs=float(diff.max()), + mean_abs=float(diff.mean()), + worst_index=[int(i) for i in torch.unravel_index(torch.tensor(worst), diff.shape)], + worst_actual=float(a.reshape(-1)[worst]), + worst_expected=float(r.reshape(-1)[worst]), + ) + if rule is None: + result["passed"] = None + return result + atol, rtol = tolerance + violations = int((diff > atol + rtol * r.abs()).sum()) + result.update(atol=atol, rtol=rtol, violations=violations) + if violations: + problems.append( + f"{violations} of {numel} elements outside atol {atol} + rtol {rtol} * |ref| " + f"(max abs {result['max_abs']:.3g})" + ) + result.update(passed=not problems, problems=problems) + return result + + +def compare_exact(name: str, actual: torch.Tensor, expected: torch.Tensor) -> dict: + """Integer selections: equal element for element and in order, never sorted first.""" + same_kind = tuple(actual.shape) == tuple(expected.shape) and actual.dtype == expected.dtype + if same_kind: + rows = (actual == expected).reshape(expected.shape[0], -1).all(dim=-1) + mismatched = [int(i) for i in torch.nonzero(~rows).flatten().cpu()] + else: + mismatched = list(range(expected.shape[0])) + problems = [] + if not same_kind: + problems.append( + f"{tuple(actual.shape)} {actual.dtype} != {tuple(expected.shape)} {expected.dtype}" + ) + elif mismatched: + problems.append(f"rows {mismatched[:8]} differ") + return { + "tensor": name, + "rule": "exact, in order", + "shape": list(expected.shape), + "mismatched_rows": mismatched, + "passed": not problems, + "problems": problems, + } + + +def output_contract(out, shape: tuple[int, ...], dtype: torch.dtype, sm: tuple[int, int]) -> dict: + """The forward returned a finite ``shape`` bf16 tensor on the sm_103 device.""" + problems = [] + if not isinstance(out, torch.Tensor): + problems.append(f"returned {type(out).__name__}") + elif tuple(out.shape) != shape or out.dtype != dtype or not out.is_cuda: + problems.append(f"returned {tuple(out.shape)} {out.dtype} on {out.device}") + elif torch.cuda.get_device_capability(out.device) != sm: + problems.append(f"on {torch.cuda.get_device_name(out.device)}") + elif not bool(torch.isfinite(out).all()): + problems.append("non-finite values") + return { + "tensor": "output_contract", + "rule": f"{list(shape)} bf16 on the sm_103 device, finite", + "passed": not problems, + "problems": problems, + } + + +def tensor_versions(module: torch.nn.Module) -> dict[str, int]: + """The in-place write counter of every parameter and buffer, by name.""" + versions = {name: p._version for name, p in module.named_parameters()} + versions.update({name: b._version for name, b in module.named_buffers()}) + return versions + + +def unchanged(x: torch.Tensor, x_before: torch.Tensor, before: dict, after: dict) -> dict: + written = sorted(name for name, version in before.items() if after.get(name) != version) + problems = [] + if not bitwise_equal(x, x_before): + problems.append("the input was modified") + if written: + problems.append(f"{len(written)} parameters or buffers were written: {written[:4]}") + return { + "tensor": "input_and_parameters", + "rule": "input bits and parameter write counters unchanged by the call", + "passed": not problems, + "problems": problems, + } + + +def bitwise_equal(a: torch.Tensor, b: torch.Tensor) -> bool: + """Same dtype, shape and bits (so -0.0 differs from +0.0).""" + if a.dtype != b.dtype or a.shape != b.shape: + return False + view = {torch.bfloat16: torch.int16, torch.float32: torch.int32}.get(a.dtype) + if view is None: + return bool(torch.equal(a, b)) + return bool(torch.equal(a.contiguous().view(view), b.contiguous().view(view))) + + +def assert_all_pass(results: list[dict]) -> None: + failed = [r for r in results if r["passed"] is False] + assert not failed, "\n".join( + f"{r['tensor']} [{r['rule']}]: {'; '.join(r['problems'])}" for r in failed + ) + + +# -------------------------------------------------- weights + + +def cheap_weights(param_shapes) -> "OrderedDict[str, torch.Tensor]": + """A valid mapping for ``param_shapes`` (name -> (shape, dtype)) that allocates nothing: + one expanded constant per dtype, so a load that copied any of it would change the + generated values it replaced.""" + constants = { + torch.float32: torch.full((), 0.5, dtype=torch.float32, device="cuda"), + torch.bfloat16: torch.full((), 0.25, dtype=torch.bfloat16, device="cuda"), + } + return OrderedDict( + (name, constants[dtype].expand(shape)) for name, (shape, dtype) in param_shapes.items() + ) + + +# -------------------------------------------------- op-path instrumentation + + +@contextlib.contextmanager +def recording( + *wrappers, + recorded_kwargs: tuple[str, ...], + experts=None, + data_ptrs: bool = False, + qkv: bool = False, +): + """Log every call of these entries, in order, with its nesting depth, the arguments that + identify the path and its output. ``experts``: an MoE class whose ``_expert`` calls are + also logged, with the expert asked for and the rows it is passed. ``data_ptrs``: the + positional tensors' data pointers. ``qkv``: the shape and tensor of a ``q`` keyword.""" + calls: list[dict] = [] + depth = [0] + saved = [] + for w in wrappers: + cls = type(w) + original = cls.__call__ + saved.append((cls, "__call__", original)) + + def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): + tensors = [a for a in args if isinstance(a, torch.Tensor)] + entry = { + "op": _name, + "depth": depth[0], + "shapes": [list(t.shape) for t in tensors], + } + if data_ptrs: + entry["data_ptrs"] = [t.data_ptr() for t in tensors] + entry.update({key: kwargs[key] for key in recorded_kwargs if key in kwargs}) + if qkv and "q" in kwargs: + entry["q_shape"] = list(kwargs["q"].shape) + entry["qkv"] = kwargs["q"] + calls.append(entry) + depth[0] += 1 + try: + out = _original(self, *args, **kwargs) + finally: + depth[0] -= 1 + entry["output"] = out + return out + + cls.__call__ = logged + if experts is not None: + original_expert = experts._expert + saved.append((experts, "_expert", original_expert)) + + def asked(self, expert_id, rows, _original=original_expert): + calls.append({"op": "expert", "depth": depth[0], "expert_id": expert_id, "rows": rows}) + return _original(self, expert_id, rows) + + experts._expert = asked + try: + yield calls + finally: + for cls, name, original in reversed(saved): + setattr(cls, name, original) + + +def loggable(calls: list[dict], keep: tuple[str, ...]) -> list[dict]: + """The recorded calls without their tensors, for the evidence record.""" + out = [] + for call in calls: + entry = {key: call[key] for key in keep if key in call} + if "rows" in call: + entry["rows"] = list(call["rows"].shape) + out.append(entry) + return out + + +@contextlib.contextmanager +def builtin_forbidden(targets): + """The built-in Beam modeling must not run while the component does: every ``(class, + method)`` in ``targets`` raises for the duration of the block.""" + saved = [(cls, name, getattr(cls, name)) for cls, name in targets] + + def refuse(self, *args, **kwargs): + raise AssertionError(f"the built-in {type(self).__name__} ran inside the component") + + for cls, name, _ in saved: + setattr(cls, name, refuse) + try: + yield + finally: + for cls, name, original in reversed(saved): + setattr(cls, name, original) + + +@contextlib.contextmanager +def no_dispatch(ops): + """Fail on any call of ``ops``: rejected inputs must be rejected before a kernel runs.""" + saved = [] + for w in ops: + cls = type(w) + saved.append((cls, cls.__call__)) + + def refuse(self, *args, _name=cls.__name__, **kwargs): + raise AssertionError(f"{_name} dispatched before the input was rejected") + + cls.__call__ = refuse + try: + yield + finally: + for cls, original in reversed(saved): + cls.__call__ = original + + +class TorchCallTrace(TorchFunctionMode): + """The name of every torch call made inside the block and the devices of its outputs.""" + + def __init__(self): + super().__init__() + self.calls: list[tuple[str, list[str]]] = [] + + def __torch_function__(self, func, types, args=(), kwargs=None): + out = func(*args, **(kwargs or {})) + outs = out if isinstance(out, (tuple, list)) else [out] + devices = [str(t.device) for t in outs if isinstance(t, torch.Tensor)] + self.calls.append((resolve_name(func) or repr(func), devices)) + return out + + +# -------------------------------------------------- one request on a fresh KV cache + + +@contextlib.contextmanager +def fresh_context( + num_tokens: int, + cached_tokens: int, + num_requests: int, + *, + max_kv_tokens: int, + max_tokens: int, + request_id: int, + mgr_args: dict, +): + """A KVCacheManagerV2 built from ``mgr_args`` and prepared TrtllmAttentionMetadata for + ``num_requests`` new requests of ``num_tokens`` after ``cached_tokens`` cached ones.""" + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + from tensorrt_llm._torch.metadata import KVCacheParams + from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 + from tensorrt_llm._torch.pyexecutor.resource_manager import CacheTypeCpp + from tensorrt_llm.llmapi.llm_args import KvCacheConfig + + mgr = KVCacheManagerV2( + KvCacheConfig(max_tokens=max_kv_tokens, enable_block_reuse=False), + CacheTypeCpp.SELF, + **mgr_args, + ) + try: + request_ids = [request_id + i for i in range(num_requests)] + mgr.add_dummy_requests(request_ids, token_nums=[cached_tokens + num_tokens] * num_requests) + md = TrtllmAttentionMetadata( + max_num_requests=num_requests, + max_num_tokens=max_tokens * num_requests, + kv_cache_manager=mgr, + num_contexts=num_requests, + kv_cache_params=KVCacheParams( + use_cache=True, num_cached_tokens_per_seq=[cached_tokens] * num_requests + ), + seq_lens=torch.tensor([num_tokens] * num_requests, dtype=torch.int), + request_ids=request_ids, + prompt_lens=[cached_tokens + num_tokens] * num_requests, + ) + md.prepare() + yield md + finally: + mgr.shutdown() + + +def cache_rows( + md: TrtllmAttentionMetadata, length: int, layer_idx: int, tokens_per_block: int +) -> tuple[torch.Tensor, torch.Tensor]: + """This request's K and V rows read back from the pool, [T, Hkv, D] each. The manager's + APIs take the logical layer and translate it to the local slot themselves.""" + mgr = md.kv_cache_manager + view = mgr.get_buffers(layer_idx, kv_layout="HND") + view = view[:: mgr.get_layer_page_index_scale(layer_idx) // 2] + keys, values = [], [] + for page in range((length + tokens_per_block - 1) // tokens_per_block): + k_off = int(md.kv_cache_block_offsets[0, 0, 0, page]) + rows = min(tokens_per_block, length - page * tokens_per_block) + keys.append(view[k_off // 2, 0, :, :rows, :].permute(1, 0, 2)) + values.append(view[k_off // 2, 1, :, :rows, :].permute(1, 0, 2)) + return torch.cat(keys), torch.cat(values) + + +# -------------------------------------------------- unsupported block inputs + +#: The message each hidden-state or position refusal of ``_attention.check_step`` raises. +#: Metadata refusals have no pattern here: they are checked by type alone. +_BAD_INPUT_MATCH = { + "fp16_hidden": "hidden_states must be a torch.bfloat16 tensor", + "fp32_hidden": "hidden_states must be a torch.bfloat16 tensor", + "hidden_5120x2": r"hidden_states must be \[1, T, 5120\]", + "batch_2": r"hidden_states must be \[1, T, 5120\]", + "too_long": "supported token counts per call are", + "empty": "supported token counts per call are", + "non_contiguous": "hidden_states must be contiguous", + "positions_offset": "its positions must be exactly those", + "float_positions": r"position_ids must be an int32/int64 tensor of shape \(1, 17\) on cuda", + "positions_shape": r"position_ids must be an int32/int64 tensor of shape \(1, 17\) on cuda", +} + + +def bad_block_input(case: str, max_tokens: int, dtype: torch.dtype): + """One unsupported call of a block at 17 tokens: ``(hidden_states, position_ids, + fresh_context kwargs, ValueError message pattern or None)``. ``metadata_type`` keeps + the valid inputs; the caller replaces the metadata itself.""" + from tensorrt_llm._torch.pyexecutor.resource_manager import DataType + + length = 17 + x = torch.randn(1, length, 5120, device="cuda").to(dtype) + positions = torch.arange(length, device="cuda").view(1, length) + context = dict(num_tokens=length) + if case == "fp16_hidden": + x = x.half() + elif case == "fp32_hidden": + x = x.float() + elif case == "hidden_5120x2": + x = torch.cat((x, x), dim=-1) + elif case == "batch_2": + x = torch.cat((x, x), dim=0) + elif case == "too_long": + length = max_tokens + 1 + x = torch.randn(1, length, 5120, device="cuda").to(dtype) + positions = torch.arange(length, device="cuda").view(1, length) + context = dict(num_tokens=17) + elif case == "empty": + x = x[:, :0] + positions = positions[:, :0] + elif case == "non_contiguous": + x = torch.randn(1, length, 2 * 5120, device="cuda").to(dtype)[..., ::2] + elif case == "positions_offset": + positions = positions + 1 + elif case == "float_positions": + positions = positions.float() + elif case == "positions_shape": + positions = positions.view(length) + elif case == "cached_prefix": + context = dict(num_tokens=length, cached_tokens=8) + elif case == "two_requests": + context = dict(num_tokens=length, num_requests=2) + elif case == "context_length_mismatch": + context = dict(num_tokens=length - 1) + elif case == "kv_cache_heads": + context = dict(num_tokens=length, num_kv_heads=4) + elif case == "kv_cache_dtype": + context = dict(num_tokens=length, dtype=DataType.HALF) + elif case == "layer0_cache": + context = dict(num_tokens=length, layer_mask=None) + return x, positions, context, _BAD_INPUT_MATCH.get(case) + + +# -------------------------------------------------- the six-block small model + +#: The checkpoint's configuration reduced to its first six layers; only these fields are read. +SMALL_MODEL_CONFIG = types.MappingProxyType( + dict( + num_hidden_layers=6, + hidden_size=5120, + intermediate_size=20480, + moe_intermediate_size=2560, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + vocab_size=151936, + rms_norm_eps=1e-6, + hidden_act="silu", + attention_bias=False, + attention_dropout=0.0, + sliding_window=512, + layer_types=("sliding_attention",) * 5 + ("full_attention",), + mlp_layer_types=("dense",) * 3 + ("sparse",) * 3, + rope_parameters={ + "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, + "full_attention": None, + }, + num_experts=256, + num_experts_per_tok=8, + n_group=8, + topk_group=4, + norm_topk_prob=True, + routed_scaling_factor=2.5, + tie_word_embeddings=False, + final_logit_softcapping=30.0, + max_position_embeddings=1048576, + ) +) + + +def small_model_config(**overrides) -> types.SimpleNamespace: + fields = dict(SMALL_MODEL_CONFIG) + fields["layer_types"] = list(fields["layer_types"]) + fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) + fields["rope_parameters"] = { + k: (dict(v) if v else v) for k, v in fields["rope_parameters"].items() + } + fields.update(overrides) + return types.SimpleNamespace(**fields) + + +def _small_model_half_width(name: str, shape: tuple, embed_name: str) -> tuple[float, float]: + """(half width, offset) of the uniform draw for one tensor: unit-variance projections, + embeddings of std 1, norm weights 1 +- 0.4, gate bias std 0.75, correction bias std 0.05.""" + if name == embed_name: + return math.sqrt(3.0), 0.0 + if name.endswith("norm.weight") or name.endswith("layernorm.weight"): + return 0.4, 1.0 + if name.endswith("gate_bias"): + return 0.75 * math.sqrt(3.0), 0.0 + if name.endswith("e_score_correction_bias"): + return 0.05 * math.sqrt(3.0), 0.0 + return math.sqrt(3.0 / shape[-1]), 0.0 + + +def generate_small_model_state(seed: int, model_module) -> "OrderedDict[str, torch.Tensor]": + """All 2394 tensors of ``model_module.weight_schema``, each drawn from its own generator + keyed by seed and name, fp32 then rounded once to the stored dtype.""" + state = OrderedDict() + schema = model_module.weight_schema(small_model_config()) + for name, (shape, dtype) in schema.items(): + g = torch.Generator(device="cuda").manual_seed(zlib.crc32(f"{seed}|{name}".encode())) + half, offset = _small_model_half_width(name, shape, model_module.EMBED) + t = torch.rand(shape, generator=g, device="cuda", dtype=torch.float32) + state[name] = (t.mul_(2.0 * half).sub_(half).add_(offset)).to(dtype) + return state diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py new file mode 100644 index 000000000000..5578100d6cdb --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py @@ -0,0 +1,501 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Plain-Torch reference for one dense sliding-attention decoder block (block 0). + +An independent restatement of the source block semantics for generated-input +component replay: sandwich RMSNorms (fp32 statistics and affine, one cast), +separate bias-free Q/K/V/gate projections in [out, in] layout, per-head Q/K +RMSNorm rounded to the activation dtype before fp32 NeoX RoPE, GQA attention +with a causal window mask, per-head sigmoid output gate in the activation +dtype, bias-free output projection, fp32 SiLU(gate) * up cast once before the +down projection, and depth-scaled residual adds in the activation dtype. + +Only torch is imported. `Variant` selects documented equivalent formulations +(valid alternatives) or deliberately wrong ones (fault witnesses); the default +is the source formulation. +""" + +from __future__ import annotations + +import contextlib +import math +from collections import OrderedDict +from dataclasses import dataclass +from typing import Callable, Iterator + +import torch +import torch.nn.functional as F + + +@dataclass(frozen=True) +class Dims: + hidden: int = 5120 + intermediate: int = 20480 + heads: int = 48 + kv_heads: int = 8 + head_dim: int = 128 + eps: float = 1e-6 + rope_theta: float = 10000.0 + rope_factor: float = 1.0 + window: int = 512 + layer_id: int = 0 + max_len: int = 129 + + +BLOCK0 = Dims() +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) + +STATE_RECIPE = { + "construction": "torch.manual_seed(1300 + seed + layer_id); fp32 kaiming_uniform_(a=sqrt(5)) on CPU for the " + "eight projection matrices in module construction order (q, k, v, gate, o, mlp gate, up, down)", + "overrides": "torch.Generator().manual_seed(10000 + seed) in state_dict order: *norm.weight = 1 + 0.25 * N(0, 1); " + "self_attn.gate_bias = 0.75 * N(0, 1)", + "dtype": "every tensor cast once to bfloat16", + "input": "torch.Generator().manual_seed(1200 + seed): 2 * N(0, 1) of shape [1, max_len, hidden], cast once to " + "bfloat16; length-T cases use the first T rows; positions arange(T)", +} + + +def depth_coefs(layer_id: int, is_attention: bool) -> tuple[float, float]: + one_based = layer_id + 1 + num_adds = 2 * one_based if is_attention else 2 * one_based + 1 + denom = math.sqrt(num_adds) + return 1.0 / denom, math.sqrt(num_adds - 1) / denom + + +def param_shapes(d: Dims = BLOCK0) -> "OrderedDict[str, tuple[int, ...]]": + """Block parameters in source state_dict order.""" + q, kv = d.heads * d.head_dim, d.kv_heads * d.head_dim + return OrderedDict( + [ + ("input_layernorm.weight", (d.hidden,)), + ("self_attn.gate_bias", (d.heads,)), + ("self_attn.q_proj.weight", (q, d.hidden)), + ("self_attn.k_proj.weight", (kv, d.hidden)), + ("self_attn.v_proj.weight", (kv, d.hidden)), + ("self_attn.gate_proj.weight", (q, d.hidden)), + ("self_attn.o_proj.weight", (d.hidden, q)), + ("self_attn.q_norm.weight", (d.head_dim,)), + ("self_attn.k_norm.weight", (d.head_dim,)), + ("pre_feedforward_layernorm.weight", (d.hidden,)), + ("mlp.gate_proj.weight", (d.intermediate, d.hidden)), + ("mlp.up_proj.weight", (d.intermediate, d.hidden)), + ("mlp.down_proj.weight", (d.hidden, d.intermediate)), + ("post_attention_layernorm.weight", (d.hidden,)), + ("post_feedforward_layernorm.weight", (d.hidden,)), + ] + ) + + +INIT_ORDER = ( + "self_attn.q_proj.weight", + "self_attn.k_proj.weight", + "self_attn.v_proj.weight", + "self_attn.gate_proj.weight", + "self_attn.o_proj.weight", + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight", +) + + +def generate_state(seed: int, d: Dims = BLOCK0) -> "OrderedDict[str, torch.Tensor]": + """Deterministic bf16 block state on CPU (see STATE_RECIPE).""" + shapes = param_shapes(d) + torch.manual_seed(1300 + seed + d.layer_id) + matrices = {} + for name in INIT_ORDER: + weight = torch.empty(shapes[name], dtype=torch.float32) + torch.nn.init.kaiming_uniform_(weight, a=math.sqrt(5)) + matrices[name] = weight + gen = torch.Generator().manual_seed(10000 + seed) + state: "OrderedDict[str, torch.Tensor]" = OrderedDict() + for name, shape in shapes.items(): + if name.endswith("norm.weight"): + value = 1.0 + 0.25 * torch.randn(shape, generator=gen) + elif name == "self_attn.gate_bias": + value = 0.75 * torch.randn(shape, generator=gen) + else: + value = matrices[name] + state[name] = value.to(torch.bfloat16).contiguous() + return state + + +def generate_input(seed: int, d: Dims = BLOCK0) -> torch.Tensor: + gen = torch.Generator().manual_seed(1200 + seed) + return (2.0 * torch.randn(1, d.max_len, d.hidden, generator=gen)).to(torch.bfloat16) + + +def state_problems(state: "dict[str, torch.Tensor]", d: Dims = BLOCK0) -> list[str]: + """Exact accounting plus the nontrivial-state properties the comparisons rely on.""" + problems = [] + shapes = param_shapes(d) + if list(state) != list(shapes): + problems.append(f"names/order {list(state)} != {list(shapes)}") + for name, shape in shapes.items(): + t = state.get(name) + if t is None: + continue + if tuple(t.shape) != shape or t.dtype != torch.bfloat16: + problems.append(f"{name}: {tuple(t.shape)} {t.dtype}, expected {shape} bfloat16") + if not bool(torch.isfinite(t.float()).all()): + problems.append(f"{name}: nonfinite") + norms = [n for n in shapes if n.endswith("norm.weight")] + for name in norms: + if name in state and bool((state[name].float() == 1.0).all()): + problems.append(f"{name}: unit affine") + for i, a in enumerate(norms): + for b in norms[i + 1 :]: + if ( + a in state + and b in state + and state[a].shape == state[b].shape + and torch.equal(state[a], state[b]) + ): + problems.append(f"{a} and {b} are identical") + bias = state.get("self_attn.gate_bias") + if bias is not None: + if not bool((bias.float() != 0).any()): + problems.append("gate_bias is all zero") + if bool((bias == bias[0]).all()): + problems.append("gate_bias does not vary across heads") + return problems + + +@dataclass(frozen=True) +class Variant: + """Source formulation by default; see module docstring.""" + + # Valid alternatives (same mathematics, different evaluation). + sdpa_backend: str | None = ( + None # None: default dispatch; MATH | EFFICIENT_ATTENTION | FLASH_ATTENTION | CUDNN_ATTENTION + ) + sdpa_form: str = "mask" # mask: explicit window mask; is_causal: causal flag (equal for T <= window); enable_gqa + gemm: str = "batched" # batched | per_row | fused | matmul_t + rmsnorm: str = "source" # source | alt_order | chunked + gate: str = "source" # source | fp32_sigmoid_then_cast + residual: str = "source" # source | torch_add + silu: str = "source" # source: F.silu | x_sigmoid: x * sigmoid(x) | exp_form: x / (1 + exp(-x)); all fp32 + # Fault witnesses (wrong semantics). + attn_coefs: tuple[float, float] | None = None + mlp_coefs: tuple[float, float] | None = None + residual_fp32_fused: bool = False + omit_post_attention_norm: bool = False + swap_post_attention_and_pre_ffn_norms: bool = False + rmsnorm_round_before_affine: bool = False + softmax_scale: float | None = None + gqa_tiled: bool = False + noncausal: bool = False + omit_k_norm: bool = False + swap_qk_norm_weights: bool = False + gate_bias: str = "source" # source | omitted | uniform_mean | heads_rolled + swiglu_bf16_activation: bool = False + rope_interleaved: bool = False + rope_positions_offset: int = 0 + normalize_v: bool = False + qk_norm_fp32_into_rope: bool = False + + +SOURCE = Variant() + +_SDPA = { + "MATH": "MATH", + "EFFICIENT_ATTENTION": "EFFICIENT_ATTENTION", + "FLASH_ATTENTION": "FLASH_ATTENTION", + "CUDNN_ATTENTION": "CUDNN_ATTENTION", +} + + +@contextlib.contextmanager +def sdpa_backend(name: str | None) -> Iterator[None]: + if name is None: + yield + return + from torch.nn.attention import SDPBackend, sdpa_kernel + + with sdpa_kernel([getattr(SDPBackend, _SDPA[name])]): + yield + + +Record = Callable[[str, torch.Tensor], None] + + +def _noop(name: str, value: torch.Tensor) -> None: + del name, value + + +def linear(x: torch.Tensor, weight: torch.Tensor, v: Variant) -> torch.Tensor: + if v.gemm == "per_row": + rows = [F.linear(x[..., i : i + 1, :], weight) for i in range(x.shape[-2])] + return torch.cat(rows, dim=-2) + if v.gemm == "matmul_t": + return torch.matmul(x, weight.t()) + return F.linear(x, weight) + + +def rms_norm( + x: torch.Tensor, weight: torch.Tensor, eps: float, v: Variant = SOURCE +) -> torch.Tensor: + in_dtype = x.dtype + x32 = x.float() + if v.rmsnorm == "alt_order": + normed = x32 / torch.sqrt(x32.pow(2).sum(-1, keepdim=True) / x32.shape[-1] + eps) + elif v.rmsnorm == "chunked": + n = x32.shape[-1] + sq = x32.pow(2).unflatten(-1, (n // 32, 32)).sum(-1).sum(-1, keepdim=True) + normed = x32 * (sq / n + eps).rsqrt() + else: + rms = x32.pow(2).mean(dim=-1, keepdim=True).add(eps).rsqrt() + normed = x32 * rms + if v.rmsnorm_round_before_affine: + return normed.to(in_dtype) * weight + return (normed * weight).to(in_dtype) + + +def rope_cos_sin(positions: torch.Tensor, d: Dims = BLOCK0) -> tuple[torch.Tensor, torch.Tensor]: + exponent = torch.arange(0, d.head_dim, 2, dtype=torch.float32, device=positions.device) + inv_freq = 1.0 / (d.rope_theta ** (exponent / d.head_dim)) + inv_freq = inv_freq / d.rope_factor + freqs = positions.float().unsqueeze(-1) * inv_freq + emb = torch.cat((freqs, freqs), dim=-1) + return emb.cos(), emb.sin() + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + +def apply_rope( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, v: Variant = SOURCE +) -> tuple[torch.Tensor, torch.Tensor]: + """q, k: (B, T, heads, head_dim); cos, sin: (B, T, head_dim) fp32.""" + q_dtype, k_dtype = q.dtype, k.dtype + cos = cos.unsqueeze(2).float() + sin = sin.unsqueeze(2).float() + q, k = q.float(), k.float() + if v.rope_interleaved: + + def rot(x: torch.Tensor) -> torch.Tensor: + half = x.shape[-1] // 2 + c, s = cos[..., :half], sin[..., :half] + even, odd = x[..., 0::2], x[..., 1::2] + return torch.stack((even * c - odd * s, even * s + odd * c), dim=-1).flatten(-2) + + return rot(q).to(q_dtype), rot(k).to(k_dtype) + q_out = q * cos + _rotate_half(q) * sin + k_out = k * cos + _rotate_half(k) * sin + return q_out.to(q_dtype), k_out.to(k_dtype) + + +def causal_window_mask(length: int, window: int | None, device: torch.device) -> torch.Tensor: + q = torch.arange(length, device=device).unsqueeze(1) + k = torch.arange(length, device=device).unsqueeze(0) + mask = q >= k + if window is not None: + mask = mask & ((q - k) < window) + return mask.unsqueeze(0).unsqueeze(0) + + +def attention_core( + q: torch.Tensor, k: torch.Tensor, val: torch.Tensor, d: Dims = BLOCK0, v: Variant = SOURCE +) -> torch.Tensor: + """q: (B, T, heads, D) post-RoPE; k, val: (B, T, kv_heads, D). Returns (B, T, heads, D).""" + length = q.shape[1] + q, k, val = q.transpose(1, 2), k.transpose(1, 2), val.transpose(1, 2) + repeat = d.heads // d.kv_heads + gqa_native = v.sdpa_form == "enable_gqa" + if not gqa_native and repeat != 1: + if v.gqa_tiled: + k, val = k.repeat(1, repeat, 1, 1), val.repeat(1, repeat, 1, 1) + else: + k, val = k.repeat_interleave(repeat, dim=1), val.repeat_interleave(repeat, dim=1) + scale = d.head_dim**-0.5 if v.softmax_scale is None else v.softmax_scale + kwargs = {"dropout_p": 0.0, "scale": scale} + if gqa_native: + kwargs["enable_gqa"] = True + with sdpa_backend(v.sdpa_backend): + if v.noncausal: + out = F.scaled_dot_product_attention(q, k, val, **kwargs) + elif v.sdpa_form == "is_causal": + out = F.scaled_dot_product_attention(q, k, val, is_causal=True, **kwargs) + else: + mask = causal_window_mask(length, d.window, q.device) + out = F.scaled_dot_product_attention(q, k, val, attn_mask=mask, **kwargs) + return out.transpose(1, 2) + + +def output_gate( + attn: torch.Tensor, gate: torch.Tensor, gate_bias: torch.Tensor, v: Variant = SOURCE +) -> torch.Tensor: + """attn, gate: (B, T, heads, D) activation dtype; gate_bias: (heads,).""" + if v.gate_bias == "omitted": + gate_bias = torch.zeros_like(gate_bias) + elif v.gate_bias == "uniform_mean": + gate_bias = torch.full_like(gate_bias, float(gate_bias.float().mean())) + elif v.gate_bias == "heads_rolled": + gate_bias = gate_bias.roll(1) + out_dtype = attn.dtype + biased = gate + gate_bias[None, None, :, None] + if v.gate == "fp32_sigmoid_then_cast": + return attn * torch.sigmoid(biased.float()).to(out_dtype) + return (attn * torch.sigmoid(biased)).to(out_dtype) + + +def attention( + x: torch.Tensor, + positions: torch.Tensor, + s: "dict[str, torch.Tensor]", + d: Dims = BLOCK0, + v: Variant = SOURCE, + record: Record = _noop, + fused: "dict[str, torch.Tensor] | None" = None, +) -> torch.Tensor: + b, t, _ = x.shape + qw, kvw = d.heads * d.head_dim, d.kv_heads * d.head_dim + if v.gemm == "fused": + assert fused is not None + q, k, val, gate = ( + y.contiguous() for y in F.linear(x, fused["qkvg"]).split([qw, kvw, kvw, qw], dim=-1) + ) + else: + q = linear(x, s["self_attn.q_proj.weight"], v) + k = linear(x, s["self_attn.k_proj.weight"], v) + val = linear(x, s["self_attn.v_proj.weight"], v) + gate = linear(x, s["self_attn.gate_proj.weight"], v) + for name, value in (("q_proj", q), ("k_proj", k), ("v_proj", val), ("gate_proj", gate)): + record(name, value) + q = q.view(b, t, d.heads, d.head_dim) + k = k.view(b, t, d.kv_heads, d.head_dim) + val = val.view(b, t, d.kv_heads, d.head_dim) + gate = gate.view(b, t, d.heads, d.head_dim) + qn_w, kn_w = s["self_attn.q_norm.weight"], s["self_attn.k_norm.weight"] + if v.swap_qk_norm_weights: + qn_w, kn_w = kn_w, qn_w + if v.qk_norm_fp32_into_rope: + q32 = q.float() + q = (q32 * q32.pow(2).mean(-1, keepdim=True).add(d.eps).rsqrt()) * qn_w + k32 = k.float() + k = (k32 * k32.pow(2).mean(-1, keepdim=True).add(d.eps).rsqrt()) * kn_w + qn, kn = q.to(torch.bfloat16), k.to(torch.bfloat16) + else: + q = rms_norm(q, qn_w, d.eps, v) + if not v.omit_k_norm: + k = rms_norm(k, kn_w, d.eps, v) + qn, kn = q, k + if v.normalize_v: + val = rms_norm(val, kn_w, d.eps, v) + record("q_norm", qn) + record("k_norm", kn) + cos, sin = rope_cos_sin(positions + v.rope_positions_offset, d) + q, k = apply_rope(q, k, cos, sin, v) + if v.qk_norm_fp32_into_rope: + q, k = q.to(torch.bfloat16), k.to(torch.bfloat16) + record("q_rope", q) + record("k_rope", k) + attn = attention_core(q, k, val, d, v) + record("attention_core", attn) + attn = output_gate(attn, gate, s["self_attn.gate_bias"], v) + attn = attn.reshape(b, t, qw) + record("gated", attn) + out = linear(attn, s["self_attn.o_proj.weight"], v) + record("attention_output", out) + return out + + +def silu_and_mul(gate: torch.Tensor, up: torch.Tensor, v: Variant = SOURCE) -> torch.Tensor: + """fp32 SiLU(gate) * up, cast once to the activation dtype.""" + if v.swiglu_bf16_activation: + return F.silu(gate) * up + g, u = gate.float(), up.float() + if v.silu == "x_sigmoid": + act = g * torch.sigmoid(g) + elif v.silu == "exp_form": + act = g / (1.0 + torch.exp(-g)) + else: + act = F.silu(g) + return (act * u).to(gate.dtype) + + +def swiglu( + x: torch.Tensor, + s: "dict[str, torch.Tensor]", + d: Dims = BLOCK0, + v: Variant = SOURCE, + record: Record = _noop, + fused: "dict[str, torch.Tensor] | None" = None, +) -> torch.Tensor: + if v.gemm == "fused": + assert fused is not None + gate, up = ( + y.contiguous() + for y in F.linear(x, fused["gate_up"]).split([d.intermediate] * 2, dim=-1) + ) + else: + gate = linear(x, s["mlp.gate_proj.weight"], v) + up = linear(x, s["mlp.up_proj.weight"], v) + record("mlp_gate", gate) + record("mlp_up", up) + act = silu_and_mul(gate, up, v) + record("mlp_act", act) + out = linear(act, s["mlp.down_proj.weight"], v) + record("mlp_output", out) + return out + + +def residual_add( + x: torch.Tensor, residual: torch.Tensor, scales: tuple[float, float], v: Variant = SOURCE +) -> torch.Tensor: + x_scale, r_scale = scales + if v.residual_fp32_fused: + return (x.float() * x_scale + residual.float() * r_scale).to(x.dtype) + if v.residual == "torch_add": + return torch.add(x * x_scale, residual * r_scale) + return x * x_scale + residual * r_scale + + +def fused_weights(s: "dict[str, torch.Tensor]") -> "dict[str, torch.Tensor]": + return { + "qkvg": torch.cat( + [s[f"self_attn.{n}_proj.weight"] for n in ("q", "k", "v", "gate")], dim=0 + ).contiguous(), + "gate_up": torch.cat( + [s["mlp.gate_proj.weight"], s["mlp.up_proj.weight"]], dim=0 + ).contiguous(), + } + + +def block( + x: torch.Tensor, + positions: torch.Tensor, + s: "dict[str, torch.Tensor]", + d: Dims = BLOCK0, + v: Variant = SOURCE, + record: Record = _noop, + fused: "dict[str, torch.Tensor] | None" = None, +) -> torch.Tensor: + """x: (1, T, hidden) bf16; positions: (1, T) int64. Returns (1, T, hidden) bf16.""" + attn_scales = v.attn_coefs or depth_coefs(d.layer_id, True) + mlp_scales = v.mlp_coefs or depth_coefs(d.layer_id, False) + post_attn_w = s["post_attention_layernorm.weight"] + pre_ffn_w = s["pre_feedforward_layernorm.weight"] + if v.swap_post_attention_and_pre_ffn_norms: + post_attn_w, pre_ffn_w = pre_ffn_w, post_attn_w + residual = x + h = rms_norm(x, s["input_layernorm.weight"], d.eps, v) + record("input_layernorm", h) + h = attention(h, positions, s, d, v, record, fused) + if not v.omit_post_attention_norm: + h = rms_norm(h, post_attn_w, d.eps, v) + record("post_attention_layernorm", h) + hidden = residual_add(h, residual, attn_scales, v) + record("attention_residual", hidden) + h = rms_norm(hidden, pre_ffn_w, d.eps, v) + record("pre_feedforward_layernorm", h) + h = swiglu(h, s, d, v, record, fused) + h = rms_norm(h, s["post_feedforward_layernorm.weight"], d.eps, v) + record("post_feedforward_layernorm", h) + out = residual_add(h, hidden, mlp_scales, v) + record("block_output", out) + return out diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py new file mode 100644 index 000000000000..9cc63a031431 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -0,0 +1,943 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam dense block 0 component: bf16 / sm_103 / tp1. + +Configuration under test, fixed below and nowhere else: block 0 (sliding attention, +dense MLP) at hidden 5120, intermediate 20480 and 48 q / 8 kv heads of 128; bf16 +weights and activations with the source model's fp32 steps; one sm_103 GPU; eager, +batch 1, one fresh context per call; generated weights; seeds {17, 41} x lengths +{1, 17, 129}. + +What is compared. ``_dense_block_reference.py`` restates the source block in plain +torch and generates its state and input deterministically on the CPU. For every case +the component's final output is compared with the reference's, and every operation of +the block is replayed on the reference's own input to that operation, so a local error +cannot hide behind the sandwich norms. Each compared tensor is held to one rule in +`RULES`. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ + python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block + +Optional environment: + +* ``BEAM_DENSE_BLOCK_SOURCE_CAPTURES``: a directory of ``seed/T.pt`` files, each a + dict of tensors captured from the source block under the reference's record names + (plus ``input`` and ``positions``). The final output and every replay are then also + compared against the captures, and the generated input must equal theirs bit for bit. +* ``BEAM_DENSE_BLOCK_EVIDENCE``: a directory that receives one JSON record per test case: + every comparison with its rule, the op calls and CUDA kernels of the forward, and the + device and software it ran on; and ``fp32-session.json``, the fp32 GEMM checks made + before and after the module's tests. + +Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable. No model files. +""" + +from __future__ import annotations + +import contextlib +import functools +import math +import os +import types +from pathlib import Path + +import pytest +import torch + +try: + import tensorrt_llm._torch.models.modeling_beam as builtin_beam +except ModuleNotFoundError as exc: + if exc.name != "tensorrt_llm._torch.models.modeling_beam": + raise + builtin_beam = None # The standard-serving implementation is a separate PR. +__extra_import_path__ = [".", "../..", "../../../.."] + +import _beam_component_support as support # noqa: E402 +import _dense_block_reference as reference # noqa: E402 +from _beam_component_support import assert_all_pass, tensor_sha256 # noqa: E402 +from _validating import validating # noqa: E402 + +from _components.dense_block import block as block_module # noqa: E501 +from _components.dense_block.block import BeamDenseBlockBf16Sm103Tp1 # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.catalog.activation.flashinfer_silu_and_mul import ( # noqa: E501 + flashinfer_silu_and_mul, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( + thop_attention, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( + beam_neox_rope, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( + beam_output_gate, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( + beam_depth_residual, +) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.resource_manager import DataType +from tensorrt_llm.mapping import Mapping + +# ----------------------------------------------------------------------------- the fixed configuration + +#: The block as the checkpoint configures it. Only these fields are read. +BLOCK_CONFIG = types.MappingProxyType( + dict( + hidden_size=5120, + intermediate_size=20480, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + rms_norm_eps=1e-6, + hidden_act="silu", + attention_bias=False, + attention_dropout=0.0, + sliding_window=512, + layer_types=("sliding_attention",), + mlp_layer_types=("dense",), + rope_parameters={ + "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, + "full_attention": None, + }, + ) +) +LAYER_IDX = 0 +DTYPE = torch.bfloat16 +SM = (10, 3) +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) +CASES = tuple((seed, length) for seed in SEEDS for length in LENGTHS) +CASE_IDS = [f"seed{seed}-T{length}" for seed, length in CASES] +#: Eager, batch 1, one fresh context: no CUDA graph, no overlap scheduler, no cached prefix. +EXECUTION = types.MappingProxyType( + dict(eager=True, batch_size=1, fresh_context=True, cuda_graph=False, overlap_scheduler=False) +) +#: The KV cache the attention op appends to: one bf16 layer, 32-token pages, 512 tokens per +#: sequence (the window), one request. +KV_CACHE = types.MappingProxyType(dict(tokens_per_block=32, max_seq_len=512, num_blocks=16)) +REQUEST_ID = 0 +ROPE_THETA = 10000.0 + +# ----------------------------------------------------------------------------- comparison rules + +_RTOL = 2.0**-7 + +#: (atol, rtol, max fraction of unequal elements or None) per kind of compared tensor. An +#: element passes when |a - r| <= atol + rtol * |r| with both widened to fp64; every element +#: of every tensor of every case must pass, and no NaN or Inf is accepted anywhere. +#: +#: Derivation: the largest atol each kind needed at rtol 2^-7 across valid evaluations of +#: the source block on these exact cases (other GEMM kernels, reduction orders, attention +#: backends and Torch builds) was 0 for the norms, RoPE, gate, residual adds and SiLU +#: product (with at most 5.7e-5 of elements one ulp apart, hence the 0.1% unequal bound), +#: 5.0e-6 for a single projection, 1.6e-4 for the dense SwiGLU output, 7.8e-4 for the +#: attention module and 0.0375 for the whole block. The bands below sit 100x, 4.9x, 3.8x +#: and 4.0x above those last four, and a zeroed final residual coefficient needs an atol +#: of 4.8. +RULES = types.MappingProxyType( + { + "rmsnorm": (1e-6, _RTOL, 1e-3), + "rope": (1e-6, _RTOL, 1e-3), + "gate": (1e-6, _RTOL, 1e-3), + "residual": (1e-6, _RTOL, 1e-3), + "swiglu_activation": (1e-6, _RTOL, 1e-3), + "projection": (5e-4, _RTOL, None), + "swiglu_output": (8e-4, _RTOL, None), + "attention_output": (3e-3, _RTOL, None), + "block_output": (0.15, _RTOL, None), + } +) + +#: The forward's op calls, in order: the composition this component claims to run. +EXPECTED_CALLS = ( + "FlashinferRmsnorm", + "CublasMm", + "CublasMm", + "CublasMm", + "CublasMm", + "FlashinferRmsnorm", + "FlashinferRmsnorm", + "BeamNeoxRope", + "ThopAttention", + "BeamOutputGate", + "CublasMm", + "FlashinferRmsnorm", + "BeamDepthResidual", + "FlashinferRmsnorm", + "CublasMm", + "CublasMm", + "FlashinferSiluAndMul", + "CublasMm", + "FlashinferRmsnorm", + "BeamDepthResidual", +) +OPS = ( + flashinfer_rmsnorm, + cublas_mm, + beam_neox_rope, + thop_attention, + beam_output_gate, + beam_depth_residual, + flashinfer_silu_and_mul, +) + +assert torch.cuda.is_available(), "the Beam dense block requires a CUDA device" + + +def block_config(**overrides) -> types.SimpleNamespace: + fields = dict(BLOCK_CONFIG) + fields["layer_types"] = list(fields["layer_types"]) + fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) + fields["rope_parameters"] = { + k: (dict(v) if v else v) for k, v in fields["rope_parameters"].items() + } + fields.update(overrides) + return types.SimpleNamespace(**fields) + + +# ----------------------------------------------------------------------------- precision and environment + + +@pytest.fixture(scope="module", autouse=True) +def ieee_fp32(): + """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both + before the module's tests and after them.""" + yield from support.ieee_fp32_session(_write_evidence) + + +def _environment(ieee: dict) -> dict: + return support.environment(ieee, EXECUTION) + + +_write_evidence = functools.partial(support.write_evidence, "BEAM_DENSE_BLOCK_EVIDENCE") + + +# ----------------------------------------------------------------------------- comparison + + +def compare(name: str, actual: torch.Tensor, expected: torch.Tensor, rule: str | None) -> dict: + """Score one tensor against one rule; `rule=None` measures without judging.""" + result = { + "tensor": name, + "rule": rule, + "shape": list(expected.shape), + "dtype": str(actual.dtype), + } + problems = [] + if tuple(actual.shape) != tuple(expected.shape): + problems.append(f"shape {tuple(actual.shape)} != {tuple(expected.shape)}") + if actual.dtype != DTYPE or expected.dtype != DTYPE: + problems.append(f"dtypes {actual.dtype}/{expected.dtype}, need {DTYPE}") + if problems: + result.update(passed=False, problems=problems) + return result + a, r = actual.double(), expected.double() + if not bool(torch.isfinite(a).all()) or not bool(torch.isfinite(r).all()): + result.update(passed=False, problems=["non-finite values"]) + return result + diff = (a - r).abs() + numel = diff.numel() + worst = int(diff.argmax()) + result.update( + numel=numel, + max_abs=float(diff.max()), + mean_abs=float(diff.mean()), + worst_index=[int(i) for i in torch.unravel_index(torch.tensor(worst), diff.shape)], + worst_actual=float(a.reshape(-1)[worst]), + worst_expected=float(r.reshape(-1)[worst]), + unequal=int((a != r).sum()), + ) + if rule is None: + result["passed"] = None + return result + atol, rtol, max_unequal = RULES[rule] + violations = int((diff > atol + rtol * r.abs()).sum()) + result.update(atol=atol, rtol=rtol, violations=violations) + if violations: + problems.append( + f"{violations} of {numel} elements outside atol {atol} + rtol {rtol} * |ref|" + ) + if max_unequal is not None: + allowed = math.floor(max_unequal * numel) + result["max_unequal"] = allowed + if result["unequal"] > allowed: + problems.append(f"{result['unequal']} unequal elements, at most {allowed} allowed") + result.update(passed=not problems, problems=problems) + return result + + +# ----------------------------------------------------------------------------- state, inputs, fresh context + +_STATE_CACHE: dict[int, dict] = {} + + +def generated(seed: int) -> dict: + """The seed's bf16 state and [1, 129, hidden] input, generated once on the CPU.""" + if seed not in _STATE_CACHE: + _STATE_CACHE.clear() # one seed's 0.8 GB of weights at a time + state = reference.generate_state(seed) + assert not reference.state_problems(state), reference.state_problems(state) + _STATE_CACHE[seed] = { + "state_cpu": state, + "state": {name: t.cuda() for name, t in state.items()}, + "input": reference.generate_input(seed).cuda(), + "block": None, + } + return _STATE_CACHE[seed] + + +def loaded_block(seed: int) -> BeamDenseBlockBf16Sm103Tp1: + entry = generated(seed) + if entry["block"] is None: + blk = BeamDenseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + blk.load_weights(entry["state"]) + entry["block"] = blk + return entry["block"] + + +def fresh_context(num_tokens: int, cached_tokens: int = 0, num_requests: int = 1, **mgr_overrides): + """A single-layer KVCacheManagerV2 and prepared TrtllmAttentionMetadata for one new request.""" + mgr_args = dict( + num_layers=1, + num_kv_heads=BLOCK_CONFIG["num_key_value_heads"], + head_dim=BLOCK_CONFIG["head_dim"], + tokens_per_block=KV_CACHE["tokens_per_block"], + max_seq_len=KV_CACHE["max_seq_len"], + max_batch_size=num_requests, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + dtype=DataType.BF16, + ) + mgr_args.update(mgr_overrides) + return support.fresh_context( + num_tokens, + cached_tokens, + num_requests, + max_kv_tokens=KV_CACHE["num_blocks"] * KV_CACHE["tokens_per_block"] * num_requests, + max_tokens=block_module.MAX_TOKENS, + request_id=REQUEST_ID, + mgr_args=mgr_args, + ) + + +def case_inputs(seed: int, length: int) -> tuple[torch.Tensor, torch.Tensor]: + x = generated(seed)["input"][:, :length].contiguous() + positions = torch.arange(length, device="cuda").view(1, length) + return x, positions + + +def reference_record(seed: int, length: int) -> dict: + """The reference block's output and every named intermediate, on the GPU.""" + x, positions = case_inputs(seed, length) + record = {"input": x, "positions": positions} + + def keep(name: str, value: torch.Tensor) -> None: + record[name] = value + + with torch.no_grad(): + out = reference.block(x, positions, generated(seed)["state"], record=keep) + assert torch.equal(out, record["block_output"]) + return record + + +def source_captures(seed: int, length: int) -> dict | None: + directory = os.environ.get("BEAM_DENSE_BLOCK_SOURCE_CAPTURES") + if not directory: + return None + captures = torch.load( + Path(directory) / f"seed{seed}" / f"T{length}.pt", map_location="cuda", weights_only=True + ) + x, positions = case_inputs(seed, length) + assert torch.equal(captures["input"], x), "the generated input differs from the captured one" + assert torch.equal( + captures["positions"].view(1, length), positions.to(captures["positions"].dtype) + ) + return captures + + +# ----------------------------------------------------------------------------- replays + + +def replay( + blk: BeamDenseBlockBf16Sm103Tp1, md: TrtllmAttentionMetadata, src: dict, label: str +) -> list[dict]: + """Each operation of the block on `src`'s own input to it, against `src`'s output of it.""" + length = src["input"].shape[1] + hidden = BLOCK_CONFIG["hidden_size"] + heads, kv_heads = BLOCK_CONFIG["num_attention_heads"], BLOCK_CONFIG["num_key_value_heads"] + d = BLOCK_CONFIG["head_dim"] + eps = BLOCK_CONFIG["rms_norm_eps"] + w = dict(blk.checkpoint_tensors()) + positions = src["positions"].view(length).cuda() + step = blk.step_args(md) + # Logical block 0 is the manager's local slot 0: the core call indexes the slot. + assert step["local_layer_idx"] == md.kv_cache_manager.layer_offsets[LAYER_IDX] == 0 + + def flat(name: str, width: int) -> torch.Tensor: + # Dense row-major: the GEMM and norm entries read their input that way. + return src[name].reshape(length, width).contiguous() + + def check(name: str, rule: str | None, actual: torch.Tensor, expected: str) -> dict: + result = compare(name, actual.reshape(src[expected].shape), src[expected], rule) + result.update(reference=label) + return result + + x = flat("input", hidden) + h = flat("input_layernorm", hidden) + out = [ + check( + "input_layernorm", + "rmsnorm", + flashinfer_rmsnorm(x, w["input_layernorm.weight"], eps), + "input_layernorm", + ) + ] + for proj in ("q_proj", "k_proj", "v_proj", "gate_proj"): + weight = w[f"self_attn.{proj}.weight"] + out.append(check(proj, "projection", cublas_mm(h, weight.t()), proj)) + q_rows = flat("q_proj", heads * d).reshape(length * heads, d) + k_rows = flat("k_proj", kv_heads * d).reshape(length * kv_heads, d) + out.append( + check( + "q_norm", + "rmsnorm", + flashinfer_rmsnorm(q_rows, w["self_attn.q_norm.weight"], eps), + "q_norm", + ) + ) + out.append( + check( + "k_norm", + "rmsnorm", + flashinfer_rmsnorm(k_rows, w["self_attn.k_norm.weight"], eps), + "k_norm", + ) + ) + q_rope, k_rope = beam_neox_rope( + src["q_norm"].reshape(length, heads, d), + src["k_norm"].reshape(length, kv_heads, d), + positions, + ROPE_THETA, + 1.0, + ) + out.append(check("q_rope", "rope", q_rope, "q_rope")) + out.append(check("k_rope", "rope", k_rope, "k_rope")) + qkv = torch.cat( + (flat("q_rope", heads * d), flat("k_rope", kv_heads * d), flat("v_proj", kv_heads * d)), + dim=-1, + ) + core = torch.empty(length, heads * d, dtype=DTYPE, device="cuda") + core = thop_attention( + q=qkv, + output=core, + num_heads=heads, + num_kv_heads=kv_heads, + head_size=d, + attention_window_size=BLOCK_CONFIG["sliding_window"], + **step, + **_attention.CALL_CONSTANTS, + ) + # No rule of its own: the attention core is judged through the attention module below. + out.append(check("attention_core", None, core, "attention_core")) + gated = beam_output_gate( + src["attention_core"].reshape(length, heads, d), + src["gate_proj"].reshape(length, heads, d), + w["self_attn.gate_bias"], + ) + out.append(check("gated", "gate", gated, "gated")) + out.append( + check( + "o_proj", + "projection", + cublas_mm(flat("gated", heads * d), w["self_attn.o_proj.weight"].t()), + "attention_output", + ) + ) + out.append( + check( + "attention_output", + "attention_output", + blk.attention(h, positions, step), + "attention_output", + ) + ) + out.append( + check( + "post_attention_layernorm", + "rmsnorm", + flashinfer_rmsnorm( + flat("attention_output", hidden), w["post_attention_layernorm.weight"], eps + ), + "post_attention_layernorm", + ) + ) + out.append( + check( + "attention_residual", + "residual", + beam_depth_residual(flat("post_attention_layernorm", hidden), x, *blk.attention_scales), + "attention_residual", + ) + ) + out.append( + check( + "pre_feedforward_layernorm", + "rmsnorm", + flashinfer_rmsnorm( + flat("attention_residual", hidden), w["pre_feedforward_layernorm.weight"], eps + ), + "pre_feedforward_layernorm", + ) + ) + h2 = flat("pre_feedforward_layernorm", hidden) + inter = BLOCK_CONFIG["intermediate_size"] + out.append( + check( + "mlp_gate", + "projection", + cublas_mm(h2, w["mlp.gate_proj.weight"].t()), + "mlp_gate", + ) + ) + out.append(check("mlp_up", "projection", cublas_mm(h2, w["mlp.up_proj.weight"].t()), "mlp_up")) + act = flashinfer_silu_and_mul( + torch.cat((flat("mlp_gate", inter), flat("mlp_up", inter)), dim=-1) + ) + out.append(check("mlp_act", "swiglu_activation", act, "mlp_act")) + out.append( + check( + "down_proj", + "projection", + cublas_mm(flat("mlp_act", inter), w["mlp.down_proj.weight"].t()), + "mlp_output", + ) + ) + out.append(check("mlp_output", "swiglu_output", blk.feed_forward(h2), "mlp_output")) + out.append( + check( + "post_feedforward_layernorm", + "rmsnorm", + flashinfer_rmsnorm( + flat("mlp_output", hidden), w["post_feedforward_layernorm.weight"], eps + ), + "post_feedforward_layernorm", + ) + ) + out.append( + check( + "block_residual", + "residual", + beam_depth_residual( + flat("post_feedforward_layernorm", hidden), + flat("attention_residual", hidden), + *blk.feed_forward_scales, + ), + "block_output", + ) + ) + return out + + +def accounting(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> dict: + """The fifteen tensors: the same names, order, shapes, dtypes and values on both sides.""" + state = generated(seed)["state"] + expected = reference.param_shapes() + loaded = blk.checkpoint_tensors() + assert list(loaded) == list(state) == list(expected), (list(loaded), list(state)) + for name, shape in expected.items(): + assert tuple(loaded[name].shape) == shape and loaded[name].dtype == DTYPE, name + assert torch.equal(loaded[name], state[name]), f"{name} differs from the reference's" + assert not reference.state_problems(generated(seed)["state_cpu"]) + record = {"tensors": len(loaded), "state_problems": []} + if os.environ.get("BEAM_DENSE_BLOCK_EVIDENCE"): + record["sha256"] = { + name: tensor_sha256(t) for name, t in generated(seed)["state_cpu"].items() + } + record["input_sha256"] = tensor_sha256(generated(seed)["input"]) + return record + + +# ----------------------------------------------------------------------------- op-path instrumentation + + +@contextlib.contextmanager +def recording(*wrappers): + """Log every call of these entries, in order, with the arguments that identify the path.""" + calls: list[dict] = [] + saved = [] + for w in wrappers: + cls = type(w) + original = cls.__call__ + saved.append((cls, original)) + + def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): + entry = {"op": _name} + tensors = [a for a in args if isinstance(a, torch.Tensor)] + entry["shapes"] = [list(t.shape) for t in tensors] + for key in ( + "num_heads", + "num_kv_heads", + "head_size", + "attention_window_size", + "max_seq_len", + "tokens_per_block", + "mask_type", + "is_fused_qkv", + "num_contexts", + "num_ctx_tokens", + "use_paged_context_fmha", + "position_embedding_type", + "local_layer_idx", + "quant_mode", + ): + if key in kwargs: + entry[key] = kwargs[key] + if "q" in kwargs: + entry["q_shape"] = list(kwargs["q"].shape) + entry["qkv"] = kwargs["q"] + calls.append(entry) + return _original(self, *args, **kwargs) + + cls.__call__ = logged + try: + yield calls + finally: + for cls, original in reversed(saved): + cls.__call__ = original + + +def builtin_beam_forbidden(): + """The built-in Beam modeling must not run while the component does.""" + if builtin_beam is None: + return support.builtin_forbidden([]) + targets = ( + builtin_beam.BeamDecoderLayer, + builtin_beam.BeamAttention, + builtin_beam.BeamSwiGLU, + builtin_beam.BeamRMSNorm, + ) + return support.builtin_forbidden([(cls, "forward") for cls in targets]) + + +def no_dispatch(): + """Fail on any op call: rejected inputs must be rejected before a kernel runs.""" + return support.no_dispatch(OPS) + + +def cache_rows(md: TrtllmAttentionMetadata, length: int) -> tuple[torch.Tensor, torch.Tensor]: + """This request's K and V rows read back from the pool, [T, Hkv, D] each.""" + return support.cache_rows(md, length, LAYER_IDX, KV_CACHE["tokens_per_block"]) + + +# ----------------------------------------------------------------------------- tests: configuration and construction + + +def test_configuration_is_the_fixed_scope() -> None: + """One configuration, no open choices: what the component accepts is what is tested.""" + assert CASES == ((17, 1), (17, 17), (17, 129), (41, 1), (41, 17), (41, 129)) + assert block_module.DEFAULT_LAYER_IDX == LAYER_IDX + for field, value in block_module.SUPPORTED_CONFIG.items(): + assert BLOCK_CONFIG[field] == value, field + assert BLOCK_CONFIG["rope_parameters"]["sliding_attention"] == block_module.SUPPORTED_ROPE + assert block_module.MAX_TOKENS == 1024 >= max(LENGTHS) + assert reference.BLOCK0.hidden == BLOCK_CONFIG["hidden_size"] + assert reference.BLOCK0.intermediate == BLOCK_CONFIG["intermediate_size"] + assert (reference.BLOCK0.heads, reference.BLOCK0.kv_heads, reference.BLOCK0.head_dim) == ( + 48, + 8, + 128, + ) + assert reference.BLOCK0.window == BLOCK_CONFIG["sliding_window"] == KV_CACHE["max_seq_len"] + assert (reference.SEEDS, reference.LENGTHS) == (SEEDS, LENGTHS) + assert torch.cuda.get_device_capability() == SM, "this suite certifies sm_103 only" + + +def test_block_constructs_alone(ieee_fp32) -> None: + """The block is built directly from its configuration, without the model around it.""" + + def refuse(self, *args, **kwargs): + raise AssertionError(f"{type(self).__name__} constructed") + + with pytest.MonkeyPatch.context() as mp: + if builtin_beam is not None: + for cls in ( + builtin_beam.BeamForCausalLM, + builtin_beam.BeamModel, + builtin_beam.BeamDecoderLayer, + ): + mp.setattr(cls, "__init__", refuse) + blk = BeamDenseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + assert isinstance(blk, torch.nn.Module) and not list(blk.children()) + assert list(blk.checkpoint_tensors()) == list(reference.param_shapes()) + for name, shape in reference.param_shapes().items(): + t = blk.checkpoint_tensors()[name] + assert tuple(t.shape) == shape and t.dtype == DTYPE and t.is_cuda, name + q_width = BLOCK_CONFIG["num_attention_heads"] * BLOCK_CONFIG["head_dim"] + kv_width = BLOCK_CONFIG["num_key_value_heads"] * BLOCK_CONFIG["head_dim"] + assert ( + blk.checkpoint_tensors()["self_attn.q_proj.weight"].shape == (q_width, 5120) == (6144, 5120) + ) + assert blk.checkpoint_tensors()["self_attn.gate_proj.weight"].shape == (6144, 5120) + assert ( + blk.checkpoint_tensors()["self_attn.k_proj.weight"].shape + == (kv_width, 5120) + == (1024, 5120) + ) + assert blk.checkpoint_tensors()["self_attn.v_proj.weight"].shape == (1024, 5120) + assert blk.attention_scales == (1 / math.sqrt(2), 1 / math.sqrt(2)) + assert blk.feed_forward_scales == (1 / math.sqrt(3), math.sqrt(2) / math.sqrt(3)) + assert "modeling_beam" not in Path(block_module.__file__).read_text() + + +# ----------------------------------------------------------------------------- tests: declared limits + +_BAD_CONFIGS = [ + ("hidden_size", dict(hidden_size=4096)), + ("intermediate_size", dict(intermediate_size=18432)), + ("num_attention_heads", dict(num_attention_heads=32)), + ("num_key_value_heads", dict(num_key_value_heads=4)), + ("head_dim", dict(head_dim=64)), + ("rms_norm_eps", dict(rms_norm_eps=1e-5)), + ("hidden_act", dict(hidden_act="gelu")), + ("attention_bias", dict(attention_bias=True)), + ("sliding_window", dict(sliding_window=1024)), + ("sliding_attention layer", dict(layer_types=["full_attention"])), + ("dense MLP", dict(mlp_layer_types=["sparse"])), + ( + "rope parameters", + dict(rope_parameters={"sliding_attention": {"rope_type": "default", "rope_theta": 1e6}}), + ), + ( + "rope parameters", + dict( + rope_parameters={ + "sliding_attention": {"rope_type": "linear", "rope_theta": 10000.0, "factor": 2.0} + } + ), + ), +] + + +@pytest.mark.parametrize( + "match,overrides", _BAD_CONFIGS, ids=[f"{m}-{i}" for i, (m, _) in enumerate(_BAD_CONFIGS)] +) +def test_rejects_unsupported_configuration(match, overrides) -> None: + with pytest.raises(ValueError, match=match): + BeamDenseBlockBf16Sm103Tp1(block_config(**overrides), LAYER_IDX, device="cuda") + + +def test_rejects_other_blocks_and_devices() -> None: + for layer_idx in (3, 5): + with pytest.raises(ValueError, match="only dense blocks"): + BeamDenseBlockBf16Sm103Tp1(block_config(), layer_idx, device="cuda") + with pytest.raises(ValueError, match="sm_103"): + BeamDenseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cpu") + + +def test_weights_must_be_exactly_the_blocks() -> None: + blk = BeamDenseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + good = { + name: torch.zeros(shape, dtype=DTYPE) for name, shape in reference.param_shapes().items() + } + missing = dict(good) + del missing["self_attn.gate_bias"] + extra = dict(good, **{"mlp.experts.0.down_proj.weight": torch.zeros(4, dtype=DTYPE)}) + transposed = dict(good, **{"self_attn.q_proj.weight": torch.zeros(5120, 6144, dtype=DTYPE)}) + fp32 = dict(good, **{"self_attn.q_norm.weight": torch.zeros(128, dtype=torch.float32)}) + for weights, match in ( + (missing, "missing"), + (extra, "unexpected"), + (transposed, "q_proj"), + (fp32, "q_norm"), + ): + with pytest.raises(ValueError, match=match): + blk.load_weights(weights) + + +_BAD_INPUTS = [ + "fp16_hidden", + "fp32_hidden", + "hidden_5120x2", + "batch_2", + "too_long", + "empty", + "non_contiguous", + "positions_offset", + "float_positions", + "positions_shape", + "metadata_type", + "cached_prefix", + "two_requests", + "context_length_mismatch", + "kv_cache_heads", + "kv_cache_dtype", +] + + +@pytest.mark.parametrize("case", _BAD_INPUTS) +def test_rejects_unsupported_inputs_before_dispatch(case) -> None: + """Each declared restriction raises ValueError on a loaded block, and no op runs first. + + The block is loaded (with zeros) so that only the input checks can refuse the call: an + unloaded block refuses every call after the checks pass, which would hide a missing + check behind the same ValueError. + """ + blk = BeamDenseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + zero = torch.zeros((), dtype=DTYPE, device="cuda") + blk.load_weights({name: zero.expand(shape) for name, shape in reference.param_shapes().items()}) + x, positions, context, match = support.bad_block_input(case, block_module.MAX_TOKENS, DTYPE) + with fresh_context(**context) as md: + if case == "metadata_type": + md = object() + with no_dispatch(), pytest.raises(ValueError, match=match): + blk(x, positions, md) + + +# ----------------------------------------------------------------------------- tests: the six cases + + +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_block_matches_reference(seed, length, ieee_fp32) -> None: + """One case: the whole block, then every operation on its own input.""" + blk = loaded_block(seed) + accounting_record = accounting(seed, blk) + ref = reference_record(seed, length) + x, positions = case_inputs(seed, length) + results = [] + with fresh_context(length) as md: + out = blk(x, positions, md) + torch.cuda.synchronize() + assert out.shape == (1, length, 5120) and out.dtype == DTYPE + final = compare("block_output", out, ref["block_output"], "block_output") + final.update(reference="reference") + results.append(final) + results.extend(replay(blk, md, ref, "reference")) + captures = source_captures(seed, length) + if captures is not None: + from_source = compare("block_output", out, captures["block_output"], "block_output") + from_source.update(reference="source_captures") + results.append(from_source) + results.extend(replay(blk, md, captures, "source_captures")) + _write_evidence( + f"block-seed{seed}-T{length}", + { + "test": "test_block_matches_reference", + "case": {"seed": seed, "length": length}, + "accounting": accounting_record, + "rules": {k: list(v) for k, v in RULES.items()}, + "results": results, + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(results) + + +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_forward_runs_the_component_path(seed, length, ieee_fp32) -> None: + """One case: the component's own composition ran, op by op, through thop attention on + this GPU, with no SDPA and no built-in Beam module on the way.""" + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + with fresh_context(length) as md: + with builtin_beam_forbidden(), validating(*OPS), recording(*OPS) as calls: + with torch.profiler.profile(activities=activities) as prof: + out = blk(x, positions, md) + torch.cuda.synchronize() + keys, values = cache_rows(md, length) + mgr = md.kv_cache_manager + cache_mapping = { + "layer_offsets": dict(mgr.layer_offsets), + "num_local_layers": mgr.num_local_layers, + "num_layers": mgr.num_layers, + } + events = prof.events() + kernels = [e.name for e in events if e.device_type == torch.autograd.DeviceType.CUDA] + cpu_ops = sorted({e.name for e in events if e.device_type == torch.autograd.DeviceType.CPU}) + names = [c["op"] for c in calls] + attention = [c for c in calls if c["op"] == "ThopAttention"] + qkv = attention[0].pop("qkv") if attention else None + for c in calls: + c.pop("qkv", None) + record = { + "test": "test_forward_runs_the_component_path", + "case": {"seed": seed, "length": length}, + "block": f"{type(blk).__module__}.{type(blk).__qualname__}", + "op_calls": calls, + "cuda_kernels": kernels, + "attention_kernels": [k for k in kernels if support.ATTENTION_KERNEL.search(k)], + "sdpa_cpu_ops": [n for n in cpu_ops if support.SDPA_OPS.search(n)], + "kv_cache": dict(KV_CACHE, manager=type(md.kv_cache_manager).__name__, **cache_mapping), + "environment": _environment(ieee_fp32), + } + _write_evidence(f"path-seed{seed}-T{length}", record) + + assert type(blk).__module__.endswith("dense_block.block") + assert tuple(names) == EXPECTED_CALLS, names + (call,) = attention + assert call["q_shape"] == [length, (48 + 2 * 8) * 128] + assert { + k: call[k] for k in ("num_heads", "num_kv_heads", "head_size", "attention_window_size") + } == { + "num_heads": 48, + "num_kv_heads": 8, + "head_size": 128, + "attention_window_size": 512, + } + assert call["mask_type"] == 1 and call["is_fused_qkv"] and call["position_embedding_type"] == 0 + assert call["num_contexts"] == 1 and call["num_ctx_tokens"] == length + assert call["use_paged_context_fmha"] is False and call["quant_mode"] == 0 + assert isinstance(md.kv_cache_manager, KVCacheManagerV2) + # Logical block 0 in a one-layer manager: slot 0, which the core call indexes. + assert cache_mapping == {"layer_offsets": {0: 0}, "num_local_layers": 1, "num_layers": 1} + assert call["local_layer_idx"] == 0 + assert record["attention_kernels"], "no attention kernel ran on the GPU" + assert not record["sdpa_cpu_ops"], record["sdpa_cpu_ops"] + assert torch.cuda.get_device_capability(x.device) == SM + assert out.shape == (1, length, 5120) and bool(torch.isfinite(out.float()).all()) + # The op appended exactly this call's post-RoPE K and V to the pool. + k_in = qkv[:, 48 * 128 : 56 * 128].reshape(length, 8, 128) + v_in = qkv[:, 56 * 128 :].reshape(length, 8, 128) + assert torch.equal(keys, k_in) and torch.equal(values, v_in) + + +def test_detects_a_wrong_residual_coefficient(ieee_fp32, monkeypatch) -> None: + """The complete-block comparison rejects a zeroed final residual coefficient and + accepts the unchanged block, on the same case and the same comparison.""" + seed, length = 17, 17 + blk = loaded_block(seed) + ref = reference_record(seed, length) + x, positions = case_inputs(seed, length) + runs = {} + for label in ("unchanged", "faulted", "restored"): + if label == "faulted": + branch_scale, _ = blk.feed_forward_scales + monkeypatch.setattr(blk, "feed_forward_scales", (branch_scale, 0.0)) + elif label == "restored": + monkeypatch.undo() + with fresh_context(length) as md: + out = blk(x, positions, md) + runs[label] = compare("block_output", out, ref["block_output"], "block_output") + runs[label]["feed_forward_scales"] = list(blk.feed_forward_scales) + _write_evidence( + f"fault-seed{seed}-T{length}", + { + "test": "test_detects_a_wrong_residual_coefficient", + "case": {"seed": seed, "length": length}, + "fault": "final residual coefficient sqrt(2)/sqrt(3) -> 0.0, fixture-scoped", + "runs": runs, + "environment": _environment(ieee_fp32), + }, + ) + assert runs["unchanged"]["passed"] is True, runs["unchanged"]["problems"] + assert runs["faulted"]["passed"] is False and runs["faulted"]["violations"] > 0 + assert runs["restored"][ + "passed" + ] is True and blk.feed_forward_scales == block_module.depth_scales(0, False) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py new file mode 100644 index 000000000000..aa0029b67cfa --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Plain-Torch reference for the MoE feed-forward of Beam decoder block 3. + +An independent restatement of the source MoE for generated-input comparison: + +* Routing, every step fp32 with TF32 off: logits ``x.float() @ gate.weight^T``, sigmoid + scores, and selection scores = scores + ``gate.e_score_correction_bias``, which only + choose. Each of the 8 groups of 32 experts is scored by the sum of its two largest + selection scores; the 4 best groups are kept, every expert outside them is excluded, + and the 8 best remaining selection scores are chosen in descending order. The mixing + weights are the chosen experts' unbiased scores over their sum plus 1e-20. +* Each selected expert, in ascending id order, runs its bias-free bf16 SwiGLU on its + tokens' rows in ascending token order: bf16 gate and up projections, fp32 SiLU and + product, one bf16 rounding, bf16 down projection. +* Each expert output times its fp32 weight is rounded to bf16, the routes are summed per + token in fp32, and the sum is scaled by the routed scaling factor 2.5 once and rounded + to bf16. + +Only torch is imported. Group exclusion is written as group membership; the two +selections stay ``topk(..., sorted=True)``, the source's own calls, because the order of +tied scores is whatever that call returns. + +State and inputs are generated on the GPU. Every tensor draws from a generator seeded +with ``crc32(f"{seed}|{name}")``, so its values do not depend on which other tensors were +generated or in which order. A device generator's draws depend on the device and the +torch build: the bytes are reproducible there, not across them. +""" + +from __future__ import annotations + +import contextlib +import math +import zlib +from collections import OrderedDict +from typing import Callable, Iterator + +import torch +import torch.nn.functional as F + +HIDDEN = 5120 +INTERMEDIATE = 2560 +NUM_EXPERTS = 256 +N_GROUP = 8 +PER_GROUP = NUM_EXPERTS // N_GROUP +TOPK_GROUP = 4 +TOP_K = 8 +NORM_EPS = 1e-20 +ROUTED_SCALING_FACTOR = 2.5 +MAX_TOKENS = 129 +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) + +#: Each expert's three tensors by name relative to ``experts..``, [out, in]. +EXPERT_SHAPES = OrderedDict( + [ + ("gate_proj.weight", (INTERMEDIATE, HIDDEN)), + ("up_proj.weight", (INTERMEDIATE, HIDDEN)), + ("down_proj.weight", (HIDDEN, INTERMEDIATE)), + ] +) + +STATE_RECIPE = { + "seeding": "one torch.Generator on the device per call, re-seeded for every tensor with " + "zlib.crc32(f'{seed}|{name}')", + "gate.weight": "fp32 N(0, 1) / sqrt(5120)", + "gate.e_score_correction_bias": "fp32 0.05 * N(0, 1)", + "experts..gate_proj.weight, experts..up_proj.weight": "N(0, 1) / sqrt(5120), drawn " + "in fp32 and rounded once to bf16", + "experts..down_proj.weight": "N(0, 1) / sqrt(2560), drawn in fp32 and rounded once to bf16", + "input": "N(0, 1) of shape [129, 5120] under the name 'input', drawn in fp32 and rounded " + "once to bf16; a T-token case uses the first T rows", +} + +#: ``record(name, value)``: receives the named intermediates of a call. +Record = Callable[[str, object], None] + + +# ----------------------------------------------------------------------------- numerics + + +@contextlib.contextmanager +def reference_math() -> Iterator[None]: + """IEEE fp32 GEMMs (TF32 off), bf16 GEMMs without reduced-precision reductions and + autocast off for the duration; the caller's settings are restored afterwards.""" + matmul = torch.backends.cuda.matmul + saved_tf32 = matmul.allow_tf32 + saved_precision = getattr(matmul, "fp32_precision", None) + saved_bf16 = matmul.allow_bf16_reduced_precision_reduction + matmul.allow_tf32 = False + if saved_precision is not None: + # Both switches: with only the legacy one moved, fp32_precision stays "tf32". + matmul.fp32_precision = "ieee" + matmul.allow_bf16_reduced_precision_reduction = False + try: + with torch.autocast("cuda", enabled=False): + yield + finally: + # The legacy switch first: its setter also moves the newer one restored after it. + matmul.allow_tf32 = saved_tf32 + if saved_precision is not None: + matmul.fp32_precision = saved_precision + matmul.allow_bf16_reduced_precision_reduction = saved_bf16 + + +# ----------------------------------------------------------------------------- state and inputs + + +def tensor_seed(seed: int, name: str) -> int: + """The generator seed of the tensor called ``name`` in case ``seed``.""" + return zlib.crc32(f"{seed}|{name}".encode()) + + +def param_shapes() -> "OrderedDict[str, tuple[tuple[int, ...], torch.dtype]]": + """The MoE's 770 tensors by checkpoint name relative to ``mlp.``, in checkpoint order: + the router's fp32 weight and correction bias, then each expert's three bf16 matrices.""" + shapes = OrderedDict( + [ + ("gate.weight", ((NUM_EXPERTS, HIDDEN), torch.float32)), + ("gate.e_score_correction_bias", ((NUM_EXPERTS,), torch.float32)), + ] + ) + for expert_id in range(NUM_EXPERTS): + for name, shape in EXPERT_SHAPES.items(): + shapes[f"experts.{expert_id}.{name}"] = (shape, torch.bfloat16) + return shapes + + +def _draw( + gen: torch.Generator, seed: int, name: str, shape: tuple[int, ...], device +) -> torch.Tensor: + """Standard normal fp32 values of ``shape`` from ``gen`` re-seeded for ``name``.""" + gen.manual_seed(tensor_seed(seed, name)) + return torch.randn(shape, generator=gen, dtype=torch.float32, device=device) + + +def generate_state( + seed: int, device: str | torch.device = "cuda" +) -> "OrderedDict[str, torch.Tensor]": + """The seed's 770 tensors on ``device``, in checkpoint order (see `STATE_RECIPE`). + + Every matrix is scaled by 1 / sqrt(its input width), the correction bias by 0.05; the + expert matrices are rounded once to bf16, the router stays fp32. + """ + gen = torch.Generator(device=device) + state: "OrderedDict[str, torch.Tensor]" = OrderedDict() + for name, (shape, dtype) in param_shapes().items(): + scale = 0.05 if len(shape) == 1 else 1.0 / math.sqrt(shape[1]) + state[name] = (_draw(gen, seed, name, shape, device) * scale).to(dtype).contiguous() + return state + + +def generate_input( + seed: int, tokens: int = MAX_TOKENS, device: str | torch.device = "cuda" +) -> torch.Tensor: + """``[tokens, 5120]`` bf16: the first ``tokens`` rows of the seed's 129-row input.""" + if not 1 <= tokens <= MAX_TOKENS: + raise ValueError(f"tokens must be in 1..{MAX_TOKENS}, got {tokens}") + gen = torch.Generator(device=device) + rows = _draw(gen, seed, "input", (MAX_TOKENS, HIDDEN), device).to(torch.bfloat16) + return rows[:tokens] + + +def state_problems(state: "dict[str, torch.Tensor]", seed: int) -> list[str]: + """Exact accounting plus the properties the comparisons rely on: finite values, a bias + that varies across experts, and no two expert matrices alike.""" + problems = [] + shapes = param_shapes() + if list(state) != list(shapes): + problems.append("names or order differ from param_shapes()") + if len({tensor_seed(seed, name) for name in shapes}) != len(shapes): + problems.append("two tensors share a generator seed") + for name, (shape, dtype) in shapes.items(): + tensor = state.get(name) + if tensor is None: + continue + if tuple(tensor.shape) != shape or tensor.dtype != dtype: + found = f"{tuple(tensor.shape)} {tensor.dtype}" + problems.append(f"{name}: {found}, expected {shape} {dtype}") + elif not bool(torch.isfinite(tensor).all()): + problems.append(f"{name}: nonfinite") + if problems: + return problems + bias = state["gate.e_score_correction_bias"] + if bool((bias == bias[0]).all()): + problems.append("the correction bias does not vary across experts") + for width, names in ( + (HIDDEN, ("gate_proj.weight", "up_proj.weight")), + (INTERMEDIATE, ("down_proj.weight",)), + ): + first_rows = torch.stack( + [state[f"experts.{e}.{n}"][0].float() for e in range(NUM_EXPERTS) for n in names] + ) + if torch.unique(first_rows, dim=0).shape[0] != first_rows.shape[0]: + problems.append(f"two expert matrices of input width {width} share their first row") + return problems + + +# ----------------------------------------------------------------------------- the MoE + + +def route( + x: torch.Tensor, weight: torch.Tensor, correction_bias: torch.Tensor +) -> dict[str, torch.Tensor]: + """The source routing of ``x [T, 5120]`` bf16, every step fp32. + + Returns ``logits``, ``scores`` and ``selection`` ``[T, 256]``, ``group_scores`` + ``[T, 8]``, ``groups`` ``[T, 4]`` int64 (descending), ``masked`` ``[T, 256]`` (-inf + outside the kept groups), ``ids`` ``[T, 8]`` int64 (descending selection score) and + ``weights`` ``[T, 8]`` fp32. + """ + with reference_math(): + logits = F.linear(x.float(), weight.float()) + scores = torch.sigmoid(logits) + selection = scores + correction_bias.float() + tokens = selection.shape[0] + group_scores = selection.view(tokens, N_GROUP, PER_GROUP).topk(2, dim=-1).values.sum(dim=-1) + groups = group_scores.topk(TOPK_GROUP, dim=-1, sorted=True).indices + expert_group = torch.arange(NUM_EXPERTS, device=x.device) // PER_GROUP + kept = (expert_group.view(1, NUM_EXPERTS, 1) == groups.unsqueeze(1)).any(dim=-1) + masked = torch.where(kept, selection, torch.full_like(selection, float("-inf"))) + ids = masked.topk(TOP_K, dim=-1, sorted=True).indices + chosen = torch.gather(scores, -1, ids) + weights = chosen / (chosen.sum(dim=-1, keepdim=True) + NORM_EPS) + return dict( + logits=logits, + scores=scores, + selection=selection, + group_scores=group_scores, + groups=groups, + masked=masked, + ids=ids, + weights=weights, + ) + + +def expert( + rows: torch.Tensor, + gate_weight: torch.Tensor, + up_weight: torch.Tensor, + down_weight: torch.Tensor, +) -> dict[str, torch.Tensor]: + """One expert on ``rows [n, 5120]`` bf16, every step: ``gate``, ``up`` (bf16), + ``silu``, ``product`` (fp32), ``act`` (bf16, the one rounding) and ``y`` ``[n, 5120]`` + bf16.""" + with reference_math(): + gate = F.linear(rows, gate_weight) + up = F.linear(rows, up_weight) + silu = F.silu(gate.float()) + product = silu * up.float() + act = product.to(torch.bfloat16) + y = F.linear(act, down_weight) + return dict(gate=gate, up=up, silu=silu, product=product, act=act, y=y) + + +def moe( + x: torch.Tensor, state: "dict[str, torch.Tensor]", record: Record | None = None +) -> torch.Tensor: + """The source MoE of ``x [T, 5120]`` bf16; returns ``[T, 5120]`` bf16. + + ``state`` holds the 770 tensors by name relative to ``mlp.``. ``record``, when given, + receives ``routing`` (the dict of `route`), ``expert_calls`` (``[(expert_id, tokens)]`` + in call order), ``accumulator`` (the fp32 sum before scaling) and ``output``. + """ + assert x.dtype == torch.bfloat16 and x.dim() == 2 and x.shape[1] == HIDDEN, ( + x.dtype, + tuple(x.shape), + ) + routing = route(x, state["gate.weight"], state["gate.e_score_correction_bias"]) + ids, weights = routing["ids"], routing["weights"] + acc = torch.zeros(x.shape[0], HIDDEN, dtype=torch.float32, device=x.device) + calls = [] + for expert_id in sorted(set(ids.reshape(-1).tolist())): + # Row-major nonzero: the expert's tokens in ascending order, each once. + tokens, slots = (ids == expert_id).nonzero(as_tuple=True) + matrices = [state[f"experts.{expert_id}.{name}"] for name in EXPERT_SHAPES] + y = expert(x[tokens], *matrices)["y"] + routed = (y.float() * weights[tokens, slots].unsqueeze(-1)).to(torch.bfloat16) + acc[tokens] += routed.float() + calls.append((expert_id, tokens)) + output = (acc * ROUTED_SCALING_FACTOR).to(torch.bfloat16) + if record is not None: + record("routing", routing) + record("expert_calls", calls) + record("accumulator", acc) + record("output", output) + return output diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py new file mode 100644 index 000000000000..646ebc604639 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -0,0 +1,871 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam MoE component of decoder block 3: bf16 / sm_103 / tp1. + +Configuration under test, fixed below and nowhere else: block 3's MoE at hidden 5120 -- +256 experts of intermediate 2560 in 8 groups of 32, 4 groups and 8 experts per token, +routed scaling factor 2.5; an fp32 router and bf16 experts with the source model's fp32 +steps; one sm_103 GPU; eager, one call per case on already-normalized bf16 hidden +states; generated weights; seeds {17, 41} x lengths {1, 17, 129}, and designed fixtures. + +What is compared. ``_moe_reference.py`` restates the source MoE in plain torch and +generates its state and input deterministically on the GPU. For every case the forward's +own routing must select exactly the reference's groups and experts, in order, and its +router steps and its output are held to the rules in `RULES`. The forward's op and +expert calls, and fixtures whose routing is fixed by design, show that each token +reaches its own experts and that expert ``e`` is the one loaded under ``experts.``. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ + python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/moe + +Optional environment: + +* ``BEAM_MOE_EVIDENCE``: a directory that receives one JSON record per compared case: + every comparison with its rule, the op and expert calls where they were recorded, and + the device and software it ran on; and ``fp32-session.json``, the fp32 GEMM checks made + before and after the module's tests. + +Dependencies: a TensorRT-LLM build for sm_103 and about 60 GiB of free device memory: +one seed's generated state and the loaded MoE hold 18.75 GiB of expert weights each, and +one case builds a second, unloaded MoE. No model files. +""" + +from __future__ import annotations + +import functools +import gc +import types +from collections import OrderedDict +from pathlib import Path + +import pytest +import torch + +try: + import tensorrt_llm._torch.models.modeling_beam as builtin_beam +except ModuleNotFoundError as exc: + if exc.name != "tensorrt_llm._torch.models.modeling_beam": + raise + builtin_beam = None # The standard-serving implementation is a separate PR. +__extra_import_path__ = [".", "../..", "../../../.."] + +import _beam_component_support as support # noqa: E402 +import _moe_reference as reference # noqa: E402 +from _beam_component_support import ( # noqa: E402 + assert_all_pass, + bitwise_equal, + compare_exact, + tensor_sha256, + tensor_versions, + unchanged, +) +from _validating import validating # noqa: E402 + +from _components.moe import moe as moe_module +from _components.moe.moe import BeamMoeBf16Sm103Tp1 +from _components.moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 # noqa: E501 +from _components.moe_router.router import BeamMoeRouterBf16Sm103Tp1 # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch._experimental.modeling_v2.models.beam import dispatch_combine # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 + swiglu_fp32, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 + beam_dispatch_combine, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 + beam_grouped_topk_router, +) + +assert torch.cuda.is_available(), "the Beam MoE tests require a CUDA device" + +pytestmark = pytest.mark.high_cuda_memory + +# -------------------------------------------------- the fixed configuration + +#: Block 3's MoE as the checkpoint configures it. Only these fields are read. +MOE_CONFIG = types.MappingProxyType( + dict( + hidden_size=5120, + moe_intermediate_size=2560, + num_experts=256, + num_experts_per_tok=8, + n_group=8, + topk_group=4, + norm_topk_prob=True, + routed_scaling_factor=2.5, + hidden_act="silu", + mlp_layer_types=("dense", "dense", "dense", "sparse"), + ) +) +LAYER_IDX = 3 +DTYPE = torch.bfloat16 +SM = (10, 3) +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) +CASES = tuple((seed, length) for seed in SEEDS for length in LENGTHS) +CASE_IDS = [f"seed{seed}-T{length}" for seed, length in CASES] +#: Eager, one GPU, one call per case on already-normalized hidden states. +EXECUTION = types.MappingProxyType(dict(eager=True, cuda_graph=False, devices=1)) + +# -------------------------------------------------- comparison rules + +_RTOL = 2.0**-7 + +#: (atol, rtol, dtype) per compared tensor. An element passes when +#: |actual - reference| <= atol + rtol * |reference|, both widened to fp64; every element +#: of every compared tensor must pass, and no NaN or Inf is accepted. rtol 2^-7 is one bf16 +#: unit at any magnitude. Routing is never excused by a tolerance: the groups and the +#: expert ids must equal the reference's exactly, in order. +RULES = types.MappingProxyType( + { + "moe_output": (0.03, _RTOL, torch.bfloat16), + "router_logits": (2e-5, 0.0, torch.float32), + "router_scores": (3e-6, 0.0, torch.float32), + "router_selection_scores": (3e-6, 0.0, torch.float32), + "router_weights": (8e-7, 0.0, torch.float32), + } +) +#: Per token, |sum(mixing weights) - 1| may not exceed this and no weight may reach +#: `MAX_WEIGHT`: the routed scaling factor 2.5 folded into the weights breaks both. +WEIGHT_SUM_ATOL = 1e-6 +MAX_WEIGHT = 1.5 + +#: Every catalog entry the MoE's forward can call. +OPS = (beam_grouped_topk_router, cublas_mm, swiglu_fp32, beam_dispatch_combine) +#: What each selected expert runs, in order, inside BeamDispatchCombine. +EXPERT_CALLS = ("expert", "CublasMm", "CublasMm", "SwigluFp32", "CublasMm") +#: Keyword arguments of the entries' calls that identify the path. +_RECORDED_KWARGS = ("n_group", "topk_group", "top_k", "eps", "routed_scaling_factor") + +# -------------------------------------------------- designed routing + +#: Two sets of eight experts, the first and last expert of four groups each: A holds groups +#: 0, 1, 4 and 7, both ends of the id range included; B holds groups 2, 3, 5 and 6. +SET_A = (0, 31, 32, 63, 128, 159, 224, 255) +SET_B = (64, 95, 96, 127, 160, 191, 192, 223) +#: Per designed token, the rank of each member of its set, in the set's order. Rank r is the +#: logit -1 + r / 2; every other expert's logit is -2. +RANKS = ( + (0, 1, 2, 3, 4, 5, 6, 7), + (7, 6, 5, 4, 3, 2, 1, 0), + (3, 7, 1, 5, 0, 4, 2, 6), + (6, 2, 4, 0, 5, 1, 7, 3), +) +#: The hidden column that carries each designed token's logits, per set. +COLUMNS_A = tuple(5 + t * (5120 // 4) for t in range(len(RANKS))) +COLUMNS_B = tuple(5 + 5120 // 8 + t * (5120 // 4) for t in range(len(RANKS))) +SPIKE = 16.0 +OTHER_LOGIT = -2.0 +FIXTURE_SEED = 17 + + +def designed_logit(rank: int) -> float: + return -1.0 + 0.5 * rank + + +def moe_config(**overrides) -> types.SimpleNamespace: + fields = dict(MOE_CONFIG) + fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) + fields.update(overrides) + return types.SimpleNamespace(**fields) + + +# -------------------------------------------------- precision and environment + + +@pytest.fixture(scope="module", autouse=True) +def ieee_fp32(): + """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both + before the module's tests and after them.""" + yield from support.ieee_fp32_session(_write_evidence) + + +@pytest.fixture(scope="module", autouse=True) +def device_memory(): + """Hand the module's cached state and MoE back to the device when its tests are done.""" + yield + release_cache() + + +def _environment(ieee: dict) -> dict: + return support.environment(ieee, EXECUTION) + + +_write_evidence = functools.partial(support.write_evidence, "BEAM_MOE_EVIDENCE") + + +# -------------------------------------------------- comparison + + +def compare(name: str, actual: torch.Tensor, expected: torch.Tensor, rule: str | None) -> dict: + """Score one tensor against one rule; `rule=None` measures without judging.""" + if rule is None: + return support.compare(name, actual, expected, None, None, expected.dtype) + atol, rtol, dtype = RULES[rule] + return support.compare(name, actual, expected, rule, (atol, rtol), dtype) + + +def compare_routes(route: dict, ref: dict) -> list[dict]: + """The router op's mapping of one call against the reference route of the same input.""" + results = [ + compare_exact("groups", route["groups"], ref["groups"]), + compare_exact("ids", route["ids"], ref["ids"]), + compare("router_logits", route["logits"], ref["logits"], "router_logits"), + compare("router_scores", route["scores"], ref["scores"], "router_scores"), + compare( + "router_selection_scores", + route["selection_scores"], + ref["selection"], + "router_selection_scores", + ), + compare("router_group_scores", route["group_scores"], ref["group_scores"], None), + compare("router_weights", route["weights"], ref["weights"], "router_weights"), + ] + same = torch.equal(torch.isinf(route["masked_selection_scores"]), torch.isinf(ref["masked"])) + results.append( + { + "tensor": "excluded_experts", + "rule": "exactly the reference's -inf exclusions", + "passed": same, + "problems": [] if same else ["the excluded experts differ"], + } + ) + return results + + +def weight_invariants(route: dict) -> dict: + """Mixing weights sum to one per token and carry no routed scaling factor.""" + weights = route["weights"] + deviation = float((weights.double().sum(dim=-1) - 1.0).abs().max()) + largest = float(weights.max()) + problems = [] + if deviation > WEIGHT_SUM_ATOL: + problems.append(f"a token's weights sum to 1 +- {deviation:.3g}") + if largest >= MAX_WEIGHT: + problems.append(f"a weight of {largest:.4g}") + return { + "tensor": "mixing_weights", + "rule": f"|sum - 1| <= {WEIGHT_SUM_ATOL:g} per token, every weight < {MAX_WEIGHT}", + "max_abs_sum_minus_one": deviation, + "max_weight": largest, + "passed": not problems, + "problems": problems, + } + + +def output_contract(out, tokens: int) -> dict: + return support.output_contract(out, (tokens, 5120), DTYPE, SM) + + +# -------------------------------------------------- state, inputs and the loaded MoE + +_STATE_CACHE: dict[int, dict] = {} + + +def release_cache() -> None: + """Drop the cached state and MoE and hand their memory back to the device.""" + _STATE_CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + + +def generated(seed: int) -> dict: + """The seed's state and [129, 5120] input, generated once on the GPU.""" + if seed not in _STATE_CACHE: + release_cache() # one seed's 18.75 GiB of expert weights, and its MoE, at a time + state = reference.generate_state(seed) + problems = reference.state_problems(state, seed) + assert not problems, problems + _STATE_CACHE[seed] = { + "state": state, + "input": reference.generate_input(seed), + "designed_state": None, + "moe": None, + "router": None, + } + return _STATE_CACHE[seed] + + +def case_input(seed: int, length: int) -> torch.Tensor: + return generated(seed)["input"][:length] + + +def designed_router() -> tuple[torch.Tensor, torch.Tensor]: + """fp32 router weight [256, 5120] and correction bias [256] for the designed rows. + + Noise of at most 2^-11 everywhere but the eight designed columns. Designed token t's + column holds (-1 + rank / 2) / 16 for each member of its set and -2 / 16 for every + other expert, so a row with 16 there and zero at every other designed column has the + designed logits plus noise of standard deviation about 0.02, far inside the 0.5 between + ranks. The bias, at most 2^-7, moves no selection. + """ + gen = torch.Generator(device="cuda") + gen.manual_seed(reference.tensor_seed(FIXTURE_SEED, "designed.gate.weight")) + weight = (2.0 * torch.rand(256, 5120, generator=gen, device="cuda") - 1.0) * 2.0**-11 + for columns, members in ((COLUMNS_A, SET_A), (COLUMNS_B, SET_B)): + for token, column in enumerate(columns): + weight[:, column] = OTHER_LOGIT / SPIKE + for expert_id, rank in zip(members, RANKS[token]): + weight[expert_id, column] = designed_logit(rank) / SPIKE + gen.manual_seed(reference.tensor_seed(FIXTURE_SEED, "designed.gate.e_score_correction_bias")) + bias = (2.0 * torch.rand(256, generator=gen, device="cuda") - 1.0) * 2.0**-7 + return weight.contiguous(), bias.contiguous() + + +def designed_rows(columns: tuple[int, ...], name: str) -> torch.Tensor: + """[4, 5120] bf16: unit normal noise, zero at every designed column but row t's own + ``columns[t]``, which holds 16.""" + gen = torch.Generator(device="cuda") + gen.manual_seed(reference.tensor_seed(FIXTURE_SEED, name)) + rows = torch.randn(len(columns), 5120, generator=gen, device="cuda") + rows[:, list(COLUMNS_A + COLUMNS_B)] = 0.0 + rows[torch.arange(len(columns)), list(columns)] = SPIKE + return rows.to(DTYPE) + + +def designed_ids(members: tuple[int, ...], token: int) -> list[int]: + """The members by descending designed logit: the routing the design forces on a token.""" + return [expert_id for _, expert_id in sorted(zip(RANKS[token], members), reverse=True)] + + +def designed_state(seed: int) -> "OrderedDict[str, torch.Tensor]": + """The seed's generated experts with the designed router.""" + entry = generated(seed) + if entry["designed_state"] is None: + weight, bias = designed_router() + router = {"gate.weight": weight, "gate.e_score_correction_bias": bias} + entry["designed_state"] = OrderedDict(entry["state"], **router) + return entry["designed_state"] + + +def loaded_moe(seed: int, router: str = "generated") -> BeamMoeBf16Sm103Tp1: + """The seed's cached MoE, loaded with the generated router or the "designed" one.""" + entry = generated(seed) + if entry["moe"] is None: + entry["moe"] = BeamMoeBf16Sm103Tp1(moe_config(), LAYER_IDX, device="cuda") + if entry["router"] != router: + entry["router"] = None # unknown until the load completes + entry["moe"].load_weights(entry["state"] if router == "generated" else designed_state(seed)) + entry["router"] = router + return entry["moe"] + + +def cheap_weights() -> "OrderedDict[str, torch.Tensor]": + """A valid 770-tensor mapping that allocates nothing (see support.cheap_weights).""" + return support.cheap_weights(reference.param_shapes()) + + +# -------------------------------------------------- op-path instrumentation + + +def recording(*wrappers, experts: bool = False): + """Log every call of these entries, in order, with its nesting depth, the arguments that + identify the path, its positional tensors' data pointers and its output; with + ``experts``, also every expert the MoE asks for and the rows it passes.""" + return support.recording( + *wrappers, + recorded_kwargs=_RECORDED_KWARGS, + experts=BeamMoeBf16Sm103Tp1 if experts else None, + data_ptrs=True, + ) + + +def loggable(calls: list[dict]) -> list[dict]: + """The recorded calls without their tensors, for the evidence record.""" + return support.loggable(calls, ("op", "depth", "shapes", "expert_id", *_RECORDED_KWARGS)) + + +def builtin_beam_forbidden(): + """The built-in Beam modeling must not run while the component does.""" + if builtin_beam is None: + return support.builtin_forbidden([]) + return support.builtin_forbidden( + [ + (builtin_beam.BeamMoE, "forward"), + (builtin_beam.BeamMoE, "combine"), + (builtin_beam.BeamRouter, "forward"), + (builtin_beam.BeamRouter, "route"), + (builtin_beam.BeamSwiGLU, "forward"), + (builtin_beam.BeamRMSNorm, "forward"), + (builtin_beam.BeamAttention, "forward"), + (builtin_beam.BeamDecoderLayer, "forward"), + ] + ) + + +def no_dispatch(): + """Fail on any op call: rejected inputs must be rejected before a kernel runs.""" + return support.no_dispatch(OPS) + + +# -------------------------------------------------- tests: configuration and construction + + +def test_moe_constructs_alone() -> None: + """The MoE is built directly from its configuration, without the model around it: a + router and 256 experts holding 770 tensors, every expert matrix its own storage.""" + + def refuse(self, *args, **kwargs): + raise AssertionError(f"{type(self).__name__} constructed") + + with pytest.MonkeyPatch.context() as mp: + if builtin_beam is not None: + for cls in ( + builtin_beam.BeamForCausalLM, + builtin_beam.BeamModel, + builtin_beam.BeamDecoderLayer, + builtin_beam.BeamMoE, + builtin_beam.BeamRouter, + builtin_beam.BeamSwiGLU, + ): + mp.setattr(cls, "__init__", refuse) + moe = BeamMoeBf16Sm103Tp1(moe_config(), LAYER_IDX, device="cuda") + assert isinstance(moe.gate, BeamMoeRouterBf16Sm103Tp1) + assert len(moe.experts) == 256 + assert all(isinstance(expert, BeamMoeExpertBf16Sm103Tp1) for expert in moe.experts) + assert [name for name, _ in moe.named_children()] == ["gate", "experts"] + tensors = moe.checkpoint_tensors() + assert list(tensors) == list(reference.param_shapes()) + for name, (shape, dtype) in reference.param_shapes().items(): + t = tensors[name] + assert tuple(t.shape) == shape and t.dtype == dtype and t.is_cuda, name + assert tensors["gate.weight"] is moe.gate.weight + assert tensors["gate.e_score_correction_bias"] is moe.gate.e_score_correction_bias + for expert_id in (0, 1, 31, 32, 128, 255): + expert = moe.experts[expert_id] + for attr in ("gate_proj", "up_proj", "down_proj"): + assert tensors[f"experts.{expert_id}.{attr}.weight"] is getattr(expert, attr) + pointers = [t.data_ptr() for name, t in tensors.items() if name.startswith("experts.")] + assert len(pointers) == 768 and len(set(pointers)) == 768 + assert "modeling_beam" not in Path(moe_module.__file__).read_text() + + +# -------------------------------------------------- tests: declared limits + +_BAD_CONFIGS = [ + ("moe_intermediate_size", dict(moe_intermediate_size=1536), LAYER_IDX, "moe_intermediate"), + ("routed_scaling_factor", dict(routed_scaling_factor=1.0), LAYER_IDX, "routed_scaling"), + ("hidden_act", dict(hidden_act="gelu"), LAYER_IDX, "unsupported hidden_act"), + ("num_experts", dict(num_experts=128), LAYER_IDX, "unsupported num_experts"), + ("dense_block_3", dict(mlp_layer_types=["dense"] * 4), LAYER_IDX, "sparse"), + ("block_2", {}, 2, "only the routers of sparse blocks"), +] + + +@pytest.mark.parametrize( + "case,overrides,layer_idx,match", _BAD_CONFIGS, ids=[c[0] for c in _BAD_CONFIGS] +) +def test_rejects_unsupported_configuration(case, overrides, layer_idx, match) -> None: + """Anything outside the one configuration is refused at construction.""" + with pytest.raises(ValueError, match=match): + BeamMoeBf16Sm103Tp1(moe_config(**overrides), layer_idx, device="cuda") + + +def test_rejects_a_host_device() -> None: + with pytest.raises(ValueError, match="sm_103"): + BeamMoeBf16Sm103Tp1(moe_config(), LAYER_IDX, device="cpu") + + +# -------------------------------------------------- tests: loading + + +def test_load_weights_maps_every_expert() -> None: + """Every checkpoint name lands in its own tensor, exactly: expert e's matrices in + ``experts[e]`` for all 256, the router's fp32 weight in ``gate.weight`` and the + selection bias in the router's ``e_score_correction_bias`` buffer.""" + seed = 17 + state = generated(seed)["state"] + moe = loaded_moe(seed) + loaded = moe.checkpoint_tensors() + assert list(loaded) == list(state) == list(reference.param_shapes()) + wrong = [] + for name, tensor in loaded.items(): + given = state[name] + if tensor.dtype != given.dtype or tensor.shape != given.shape: + wrong.append(f"{name}: {tuple(tensor.shape)} {tensor.dtype}") + elif not torch.equal(tensor, given): + wrong.append(f"{name}: values differ") + assert tensor.data_ptr() != given.data_ptr(), f"{name} aliases the given tensor" + assert not wrong, wrong[:8] + for expert_id, expert in enumerate(moe.experts): + for attr in ("gate_proj", "up_proj", "down_proj"): + given = state[f"experts.{expert_id}.{attr}.weight"] + assert torch.equal(getattr(expert, attr), given), (expert_id, attr) + router = moe.gate + assert router.weight.dtype == torch.float32 + assert torch.equal(router.weight, state["gate.weight"]) + assert [name for name, _ in router.named_buffers()] == ["e_score_correction_bias"] + assert [name for name, _ in router.named_parameters()] == ["weight"] + bias = router.e_score_correction_bias + assert bias.dtype == torch.float32 + assert torch.equal(bias, state["gate.e_score_correction_bias"]) + + +_BAD_LOADS = [ + ("last_down_fp32", TypeError, r"experts\.255\.down_proj\.weight must be torch\.bfloat16"), + ("last_gate_transposed", ValueError, r"experts\.255\.gate_proj\.weight must be \[2560, 5120\]"), + ("router_weight_bf16", TypeError, r"gate\.weight must be torch\.float32"), + ("missing_bias", ValueError, r"missing \['gate\.e_score_correction_bias'\]"), + ("unexpected_name", ValueError, r"unexpected \['experts\.256\.gate_proj\.weight'\]"), +] + + +@pytest.mark.parametrize("case,error,match", _BAD_LOADS, ids=[c for c, _, _ in _BAD_LOADS]) +def test_rejected_load_changes_nothing(case, error, match) -> None: + """Every tensor is checked before any is copied: a rejected load writes no parameter or + buffer, leaves every value as it was, and the MoE still runs.""" + seed = 17 + entry = generated(seed) + moe = loaded_moe(seed) + bad = cheap_weights() + if case == "last_down_fp32": + bad["experts.255.down_proj.weight"] = torch.zeros((), device="cuda").expand(5120, 2560) + elif case == "last_gate_transposed": + bad["experts.255.gate_proj.weight"] = bad["experts.255.gate_proj.weight"].t() + elif case == "router_weight_bf16": + bad["gate.weight"] = bad["gate.weight"].to(DTYPE) + elif case == "missing_bias": + del bad["gate.e_score_correction_bias"] + elif case == "unexpected_name": + bad["experts.256.gate_proj.weight"] = bad["experts.255.gate_proj.weight"] + names = ("gate.weight", "gate.e_score_correction_bias") + tuple( + f"experts.255.{n}" for n in reference.EXPERT_SHAPES + ) + snapshots = {name: moe.checkpoint_tensors()[name].clone() for name in names} + versions = tensor_versions(moe) + try: + with no_dispatch(), pytest.raises(error, match=match): + moe.load_weights(bad) + finally: + written = sorted(n for n, v in tensor_versions(moe).items() if v != versions[n]) + if written: + entry["router"] = None # reload the generated state before the MoE is used again + current = moe.checkpoint_tensors() + changed = [name for name, value in snapshots.items() if not torch.equal(current[name], value)] + assert not written, f"{len(written)} tensors written by a rejected load: {written[:8]}" + assert not changed, changed + out = moe(case_input(seed, 1)) + assert out.shape == (1, 5120) and bool(torch.isfinite(out).all()) + + +_BAD_INPUTS = [ + ("fp32_input", TypeError, "must be torch.bfloat16"), + ("fp16_input", TypeError, "must be torch.bfloat16"), + ("batched_3d", ValueError, r"must be \[tokens, 5120\]"), + ("width_5119", ValueError, r"must be \[tokens, 5120\]"), + ("zero_tokens", ValueError, "at least one token"), + ("host_input", ValueError, "MoE's device"), + ("not_a_tensor", TypeError, "must be a torch.Tensor"), + ("not_loaded", RuntimeError, "load_weights must be called"), +] + + +@pytest.mark.parametrize("case,error,match", _BAD_INPUTS, ids=[c for c, _, _ in _BAD_INPUTS]) +def test_rejects_unsupported_inputs_before_dispatch(case, error, match) -> None: + """Each declared restriction raises before any op runs and before any expert is asked + for.""" + if case == "not_loaded": + moe = BeamMoeBf16Sm103Tp1(moe_config(), LAYER_IDX, device="cuda") + x = torch.randn(17, 5120, device="cuda").to(DTYPE) + else: + moe = loaded_moe(17) + x = case_input(17, 17) + bad = { + "fp32_input": lambda: x.float(), + "fp16_input": lambda: x.half(), + "batched_3d": lambda: x.unsqueeze(0), + "width_5119": lambda: x[:, :5119], + "zero_tokens": lambda: x[:0], + "host_input": lambda: x.cpu(), + "not_a_tensor": lambda: x.tolist(), + "not_loaded": lambda: x, + }[case]() + with recording(experts=True) as asked, no_dispatch(), pytest.raises(error, match=match): + moe(bad) + assert asked == [], loggable(asked) + + +# -------------------------------------------------- tests: the six cases + + +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_moe_matches_reference(seed, length, ieee_fp32) -> None: + """One case: the forward's own routing (the router op's mapping of that call) selects + exactly the reference's groups and experts in order, its fp32 steps and the output are + within their rules, and the call leaves its input and the parameters untouched.""" + moe = loaded_moe(seed) + state = generated(seed)["state"] + x = case_input(seed, length) + x_before = x.clone() + versions = tensor_versions(moe) + with recording(beam_grouped_topk_router) as calls: + out = moe(x) + torch.cuda.synchronize() + after = tensor_versions(moe) + (router_call,) = calls + route = router_call["output"] + ref_route = reference.route(x, state["gate.weight"], state["gate.e_score_correction_bias"]) + ref_out = reference.moe(x, state) + results = [output_contract(out, length)] + results.append(compare("moe_output", out, ref_out, "moe_output")) + results.extend(compare_routes(route, ref_route)) + results.append(weight_invariants(route)) + results.append(unchanged(x, x_before, versions, after)) + _write_evidence( + f"moe-seed{seed}-T{length}", + { + "test": "test_moe_matches_reference", + "case": {"seed": seed, "length": length}, + "rules": {k: [v[0], v[1], str(v[2])] for k, v in RULES.items()}, + "results": results, + "selected_experts": sorted(set(route["ids"].reshape(-1).tolist())), + "input_sha256": tensor_sha256(x), + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(results) + + +# -------------------------------------------------- tests: the component path + + +def test_forward_runs_the_component_path(ieee_fp32) -> None: + """One case: the router op once, then BeamDispatchCombine, inside which every selected + expert -- ascending, once each -- receives exactly its tokens' rows in token order and + runs its own three projections and SwiGLU; every op's guard admits its arguments and no + built-in Beam module runs.""" + seed, length = 17, 17 + moe = loaded_moe(seed) + x = case_input(seed, length) + with builtin_beam_forbidden(), validating(*OPS), recording(*OPS, experts=True) as calls: + out = moe(x) + torch.cuda.synchronize() + _write_evidence( + f"path-seed{seed}-T{length}", + { + "test": "test_forward_runs_the_component_path", + "case": {"seed": seed, "length": length}, + "moe": f"{type(moe).__module__}.{type(moe).__qualname__}", + "op_calls": loggable(calls), + "environment": _environment(ieee_fp32), + }, + ) + + assert type(moe).__module__.endswith("moe.moe") + assert [c["op"] for c in calls[:2]] == ["BeamGroupedTopkRouter", "BeamDispatchCombine"] + router, dispatch = calls[:2] + assert router["depth"] == dispatch["depth"] == 0 + assert {k: router[k] for k in ("n_group", "topk_group", "top_k", "eps")} == { + "n_group": 8, + "topk_group": 4, + "top_k": 8, + "eps": 1e-20, + } + assert router["shapes"] == [[length, 5120], [256, 5120], [256]] + assert dispatch["shapes"] == [[length, 5120], [length, 8], [length, 8]] + assert dispatch["routed_scaling_factor"] == 2.5 + assert dispatch["output"] is out + # Everything after the dispatch entry ran inside it. + nested = calls[2:] + assert all(c["depth"] == 1 for c in nested), loggable(nested) + ids = router["output"]["ids"] + selected = sorted(set(ids.reshape(-1).tolist())) + assert [c["op"] for c in nested] == list(EXPERT_CALLS) * len(selected), loggable(nested) + rows_total = 0 + for i, expert_id in enumerate(selected): + asked, gate, up, act, down = nested[len(EXPERT_CALLS) * i : len(EXPERT_CALLS) * (i + 1)] + assert asked["expert_id"] == expert_id + tokens = (ids == expert_id).any(dim=-1).nonzero().squeeze(-1) + n = int(tokens.numel()) + assert n >= 1 and bitwise_equal(asked["rows"], x[tokens]), expert_id + assert gate["shapes"] == up["shapes"] == [[n, 5120], [5120, 2560]], expert_id + assert act["shapes"] == [[n, 2560], [n, 2560]], expert_id + assert down["shapes"] == [[n, 2560], [2560, 5120]], expert_id + # Expert e's own stored matrices, read in place, on the rows it was given. + expert = moe.experts[expert_id] + assert gate["data_ptrs"] == [asked["rows"].data_ptr(), expert.gate_proj.data_ptr()] + assert up["data_ptrs"] == [asked["rows"].data_ptr(), expert.up_proj.data_ptr()] + assert down["data_ptrs"][1] == expert.down_proj.data_ptr() + rows_total += n + assert rows_total == 8 * length + assert out.shape == (length, 5120) and out.dtype == DTYPE and out.is_cuda + assert torch.cuda.get_device_capability(out.device) == SM + + +# -------------------------------------------------- tests: designed fixtures + + +def test_expert_identity_fixture(ieee_fp32, monkeypatch) -> None: + """Designed logits route four tokens to the same eight experts -- both ends of the id + range and of four groups -- in four different orders. Every one of the eight gets all + four rows, the other 248 are never asked for, the output matches the reference, and + putting expert 0 in expert 255's place fails the same comparison.""" + seed = 17 + moe = loaded_moe(seed, router="designed") + state = designed_state(seed) + x = designed_rows(COLUMNS_A, "designed.rows_a") + ref_route = reference.route(x, state["gate.weight"], state["gate.e_score_correction_bias"]) + ref_out = reference.moe(x, state) + with recording(beam_grouped_topk_router, experts=True) as calls: + out = moe(x) + route = calls[0]["output"] + asked = [c for c in calls if c["op"] == "expert"] + designed = [designed_ids(SET_A, token) for token in range(len(RANKS))] + runs = {"unchanged": compare("moe_output", out, ref_out, "moe_output")} + # experts[255] = experts[0], as ModuleList item assignment sets it, undone below. + monkeypatch.setattr(moe.experts, "255", moe.experts[0]) + assert moe.experts[255] is moe.experts[0] + runs["expert_255_is_expert_0"] = compare("moe_output", moe(x), ref_out, "moe_output") + monkeypatch.undo() + assert moe.experts[255] is not moe.experts[0] + runs["restored"] = compare("moe_output", moe(x), ref_out, "moe_output") + _write_evidence( + "fixture-expert-identity", + { + "test": "test_expert_identity_fixture", + "designed_ids": designed, + "ids": route["ids"].tolist(), + "weights": route["weights"].tolist(), + "expert_calls": loggable(asked), + "runs": runs, + "environment": _environment(ieee_fp32), + }, + ) + assert route["ids"].tolist() == designed, route["ids"].tolist() + assert ref_route["ids"].tolist() == designed, ref_route["ids"].tolist() + assert torch.equal(route["groups"], ref_route["groups"]) + assert sorted(route["groups"][0].tolist()) == [0, 1, 4, 7] + assert [c["expert_id"] for c in asked] == sorted(SET_A) + for call in asked: + assert bitwise_equal(call["rows"], x), call["expert_id"] + assert runs["unchanged"]["passed"] is True, runs["unchanged"]["problems"] + faulted = runs["expert_255_is_expert_0"] + assert faulted["passed"] is False and faulted["violations"] > 0, faulted + assert runs["restored"]["passed"] is True, runs["restored"]["problems"] + + +def test_changed_routing_between_calls(ieee_fp32) -> None: + """Rows routed to expert set A, then rows routed to the disjoint set B, then A again: + B's call routes as designed and asks for no expert of A, and the repeated A call + reproduces the first one bit for bit -- nothing of one call's routing survives into + the next.""" + seed = 17 + moe = loaded_moe(seed, router="designed") + state = designed_state(seed) + rows_a = designed_rows(COLUMNS_A, "designed.rows_a") + rows_b = designed_rows(COLUMNS_B, "designed.rows_b") + runs = [] + for x in (rows_a, rows_b, rows_a): + with recording(beam_grouped_topk_router, experts=True) as calls: + out = moe(x) + runs.append( + { + "out": out, + "ids": calls[0]["output"]["ids"], + "experts": [c["expert_id"] for c in calls if c["op"] == "expert"], + "vs_reference": compare("moe_output", out, reference.moe(x, state), "moe_output"), + } + ) + first, second, third = runs + _write_evidence( + "fixture-changed-routing", + { + "test": "test_changed_routing_between_calls", + "calls": [ + {"ids": r["ids"].tolist(), "experts": r["experts"], "result": r["vs_reference"]} + for r in runs + ], + "environment": _environment(ieee_fp32), + }, + ) + assert first["ids"].tolist() == [designed_ids(SET_A, t) for t in range(len(RANKS))] + assert second["ids"].tolist() == [designed_ids(SET_B, t) for t in range(len(RANKS))] + assert first["experts"] == third["experts"] == sorted(SET_A) + assert second["experts"] == sorted(SET_B) + assert not set(second["experts"]) & set(SET_A) + assert torch.equal(third["ids"], first["ids"]) + assert bitwise_equal(third["out"], first["out"]) + for run in runs: + assert run["vs_reference"]["passed"] is True, run["vs_reference"]["problems"] + + +def test_zero_input_gives_exact_zero() -> None: + """Zero hidden states: every score is 1/2, so every mixing weight is exactly 1/8, every + selected expert runs on zero rows, and the output is exactly zero -- no residual, bias + or other term reaches it.""" + moe = loaded_moe(17) + x = torch.zeros(17, 5120, dtype=DTYPE, device="cuda") + with recording(beam_grouped_topk_router, experts=True) as calls: + out = moe(x) + route = calls[0]["output"] + asked = [c for c in calls if c["op"] == "expert"] + assert out.shape == (17, 5120) and out.dtype == DTYPE + assert bool((out == 0).all()) + assert bool((route["scores"] == 0.5).all()) + assert bool((route["weights"] == 0.125).all()), route["weights"] + assert [c["expert_id"] for c in asked] == sorted(set(route["ids"].reshape(-1).tolist())) + + +def test_token_permutation_preserves_correspondence(ieee_fp32) -> None: + """Reversing the tokens reverses the routing and the output: each token keeps its own + experts and its own output row.""" + seed, length = 17, 17 + moe = loaded_moe(seed) + state = generated(seed)["state"] + x = case_input(seed, length) + perm = torch.arange(length - 1, -1, -1, device="cuda") + inverse = torch.argsort(perm) + outputs, routes = [], [] + for rows in (x, x[perm]): + with recording(beam_grouped_topk_router) as calls: + outputs.append(moe(rows)) + routes.append(calls[0]["output"]) + base, permuted = outputs + ref_out = reference.moe(x, state) + results = [ + compare_exact("ids_back_in_token_order", routes[1]["ids"][inverse], routes[0]["ids"]), + compare("output_back_in_token_order", permuted[inverse], base, "moe_output"), + compare("output_vs_reference", base, ref_out, "moe_output"), + compare("permuted_output_vs_reference", permuted, ref_out[perm], "moe_output"), + ] + _write_evidence( + f"permutation-seed{seed}-T{length}", + { + "test": "test_token_permutation_preserves_correspondence", + "results": results, + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(results) + + +def test_detects_routed_scaling_faults(ieee_fp32, monkeypatch) -> None: + """The output comparison rejects the combination scaled by 1.0 or by 2.5 twice, and + accepts the unchanged and the restored MoE, on the same case and the same rule.""" + seed, length = 17, 17 + moe = loaded_moe(seed) + x = case_input(seed, length) + ref_out = reference.moe(x, generated(seed)["state"]) + runs = {"unchanged": compare("moe_output", moe(x), ref_out, "moe_output")} + for factor in (1.0, 6.25): + monkeypatch.setattr(dispatch_combine, "ROUTED_SCALING_FACTOR", factor) + runs[f"scaled_by_{factor}"] = compare("moe_output", moe(x), ref_out, "moe_output") + monkeypatch.undo() + runs["restored"] = compare("moe_output", moe(x), ref_out, "moe_output") + _write_evidence( + f"fault-scaling-seed{seed}-T{length}", + { + "test": "test_detects_routed_scaling_faults", + "runs": runs, + "environment": _environment(ieee_fp32), + }, + ) + assert dispatch_combine.ROUTED_SCALING_FACTOR == 2.5 + assert runs["unchanged"]["passed"] is True, runs["unchanged"]["problems"] + for factor in (1.0, 6.25): + faulted = runs[f"scaled_by_{factor}"] + assert faulted["passed"] is False and faulted["violations"] > 0, factor + assert runs["restored"]["passed"] is True, runs["restored"]["problems"] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/_expert_cases.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/_expert_cases.py index 76d0d76333d9..dc98f739e4e6 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/_expert_cases.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/_expert_cases.py @@ -12,7 +12,6 @@ import contextlib import math -import os import torch import torch.nn.functional as F @@ -57,8 +56,6 @@ "y": (0.0, 0.0), } -_TF32_OVERRIDES = ("TORCH_ALLOW_TF32_CUBLAS_OVERRIDE", "NVIDIA_TF32_OVERRIDE") - # ----------------------------------------------------------------------------- numerics @@ -88,10 +85,6 @@ def numerical_state() -> dict: def assert_numerical_controls() -> None: - for var in _TF32_OVERRIDES: - assert var not in os.environ, ( - f"{var} substitutes TF32 for fp32 GEMMs; run with env -u {var}" - ) state = numerical_state() assert state == _CONTROLLED, f"numerical controls not in force: {state}" diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py new file mode 100644 index 000000000000..0e951239b020 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for one Beam MoE routed expert of block 3: bf16 / sm_103 / tp1. + +Configuration under test, fixed in ``_expert_cases.py``: hidden 5120, intermediate 2560, +bias-free BF16 weights [out, in], BF16 already-normalized input; one sm_103 GPU; eager; +generated parameters; seeds {17, 41} x tokens {1, 17, 129}, and exact fixtures. + +What is compared. Every call goes through the component's ``forward``. The three +``cublas_mm`` results and the activation op's mapping of that same call are captured, and +every step -- gate, up, fp32 SiLU, fp32 product, bf16 activation, output -- is compared +with the source expert written in plain torch under ``_expert_cases.MATRIX_RULES``, or +exactly on the fixtures built so that every step is known exactly. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ + python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert + +Dependencies: a TensorRT-LLM build for sm_103. No model files. +""" + +from __future__ import annotations + +import contextlib +import gc + +import pytest +import torch + +__extra_import_path__ = [".", "../..", "../../../.."] + +import _expert_cases as cases # noqa: E402 +from _beam_component_support import TorchCallTrace # noqa: E402 +from _validating import validating # noqa: E402 + +from _components.moe_expert.expert import WEIGHT_SHAPES, BeamMoeExpertBf16Sm103Tp1 # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import CublasMm, cublas_mm +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 + SwigluFp32, + swiglu_fp32, +) + +assert torch.cuda.is_available(), "the Beam MoE expert tests require a CUDA device" + +BF16 = torch.bfloat16 +_WEIGHT_BYTES = sum(2 * rows * cols for rows, cols in WEIGHT_SHAPES.values()) + + +@pytest.fixture(scope="module", autouse=True) +def controls(): + """IEEE fp32, no reduced-precision bf16 reduction and no autocast for the whole module.""" + with cases.numerical_controls(): + yield + + +# ----------------------------------------------------------------------------- observing a call + + +@contextlib.contextmanager +def observed_calls(): + """Every cublas_mm result and activation mapping made inside the block, unchanged.""" + seen = {"mm": [], "act": []} + mm_call, act_call = CublasMm.__call__, SwigluFp32.__call__ + + def mm(self, *args, **kwargs): + out = mm_call(self, *args, **kwargs) + seen["mm"].append((args, out)) + return out + + def act(self, *args, **kwargs): + out = act_call(self, *args, **kwargs) + seen["act"].append((args, out)) + return out + + CublasMm.__call__, SwigluFp32.__call__ = mm, act + try: + yield seen + finally: + CublasMm.__call__, SwigluFp32.__call__ = mm_call, act_call + + +def run(expert: BeamMoeExpertBf16Sm103Tp1, x: torch.Tensor) -> dict: + """Every step of one forward call, by name; ``y`` is what forward returned.""" + with validating(cublas_mm, swiglu_fp32), observed_calls() as seen: + y = expert(x) + assert len(seen["mm"]) == 3 and len(seen["act"]) == 1, seen + (gate_args, gate), (up_args, up), (down_args, out) = seen["mm"] + ((act_args, steps),) = seen["act"] + for args, weight in ( + (gate_args, expert.gate_proj), + (up_args, expert.up_proj), + (down_args, expert.down_proj), + ): + # The stored [out, in] weight itself, read column-major: no copy, no transpose kernel. + operand = args[1] + assert operand.data_ptr() == weight.data_ptr() and operand.stride() == (1, weight.shape[1]) + assert act_args[0] is gate and act_args[1] is up and down_args[0] is steps["act"] + assert out is y + return dict(gate=gate, up=up, **{k: steps[k] for k in ("silu", "product", "act")}, y=y) + + +def loaded(weights: dict) -> BeamMoeExpertBf16Sm103Tp1: + expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") + expert.load_weights(weights) + return expert + + +def bitwise_equal(a: torch.Tensor, b: torch.Tensor) -> bool: + return ( + a.dtype == b.dtype + and a.shape == b.shape + and torch.equal(a.view(torch.int16), b.view(torch.int16)) + ) + + +# ----------------------------------------------------------------------------- state + + +def _requested_cuda_bytes() -> int: + """Bytes the live tensors asked for; unlike memory_allocated, free of block rounding.""" + return torch.cuda.memory_stats()["requested_bytes.all.current"] + + +def test_construction_holds_one_expert_and_nothing_else(): + gc.collect() + torch.cuda.synchronize() + before = _requested_cuda_bytes() + expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") + torch.cuda.synchronize() + assert _requested_cuda_bytes() - before == _WEIGHT_BYTES == 78_643_200 + inventory = { + name: (tuple(p.shape), p.dtype, p.device.type, p.requires_grad) + for name, p in expert.named_parameters() + } + assert inventory == { + "gate_proj": ((2560, 5120), BF16, "cuda", False), + "up_proj": ((2560, 5120), BF16, "cuda", False), + "down_proj": ((5120, 2560), BF16, "cuda", False), + } + assert sum(p.numel() for p in expert.parameters()) == 39_321_600 + assert list(expert.buffers()) == [] and list(expert.children()) == [] + assert list(expert.checkpoint_tensors()) == list(WEIGHT_SHAPES) + + +def test_construction_rejects_a_host_device(): + with pytest.raises(ValueError, match="sm_103 CUDA devices"): + BeamMoeExpertBf16Sm103Tp1(device="cpu") + + +@pytest.mark.parametrize("source_device", ["cpu", "cuda"]) +def test_load_copies_exact_values(source_device): + weights = {k: v.to(source_device) for k, v in cases.matrix_case(17)[0].items()} + expert = loaded(weights) + for name, tensor in expert.checkpoint_tensors().items(): + assert tensor.device.type == "cuda" and tensor.dtype == BF16 + assert tuple(tensor.shape) == WEIGHT_SHAPES[name] + assert bitwise_equal(tensor.cpu(), weights[name].cpu()), name + assert tensor.data_ptr() != weights[name].data_ptr() + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("not_loaded", RuntimeError, "load_weights must be called"), + ("rank_1", ValueError, r"must be \[tokens, 5120\]"), + ("rank_3", ValueError, r"must be \[tokens, 5120\]"), + ("width_5119", ValueError, r"must be \[tokens, 5120\]"), + ("zero_tokens", ValueError, "at least one token"), + ("fp32", TypeError, "must be torch.bfloat16"), + ("fp16", TypeError, "must be torch.bfloat16"), + ("host", ValueError, "expert's device"), + ("not_a_tensor", TypeError, "must be a torch.Tensor"), + ], +) +def test_rejects_unsupported_inputs_before_any_kernel(case, error, match): + weights, x_all = cases.matrix_case(17) + expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") + if case != "not_loaded": + expert.load_weights(weights) + x = x_all[:17].cuda() + bad = { + "not_loaded": x, + "rank_1": x[0], + "rank_3": x.unsqueeze(0), + "width_5119": x[:, :5119], + "zero_tokens": x[:0], + "fp32": x.float(), + "fp16": x.half(), + "host": x.cpu(), + "not_a_tensor": x.tolist(), + }[case] + with observed_calls() as seen, pytest.raises(error, match=match): + expert(bad) + assert seen["mm"] == [] and seen["act"] == [] + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("down_fp32", TypeError, "down_proj.weight must be torch.bfloat16"), + ("down_transposed", ValueError, r"down_proj.weight must be \[5120, 2560\]"), + ("gate_fp16", TypeError, "gate_proj.weight must be torch.bfloat16"), + ("up_transposed", ValueError, r"up_proj.weight must be \[2560, 5120\]"), + ("missing_down", ValueError, r"missing \['down_proj.weight'\]"), + ("unexpected_bias", ValueError, r"unexpected \['down_proj.bias'\]"), + ("down_not_a_tensor", TypeError, "down_proj.weight must be a torch.Tensor"), + ("down_meta", ValueError, "dense host or CUDA tensor"), + ("not_a_mapping", TypeError, "weights must be a mapping"), + ], +) +def test_rejected_load_leaves_the_expert_unchanged(case, error, match): + """Every other tensor is a valid replacement; a bad down tensor is the last one checked.""" + weights, x_all = cases.matrix_case(17) + expert = loaded(weights) + x = x_all[:17].cuda() + before = {k: v.clone() for k, v in expert.checkpoint_tensors().items()} + y_before = expert(x) + valid = cases.matrix_case(41)[0] + gate, up, down = valid.values() + replacement = { + "down_fp32": dict(valid, **{"down_proj.weight": down.float()}), + "down_transposed": dict(valid, **{"down_proj.weight": down.reshape(2560, 5120)}), + "gate_fp16": dict(valid, **{"gate_proj.weight": gate.half()}), + "up_transposed": dict(valid, **{"up_proj.weight": up.reshape(5120, 2560)}), + "missing_down": {k: v for k, v in valid.items() if k != "down_proj.weight"}, + "unexpected_bias": dict(valid, **{"down_proj.bias": down[:, 0]}), + "down_not_a_tensor": dict(valid, **{"down_proj.weight": down.float().numpy()}), + "down_meta": dict(valid, **{"down_proj.weight": torch.empty_like(down, device="meta")}), + "not_a_mapping": list(valid.items()), + }[case] + with pytest.raises(error, match=match): + expert.load_weights(replacement) + for name, tensor in expert.checkpoint_tensors().items(): + assert bitwise_equal(tensor, before[name]), name + assert bitwise_equal(expert(x), y_before) + + +def test_rejected_first_load_leaves_the_expert_unloaded(): + weights = dict(cases.matrix_case(17)[0]) + weights["down_proj.weight"] = weights["down_proj.weight"].float() + expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") + with pytest.raises(TypeError): + expert.load_weights(weights) + with pytest.raises(RuntimeError, match="load_weights must be called"): + expert(torch.zeros(1, 5120, dtype=BF16, device="cuda")) + + +# ----------------------------------------------------------------------------- the matrix + + +@pytest.mark.parametrize("tokens", cases.TOKENS) +@pytest.mark.parametrize("seed", cases.SEEDS) +def test_matrix_every_step_matches_the_source_expert(seed, tokens): + weights, x_all = cases.matrix_case(seed) + expert = loaded(weights) + x = x_all[:tokens].cuda() + x_before = x.clone() + params_before = {k: v.clone() for k, v in expert.checkpoint_tensors().items()} + + steps = run(expert, x) + y = steps["y"] + assert y.shape == (tokens, 5120) and y.dtype == BF16 and y.device == x.device + assert bool(torch.isfinite(y).all()) + assert bitwise_equal(x, x_before) + for name, tensor in expert.checkpoint_tensors().items(): + assert bitwise_equal(tensor, params_before[name]), name + # Observing the call does not change it. + assert bitwise_equal(expert(x), y) + + gold = cases.golden(x, *(t.cuda() for t in weights.values())) + cases.assert_stages(steps, gold, cases.MATRIX_RULES, f"seed {seed}, {tokens} tokens") + + +# ----------------------------------------------------------------------------- fixtures + + +def test_zero_input_gives_exact_zero(): + expert = loaded(cases.matrix_case(17)[0]) + steps = run(expert, torch.zeros(17, 5120, dtype=BF16, device="cuda")) + for stage, tensor in steps.items(): + assert tensor.dtype == cases.STAGE_DTYPES[stage] + assert bool((tensor == 0).all()), stage + + +def test_signed_gates_exercise_silu_exactly(): + weights, x, expected = cases.signed_gate_fixture() + gates = expected["gate"].double() + assert int((gates < 0).sum()) >= 2000 and int((gates > 0).sum()) >= 2000 + assert int((gates == 0).sum()) > 0 and int((gates.abs() >= 24).sum()) > 0 + steps = run(loaded(weights), x.cuda()) + cases.assert_stages(steps, expected, cases.EXACT_RULES, "signed gates") + # SiLU is not linear on these gates: it is neither the gate nor half of it. + silu = steps["silu"].double().cpu() + assert not torch.equal(silu, gates) and not torch.equal(silu, gates / 2) + + +def test_precision_fixture_tells_fp32_activation_from_bf16_steps(): + weights, x, expected, premature_act = cases.precision_fixture() + steps = run(loaded(weights), x.cuda()) + cases.assert_stages(steps, expected, cases.EXACT_RULES, "precision fixture") + # Every element of the exact activation differs from the twice-rounded one. + assert bool((steps["act"].cpu().double() != premature_act.double()).all()) + + +def test_sparse_mapping_fixes_orientation_and_feature_maps(): + weights, x, expected = cases.sparse_mapping_fixture() + steps = run(loaded(weights), x.cuda()) + cases.assert_stages(steps, expected, cases.EXACT_RULES, "sparse mapping") + # The fixture separates the maps it is meant to check: each wrong reading moves y. + gate_w, up_w, down_w = (t.cuda() for t in weights.values()) + xd = x.cuda() + wrong = { + "gate/up swapped": cases.golden(xd, up_w, gate_w, down_w), + "gate read transposed": cases.golden(xd, gate_w.reshape(5120, 2560).t(), up_w, down_w), + "down read transposed": cases.golden(xd, gate_w, up_w, down_w.reshape(2560, 5120).t()), + } + for name, gold in wrong.items(): + assert not torch.equal(gold["y"].cpu(), expected["y"]), name + + +def test_row_permutation_keeps_tokens_apart(): + weights, x_all = cases.matrix_case(17) + expert = loaded(weights) + x = x_all.cuda() + perm = torch.randperm(x.shape[0], generator=torch.Generator().manual_seed(20261019)).cuda() + inverse = torch.argsort(perm) + base = run(expert, x) + permuted = run(expert, x[perm]) + back = {stage: t[inverse] for stage, t in permuted.items()} + cases.assert_stages(back, base, cases.MATRIX_RULES, "row permutation") + # A token-mixing expert would fail the same rule. + mixed = torch.roll(permuted["y"], 1, dims=0)[inverse] + assert cases.violations(mixed, base["y"], *cases.MATRIX_RULES["y"])["failing"] > 0 + + +def test_no_residual_weighting_or_collective(): + """Zero down weights with a nonzero input: an added residual, bias or any other term + would leave the output nonzero. Exact outputs on the fixtures above rule out a mixing + weight or the routed scaling factor.""" + weights = dict(cases.matrix_case(17)[0]) + weights["down_proj.weight"] = torch.zeros_like(weights["down_proj.weight"]) + expert = loaded(weights) + x = cases.matrix_case(17)[1][:17].cuda() + with TorchCallTrace() as trace: + steps = run(expert, x) + assert bool((steps["y"] == 0).all()) + assert all(bool((steps[s] != 0).any()) for s in ("gate", "up", "act")) + names = [name for name, _ in trace.calls] + assert not any("reduce" in n or "all_gather" in n or "distributed" in n for n in names), names + + +def test_forward_runs_on_the_gpu(): + weights, x_all = cases.matrix_case(41) + expert = loaded(weights) + x = x_all[:129].cuda() + with TorchCallTrace() as trace: + expert(x) + names = [name for name, _ in trace.calls] + assert sum("cublas_mm" in name for name in names) == 3, names + assert "torch.nn.functional.silu" in names, names + produced = [device for _, devices in trace.calls for device in devices] + assert produced and all(device == str(x.device) for device in produced), trace.calls diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py new file mode 100644 index 000000000000..b59ac880b367 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -0,0 +1,898 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam MoE router component of block 3: bf16 / sm_103 / tp1. + +Configuration under test, fixed in ``_router_cases.py`` and nowhere else: 256 experts in +8 groups of 32, 4 groups kept, 8 experts per token, hidden 5120; FP32 router weight and +correction bias, BF16 already-normalized input; one sm_103 GPU; eager; generated +parameters; seeds {17, 41} x tokens {1, 17, 129}, and the controlled fixtures there. + +What is compared. Every call goes through the component's ``forward``; the op's mapping +of that same call is captured, and each of its steps is compared with the op's own +``reference`` on the same input under the rules in ``_router_cases.RULES``. Groups and +expert ids are compared exactly, in order. Each fixture also states the decision it was +built to force, independently of either implementation. + +Invocation (the fixture enforces and probes IEEE fp32):: + + python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router + +Optional environment: + +* ``BEAM_MOE_ROUTER_SOURCE_CAPTURES``: a directory of routes captured from the source + model's router: ``seed/params.pt`` (``weight``, ``bias``), ``seed/T.pt`` + (``hidden`` and ``native``, a dict of ``logits``, ``scores``, ``corrected``, + ``group_score``, ``top_groups``, ``masked``, ``ids``, ``weights``) and + ``fixtures/.pt`` (``hidden``, ``bias`` and the same names). The generated inputs + must then equal the captured ones bit for bit, the op's ``reference`` must reproduce + every captured tensor bit for bit, and the component is compared with the captures + under the same rules as with the reference. +* ``BEAM_MOE_ROUTER_EVIDENCE``: a directory that receives one JSON record per test case: + every comparison with its rule, hashes of the routed tensors, the torch calls and CUDA + kernels of the forward, and the device and software it ran on. + +Dependencies: a TensorRT-LLM build for sm_103. No model files. +""" + +from __future__ import annotations + +import contextlib +import functools +import os +import platform +import sys +import types +from pathlib import Path + +import pytest +import torch +from torch.overrides import TorchFunctionMode, resolve_name + +import tensorrt_llm + +try: + import tensorrt_llm._torch.models.modeling_beam as builtin_beam +except ModuleNotFoundError as exc: + if exc.name != "tensorrt_llm._torch.models.modeling_beam": + raise + builtin_beam = None # The standard-serving implementation is a separate PR. +__extra_import_path__ = [".", "../..", "../../../.."] + +import _beam_component_support as support # noqa: E402 +import _router_cases as cases # noqa: E402 +from _beam_component_support import tensor_sha256 # noqa: E402 +from _precision import true_fp32_matmul # noqa: E402 +from _validating import validating # noqa: E402 + +from _components.moe_router.router import NORM_EPS, BeamMoeRouterBf16Sm103Tp1 # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 + BeamGroupedTopkRouter, + beam_grouped_topk_router, +) + +assert torch.cuda.is_available(), "the Beam MoE router tests require a CUDA device" + +#: The router as the checkpoint configures it. Only these fields are read. +ROUTER_CONFIG = types.MappingProxyType( + dict( + hidden_size=5120, + num_experts=256, + num_experts_per_tok=8, + n_group=8, + topk_group=4, + norm_topk_prob=True, + mlp_layer_types=("dense", "dense", "dense", "sparse"), + ) +) +LAYER_IDX = 3 +SM = (10, 3) +#: Eager, one GPU, one call per case. +EXECUTION = types.MappingProxyType(dict(eager=True, cuda_graph=False, devices=1)) +OP_KWARGS = types.MappingProxyType( + dict(n_group=cases.GROUPS, topk_group=cases.TOPK_GROUP, top_k=cases.TOP_K, eps=cases.NORM_EPS) +) +#: Op names -> the names a source capture stores them under. +CAPTURE_NAMES = types.MappingProxyType( + dict( + logits="logits", + scores="scores", + selection_scores="corrected", + group_scores="group_score", + groups="top_groups", + masked_selection_scores="masked", + ids="ids", + weights="weights", + ) +) +#: Calls that move token data to the host, and with it a routing decision. +HOST_TRANSFERS = frozenset( + "torch.Tensor." + name + for name in ("cpu", "item", "tolist", "numpy", "__bool__", "__int__", "__float__", "__index__") +) + + +def router_config(**overrides) -> types.SimpleNamespace: + fields = dict(ROUTER_CONFIG) + fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) + fields.update(overrides) + return types.SimpleNamespace(**fields) + + +@pytest.fixture(scope="module", autouse=True) +def ieee_fp32(): + """Enforce IEEE fp32 and check the actual GEMM precision.""" + yield from support.ieee_fp32_session(_write_evidence) + + +# ----------------------------------------------------------------------------- evidence + + +def _environment() -> dict: + device = torch.cuda.current_device() + return { + "device_name": torch.cuda.get_device_name(device), + "device_capability": list(torch.cuda.get_device_capability(device)), + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "tensorrt_llm": tensorrt_llm.__version__, + "tensorrt_llm_file": tensorrt_llm.__file__, + "python": sys.version.split()[0], + "machine": platform.machine(), + "argv": sys.argv, + "pid": os.getpid(), + "slurm_job_id": os.environ.get("SLURM_JOB_ID"), + "slurm_step_id": os.environ.get("SLURM_STEP_ID"), + "execution": dict(EXECUTION), + "ambient_math": { + "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, + }, + "source_captures": os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES"), + } + + +_write_evidence = functools.partial(support.write_evidence, "BEAM_MOE_ROUTER_EVIDENCE") + + +def route_hashes(route: dict) -> dict: + return {name: tensor_sha256(t) for name, t in sorted(route.items())} + + +# ----------------------------------------------------------------------------- source captures + + +def _captures_dir() -> Path | None: + directory = os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES") + return Path(directory) if directory else None + + +def _as_route(captured: dict) -> dict: + return {name: captured[key] for name, key in CAPTURE_NAMES.items()} + + +def matrix_captures(seed: int, tokens: int) -> dict | None: + root = _captures_dir() + if root is None: + return None + params = torch.load(root / f"seed{seed}" / "params.pt", weights_only=True) + blob = torch.load(root / f"seed{seed}" / f"T{tokens}.pt", weights_only=True) + return dict( + weight=params["weight"], + bias=params["bias"], + hidden=blob["hidden"], + route=_as_route(blob["native"]), + ) + + +def fixture_captures(name: str) -> dict | None: + root = _captures_dir() + if root is None: + return None + blob = torch.load(root / "fixtures" / f"{name}.pt", weights_only=True) + return dict(bias=blob["bias"], hidden=blob["hidden"], route=_as_route(blob)) + + +def check_against_captures( + captured: dict | None, inputs: dict, route: dict, reference: dict +) -> list: + """With captures: same input bytes, the reference reproduces them bitwise, and the + component meets the rules against them. Without: nothing.""" + if captured is None: + return [] + results = [] + for name, tensor in inputs.items(): + same = torch.equal(tensor.cpu(), captured[name].cpu()) + results.append( + { + "tensor": f"input:{name}", + "against": "source_captures", + "rule": "bitwise", + "passed": same, + "problems": [] if same else [f"generated {name} differs from the capture"], + } + ) + for name, tensor in captured["route"].items(): + same = torch.equal(reference[name].cpu(), tensor.cpu()) + results.append( + { + "tensor": f"reference:{name}", + "against": "source_captures", + "rule": "bitwise", + "passed": same, + "problems": [] if same else [f"reference {name} is not the captured {name}"], + } + ) + device_route = {k: v.to(route["ids"].device) for k, v in captured["route"].items()} + results.extend(cases.compare_routes(route, device_route, "source_captures")) + return results + + +# ----------------------------------------------------------------------------- observing a call + + +@contextlib.contextmanager +def recording_op_calls(): + """Capture the mapping each op call returns, without changing what it returns.""" + calls: list[dict] = [] + original = BeamGroupedTopkRouter.__call__ + + def recorded(self, *args, **kwargs): + out = original(self, *args, **kwargs) + calls.append(out) + return out + + BeamGroupedTopkRouter.__call__ = recorded + try: + yield calls + finally: + BeamGroupedTopkRouter.__call__ = original + + +@contextlib.contextmanager +def builtin_router_forbidden(): + """The built-in Beam router must not compute any part of this component's routing.""" + if builtin_beam is None: + yield + return + saved = {name: getattr(builtin_beam.BeamRouter, name) for name in ("forward", "route")} + + def refuse(*args, **kwargs): + raise AssertionError("the built-in BeamRouter was called") + + for name in saved: + setattr(builtin_beam.BeamRouter, name, refuse) + try: + yield + finally: + for name, fn in saved.items(): + setattr(builtin_beam.BeamRouter, name, fn) + + +def _tensors_of(value): + if isinstance(value, torch.Tensor): + return [value] + if isinstance(value, (tuple, list)): + return [t for v in value for t in _tensors_of(v)] + if isinstance(value, dict): + return [t for v in value.values() for t in _tensors_of(v)] + return [] + + +class TorchCallTrace(TorchFunctionMode): + """Every torch call made inside the block, with the device and dtype of what it returned, + and the fp32 matmul and autocast state at each projection.""" + + def __init__(self): + super().__init__() + self.calls: list[dict] = [] + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + name = resolve_name(func) or getattr(func, "__qualname__", None) or repr(func) + entry = {"func": name} + if func is torch.ops.trtllm.cublas_mm: + entry["math"] = { + "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, + "autocast_cuda": torch.is_autocast_enabled("cuda"), + } + entry["operand_dtypes"] = [str(t.dtype) for t in _tensors_of(args)] + out = func(*args, **kwargs) + produced = _tensors_of(out) + if produced: + entry["outputs"] = [[str(t.device), str(t.dtype), list(t.shape)] for t in produced] + self.calls.append(entry) + return out + + +def loaded_router(weight: torch.Tensor, bias: torch.Tensor) -> BeamMoeRouterBf16Sm103Tp1: + router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") + router.load_weights(weight.cuda(), bias.cuda()) + return router + + +def routed(router: BeamMoeRouterBf16Sm103Tp1, hidden: torch.Tensor) -> tuple: + """(weights, ids, the op's mapping of that same forward call).""" + with builtin_router_forbidden(), validating(beam_grouped_topk_router): + with recording_op_calls() as calls: + weights, ids = router(hidden) + assert len(calls) == 1, f"forward made {len(calls)} op calls, expected 1" + (route,) = calls + assert weights is route["weights"] and ids is route["ids"], ( + "forward did not return the op call's own weights and ids" + ) + return weights, ids, route + + +def reference_route(router: BeamMoeRouterBf16Sm103Tp1, hidden: torch.Tensor) -> dict: + with true_fp32_matmul(): + return beam_grouped_topk_router.reference( + hidden, router.weight, router.e_score_correction_bias, **OP_KWARGS + ) + + +def check_output_contract(weights, ids, hidden) -> list: + tokens = hidden.shape[0] + problems = [] + if tuple(weights.shape) != (tokens, 8) or weights.dtype != torch.float32: + problems.append(f"weights {tuple(weights.shape)} {weights.dtype}") + if tuple(ids.shape) != (tokens, 8) or ids.dtype != torch.int64: + problems.append(f"ids {tuple(ids.shape)} {ids.dtype}") + if weights.device != hidden.device or ids.device != hidden.device: + problems.append(f"outputs on {weights.device}/{ids.device}, input on {hidden.device}") + return [ + { + "tensor": "output_contract", + "against": "contract", + "rule": "[T, 8] float32 weights and int64 ids on the input device", + "passed": not problems, + "problems": problems, + } + ] + + +def check_invariants(route: dict, sum_to_one: bool) -> list: + """Per row: eight distinct ids in range, each in a selected group; finite, nonnegative + weights; with `sum_to_one`, |sum(weights) - 1| <= NORM_ATOL.""" + ids, groups, weights = route["ids"], route["groups"], route["weights"] + distinct = (ids.sort(dim=-1).values.diff(dim=-1) != 0).all(dim=-1) + in_range = ((ids >= 0) & (ids < cases.EXPERTS)).all(dim=-1) + member = ((ids // cases.PER_GROUP).unsqueeze(-1) == groups.unsqueeze(1)).any(dim=-1).all(-1) + excluded = route["masked_selection_scores"] == float("-inf") + expert_selected_group = ( + (torch.arange(cases.EXPERTS, device=ids.device) // cases.PER_GROUP).view(1, -1, 1) + == groups.unsqueeze(1) + ).any(dim=-1) + mask_exact = (excluded == ~expert_selected_group).all(dim=-1) + finite = (torch.isfinite(weights) & (weights >= 0)).all(dim=-1) + deviation = (weights.sum(dim=-1).double() - 1.0).abs() + checks = { + "eight_distinct_ids": distinct, + "ids_in_range": in_range, + "ids_in_selected_groups": member, + "excluded_exactly_outside_selected_groups": mask_exact, + "weights_finite_nonnegative": finite, + } + if sum_to_one: + checks["weights_sum_to_one"] = deviation <= cases.NORM_ATOL + results = [] + for name, ok in checks.items(): + bad = [int(i) for i in torch.nonzero(~ok).flatten().cpu()] + results.append( + { + "tensor": name, + "against": "invariant", + "rule": f"|sum - 1| <= {cases.NORM_ATOL:g}" + if name == "weights_sum_to_one" + else name, + "passed": not bad, + "problems": [f"rows {bad[:8]}"] if bad else [], + } + ) + results[-1]["max_abs_sum_minus_one"] = float(deviation.max()) + return results + + +# ----------------------------------------------------------------------------- construction and loading + + +def test_construction_allocates_only_the_router_state() -> None: + """Construction allocates exactly the two FP32 router tensors: no expert, attention, + cache or model state, and no submodule.""" + torch.cuda.synchronize() + before = torch.cuda.memory_allocated() + router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") + torch.cuda.synchronize() + allocated = torch.cuda.memory_allocated() - before + params = {n: (tuple(p.shape), p.dtype, p.requires_grad) for n, p in router.named_parameters()} + buffers = {n: (tuple(b.shape), b.dtype) for n, b in router.named_buffers()} + elements = sum(p.numel() for p in router.parameters()) + sum( + b.numel() for b in router.buffers() + ) + record = { + "test": "test_construction_allocates_only_the_router_state", + "parameters": params, + "buffers": buffers, + "submodules": [n for n, _ in router.named_modules() if n], + "elements": elements, + "cuda_bytes_allocated": allocated, + "environment": _environment(), + } + _write_evidence("construction", record) + assert params == {"weight": ((256, 5120), torch.float32, False)} + assert buffers == {"e_score_correction_bias": ((256,), torch.float32)} + assert record["submodules"] == [] + assert elements == 256 * 5120 + 256 + assert allocated == (256 * 5120 + 256) * 4, allocated + assert router.weight.device.type == "cuda" + assert torch.cuda.get_device_capability(router.weight.device) == SM + + +@pytest.mark.parametrize( + "overrides,layer_idx,match", + [ + (dict(n_group=4), LAYER_IDX, "unsupported n_group"), + (dict(topk_group=3), LAYER_IDX, "unsupported topk_group"), + (dict(num_experts_per_tok=6), LAYER_IDX, "unsupported num_experts_per_tok"), + (dict(num_experts=128), LAYER_IDX, "unsupported num_experts"), + (dict(hidden_size=4096), LAYER_IDX, "unsupported hidden_size"), + (dict(norm_topk_prob=False), LAYER_IDX, "unsupported norm_topk_prob"), + (dict(mlp_layer_types=["dense"] * 4), LAYER_IDX, "sparse"), + ({}, 0, "only the routers of sparse blocks"), + ], +) +def test_construction_rejects_unsupported_configs(overrides, layer_idx, match) -> None: + with pytest.raises(ValueError, match=match): + BeamMoeRouterBf16Sm103Tp1(router_config(**overrides), layer_idx, device="cuda") + + +def test_construction_rejects_a_cpu_device() -> None: + with pytest.raises(ValueError, match="sm_103 CUDA devices"): + BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cpu") + + +@pytest.mark.parametrize("seed", cases.SEEDS) +def test_loading_copies_both_tensors_exactly(seed) -> None: + weight, bias, _ = cases.matrix_case(seed, 1) + router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") + router.load_weights(weight, bias) + _write_evidence( + f"loading-seed{seed}", + { + "test": "test_loading_copies_both_tensors_exactly", + "weight": [ + list(router.weight.shape), + str(router.weight.dtype), + tensor_sha256(router.weight), + ], + "bias": [ + list(router.e_score_correction_bias.shape), + str(router.e_score_correction_bias.dtype), + tensor_sha256(router.e_score_correction_bias), + ], + "generated": [tensor_sha256(weight), tensor_sha256(bias)], + }, + ) + assert router.weight.dtype == torch.float32 and tuple(router.weight.shape) == (256, 5120) + assert router.e_score_correction_bias.dtype == torch.float32 + assert tuple(router.e_score_correction_bias.shape) == (256,) + assert torch.equal(router.weight.cpu(), weight) + assert torch.equal(router.e_score_correction_bias.cpu(), bias) + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("weight_rows", ValueError, r"weight must be \[256, 5120\]"), + ("weight_width", ValueError, r"weight must be \[256, 5120\]"), + ("weight_transposed", ValueError, r"weight must be \[256, 5120\]"), + ("bias_length", ValueError, r"correction_bias must be \[256\]"), + ("bias_length_one", ValueError, r"correction_bias must be \[256\]"), + ("weight_bf16", TypeError, "weight must be torch.float32"), + ("bias_fp64", TypeError, "correction_bias must be torch.float32"), + ("weight_not_tensor", TypeError, "weight must be a torch.Tensor"), + ], +) +def test_loading_rejects_invalid_parameters(case, error, match) -> None: + """A rejected load raises before copying either tensor: the router keeps its state.""" + weight, bias, _ = cases.matrix_case(17, 1) + router = loaded_router(weight, bias) + good_weight, good_bias = router.weight.clone(), router.e_score_correction_bias.clone() + other_weight, other_bias, _ = cases.matrix_case(41, 1) + bad = { + "weight_rows": (other_weight[:255], other_bias), + "weight_width": (other_weight[:, :5119], other_bias), + "weight_transposed": (other_weight.t().contiguous(), other_bias), + "bias_length": (other_weight, other_bias[:255]), + "bias_length_one": (other_weight, other_bias[:1]), + "weight_bf16": (other_weight.to(torch.bfloat16), other_bias), + "bias_fp64": (other_weight, other_bias.double()), + "weight_not_tensor": ([0.0] * 5120, other_bias), + }[case] + with pytest.raises(error, match=match): + router.load_weights(*bad) + assert torch.equal(router.weight, good_weight) + assert torch.equal(router.e_score_correction_bias, good_bias) + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("width_5119", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("width_5121", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("batched_3d", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("one_dim", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("fp32_input", TypeError, "hidden_states must be torch.bfloat16"), + ("fp16_input", TypeError, "hidden_states must be torch.bfloat16"), + ("no_tokens", ValueError, "at least one token"), + ("cpu_input", ValueError, "router's device"), + ("not_a_tensor", TypeError, "hidden_states must be a torch.Tensor"), + ("not_loaded", RuntimeError, "load_weights must be called"), + ], +) +def test_forward_rejects_invalid_input(case, error, match) -> None: + weight, bias, hidden = cases.matrix_case(17, 17) + if case == "not_loaded": + router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") + else: + router = loaded_router(weight, bias) + hidden = hidden.cuda() + bad = { + "width_5119": hidden[:, :5119], + "width_5121": torch.cat([hidden, hidden[:, :1]], dim=1), + "batched_3d": hidden.unsqueeze(0), + "one_dim": hidden[0], + "fp32_input": hidden.float(), + "fp16_input": hidden.half(), + "no_tokens": hidden[:0], + "cpu_input": hidden.cpu(), + "not_a_tensor": [0.0] * 5120, + "not_loaded": hidden, + }[case] + with pytest.raises(error, match=match): + router(bad) + + +# ----------------------------------------------------------------------------- precision + + +def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: + """Projection, sigmoid, selection scores, group scores and normalization are fp32, the + projection is IEEE fp32 even when the caller has TF32 or bf16 autocast on, and the + caller's global precision policy remains unchanged.""" + weight, bias, hidden = cases.matrix_case(17, 17) + router = loaded_router(weight, bias) + hidden = hidden.cuda() + plain_weights, plain_ids, plain = routed(router, hidden) + + saved = torch.backends.cuda.matmul.fp32_precision + torch.backends.cuda.matmul.fp32_precision = "tf32" + try: + ambient = torch.backends.cuda.matmul.fp32_precision + with torch.autocast("cuda", dtype=torch.bfloat16), TorchCallTrace() as trace: + weights, ids, route = routed(router, hidden) + after = torch.backends.cuda.matmul.fp32_precision + finally: + torch.backends.cuda.matmul.fp32_precision = saved + + # TF32 operands are ~1e-3 off; an IEEE fp32 product of these operands is ~1e-6 off fp64. + fp64_logits = hidden.double() @ router.weight.double().t() + fp64_error = float((route["logits"].double() - fp64_logits).abs().max()) + projection = [c for c in trace.calls if c["func"] == "trtllm.cublas_mm"] + dtypes = {name: str(t.dtype) for name, t in route.items()} + _write_evidence( + "precision", + { + "test": "test_precision_path_is_fp32_without_tf32_or_autocast", + "ambient_during_call": {"fp32_precision": ambient, "autocast_bf16": True}, + "ambient_after_call": after, + "projection_calls": projection, + "route_dtypes": dtypes, + "logits_vs_fp64_max_abs": fp64_error, + "bitwise_equal_to_plain_call": {k: torch.equal(route[k], plain[k]) for k in route}, + "torch_calls": [c["func"] for c in trace.calls], + "environment": _environment(), + }, + ) + assert ambient == after == "tf32", (ambient, after) + assert len(projection) == 1, projection + assert projection[0]["math"] == { + "cuda_matmul_fp32_precision": "tf32", + "autocast_cuda": False, + }, projection[0] + assert projection[0]["operand_dtypes"] == ["torch.float32", "torch.float32"], projection[0] + assert projection[0]["outputs"][0][1] == "torch.float32" + for name in ("logits", "scores", "selection_scores", "group_scores", "weights"): + assert dtypes[name] == "torch.float32", (name, dtypes[name]) + assert dtypes["masked_selection_scores"] == "torch.float32" + assert dtypes["ids"] == dtypes["groups"] == "torch.int64" + assert fp64_error <= 1e-5, f"projection is not IEEE fp32: {fp64_error:.3e} from fp64" + for name in route: + assert torch.equal(route[name], plain[name]), f"{name} changed under TF32/autocast" + assert torch.equal(weights, plain_weights) and torch.equal(ids, plain_ids) + + +# ----------------------------------------------------------------------------- the fixed matrix + + +@pytest.mark.parametrize("seed,tokens", cases.CASES, ids=cases.CASE_IDS) +def test_matrix(seed, tokens) -> None: + """One case of the fixed matrix: the output contract, every step against the reference + (and the source captures when given), the routing invariants, and the evidence that + the routing ran on the GPU in this component's op with no host synchronization.""" + weight, bias, hidden_cpu = cases.matrix_case(seed, tokens) + router = loaded_router(weight, bias) + hidden = hidden_cpu.cuda() + torch.cuda.synchronize() + + activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + with torch.profiler.profile(activities=activities) as prof: + with TorchCallTrace() as trace: + previous = torch.cuda.get_sync_debug_mode() + torch.cuda.set_sync_debug_mode("error") + try: + weights, ids, route = routed(router, hidden) + finally: + torch.cuda.set_sync_debug_mode(previous) + torch.cuda.synchronize() + kernels = [e.name for e in prof.events() if e.device_type == torch.autograd.DeviceType.CUDA] + + reference = reference_route(router, hidden) + results = check_output_contract(weights, ids, hidden) + results += cases.compare_routes(route, reference, "reference") + results += check_invariants(route, sum_to_one=True) + results += check_against_captures( + matrix_captures(seed, tokens), + {"weight": weight, "bias": bias, "hidden": hidden_cpu}, + route, + reference, + ) + + host = [c["func"] for c in trace.calls if c["func"] in HOST_TRANSFERS] + off_device = [ + c for c in trace.calls for o in c.get("outputs", []) if not o[0].startswith("cuda") + ] + called = {c["func"] for c in trace.calls} + routing_calls = { + "trtllm.cublas_mm", + "torch.Tensor.sigmoid", + "torch.Tensor.topk", + "torch.Tensor.scatter_", + "torch.Tensor.masked_fill", + "torch.Tensor.gather", + "torch.Tensor.sum", + } + _write_evidence( + f"matrix-seed{seed}-T{tokens}", + { + "test": "test_matrix", + "case": {"seed": seed, "tokens": tokens}, + "component": f"{type(router).__module__}.{type(router).__qualname__}", + "op": f"{BeamGroupedTopkRouter.__module__}.{BeamGroupedTopkRouter.__qualname__}", + "rules": dict(cases.RULES, norm_atol=cases.NORM_ATOL), + "results": results, + "hashes": route_hashes(route), + "inputs": { + "weight": tensor_sha256(weight), + "bias": tensor_sha256(bias), + "hidden": tensor_sha256(hidden_cpu), + }, + "ids": route["ids"].tolist(), + "groups": route["groups"].tolist(), + "torch_calls": trace.calls, + "host_transfers": host, + "off_device_outputs": off_device, + "sync_debug_mode": "error", + "cuda_kernels": kernels, + "environment": _environment(), + }, + ) + assert type(router).__module__.endswith("moe_router.router") + assert torch.cuda.get_device_capability(hidden.device) == SM + assert not host, f"routing moved token data to the host: {host}" + assert not off_device, f"routing produced tensors off the GPU: {off_device[:3]}" + assert routing_calls <= called, sorted(routing_calls - called) + assert kernels, "no CUDA kernel ran" + cases.assert_all_pass(results) + + +# ----------------------------------------------------------------------------- controlled fixtures + + +def route_fixture(name: str) -> tuple[dict, dict, list]: + """(component route, reference route, comparisons) for one fixture, captures included.""" + spec = cases.fixtures()[name] + weight, bias, hidden_cpu = cases.fixture_tensors(spec) + router = loaded_router(weight, bias) + hidden = hidden_cpu.cuda() + _, _, route = routed(router, hidden) + reference = reference_route(router, hidden) + results = cases.compare_routes(route, reference, "reference") + results += check_invariants(route, sum_to_one=False) + results += check_against_captures( + fixture_captures(name), {"bias": bias, "hidden": hidden_cpu}, route, reference + ) + exact = torch.equal(route["logits"].cpu(), spec["logits"].float()) + results.append( + { + "tensor": "fixture_logits", + "against": "fixture", + "rule": "the one-hot rows reproduce the fixture's logits exactly", + "passed": exact, + "problems": [] if exact else ["logits differ from the fixture's"], + } + ) + return route, reference, results + + +def _record_fixture(name: str, test: str, route: dict, results: list, extra: dict) -> None: + _write_evidence( + f"fixture-{name}", + { + "test": test, + "fixture": name, + "intent": cases.fixtures()[name]["intent"], + "rows": list(cases.fixtures()[name]["rows"]), + "ids": route["ids"].tolist(), + "groups": route["groups"].tolist(), + "weights": route["weights"].tolist(), + "row_sums": route["weights"].sum(dim=-1).tolist(), + "hashes": route_hashes(route), + "results": results, + **extra, + "environment": _environment(), + }, + ) + + +def test_bias_selects_but_never_weights() -> None: + """The correction bias changes which experts are chosen, and the returned weights are + still the normalized unbiased scores of the chosen experts.""" + zero, _, zero_results = route_fixture("bias_zero") + shift, _, shift_results = route_fixture("bias_shift") + ids, scores = shift["ids"], shift["scores"] + unbiased = scores.gather(-1, ids) + unbiased = unbiased / (unbiased.sum(dim=-1, keepdim=True) + NORM_EPS) + biased = shift["selection_scores"].gather(-1, ids) + biased = biased / (biased.sum(dim=-1, keepdim=True) + NORM_EPS) + selected_bias = cases.fixtures()["bias_shift"]["bias"][ids[0].cpu()] + leak_gap = float((biased - shift["weights"]).abs().max()) + extra = { + "ids_without_bias": zero["ids"].tolist(), + "ids_with_bias": ids.tolist(), + "groups_with_bias": shift["groups"].tolist(), + "selected_bias": selected_bias.tolist(), + "weights_vs_normalized_unbiased": cases.compare( + "weights", shift["weights"], unbiased, cases.RULES["weights"] + ), + "weights_vs_normalized_biased_max_abs": leak_gap, + } + _record_fixture("bias_zero", "test_bias_selects_but_never_weights", zero, zero_results, {}) + _record_fixture( + "bias_shift", "test_bias_selects_but_never_weights", shift, shift_results, extra + ) + cases.assert_all_pass(zero_results + shift_results) + assert zero["groups"].tolist() == [[0, 1, 2, 3]] + assert zero["ids"].tolist() == [[0, 1, 32, 33, 64, 65, 96, 97]] + assert shift["groups"].tolist() == [[6, 0, 1, 2]] + assert ids.tolist() == [[193, 192, 0, 1, 32, 33, 64, 65]] + assert torch.equal(zero["logits"], shift["logits"]) and torch.equal(zero["scores"], scores) + assert bool((selected_bias != 0).any()) and len(set(selected_bias.tolist())) > 1 + assert extra["weights_vs_normalized_unbiased"]["passed"], extra[ + "weights_vs_normalized_unbiased" + ] + assert leak_gap > cases.RULES["weights"], f"bias in the weights would be invisible: {leak_gap}" + + +def test_group_boundary_moves_with_the_top_two_sum() -> None: + """Only the challenger group's second expert moves, and the group crosses the 4th/5th + boundary; the group holding the single best expert stays out.""" + route, _, results = route_fixture("group_boundary") + group_scores = route["group_scores"] + _record_fixture( + "group_boundary", + "test_group_boundary_moves_with_the_top_two_sum", + route, + results, + {"group_scores": group_scores.tolist()}, + ) + cases.assert_all_pass(results) + assert route["groups"].tolist() == [[0, 1, 2, 3], [0, 1, 2, 4]] + assert route["ids"].tolist() == [ + [0, 1, 32, 33, 64, 65, 96, 97], + [0, 1, 32, 33, 64, 65, 129, 128], + ] + # Group 4 scores 0.66 + 0.62 then 0.66 + 0.67 against group 3's 0.655 + 0.645. + assert float(group_scores[0, 4]) < float(group_scores[0, 3]) < float(group_scores[1, 4]) + assert 5 not in route["groups"].flatten().tolist() + + +def test_exclusion_masks_every_expert_of_dropped_groups() -> None: + """The best single expert sits in a group that loses; it and every other expert of the + four dropped groups are -inf, although every retained selection score is negative.""" + route, _, results = route_fixture("exclusion") + ids = route["ids"] + selected_scores = route["selection_scores"].gather(-1, ids) + spike_score = float(route["selection_scores"][0, cases.EXCLUSION_SPIKE]) + _record_fixture( + "exclusion", + "test_exclusion_masks_every_expert_of_dropped_groups", + route, + results, + {"spike": cases.EXCLUSION_SPIKE, "spike_selection_score": spike_score}, + ) + cases.assert_all_pass(results) + assert sorted(route["groups"][0].tolist()) == [0, 1, 2, 3] + assert cases.EXCLUSION_SPIKE not in ids[0].tolist() + assert spike_score == float(route["selection_scores"].max()) + assert bool((selected_scores < 0).all()) + assert all(e // cases.PER_GROUP in route["groups"][0].tolist() for e in ids[0].tolist()) + + +def test_expert_boundary_moves_one_expert() -> None: + """Expert 2 crosses the 8th/9th boundary within an unchanged group selection: it enters + last and expert 97 leaves, every other position unchanged.""" + route, _, results = route_fixture("expert_boundary") + _record_fixture("expert_boundary", "test_expert_boundary_moves_one_expert", route, results, {}) + cases.assert_all_pass(results) + assert route["groups"].tolist() == [[0, 1, 2, 3], [0, 1, 2, 3]] + assert route["ids"].tolist() == [ + [0, 1, 32, 33, 64, 65, 96, 97], + [0, 1, 32, 33, 64, 65, 96, 2], + ] + + +def test_ties_follow_the_pinned_topk_order() -> None: + """Exactly tied scores are routed in the order `torch.topk(..., sorted=True)` returns + -- the source's order, stated in `_router_cases.TIE_ROUTES` -- whether a row is routed + in a batch or alone. Ids are compared in order, never sorted.""" + route, _, results = route_fixture("ties") + spec = cases.fixtures()["ties"] + weight, bias, hidden = cases.fixture_tensors(spec) + router = loaded_router(weight, bias) + alone = {} + for i, name in enumerate(spec["rows"]): + _, _, single = routed(router, hidden[i : i + 1].cuda()) + alone[name] = (single["groups"][0].tolist(), single["ids"][0].tolist()) + batched = { + name: (route["groups"][i].tolist(), route["ids"][i].tolist()) + for i, name in enumerate(spec["rows"]) + } + _record_fixture( + "ties", + "test_ties_follow_the_pinned_topk_order", + route, + results, + {"batched": batched, "alone": alone, "pinned": cases.TIE_ROUTES}, + ) + cases.assert_all_pass(results) + for name, pinned in cases.TIE_ROUTES.items(): + assert batched[name] == (list(pinned[0]), list(pinned[1])), (name, batched[name]) + assert alone[name] == batched[name], (name, alone[name]) + + +def test_epsilon_is_the_source_denominator() -> None: + """Weights divide by sum + 1e-20: a row whose scores underflow to 0 gets zero weights, + not NaN, and a row whose selected scores sum to ~1e-20 sums to S / (S + 1e-20), well + below 1 -- a denominator without the term, or with 1e-12 or 1e-6, gives 1 or ~0.""" + route, _, results = route_fixture("epsilon") + selected = route["scores"].gather(-1, route["ids"]) + exact = torch.sigmoid(route["logits"].double()).gather(-1, route["ids"]).sum(dim=-1) + expected_sum = (exact / (exact + 1e-20)).tolist() + sums = route["weights"].sum(dim=-1).double().tolist() + _record_fixture( + "epsilon", + "test_epsilon_is_the_source_denominator", + route, + results, + { + "selected_score_sums": selected.sum(dim=-1).tolist(), + "expected_row_sums_fp64": expected_sum, + }, + ) + cases.assert_all_pass(results) + assert route["ids"].tolist() == [cases.EPSILON_IDS, cases.EPSILON_IDS] + assert route["groups"].tolist() == [[0, 1, 2, 3], [0, 1, 2, 3]] + assert bool((route["weights"][0] == 0).all()), route["weights"][0] + assert float(selected[1].sum()) > 0 + assert sums[1] < 0.9, sums[1] + assert abs(sums[1] - expected_sum[1]) <= cases.NORM_ATOL, (sums[1], expected_sum[1]) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py new file mode 100644 index 000000000000..331c5c0c2163 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py @@ -0,0 +1,759 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for cached greedy generation and prefix reuse on the Beam six-block model. + +Configuration under test: the six-block model of ``test_small_model_prefill.py`` (bf16, sm_103, +eager TP1, generated weights) driven by ``generation.BeamSmallModelGenerator`` -- one +``KVCacheManagerV2`` with whole-page block reuse, 32-token pages and a 1024-token capacity, +TRTLLM attention through the catalog ``thop_attention`` op, and the existing ``TorchSampler``. + +What is checked, without model files or captured data: + +* the request lifecycle -- create, prefill, decode, sampled tokens and finish reason, + release -- with its counters, and that nothing runs after the finishing token; +* that a decode step appends exactly its own keys and values, one row per layer at the + token's absolute position, and changes no other written slot; +* prefix reuse through the manager: a matching prefix is claimed page by page and only the + rest of the prompt is computed, a different first token misses, and two live requests + sharing a prefix never write each other's slots; with stable expert selection, reused + and freshly computed prefixes produce matching final logits within BF16 rounding bounds; +* exhausted compute-only requests run no further attention, and a failed prefill or decode + cannot be retried but can still be released; +* the sampler's greedy choice on identical logits (lowest index among exact maxima); +* the sliding window at the cache level: at query position 512 the sliding layer excludes + key 0 exactly and the full layer keeps it; +* rejections before any kernel runs: malformed or foreign request state, capacity overflow, + positions that do not continue the cache, a step that does not begin where the request's + cache history ends (rewound into cached tokens -- a prefix reused from another request + included -- or leaving a gap), a cached prefix without paged-context FMHA and a block-offset + table that does not address the request's pages, each leaving every written slot unchanged; + and weights that change -- in place or by replacing a parameter -- while a generator holds a + cache computed from them, refused before the step looks up a prefix or allocates. + +Numerical agreement with the source model is established separately. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported; the +generation-phase attention kernels compile on first use and need the installation's ``pip`` +on ``PATH``):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ + tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model + +Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 130 GiB of +free device memory (generated state and loaded model). No model files. +""" + +from __future__ import annotations + +import contextlib +import gc +import zlib +from collections import OrderedDict + +import numpy as np +import pytest +import torch + +__extra_import_path__ = ["../.."] + +import _beam_component_support as support # noqa: E402 +from _beam_component_support import bitwise_equal # noqa: E402 + +from _components.six_layer_model import generation # noqa: E501 +from _components.six_layer_model import model as model_module # noqa: E501 +from _components.six_layer_model.generation import BeamSmallModelGenerator, Lifecycle # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( + thop_attention, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention +from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.metadata import KVCacheParams +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState +from tensorrt_llm._torch.pyexecutor.sampler import TorchSampler +from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm._torch.pyexecutor.seq_slot_manager import SeqSlotManager +from tensorrt_llm.bindings.executor import FinishReason + +assert torch.cuda.is_available(), "the Beam six-block model requires a CUDA device" + +pytestmark = pytest.mark.high_cuda_memory + +VOCAB = 151936 +TPB = 32 + +#: The six-block configuration of ``test_small_model_prefill.py``. +model_config = support.small_model_config + + +def generate_state(seed: int) -> "OrderedDict[str, torch.Tensor]": + """All 2394 tensors, each from its own generator keyed by seed and name, fp32 rounded once.""" + return support.generate_small_model_state(seed, model_module) + + +def token_stream(name: str, length: int) -> list: + g = torch.Generator(device="cuda").manual_seed(zlib.crc32(f"17|{name}".encode())) + return torch.randint(0, VOCAB, (length,), generator=g, device="cuda").tolist() + + +_CACHE: dict = {} + + +@pytest.fixture(scope="module", autouse=True) +def device_memory(): + """Release the cached model before the next module runs.""" + yield + _CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + + +def loaded() -> model_module.BeamSixLayerBf16Sm103Tp1: + """The seed-17 model; its generated state is dropped once loaded.""" + if "model" not in _CACHE: + state = generate_state(17) + model = model_module.BeamSixLayerBf16Sm103Tp1(model_config(), device="cuda") + model.load_weights(state) + del state + torch.cuda.empty_cache() + _CACHE["model"] = model + return _CACHE["model"] + + +@contextlib.contextmanager +def counting_attention_calls(): + """Count thop attention calls (and their row counts) while letting them run.""" + cls = type(thop_attention) + original = cls.__call__ + calls: list = [] + + def counted(self, *args, **kwargs): + calls.append( + (kwargs["local_layer_idx"], kwargs["q"].shape[0], kwargs["q"].detach().clone()) + ) + return original(self, *args, **kwargs) + + cls.__call__ = counted + try: + yield calls + finally: + cls.__call__ = original + + +def read_slot(mgr, request_id: int, layer: int, position: int): + """(K [8, 128], V [8, 128]) of one token slot, through the manager's own addressing.""" + pool = int(mgr.impl.get_layer_group_id(model_module.LayerId(mgr.layer_offsets[layer]))) + cache = mgr.kv_cache_map[request_id] + pages = np.frombuffer( + cache.get_base_page_indices(pool), dtype=np.int32, count=int(cache.num_blocks) + ) + buf = mgr.get_buffers(layer, kv_layout="HND") + row = int(pages[position // TPB]) * mgr.get_layer_page_index_scale(layer) // 2 + return buf[row, 0, :, position % TPB, :], buf[row, 1, :, position % TPB, :] + + +def written_bytes(mgr, request_id: int, length: int) -> torch.Tensor: + """Every written K/V slot of a request, all six layers, as one int16 tensor.""" + rows = [] + for layer in range(6): + for p in range(length): + k, v = read_slot(mgr, request_id, layer, p) + rows.append(torch.cat((k, v)).view(torch.int16)) + return torch.stack(rows).cpu() + + +# ------------------------------------------------------------------ lifecycle and stopping + + +def test_generation_lifecycle_counters_and_length_stop() -> None: + """create -> prefill -> decode until LENGTH -> release, with counters and call counts.""" + model = loaded() + prompt = token_stream("generation_prompt", 17) + caller = list(prompt) + with BeamSmallModelGenerator(model) as gen, counting_attention_calls() as calls: + state = gen.create_request(prompt, max_new_tokens=5) + assert state.lifecycle is Lifecycle.NEW + first = gen.prefill(state) + assert state.lifecycle is Lifecycle.PREFILLED and state.consumed == 17 + assert first.logits.shape == (VOCAB,) and first.logits.dtype == torch.float32 + assert first.token_id == int(torch.argmax(first.logits)) == state.pending_token + assert [rows for _, rows, _ in calls] == [17] * 6 + steps = [first] + while state.lifecycle is not Lifecycle.FINISHED: + steps.append(gen.decode(state)) + assert state.finish_reason == FinishReason.LENGTH + assert len(state.output_token_ids) == 5 and steps[-1].finish_reason == FinishReason.LENGTH + # The last sampled token is returned but never consumed: P + G - 1 cached tokens. + assert state.consumed == 17 + 5 - 1 + assert [s.position for s in steps] == [16, 17, 18, 19, 20] + assert len(calls) == 6 * 5 and all(rows == 1 for _, rows, _ in calls[6:]) + for step in steps: + assert ( + bool(torch.isfinite(step.logits).all()) and float(step.logits.abs().max()) <= 30.0 + ) + assert step.token_id == int(torch.argmax(step.logits)) + with pytest.raises(ValueError, match="finished"): + gen.decode(state) + assert len(calls) == 30 + gen.release(state) + assert state.lifecycle is Lifecycle.RELEASED + for call in (gen.decode, gen.prefill, gen.release): + with pytest.raises(ValueError, match="released"): + call(state) + assert prompt == caller + + +def test_end_id_stops_before_the_budget() -> None: + """END_ID at the first sampled token after the first that is new to the run: exactly j + tokens, END_ID, and no model call after the j-th; END_ID also wins at the token limit.""" + model = loaded() + prompt = token_stream("generation_prompt", 17) + with BeamSmallModelGenerator(model) as gen: + free = gen.generate(gen.create_request(prompt, max_new_tokens=8, reuse=False)) + assert free.finish_reason == FinishReason.LENGTH and len(free.token_ids) == 8 + tokens = free.token_ids + j = next(j for j in range(2, 9) if tokens[j - 1] not in tokens[: j - 1]) + for state in list(gen.live_requests): + gen.release(state) + for budget in (8, j): + with counting_attention_calls() as calls: + state = gen.create_request( + prompt, max_new_tokens=budget, end_id=tokens[j - 1], reuse=False + ) + out = gen.generate(state) + assert out.token_ids == tokens[:j] and out.finish_reason == FinishReason.END_ID + assert len(calls) == 6 * j and state.consumed == 17 + j - 1 + gen.release(state) + + +@pytest.mark.parametrize( + "case", + [ + "empty", + "token_out_of_range", + "zero_budget", + "over_capacity", + "bad_end_id", + "end_id_no_sampling", + ], +) +def test_create_request_rejects_invalid_requests(case) -> None: + model = loaded() + with BeamSmallModelGenerator(model) as gen: + kwargs = dict(max_new_tokens=2) + prompt = [1, 2, 3] + if case == "empty": + prompt = [] + elif case == "token_out_of_range": + prompt = [VOCAB] + elif case == "zero_budget": + kwargs["max_new_tokens"] = 0 + elif case == "over_capacity": + prompt, kwargs["max_new_tokens"] = [1] * 1000, 25 + elif case == "bad_end_id": + kwargs["end_id"] = -1 + else: + kwargs.update(end_id=5, sample=False) + with pytest.raises(ValueError): + gen.create_request(prompt, **kwargs) + assert gen.live_requests == () + + +def test_handles_are_owned_and_bounded() -> None: + """A third live state, a foreign handle and a decode before prefill are refused.""" + model = loaded() + with BeamSmallModelGenerator(model) as gen, BeamSmallModelGenerator(model) as other: + a = gen.create_request([1, 2, 3], max_new_tokens=1) + b = gen.create_request([4, 5, 6], max_new_tokens=1) + with pytest.raises(ValueError, match="live"): + gen.create_request([7], max_new_tokens=1) + foreign = other.create_request([1, 2, 3], max_new_tokens=1) + with counting_attention_calls() as calls: + with pytest.raises(ValueError, match="another generator"): + gen.prefill(foreign) + with pytest.raises(ValueError, match="new"): + gen.decode(a) + assert calls == [] + for state in (a, b): + gen.release(state) + other.release(foreign) + + +@pytest.mark.parametrize("step", ["prefill", "decode"]) +def test_failed_step_cannot_be_retried(monkeypatch: pytest.MonkeyPatch, step: str) -> None: + model = loaded() + with BeamSmallModelGenerator(model) as gen: + state = gen.create_request([1, 2, 3], max_new_tokens=2, sample=False) + if step == "decode": + gen.prefill(state) + error = RuntimeError("injected forward failure") + calls = 0 + + def fail_forward(*args, **kwargs): + nonlocal calls + calls += 1 + raise error + + monkeypatch.setattr(gen, "_forward", fail_forward) + with pytest.raises(RuntimeError, match="injected forward failure") as caught: + if step == "prefill": + gen.prefill(state) + else: + gen.decode(state, 7) + assert caught.value is error and state.lifecycle is Lifecycle.FAILED + with pytest.raises(ValueError, match="failed"): + gen.prefill(state) + with pytest.raises(ValueError, match="failed"): + gen.decode(state, 7) + assert calls == 1 + gen.release(state) + assert state.lifecycle is Lifecycle.RELEASED and gen.live_requests == () + assert state.request_id not in gen.manager.kv_cache_map + + +# ------------------------------------------------------------------ cache contents + + +def test_decode_appends_exactly_its_own_keys_and_values() -> None: + """Each explicit token lands at its absolute position in all six layers, as the packed + K/V the step passed to attention, and no earlier slot changes.""" + model = loaded() + prompt = token_stream("append_prompt", 30) + tokens = token_stream("append_tokens", 4) + with BeamSmallModelGenerator(model) as gen: + state = gen.create_request(prompt, max_new_tokens=4, sample=False) + gen.prefill(state) + for token in tokens: + position = state.consumed + before = written_bytes(gen.manager, state.request_id, position) + with counting_attention_calls() as calls: + gen.decode(state, token) + assert [layer for layer, _, _ in calls] == list(range(6)) + for layer, rows, qkv in calls: + assert rows == 1 + k, v = read_slot(gen.manager, state.request_id, layer, position) + assert bitwise_equal(k, qkv[0, 48 * 128 : 56 * 128].view(8, 128)) + assert bitwise_equal(v, qkv[0, 56 * 128 :].view(8, 128)) + assert torch.equal(written_bytes(gen.manager, state.request_id, position), before) + assert state.consumed == position + 1 + record = state.ledger[-1] + assert ( + record["positions"] == [position, position] and record["input"] == "teacher_forced" + ) + assert record["capacity"] >= position + 1 and record["history"] == position + 1 + before = written_bytes(gen.manager, state.request_id, state.consumed) + with counting_attention_calls() as calls: + with pytest.raises(ValueError, match="budget.*spent"): + gen.decode(state, tokens[-1]) + assert calls == [] + assert state.explicit_tokens == len(tokens) and state.consumed == len(prompt) + len(tokens) + assert gen.manager.kv_cache_map[state.request_id].history_length == state.consumed + assert torch.equal(written_bytes(gen.manager, state.request_id, state.consumed), before) + gen.release(state) + + +def test_prefix_reuse_claims_whole_pages_and_computes_the_rest() -> None: + """B claims A's two common pages and computes 17 rows; the claimed slots are A's bytes and + equal what a reuse-disabled control computes; a different first token misses.""" + model = loaded() + common = token_stream("reuse_common", 64) + a_prompt = common + token_stream("reuse_suffix_a", 17) + b_prompt = common + token_stream("reuse_suffix_b", 17) + miss = [(common[0] + VOCAB // 2) % VOCAB] + common[1:] + token_stream("reuse_suffix_b", 17) + with BeamSmallModelGenerator(model) as gen: + a = gen.create_request(a_prompt, max_new_tokens=1, sample=False) + gen.prefill(a) + b = gen.create_request(b_prompt, max_new_tokens=1, sample=False) + with counting_attention_calls() as calls: + gen.prefill(b) + assert b.reused == 64 and b.computed_prompt_tokens == 17 + assert [rows for _, rows, _ in calls] == [17] * 6 + assert list(gen.manager.kv_cache_map[b.request_id].committed_tokens[:64]) == common + claimed = written_bytes(gen.manager, b.request_id, 64) + assert torch.equal(claimed, written_bytes(gen.manager, a.request_id, 64)) + # Layers 0-3 of the common tokens depend on the dense blocks only, whose arithmetic per + # token does not depend on the rest of the prompt: a recomputation reproduces them bitwise. + dense = claimed[: 4 * 64] + gen.release(b) + gen.release(a) + m = gen.create_request(miss, max_new_tokens=1, sample=False) + gen.prefill(m) + assert m.reused == 0 and m.computed_prompt_tokens == 81 + gen.release(m) + with BeamSmallModelGenerator(model) as gen: + control = gen.create_request(b_prompt, max_new_tokens=1, sample=False, reuse=False) + gen.prefill(control) + assert control.reused == 0 and control.computed_prompt_tokens == 81 + assert torch.equal(written_bytes(gen.manager, control.request_id, 64)[: 4 * 64], dense) + gen.release(control) + + +def test_prefix_reuse_logits_match_fresh_prefill_with_stable_routing() -> None: + """Compare actual cached and fresh forwards without near-tie expert-selection changes.""" + model = loaded() + biases = [ + tensor + for name, tensor in model.checkpoint_tensors().items() + if name.endswith("e_score_correction_bias") + ] + assert len(biases) == 3 + originals = [bias.clone() for bias in biases] + common = token_stream("reuse_common", 64) + prompt = common + token_stream("reuse_suffix_b", 17) + try: + # Sigmoid scores lie in [0, 1]. Spacing the selection-only bias by 2 fixes + # expert ordering across both paths while leaving mixing weights unbiased. + with torch.no_grad(): + for bias in biases: + bias.copy_(2 * torch.arange(bias.numel(), device=bias.device, dtype=bias.dtype)) + with BeamSmallModelGenerator(model) as gen: + source = gen.create_request( + common + token_stream("reuse_suffix_a", 17), max_new_tokens=1, sample=False + ) + gen.prefill(source) + gen.release(source) + reused = gen.create_request(prompt, max_new_tokens=1, sample=False) + reused_logits = gen.prefill(reused).logits + assert reused.reused == 64 and reused.computed_prompt_tokens == 17 + assert reused.consumed == len(prompt) + gen.release(reused) + fresh = gen.create_request(prompt, max_new_tokens=1, sample=False, reuse=False) + fresh_logits = gen.prefill(fresh).logits + assert fresh.reused == 0 and fresh.computed_prompt_tokens == len(prompt) + assert fresh.consumed == len(prompt) + # Fixed-route, six-layer BF16 comparison; paged and fresh attention can + # differ in rounding, so bitwise equality is not the numerical contract. + torch.testing.assert_close( + reused_logits.double(), fresh_logits.double(), atol=0.3, rtol=2**-7 + ) + gen.release(fresh) + finally: + # Restore only after all caches bound to the controlled weights have closed. + with torch.no_grad(): + for bias, original in zip(biases, originals): + bias.copy_(original) + + +def test_two_live_requests_stay_isolated() -> None: + """A and B share a reused prefix and advance alternately; neither step changes any slot the + other has written, shared or private.""" + model = loaded() + common = token_stream("reuse_common", 64) + with BeamSmallModelGenerator(model) as gen: + a = gen.create_request(common + token_stream("iso_a", 17), max_new_tokens=3, sample=False) + b = gen.create_request(common + token_stream("iso_b", 17), max_new_tokens=3, sample=False) + gen.prefill(a) + gen.prefill(b) + assert b.reused == 64 + for token_a, token_b in zip(token_stream("iso_ta", 3), token_stream("iso_tb", 3)): + for state, other, token in ((a, b, token_a), (b, a, token_b)): + before = written_bytes(gen.manager, other.request_id, other.consumed) + gen.decode(state, token) + assert torch.equal( + written_bytes(gen.manager, other.request_id, other.consumed), before + ) + gen.release(a) + gen.release(b) + + +def test_weights_cannot_change_under_an_open_cache() -> None: + """load_weights is refused while a generator is open. A parameter replaced by another tensor + (``load_state_dict(assign=True)``) or changed in place is detected before the next step: a + prefill that would reuse a released request's prefix claims nothing and calls no attention, + and a live request's decode allocates nothing. Restoring the bound parameter makes the cache + usable again; after close a new generator starts from an empty cache.""" + model = loaded() + tensors = model.checkpoint_tensors() + original = model.layers[0].self_attn_k_proj + assert tensors["model.layers.0.self_attn.k_proj.weight"] is original + prompt = token_stream("reuse_common", 64) + token_stream("reuse_suffix_a", 17) + + def assign_k_proj(tensor: torch.Tensor) -> None: + model.load_state_dict({"layers.0.self_attn_k_proj": tensor}, strict=False, assign=True) + + with BeamSmallModelGenerator(model) as gen: + released = gen.create_request(prompt, max_new_tokens=1, sample=False) + gen.prefill(released) + gen.release(released) + live = gen.create_request( + token_stream("generation_prompt", 17), max_new_tokens=2, sample=False + ) + gen.prefill(live) + versions = {n: t._version for n, t in tensors.items()} + with pytest.raises(RuntimeError, match="open generator"): + model.load_weights(tensors) + assert {n: t._version for n, t in tensors.items()} == versions + state = gen.create_request(prompt, max_new_tokens=1, sample=False) + capacity = gen.manager.kv_cache_map[live.request_id].capacity + try: + assign_k_proj(torch.zeros_like(original)) + assert model.layers[0].self_attn_k_proj is not original + assert original._version == versions["model.layers.0.self_attn.k_proj.weight"] + with counting_attention_calls() as calls: + with pytest.raises(ValueError, match="changed"): + gen.prefill(state) + with pytest.raises(ValueError, match="changed"): + gen.decode(live, 7) + assert calls == [] and state.request_id not in gen.manager.kv_cache_map + assert state.lifecycle is Lifecycle.NEW and state.reused == 0 + assert live.lifecycle is Lifecycle.PREFILLED and live.consumed == 17 + assert gen.manager.kv_cache_map[live.request_id].capacity == capacity + finally: + assign_k_proj(original) + assert model.layers[0].self_attn_k_proj is original + gen.prefill(state) + assert state.reused == 64 + gen.decode(live, 7) + gen.release(state) + state = gen.create_request(prompt, max_new_tokens=1, sample=False) + with torch.no_grad(): + original.mul_(1.0) + with counting_attention_calls() as calls: + with pytest.raises(ValueError, match="changed"): + gen.prefill(state) + with pytest.raises(ValueError, match="changed"): + gen.decode(live, 7) + assert calls == [] + assert state.lifecycle is Lifecycle.NEW and state.request_id not in gen.manager.kv_cache_map + model.load_weights(tensors) + with BeamSmallModelGenerator(model) as gen: + state = gen.create_request(prompt, max_new_tokens=1, sample=False) + gen.prefill(state) + assert state.reused == 0 + gen.release(state) + + +# ------------------------------------------------------------------ guards before dispatch + + +@contextlib.contextmanager +def step_metadata(gen, state, num_tokens: int, past: int, context: bool, **overrides): + md = TrtllmAttentionMetadata( + max_num_requests=1, + max_num_tokens=model_module.MAX_TOKENS, + kv_cache_manager=gen.manager, + num_contexts=1 if context else 0, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[past]), + seq_lens=torch.tensor([num_tokens], dtype=torch.int), + request_ids=[state.request_id], + prompt_lens=[num_tokens if context else state.prompt_len], + runtime_features=overrides.pop( + "runtime_features", AttentionRuntimeFeatures(cache_reuse=True) + ), + ) + md.prepare() + yield md + + +_BAD_STEPS = [ + "positions_restart", + "two_token_generation", + "cached_context_without_paged_fmha", + "unknown_request", + "beyond_capacity", + "block_offset_corrupted", + "rewound_into_history", + "gap_after_history", +] + + +@pytest.mark.parametrize("case", _BAD_STEPS) +def test_model_rejects_inconsistent_steps_before_dispatch(case) -> None: + """A step whose positions, metadata or block table do not describe the request's own cache + is refused with ValueError before the embedding or any attention call, and no written slot + changes. That includes a generation step that does not begin where the cache's history ends + -- one token back, into the cached history, or one token past it -- although the pages and + capacity it addresses are valid.""" + model = loaded() + prompt = token_stream("guard_prompt", 40) + with BeamSmallModelGenerator(model) as gen: + state = gen.create_request(prompt, max_new_tokens=2, sample=False) + gen.prefill(state) + request = state._request + request.add_new_token(7, 0) + # Capacity for the next token: none beyond the history for "beyond_capacity", one more + # token than usual for "gap_after_history", so that only the history check refuses it. + for _ in range({"beyond_capacity": 0, "gap_after_history": 2}.get(case, 1)): + assert gen.manager.try_allocate_generation(request) + past, ids = state.consumed, torch.tensor([[7]], device="cuda") + assert gen.manager.kv_cache_map[state.request_id].history_length == past == 40 + positions = torch.tensor([[past]], device="cuda") + md_args = dict(num_tokens=1, past=past, context=False) + if case == "positions_restart": + positions = torch.tensor([[0]], device="cuda") + elif case == "two_token_generation": + ids, positions = ( + torch.tensor([[7, 8]], device="cuda"), + torch.tensor([[past, past + 1]], device="cuda"), + ) + md_args = dict(num_tokens=2, past=past - 1, context=False) + elif case == "cached_context_without_paged_fmha": + md_args = dict( + num_tokens=1, past=past, context=True, runtime_features=AttentionRuntimeFeatures() + ) + elif case in ("rewound_into_history", "gap_after_history"): + start = past - 1 if case == "rewound_into_history" else past + 1 + md_args = dict(num_tokens=1, past=start, context=False) + positions = torch.tensor([[start]], device="cuda") + written = written_bytes(gen.manager, state.request_id, past) + with step_metadata(gen, state, **md_args) as md: + if case == "block_offset_corrupted": + md.kv_cache_block_offsets[0, 0, 0, 0] += 2 + elif case == "unknown_request": + md.request_ids = [state.request_id + 1000] + with counting_attention_calls() as calls, pytest.raises(ValueError): + model(ids, positions, md) + assert calls == [] + assert torch.equal(written_bytes(gen.manager, state.request_id, past), written) + gen.release(state) + + +def test_model_rejects_a_context_step_rewound_over_a_reused_prefix() -> None: + """B claims A's two committed pages and holds 64 tokens. A context step for B that starts at + token 32 -- inside those shared pages, on valid pages with capacity and positions that match + it -- is refused before any attention call, and the shared pages keep A's bytes.""" + model = loaded() + common = token_stream("reuse_common", 64) + a_prompt = common + token_stream("reuse_suffix_a", 17) + b_prompt = common + token_stream("reuse_suffix_b", 17) + with BeamSmallModelGenerator(model) as gen: + a = gen.create_request(a_prompt, max_new_tokens=1, sample=False) + gen.prefill(a) + b = gen.create_request(b_prompt, max_new_tokens=1, sample=False) + # The steps prefill takes before its model call: claim the prefix, size the cache. + request = b._request + batch = ScheduledRequests() + batch.context_requests_last_chunk = [request] + gen.seq_slots.prepare_resources(batch) + assert gen.manager.prepare_context(request) and request.context_current_position == 64 + assert gen.manager.resize_context(request, request.context_remaining_length) + cache = gen.manager.kv_cache_map[b.request_id] + assert cache.history_length == 64 and cache.capacity == 81 + shared = written_bytes(gen.manager, a.request_id, 81) + ids = torch.tensor([b_prompt[32:]], device="cuda") + positions = torch.arange(32, 81, device="cuda").view(1, -1) + with step_metadata(gen, b, num_tokens=49, past=32, context=True) as md: + with counting_attention_calls() as calls, pytest.raises(ValueError, match="holds 64"): + model(ids, positions, md) + assert calls == [] + assert torch.equal(written_bytes(gen.manager, a.request_id, 81), shared) + gen.release(b) + gen.release(a) + + +# ------------------------------------------------------------------ sampler and window + + +def test_torch_sampler_greedy_choice_on_identical_logits() -> None: + """The generator's sampler step picks the lowest index among exact maxima, on a context row + and on a generation row.""" + sampler = TorchSampler(generation.SAMPLER_ARGS) + slots = SeqSlotManager(generation.GENERATOR_CACHE["max_batch_size"]) + request = LlmRequest( + request_id=1, + max_new_tokens=4, + input_tokens=[1, 2, 3], + sampling_config=generation.greedy_sampling_config(), + is_streaming=False, + ) + batch = ScheduledRequests() + batch.context_requests_last_chunk = [request] + slots.prepare_resources(batch) + request.context_chunk_size = 3 + rows = [] + tied = torch.zeros(VOCAB, device="cuda") + tied[[90000, 1234, 151935]] = 2.5 + rows.append((tied, 1234)) + distinct = torch.linspace(-30.0, 30.0, VOCAB, device="cuda").flip(0) + distinct[77] = 30.0 + rows.append((distinct, 0)) + for step, (logits, expected) in enumerate(rows): + generation.sample_step(sampler, request, logits, is_context=step == 0) + assert request.get_last_tokens(0) == expected == int(torch.argmax(logits)) + if step == 0: + assert request.state == LlmRequestState.GENERATION_IN_PROGRESS + slots.free_resources(request) + + +def test_sliding_window_excludes_exactly_the_aged_key() -> None: + """Controlled keys through the generator's cache manager and the catalog attention op: at + query position 512 the sliding layer's output ignores key 0 bit for bit and depends on key + 1, while the full layer depends on key 0.""" + mgr = generation.build_cache_manager() + g = torch.Generator(device="cuda").manual_seed(512) + q = torch.randn(1, 48 * 128, generator=g, device="cuda").to(torch.bfloat16) + kv = torch.randn(513, 2 * 8 * 128, generator=g, device="cuda").to(torch.bfloat16) + request_ids = iter(range(100, 200)) + + def attend(layer: int, kv_rows: torch.Tensor) -> torch.Tensor: + rid = next(request_ids) + tokens = [(rid * 7919 + j * 104729) % VOCAB for j in range(512)] + request = LlmRequest( + request_id=rid, + max_new_tokens=1, + input_tokens=tokens, + sampling_config=generation.greedy_sampling_config(), + is_streaming=False, + ) + try: + assert mgr.prepare_context_cache(request, reuse_limit=0) == 0 + assert mgr.prepare_context(request) and mgr.resize_context(request, 512) + window = 512 if layer < 5 else mgr.max_seq_len + outputs = [] + zero_q = torch.zeros(512, 48 * 128, dtype=torch.bfloat16, device="cuda") + for start, count, context, rows in ( + (0, 512, True, torch.cat((zero_q, kv_rows[:512]), 1)), + (512, 1, False, torch.cat((q, kv_rows[512:513]), 1)), + ): + if not context: + request.move_to_next_context_chunk() + request.state = LlmRequestState.GENERATION_IN_PROGRESS + batch = ScheduledRequests() + batch.context_requests_last_chunk = [request] + mgr.update_context_resources(batch) + request.add_new_token(5, 0) + assert mgr.try_allocate_generation(request) + md = TrtllmAttentionMetadata( + max_num_requests=1, + max_num_tokens=1024, + kv_cache_manager=mgr, + num_contexts=1 if context else 0, + kv_cache_params=KVCacheParams( + use_cache=True, num_cached_tokens_per_seq=[start] + ), + seq_lens=torch.tensor([count], dtype=torch.int), + request_ids=[rid], + prompt_lens=[count if context else 512], + runtime_features=AttentionRuntimeFeatures(cache_reuse=True), + ) + md.prepare() + out = torch.empty(count, 48 * 128, dtype=torch.bfloat16, device="cuda") + thop_attention( + q=rows.contiguous(), + output=out, + num_heads=48, + num_kv_heads=8, + head_size=128, + attention_window_size=window, + **_attention.step_args(md, layer), + **_attention.CALL_CONSTANTS, + ) + outputs.append(out) + torch.cuda.synchronize() + return outputs[-1].clone() + finally: + mgr.free_resources(request) + + def perturbed(position: int) -> torch.Tensor: + rows = kv.clone() + rows[position, : 8 * 128] = -rows[position, : 8 * 128] + rows[position, 8 * 128 :] = rows[position, 8 * 128 :] * 4.0 + 1.0 + return rows + + try: + sliding, full = attend(0, kv), attend(5, kv) + assert bitwise_equal(attend(0, perturbed(0)), sliding) + assert not bitwise_equal(attend(0, perturbed(1)), sliding) + assert not bitwise_equal(attend(5, perturbed(0)), full) + finally: + mgr.shutdown() diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py new file mode 100644 index 000000000000..e945889e7f7e --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -0,0 +1,716 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam six-block small model: bf16 / sm_103 / tp1. + +Configuration under test, fixed below and nowhere else: the first six logical layers of the +checkpoint's schedule -- dense sliding-attention blocks 0-2, sparse sliding-attention blocks +3-4 and the sparse full-attention block 5 without position embedding -- at hidden 5120 with +48 q / 8 kv heads of 128, 256 experts of 2560 per sparse block, vocabulary 151936, untied +embedding and head and a final-logit softcap of 30; bf16 weights and activations with fp32 +routers; one sm_103 GPU; eager, batch 1, one fresh context per call on a six-layer +KVCacheManagerV2; generated weights; seeds {17, 41} x lengths {1, 17, 129}. + +What is checked. Construction, the 2394-tensor loading contract and its rejections, the +input and cache contracts (rejected before any op runs), and on every case the composition +itself, observed read-only at the catalog ops of the forward: the six blocks run once each +in order, each on its predecessor's output, with its own residual coefficients, attention +kind, window and cache slot; layer 5's q takes the query scale and its k is the normalized +k unrotated; the final norm, head and softcap run once each, and the returned logits are +exactly the softcap of the head output. Numerical agreement with the source model is +established separately; these tests generate everything at runtime and need no model files. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ + tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model + +Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 200 GiB +of free device memory: the generated state and the loaded model hold 62 GiB each, and one +test constructs a second model. No model files. +""" + +from __future__ import annotations + +import contextlib +import gc +import math +from collections import OrderedDict + +import pytest +import torch + +__extra_import_path__ = ["../.."] + +import _beam_component_support as support # noqa: E402 +from _beam_component_support import bitwise_equal # noqa: E402 + +from _components.six_layer_model import model as model_module # noqa: E501 +from _components.six_layer_model.model import ( # noqa: E501 + BeamSixLayerBf16Sm103Tp1, + prefill_context, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.activation.flashinfer_silu_and_mul import ( # noqa: E501 + flashinfer_silu_and_mul, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( + thop_attention, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( # noqa: E501 + beam_logit_softcap, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 + swiglu_fp32, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( + beam_neox_rope, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_nope_query_scale import ( # noqa: E501 + beam_nope_query_scale, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( + beam_output_gate, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 + beam_dispatch_combine, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 + beam_grouped_topk_router, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( + beam_depth_residual, +) +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.resource_manager import CacheTypeCpp, DataType +from tensorrt_llm.llmapi.llm_args import KvCacheConfig +from tensorrt_llm.mapping import Mapping + +assert torch.cuda.is_available(), "the Beam six-block model requires a CUDA device" + +pytestmark = pytest.mark.high_cuda_memory + +# -------------------------------------------------- the fixed configuration + +#: The checkpoint's configuration reduced to its first six layers; only these fields are read. +CONFIG = support.SMALL_MODEL_CONFIG +model_config = support.small_model_config +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) +CASES = tuple((seed, length) for seed in SEEDS for length in LENGTHS) +CASE_IDS = [f"seed{seed}-T{length}" for seed, length in CASES] +VOCAB = CONFIG["vocab_size"] +HIDDEN = CONFIG["hidden_size"] + +#: Every catalog entry the forward can call. +OPS = ( + flashinfer_rmsnorm, + cublas_mm, + beam_neox_rope, + beam_nope_query_scale, + thop_attention, + beam_output_gate, + beam_depth_residual, + flashinfer_silu_and_mul, + beam_grouped_topk_router, + beam_dispatch_combine, + swiglu_fp32, + beam_logit_softcap, +) + + +def depth_scales(layer_idx: int, is_attention: bool) -> tuple[float, float]: + """The source model's depth residual coefficients: the n-th add of the residual stream + scales the branch by 1 / sqrt(n) and the stream by sqrt(n - 1) / sqrt(n).""" + n = 2 * (layer_idx + 1) + (0 if is_attention else 1) + return 1.0 / math.sqrt(n), math.sqrt(n - 1) / math.sqrt(n) + + +# -------------------------------------------------- generated state and inputs + + +def generate_state(seed: int) -> "OrderedDict[str, torch.Tensor]": + """All 2394 tensors, each drawn from its own generator keyed by seed and name, fp32 then + rounded once to the stored dtype.""" + return support.generate_small_model_state(seed, model_module) + + +def generate_ids(seed: int, length: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randint(0, VOCAB, (1, max(LENGTHS)), generator=g, device="cuda")[:, :length] + + +_CACHE: dict = {} + + +@pytest.fixture(scope="module", autouse=True) +def device_memory(): + """Release the generated weights and model before the next module runs.""" + yield + _CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + + +def loaded(seed: int) -> tuple[BeamSixLayerBf16Sm103Tp1, dict]: + """The seed's model, loaded with the seed's generated state; one seed at a time.""" + if _CACHE.get("seed") != seed: + _CACHE.clear() + torch.cuda.empty_cache() + state = generate_state(seed) + model = BeamSixLayerBf16Sm103Tp1(model_config(), device="cuda") + model.load_weights(state) + _CACHE.update(seed=seed, state=state, model=model) + return _CACHE["model"], _CACHE["state"] + + +def positions_of(length: int) -> torch.Tensor: + return torch.arange(length, device="cuda").view(1, length) + + +def run(model, ids: torch.Tensor) -> torch.Tensor: + length = ids.shape[1] + with prefill_context(length) as md: + out = model(ids, positions_of(length), md) + torch.cuda.synchronize() + return out + + +# -------------------------------------------------- op-path instrumentation + + +@contextlib.contextmanager +def recording(model): + """Log every catalog op call of the forward in order, with the block it ran in, its + activation arguments and its output; and every block's entry and exit.""" + calls: list[dict] = [] + events: list = [] + current = [None] + saved = [] + weight_args_from = {"CublasMm": 1, "FlashinferRmsnorm": 1, "BeamGroupedTopkRouter": 1} + for w in OPS: + cls = type(w) + original = cls.__call__ + saved.append((cls, original)) + + def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): + keep = weight_args_from.get(_name, len(args)) + entry = { + "op": _name, + "block": current[0], + "args": [a if j < keep else None for j, a in enumerate(args)], + "kwargs": kwargs, + } + if isinstance(kwargs.get("q"), torch.Tensor): + entry["qkv"] = kwargs["q"].clone() # before the op appends from it + calls.append(entry) + entry["output"] = _original(self, *args, **kwargs) + return entry["output"] + + cls.__call__ = logged + handles = [] + for i, block in enumerate(model.layers): + + def pre(module, args, _i=i): + current[0] = _i + events.append(("enter", _i, args[0])) + + def post(module, args, out, _i=i): + events.append(("exit", _i, out)) + current[0] = None + + handles.append(block.register_forward_pre_hook(pre)) + handles.append(block.register_forward_hook(post)) + try: + yield calls, events + finally: + for h in handles: + h.remove() + for cls, original in reversed(saved): + cls.__call__ = original + + +@contextlib.contextmanager +def no_dispatch(): + """Fail on any op call or embedding lookup: rejected inputs are rejected before either.""" + saved = [] + for w in OPS: + cls = type(w) + saved.append((cls, cls.__call__)) + + def refuse(self, *args, _name=cls.__name__, **kwargs): + raise AssertionError(f"{_name} dispatched before the input was rejected") + + cls.__call__ = refuse + embedding = torch.nn.functional.embedding + + def refuse_embedding(*args, **kwargs): + raise AssertionError("the embedding ran before the input was rejected") + + torch.nn.functional.embedding = refuse_embedding + try: + yield + finally: + torch.nn.functional.embedding = embedding + for cls, original in reversed(saved): + cls.__call__ = original + + +@contextlib.contextmanager +def manager_context(length: int, **overrides): + """A KVCacheManagerV2 other than the model's own, with prepared metadata for one request.""" + windows = overrides.pop("max_attention_window", list(model_module.ATTENTION_WINDOWS)) + args = dict( + num_layers=6, + num_kv_heads=8, + head_dim=128, + tokens_per_block=32, + max_seq_len=1024, + max_batch_size=1, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + dtype=DataType.BF16, + ) + args.update(overrides) + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + from tensorrt_llm._torch.metadata import KVCacheParams + + mgr = KVCacheManagerV2( + KvCacheConfig(max_tokens=1024, enable_block_reuse=False, max_attention_window=windows), + CacheTypeCpp.SELF, + **args, + ) + try: + mgr.add_dummy_requests([0], token_nums=[length]) + md = TrtllmAttentionMetadata( + max_num_requests=1, + max_num_tokens=model_module.MAX_TOKENS, + kv_cache_manager=mgr, + num_contexts=1, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]), + seq_lens=torch.tensor([length], dtype=torch.int), + request_ids=[0], + prompt_lens=[length], + ) + md.prepare() + yield md + finally: + mgr.shutdown() + + +# -------------------------------------------------- tests: configuration and construction + + +def test_configuration_is_the_fixed_scope() -> None: + assert model_module.NUM_LAYERS == 6 + assert model_module.LAYER_TYPES == tuple(CONFIG["layer_types"]) + assert model_module.MLP_LAYER_TYPES == tuple(CONFIG["mlp_layer_types"]) + assert model_module.ATTENTION_WINDOWS == (512,) * 5 + (1024,) + assert model_module.MAX_TOKENS == model_module.KV_CACHE["max_seq_len"] == 1024 + for field, value in model_module.SUPPORTED_CONFIG.items(): + assert CONFIG[field] == value, field + schema = model_module.weight_schema(model_config()) + assert len(schema) == 2394 + assert sum(1 for n in schema if ".mlp.experts." in n) == 3 * 256 * 3 + fp32 = sorted(n for n, (_, dtype) in schema.items() if dtype == torch.float32) + assert fp32 == sorted( + f"model.layers.{i}.mlp.gate.{t}" + for i in (3, 4, 5) + for t in ("weight", "e_score_correction_bias") + ) + + +@pytest.mark.parametrize( + "overrides,match", + [ + (dict(num_hidden_layers=52), "num_hidden_layers"), + (dict(tie_word_embeddings=True), "tie_word_embeddings"), + (dict(final_logit_softcapping=None), "final_logit_softcapping"), + (dict(vocab_size=151552), "vocab_size"), + (dict(layer_types=["sliding_attention"] * 6), "layer_types"), + (dict(mlp_layer_types=["dense"] * 4 + ["sparse"] * 2), "mlp_layer_types"), + ( + dict( + rope_parameters={"sliding_attention": {"rope_type": "default", "rope_theta": 1e4}} + ), + "full_attention", + ), + ], +) +def test_rejects_unsupported_configuration(overrides, match) -> None: + with pytest.raises(ValueError, match=match): + BeamSixLayerBf16Sm103Tp1(model_config(**overrides), device="cuda") + + +def test_constructs_six_distinct_blocks() -> None: + model, _ = loaded(17) + assert [type(b).__name__ for b in model.layers] == ["BeamDenseBlockBf16Sm103Tp1"] * 3 + [ + "BeamSparseBlockBf16Sm103Tp1" + ] * 3 + assert [b.layer_idx for b in model.layers] == list(range(6)) + for i, block in enumerate(model.layers): + assert block.attention_scales == depth_scales(i, True) + assert block.feed_forward_scales == depth_scales(i, False) + assert [getattr(b, "is_sliding", True) for b in model.layers] == [True] * 5 + [False] + assert [b.rope_theta for b in model.layers] == [10000.0] * 5 + [None] + for i in (3, 4, 5): + assert len(model.layers[i].mlp.experts) == 256 + assert model.layers[i].mlp.gate.weight.dtype == torch.float32 + tensors = model.checkpoint_tensors() + assert len(tensors) == 2394 + spans = sorted( + (t.data_ptr(), t.data_ptr() + t.numel() * t.element_size()) for t in tensors.values() + ) + assert all(a[1] <= b[0] for a, b in zip(spans, spans[1:])), "two tensors share storage" + + +def test_unloaded_model_refuses_to_run() -> None: + loaded(17) + fresh = BeamSixLayerBf16Sm103Tp1(model_config(), device="cuda") + with prefill_context(1) as md, no_dispatch(), pytest.raises(ValueError, match="load_weights"): + fresh(generate_ids(17, 1), positions_of(1), md) + del fresh + torch.cuda.empty_cache() + + +# -------------------------------------------------- tests: loading + + +def test_load_weights_maps_every_tensor() -> None: + """Every tensor lands under its own name, value, dtype and [out, in] orientation.""" + model, state = loaded(17) + tensors = model.checkpoint_tensors() + assert list(tensors) == list(state) + for name, value in state.items(): + assert bitwise_equal(tensors[name], value), name + assert tensors[model_module.EMBED].data_ptr() != tensors[model_module.HEAD].data_ptr() + assert not torch.equal(tensors[model_module.EMBED], tensors[model_module.HEAD]) + + +_BAD_LOADS = ["missing_head", "unexpected_name", "late_expert_fp32", "late_expert_transposed"] +_BAD_LOADS += ["router_bf16", "head_not_a_tensor"] + + +@pytest.mark.parametrize("case", _BAD_LOADS) +def test_rejected_load_changes_nothing(case) -> None: + """A malformed mapping is rejected before any tensor is copied, even when only its last + tensors are wrong, and the model keeps running on what it held.""" + model, state = loaded(17) + before = run(model, generate_ids(17, 1)) + versions = {n: t._version for n, t in model.checkpoint_tensors().items()} + bad = OrderedDict(state) + late = "model.layers.5.mlp.experts.255.down_proj.weight" + if case == "missing_head": + del bad[model_module.HEAD] + elif case == "unexpected_name": + bad["model.layers.6.input_layernorm.weight"] = state[model_module.NORM] + elif case == "late_expert_fp32": + bad[late] = state[late].float() + elif case == "late_expert_transposed": + bad[late] = state[late].t() + elif case == "router_bf16": + bad["model.layers.5.mlp.gate.weight"] = state["model.layers.5.mlp.gate.weight"].bfloat16() + else: + bad[model_module.HEAD] = state[model_module.HEAD].tolist + with pytest.raises((ValueError, TypeError)): + model.load_weights(bad) + after = {n: t._version for n, t in model.checkpoint_tensors().items()} + assert after == versions + assert bitwise_equal(run(model, generate_ids(17, 1)), before) + + +def test_failed_copy_leaves_the_model_unavailable(monkeypatch) -> None: + model, state = loaded(17) + before = run(model, generate_ids(17, 17)) + + def broken(weights): + raise RuntimeError("copy failed") + + monkeypatch.setattr(model.layers[4], "load_weights", broken) + with pytest.raises(RuntimeError, match="copy failed"): + model.load_weights(state) + with prefill_context(1) as md, no_dispatch(), pytest.raises(ValueError, match="load_weights"): + model(generate_ids(17, 1), positions_of(1), md) + monkeypatch.undo() + model.load_weights(state) + assert bitwise_equal(run(model, generate_ids(17, 17)), before) + + +# -------------------------------------------------- tests: input and cache contracts + +_BAD_INPUTS = [ + "float_ids", + "bool_ids", + "int16_ids", + "negative_id", + "id_vocab_size", + "batch_2", + "flat_ids", + "empty", + "too_long", + "host_ids", + "non_contiguous_ids", + "positions_offset", + "float_positions", + "positions_shape", + "host_positions", + "metadata_type", + "cached_prefix", + "context_length_mismatch", + "one_layer_cache", + "seven_layer_cache", + "duplicate_slots", + "slot_out_of_range", + "missing_layer", + "all_sliding_windows", + "no_sliding_windows", + "fp16_cache", + "four_kv_heads", +] +#: Host pool tables replaced after the manager built them: rows sharing a slot's storage or +#: swapped between slots, an offset outside its pool, a fractional, int64 or misshapen +#: mapping, and pointer rows other than the manager's. +_BAD_POOL_TABLES = [ + "aliased_pool_rows", + "swapped_pool_offsets", + "pool_offset_out_of_range", + "fractional_pool_offsets", + "int64_pool_mapping", + "extra_pool_mapping_row", + "aliased_pool_pointers", + "secondary_pool_pointer", +] +_BAD_INPUTS += _BAD_POOL_TABLES + ["swa_scratch_cache"] + + +@pytest.mark.parametrize("case", _BAD_INPUTS) +def test_rejects_unsupported_inputs_before_dispatch(case, monkeypatch) -> None: + """Each declared restriction raises ValueError before any op or the embedding runs.""" + model, _ = loaded(17) + length = 17 + ids = generate_ids(17, length) + positions = positions_of(length) + context = lambda: prefill_context(length) # noqa: E731 + if case == "float_ids": + ids = ids.float() + elif case == "bool_ids": + ids = ids > 0 + elif case == "int16_ids": + ids = ids.to(torch.int16) + elif case == "negative_id": + ids = ids.clone() + ids[0, 5] = -1 + elif case == "id_vocab_size": + ids = ids.clone() + ids[0, -1] = VOCAB + elif case == "batch_2": + ids = torch.cat((ids, ids)) + positions = torch.cat((positions, positions)) + elif case == "flat_ids": + ids = ids.view(length) + elif case == "empty": + ids, positions = ids[:, :0], positions[:, :0] + elif case == "too_long": + ids = generate_ids(17, 129).repeat(1, 8)[:, : model_module.MAX_TOKENS + 1].contiguous() + positions = positions_of(model_module.MAX_TOKENS + 1) + elif case == "host_ids": + ids = ids.cpu() + elif case == "non_contiguous_ids": + ids = generate_ids(17, 129).repeat(1, 2)[:, : 2 * length : 2] + elif case == "positions_offset": + positions = positions + 1 + elif case == "float_positions": + positions = positions.float() + elif case == "positions_shape": + positions = positions.view(length) + elif case == "host_positions": + positions = positions.cpu() + elif case == "cached_prefix": + context = lambda: manager_context(length, max_seq_len=1024) # noqa: E731 + elif case == "context_length_mismatch": + context = lambda: prefill_context(length - 1) # noqa: E731 + elif case == "one_layer_cache": + context = lambda: manager_context(length, num_layers=1, max_attention_window=[512]) # noqa: E731 + elif case == "seven_layer_cache": + context = lambda: manager_context( # noqa: E731 + length, num_layers=7, max_attention_window=[512] * 5 + [1024, 512] + ) + elif case == "all_sliding_windows": + context = lambda: manager_context(length, max_attention_window=[512] * 6) # noqa: E731 + elif case == "no_sliding_windows": + context = lambda: manager_context(length, max_attention_window=[1024] * 6) # noqa: E731 + elif case == "fp16_cache": + context = lambda: manager_context(length, dtype=DataType.HALF) # noqa: E731 + elif case == "four_kv_heads": + context = lambda: manager_context(length, num_kv_heads=4) # noqa: E731 + with context() as md: + if case == "metadata_type": + md = object() + elif case == "cached_prefix": + md.kv_lens_runtime[0] = length + 8 + elif case in ("duplicate_slots", "slot_out_of_range", "missing_layer"): + offsets = dict(md.kv_cache_manager.layer_offsets) + if case == "duplicate_slots": + offsets[5] = offsets[4] + elif case == "slot_out_of_range": + offsets[5] = 6 + else: + del offsets[5] + monkeypatch.setattr(md.kv_cache_manager, "layer_offsets", offsets) + elif case in _BAD_POOL_TABLES: + mgr = md.kv_cache_manager + mapping = mgr.kv_cache_pool_mapping.clone() + pointers = mgr.kv_cache_pool_pointers.clone() + if case == "aliased_pool_rows": + mapping[4] = mapping[3] + elif case == "swapped_pool_offsets": + mapping[[0, 1]] = mapping[[1, 0]] + elif case == "pool_offset_out_of_range": + mapping[5, 1] = 1 << 20 + elif case == "fractional_pool_offsets": + mapping = mapping.float() + mapping[5, 1] = 0.5 + elif case == "int64_pool_mapping": + mapping = mapping.long() + elif case == "extra_pool_mapping_row": + mapping = torch.cat((mapping, mapping[5:])) + elif case == "aliased_pool_pointers": + pointers[1, 0] = pointers[0, 0] + else: + pointers[0, 1] = pointers[0, 0] + monkeypatch.setattr(mgr, "kv_cache_pool_mapping", mapping) + monkeypatch.setattr(mgr, "kv_cache_pool_pointers", pointers) + elif case == "swa_scratch_cache": + monkeypatch.setattr(md.kv_cache_manager, "enable_swa_scratch_reuse", True) + with no_dispatch(), pytest.raises(ValueError): + model(ids, positions, md) + + +def test_cache_slots_are_the_managers_own_buffers() -> None: + """The prepared cache passes check_cache, and each layer's pool row addresses the buffer + the manager itself reports for that layer: six distinct buffers, as allocated.""" + with prefill_context(17) as md: + model_module.check_cache(md, 17) + mgr = md.kv_cache_manager + mapping, pointers = md.host_kv_cache_pool_mapping, md.host_kv_cache_pool_pointers + assert mapping.dtype == torch.int32 and pointers.dtype == torch.int64 + page = 32 * 8 * 128 * 2 + addresses = [] + for layer_idx in range(6): + pool, offset = mapping[mgr.layer_offsets[layer_idx]].tolist() + addresses.append(int(pointers[pool, 0]) + offset * 2 * page) + assert addresses[-1] == mgr.get_buffers(layer_idx).data_ptr(), layer_idx + assert len(set(addresses)) == 6 + + +# -------------------------------------------------- tests: the composed forward + + +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_forward_composes_the_six_blocks(seed, length) -> None: + """One plain call and one observed call: the same logits, every element of them, and the + composition the observation shows.""" + model, _ = loaded(seed) + ids = generate_ids(seed, length) + ids_before = ids.clone() + versions = {n: t._version for n, t in model.checkpoint_tensors().items()} + plain = run(model, ids) + assert plain.shape == (1, length, VOCAB) and plain.dtype == torch.float32 + assert plain.device == model.device and bool(torch.isfinite(plain).all()) + assert float(plain.abs().max()) <= 30.0 + with prefill_context(length) as md, recording(model) as (calls, events): + observed = model(ids, positions_of(length), md) + torch.cuda.synchronize() + assert bitwise_equal(observed, plain) + assert torch.equal(ids, ids_before) + assert {n: t._version for n, t in model.checkpoint_tensors().items()} == versions + # Blocks 0..5 once each, in order, each on its predecessor's output. + assert [(kind, i) for kind, i, _ in events] == [ + (kind, i) for i in range(6) for kind in ("enter", "exit") + ] + for i in range(1, 6): + assert bitwise_equal(events[2 * i][2], events[2 * i - 1][2]) + embedded = events[0][2] + assert embedded.shape == (1, length, HIDDEN) and embedded.dtype == torch.bfloat16 + assert bitwise_equal(embedded[0], model.embed_tokens[ids[0]]) + for i in range(6): + block = [c for c in calls if c["block"] == i and c["op"] != "expert"] + attention = [c for c in block if c["op"] == "ThopAttention"] + (att,) = attention + assert att["kwargs"]["local_layer_idx"] == i + assert att["kwargs"]["attention_window_size"] == (512 if i < 5 else 1024) + residuals = [c for c in block if c["op"] == "BeamDepthResidual"] + assert [tuple(c["args"][2:4]) for c in residuals] == [ + depth_scales(i, True), + depth_scales(i, False), + ] + qkv = att["qkv"] + k_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][2]["output"] + q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][1]["output"] + packed_q = qkv[:, : 48 * 128].reshape(length, 48, 128) + packed_k = qkv[:, 48 * 128 : 56 * 128].reshape(length, 8, 128) + if i < 5: + assert [c["op"] for c in block].count("BeamNeoxRope") == 1 + assert "BeamNopeQueryScale" not in [c["op"] for c in block] + else: + assert "BeamNeoxRope" not in [c["op"] for c in block] + assert bitwise_equal(packed_k, k_norm.view(length, 8, 128)) + assert bitwise_equal( + packed_q, + beam_nope_query_scale(q_norm.view(length, 48, 128), positions_of(length)[0]), + ) + top = [c for c in calls if c["block"] is None] + assert [c["op"] for c in top] == ["FlashinferRmsnorm", "CublasMm", "BeamLogitSoftcap"] + norm, head, softcap = top + assert bitwise_equal(norm["args"][0], events[-1][2].view(length, HIDDEN)) + assert bitwise_equal(head["args"][0], norm["output"]) + assert head["output"].dtype == torch.bfloat16 and head["output"].shape == (length, VOCAB) + # The softcap, exactly once: the returned logits are this expression of the head output. + once = (head["output"].float() / 30.0).tanh() * 30.0 + assert bitwise_equal(plain.view(length, VOCAB), once) + assert bitwise_equal(softcap["args"][0], head["output"]) and softcap["args"][1] == 30.0 + + +def test_layer5_query_scale_is_visible() -> None: + """At 129 tokens the query scale changes block 5's q at later positions, so an omitted + scale would be a different call.""" + model, _ = loaded(17) + with prefill_context(129) as md, recording(model) as (calls, _): + model(generate_ids(17, 129), positions_of(129), md) + torch.cuda.synchronize() + block = [c for c in calls if c["block"] == 5] + q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][1]["output"].view(129, 48, 128) + (att,) = [c for c in block if c["op"] == "ThopAttention"] + packed_q = att["qkv"][:, : 48 * 128].reshape(129, 48, 128) + changed = (packed_q != q_norm).flatten(1).any(dim=1) + assert not bool(changed[0]) and bool(changed[-1]) + + +def test_causal_prefix_and_request_isolation() -> None: + """Changing the tokens after the first 17 leaves the dense blocks' prefix rows and block + 3's input to its feed-forward unchanged, bit for bit; and a request between two identical + requests changes nothing of the second.""" + model, _ = loaded(17) + full = generate_ids(17, 129) + other = full.clone() + other[:, 17:] = generate_ids(41, 129)[:, 17:] + + def observe(ids): + with prefill_context(ids.shape[1]) as md, recording(model) as (calls, events): + out = model(ids, positions_of(ids.shape[1]), md) + torch.cuda.synchronize() + pre_ff = [c["output"] for c in calls if c["block"] == 3 and c["op"] == "FlashinferRmsnorm"][ + 4 + ] + return out, [x for kind, _, x in events if kind == "exit"], pre_ff + + out_a, exits_a, pre_ff_a = observe(full) + out_b, exits_b, pre_ff_b = observe(other) + for i in range(3): + assert bitwise_equal(exits_a[i][:, :17], exits_b[i][:, :17]) + assert bitwise_equal(pre_ff_a[:17], pre_ff_b[:17]) + assert not torch.equal(out_a[:, 17:], out_b[:, 17:]) + first = run(model, full[:, :17]) + run(model, generate_ids(41, 17)) + assert bitwise_equal(run(model, full[:, :17]), first) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py new file mode 100644 index 000000000000..8a93c5c0adc3 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Plain-Torch reference for the sparse sliding-attention decoder block 3 of Beam. + +An independent restatement of the source block for generated-input comparison: sandwich +RMSNorms (fp32 statistics and affine, one bf16 rounding); separate bias-free q/k/v/gate +projections in [out, in] layout; per-head q/k RMSNorm rounded to bf16 before an fp32 NeoX +RoPE rounded once; causal GQA attention, 6 q heads per kv head, scale 1/sqrt(128), window +512; the per-head sigmoid output gate in bf16 steps; the output projection; the MoE of +``../moe/_moe_reference.py``; and block 3's depth-scaled residual adds, +(1/sqrt(8), sqrt(7)/sqrt(8)) after attention and (1/3, sqrt(8)/3) after the feed-forward, +each scaled term and each sum rounded to bf16. + +Only torch and that MoE reference are imported. The attention core is torch's SDPA, which +the component never runs. State and input are generated on the GPU as the MoE reference +generates its own, one seeded generator per tensor; the block's 770 ``mlp.`` tensors are +exactly the MoE reference's state of the same seed. +""" + +from __future__ import annotations + +import math +from collections import OrderedDict + +import torch +import torch.nn.functional as F + +__extra_import_path__ = ["../moe"] +import _moe_reference as moe_reference # noqa: E402 + +HIDDEN = 5120 +HEADS = 48 +KV_HEADS = 8 +HEAD_DIM = 128 +EPS = 1e-6 +ROPE_THETA = 10000.0 +WINDOW = 512 +LAYER_IDX = 3 +MAX_TOKENS = 129 +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) +#: The MoE's tensors carry this prefix among the block's names. +MLP_PREFIX = "mlp." + +STATE_RECIPE = { + "seeding": "one torch.Generator on the device per call, re-seeded for every tensor with " + "zlib.crc32(f'{seed}|{name}'); the mlp. tensors are the MoE reference's state of the " + "seed, drawn under their names relative to mlp.", + "*norm.weight": "1 + 0.25 * N(0, 1), drawn in fp32 and rounded once to bf16", + "self_attn.gate_bias": "0.75 * N(0, 1), drawn in fp32 and rounded once to bf16", + "projections": "N(0, 1) / sqrt(input width), drawn in fp32 and rounded once to bf16", + "input": "N(0, 1) of shape [1, 129, 5120] under the name 'block_input', drawn in fp32 " + "and rounded once to bf16; a T-token case uses the first T tokens, positions arange(T)", +} + + +def depth_coefs(layer_idx: int, is_attention: bool) -> tuple[float, float]: + """(branch, stream) scales of the n-th residual add, 1 / sqrt(n) and sqrt(n - 1) / + sqrt(n); block i's attention add is the 2(i + 1)-th, its feed-forward add the next.""" + num_adds = 2 * (layer_idx + 1) + (0 if is_attention else 1) + denom = math.sqrt(num_adds) + return 1.0 / denom, math.sqrt(num_adds - 1) / denom + + +ATTENTION_SCALES = depth_coefs(LAYER_IDX, True) +FEED_FORWARD_SCALES = depth_coefs(LAYER_IDX, False) + + +def _ignore(name: str, value: object) -> None: + del name, value + + +# ----------------------------------------------------------------------------- state and inputs + + +def own_shapes() -> "OrderedDict[str, tuple[int, ...]]": + """The block's twelve bf16 attention and norm tensors by checkpoint name, [out, in].""" + q, kv = HEADS * HEAD_DIM, KV_HEADS * HEAD_DIM + return OrderedDict( + [ + ("input_layernorm.weight", (HIDDEN,)), + ("self_attn.gate_bias", (HEADS,)), + ("self_attn.q_proj.weight", (q, HIDDEN)), + ("self_attn.k_proj.weight", (kv, HIDDEN)), + ("self_attn.v_proj.weight", (kv, HIDDEN)), + ("self_attn.gate_proj.weight", (q, HIDDEN)), + ("self_attn.o_proj.weight", (HIDDEN, q)), + ("self_attn.q_norm.weight", (HEAD_DIM,)), + ("self_attn.k_norm.weight", (HEAD_DIM,)), + ("pre_feedforward_layernorm.weight", (HIDDEN,)), + ("post_attention_layernorm.weight", (HIDDEN,)), + ("post_feedforward_layernorm.weight", (HIDDEN,)), + ] + ) + + +def param_shapes() -> "OrderedDict[str, tuple[tuple[int, ...], torch.dtype]]": + """The block's 782 tensors by checkpoint name: its own twelve, then the MoE's under + ``mlp.``, with their dtypes.""" + shapes = OrderedDict((name, (shape, torch.bfloat16)) for name, shape in own_shapes().items()) + for name, shape_dtype in moe_reference.param_shapes().items(): + shapes[MLP_PREFIX + name] = shape_dtype + return shapes + + +def generate_state( + seed: int, device: str | torch.device = "cuda" +) -> "OrderedDict[str, torch.Tensor]": + """The seed's 782 tensors on ``device``, in checkpoint order (see `STATE_RECIPE`).""" + gen = torch.Generator(device=device) + state: "OrderedDict[str, torch.Tensor]" = OrderedDict() + for name, shape in own_shapes().items(): + gen.manual_seed(moe_reference.tensor_seed(seed, name)) + draw = torch.randn(shape, generator=gen, dtype=torch.float32, device=device) + if name.endswith("norm.weight"): + value = 1.0 + 0.25 * draw + elif name == "self_attn.gate_bias": + value = 0.75 * draw + else: + value = draw / math.sqrt(shape[1]) + state[name] = value.to(torch.bfloat16).contiguous() + for name, tensor in moe_reference.generate_state(seed, device).items(): + state[MLP_PREFIX + name] = tensor + return state + + +def moe_state(state: "dict[str, torch.Tensor]") -> "OrderedDict[str, torch.Tensor]": + """The MoE's tensors of a block state by name relative to ``mlp.``; nothing is copied.""" + return OrderedDict( + (name[len(MLP_PREFIX) :], tensor) + for name, tensor in state.items() + if name.startswith(MLP_PREFIX) + ) + + +def generate_input(seed: int, device: str | torch.device = "cuda") -> torch.Tensor: + """``[1, 129, 5120]`` bf16: the residual stream entering block 3 (see `STATE_RECIPE`).""" + gen = torch.Generator(device=device) + gen.manual_seed(moe_reference.tensor_seed(seed, "block_input")) + x = torch.randn(1, MAX_TOKENS, HIDDEN, generator=gen, dtype=torch.float32, device=device) + return x.to(torch.bfloat16) + + +def state_problems(state: "dict[str, torch.Tensor]", seed: int) -> list[str]: + """Exact accounting plus the nontrivial-state properties the comparisons rely on.""" + shapes = param_shapes() + if list(state) != list(shapes): + return ["names or order differ from param_shapes()"] + problems = [] + own = own_shapes() + for name, shape in own.items(): + tensor = state[name] + if tuple(tensor.shape) != shape or tensor.dtype != torch.bfloat16: + problems.append(f"{name}: {tuple(tensor.shape)} {tensor.dtype}, expected {shape}") + elif not bool(torch.isfinite(tensor).all()): + problems.append(f"{name}: nonfinite") + if problems: + return problems + names = list(own) + list(moe_reference.param_shapes()) + if len({moe_reference.tensor_seed(seed, name) for name in names}) != len(names): + problems.append("two tensors share a generator seed") + norms = [name for name in own if name.endswith("norm.weight")] + for name in norms: + if bool((state[name] == 1).all()): + problems.append(f"{name}: unit affine") + for i, a in enumerate(norms): + for b in norms[i + 1 :]: + if state[a].shape == state[b].shape and torch.equal(state[a], state[b]): + problems.append(f"{a} and {b} are identical") + bias = state["self_attn.gate_bias"] + if bool((bias == bias[0]).all()): + problems.append("self_attn.gate_bias does not vary across heads") + problems.extend(f"MoE: {p}" for p in moe_reference.state_problems(moe_state(state), seed)) + return problems + + +# ----------------------------------------------------------------------------- the block + + +def rms_norm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """Over the last dim: fp32 statistics, normalization and affine, rounded once.""" + x32 = x.float() + normed = x32 * torch.rsqrt(x32.pow(2).mean(dim=-1, keepdim=True) + EPS) + return (normed * weight.float()).to(x.dtype) + + +def rope( + q: torch.Tensor, k: torch.Tensor, positions: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """NeoX half rotation of ``q [T, 48, 128]`` and ``k [T, 8, 128]`` by their token's + position: fp32 angles ``position / 10000^(2i / 128)``, ``x cos + rotate_half(x) sin`` + in fp32, rounded once.""" + exponent = torch.arange(0, HEAD_DIM, 2, dtype=torch.float32, device=positions.device) + inv_freq = 1.0 / (ROPE_THETA ** (exponent / HEAD_DIM)) + angles = positions.float().unsqueeze(-1) * inv_freq + emb = torch.cat((angles, angles), dim=-1) + cos, sin = emb.cos().unsqueeze(1), emb.sin().unsqueeze(1) + half = HEAD_DIM // 2 + + def rotate(x: torch.Tensor) -> torch.Tensor: + x32 = x.float() + rotated = torch.cat((-x32[..., half:], x32[..., :half]), dim=-1) + return (x32 * cos + rotated * sin).to(x.dtype) + + return rotate(q), rotate(k) + + +def attention_core(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + """Causal windowed GQA over one fresh context: ``q [T, 48, 128]``, ``k`` and ``v`` + ``[T, 8, 128]`` bf16, q head h reading kv head h // 6; returns ``[T, 48, 128]`` bf16.""" + tokens = q.shape[0] + repeat = HEADS // KV_HEADS + k = k.repeat_interleave(repeat, dim=1) + v = v.repeat_interleave(repeat, dim=1) + position = torch.arange(tokens, device=q.device) + distance = position.unsqueeze(1) - position.unsqueeze(0) + keep = (distance >= 0) & (distance < WINDOW) + out = F.scaled_dot_product_attention( + q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), + v.transpose(0, 1).unsqueeze(0), + attn_mask=keep, + dropout_p=0.0, + scale=HEAD_DIM**-0.5, + ) + return out.squeeze(0).transpose(0, 1) + + +def output_gate(core: torch.Tensor, gate: torch.Tensor, gate_bias: torch.Tensor) -> torch.Tensor: + """``core * sigmoid(gate + gate_bias)`` per head, ``[T, 48, 128]``: the add, the + sigmoid and the product each rounded to bf16.""" + biased = (gate.float() + gate_bias.float().view(1, HEADS, 1)).to(torch.bfloat16) + weight = torch.sigmoid(biased.float()).to(torch.bfloat16) + return (core.float() * weight.float()).to(torch.bfloat16) + + +def attention( + h: torch.Tensor, positions: torch.Tensor, state: "dict[str, torch.Tensor]", record +) -> torch.Tensor: + """Normalized input ``[T, 5120]`` -> attention output ``[T, 5120]``, before the + post-attention norm.""" + tokens = h.shape[0] + q = F.linear(h, state["self_attn.q_proj.weight"]) + k = F.linear(h, state["self_attn.k_proj.weight"]) + v = F.linear(h, state["self_attn.v_proj.weight"]) + gate = F.linear(h, state["self_attn.gate_proj.weight"]) + for name, value in (("q_proj", q), ("k_proj", k), ("v_proj", v), ("gate_proj", gate)): + record(name, value) + q = rms_norm(q.view(tokens, HEADS, HEAD_DIM), state["self_attn.q_norm.weight"]) + k = rms_norm(k.view(tokens, KV_HEADS, HEAD_DIM), state["self_attn.k_norm.weight"]) + record("q_norm", q) + record("k_norm", k) + q, k = rope(q, k, positions) + record("q_rope", q) + record("k_rope", k) + core = attention_core(q, k, v.view(tokens, KV_HEADS, HEAD_DIM)) + record("attention_core", core) + gated = output_gate(core, gate.view(tokens, HEADS, HEAD_DIM), state["self_attn.gate_bias"]) + record("gated", gated) + out = F.linear(gated.reshape(tokens, HEADS * HEAD_DIM), state["self_attn.o_proj.weight"]) + record("attention_output", out) + return out + + +def residual_add( + branch: torch.Tensor, stream: torch.Tensor, scales: tuple[float, float] +) -> torch.Tensor: + """``branch * scales[0] + stream * scales[1]``: each product and the sum computed in + fp32 and rounded to bf16 in turn.""" + branch_scale, stream_scale = scales + scaled_branch = (branch.float() * branch_scale).to(torch.bfloat16) + scaled_stream = (stream.float() * stream_scale).to(torch.bfloat16) + return (scaled_branch.float() + scaled_stream.float()).to(torch.bfloat16) + + +def block( + x: torch.Tensor, + positions: torch.Tensor, + state: "dict[str, torch.Tensor]", + record=None, +) -> torch.Tensor: + """``x [1, T, 5120]`` bf16 and integer ``positions [1, T]`` -> ``[1, T, 5120]`` bf16. + + ``record(name, value)``, when given, receives every intermediate ``[T, ...]``, the + MoE's under ``moe.`` (``moe.routing`` is its route), and ``block_output`` last. + """ + keep = record if record is not None else _ignore + tokens = x.shape[1] + stream = x.reshape(tokens, HIDDEN) + with moe_reference.reference_math(): + h = rms_norm(stream, state["input_layernorm.weight"]) + keep("input_layernorm", h) + a = attention(h, positions.reshape(tokens), state, keep) + a = rms_norm(a, state["post_attention_layernorm.weight"]) + keep("post_attention_layernorm", a) + stream = residual_add(a, stream, ATTENTION_SCALES) + keep("attention_residual", stream) + h = rms_norm(stream, state["pre_feedforward_layernorm.weight"]) + keep("pre_feedforward_layernorm", h) + m = moe_reference.moe(h, moe_state(state), lambda name, value: keep(f"moe.{name}", value)) + keep("moe_output", m) + m = rms_norm(m, state["post_feedforward_layernorm.weight"]) + keep("post_feedforward_layernorm", m) + out = residual_add(m, stream, FEED_FORWARD_SCALES) + keep("block_output", out) + return out.view(1, tokens, HIDDEN) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py new file mode 100644 index 000000000000..03d15147fc46 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -0,0 +1,1072 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam sparse decoder block 3 component: bf16 / sm_103 / tp1. + +Configuration under test, fixed below and nowhere else: block 3 of the 52-block model +(sliding attention, MoE feed-forward) at hidden 5120 with 48 q / 8 kv heads of 128, and +block 3's MoE: 256 experts of intermediate 2560, 8 per token from 4 of 8 groups, routed +scaling factor 2.5; bf16 weights and activations with the source model's fp32 steps and +an fp32 router; one sm_103 GPU; eager, batch 1, one fresh context per call on a +KVCacheManagerV2 whose layer mask selects logical layer 3 alone; generated weights; +seeds {17, 41} x lengths {1, 17, 129}. + +What is compared. ``_sparse_block_reference.py`` restates the source block in plain +torch, with the MoE of ``../moe/_moe_reference.py``, and generates its +state and input deterministically on the GPU. Routing is judged apart from numerical +error. On the component's own feed-forward input, captured from the forward, the +reference must route exactly as the component did. Against the whole reference block, +whose feed-forward input differs from the component's by rounding, every routing +decision the two take differently must sit on a near tie of the reference's own scores; +the block output is compared on the tokens routed to the same experts, and the MoE +output on the component's own feed-forward input. Each compared tensor is held to one +rule in `RULES`. + +Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ + tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block + +Optional environment: + +* ``BEAM_SPARSE_BLOCK_EVIDENCE``: a directory that receives one JSON record per test + case: every comparison with its rule, the op calls and CUDA kernels of the forward + where they were recorded, and the device and software it ran on; and + ``fp32-session.json``, the fp32 GEMM checks made before and after the module's tests. + +Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 60 GiB +of free device memory: one seed's generated state and the loaded block hold 18.75 GiB of +expert weights each, and one test builds a second block. No model files. +""" + +from __future__ import annotations + +import functools +import gc +import math +import types +from collections import OrderedDict +from pathlib import Path + +import pytest +import torch + +try: + import tensorrt_llm._torch.models.modeling_beam as builtin_beam +except ModuleNotFoundError as exc: + if exc.name != "tensorrt_llm._torch.models.modeling_beam": + raise + builtin_beam = None # The standard-serving implementation is a separate PR. +__extra_import_path__ = [".", "../..", "../moe", "../../../.."] + +import _beam_component_support as support # noqa: E402 +import _moe_reference as moe_reference # noqa: E402 +import _sparse_block_reference as reference # noqa: E402 +from _beam_component_support import ( # noqa: E402 + assert_all_pass, + bitwise_equal, + compare_exact, + tensor_sha256, + tensor_versions, + unchanged, +) +from _validating import validating # noqa: E402 + +from _components.moe.moe import BeamMoeBf16Sm103Tp1 +from _components.sparse_block import block as block_module # noqa: E501 +from _components.sparse_block.block import BeamSparseBlockBf16Sm103Tp1 # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( + thop_attention, +) +from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 + swiglu_fp32, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( + beam_neox_rope, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( + beam_output_gate, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 + beam_dispatch_combine, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 + beam_grouped_topk_router, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( + beam_depth_residual, +) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.resource_manager import DataType +from tensorrt_llm.mapping import Mapping + +assert torch.cuda.is_available(), "the Beam sparse block requires a CUDA device" + +pytestmark = pytest.mark.high_cuda_memory + +# -------------------------------------------------- the fixed configuration + +NUM_LAYERS = 52 +LAYER_IDX = 3 +_FULL_ATTENTION_LAYERS = frozenset({5, 11, 17, 23, 29, 35, 41, 47, 51}) + +#: The block as the checkpoint configures it: the 52-block model's layer pattern and +#: block 3's attention and MoE. Only these fields are read. +BLOCK_CONFIG = types.MappingProxyType( + dict( + hidden_size=5120, + num_attention_heads=48, + num_key_value_heads=8, + head_dim=128, + rms_norm_eps=1e-6, + attention_bias=False, + attention_dropout=0.0, + sliding_window=512, + num_hidden_layers=NUM_LAYERS, + layer_types=tuple( + "full_attention" if i in _FULL_ATTENTION_LAYERS else "sliding_attention" + for i in range(NUM_LAYERS) + ), + mlp_layer_types=("dense",) * 3 + ("sparse",) * (NUM_LAYERS - 3), + rope_parameters={ + "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, + "full_attention": None, + }, + moe_intermediate_size=2560, + num_experts=256, + num_experts_per_tok=8, + n_group=8, + topk_group=4, + norm_topk_prob=True, + routed_scaling_factor=2.5, + hidden_act="silu", + ) +) +DTYPE = torch.bfloat16 +SM = (10, 3) +SEEDS = (17, 41) +LENGTHS = (1, 17, 129) +CASES = tuple((seed, length) for seed in SEEDS for length in LENGTHS) +CASE_IDS = [f"seed{seed}-T{length}" for seed, length in CASES] +#: Eager, batch 1, one fresh context: no CUDA graph, no overlap scheduler, no cached prefix. +EXECUTION = types.MappingProxyType( + dict(eager=True, batch_size=1, fresh_context=True, cuda_graph=False, overlap_scheduler=False) +) +#: The KV cache the attention op appends to: logical layer 3 of 52 selected by the layer +#: mask, bf16, 32-token pages, 512 tokens per sequence (the window), one request. +KV_CACHE = types.MappingProxyType(dict(tokens_per_block=32, max_seq_len=512, num_blocks=16)) +LAYER_MASK = tuple(i == LAYER_IDX for i in range(NUM_LAYERS)) +REQUEST_ID = 0 + +# -------------------------------------------------- comparison rules + +_RTOL = 2.0**-7 + +#: (atol, rtol) per kind of compared bf16 tensor. An element passes when +#: |actual - reference| <= atol + rtol * |reference|, both widened to fp64; every element of +#: every compared tensor must pass, and no NaN or Inf is accepted. rtol 2^-7 is one bf16 unit +#: at any magnitude. "block_output" holds the block output on the tokens routed to the same +#: experts as in the reference block, and the two prefix-identical inputs' common rows; +#: "moe_output" holds the MoE output against the reference MoE on the same input. +RULES = types.MappingProxyType({"block_output": (0.1, _RTOL), "moe_output": (0.04, _RTOL)}) +#: A routing decision the component takes differently from the whole reference block -- a +#: group kept or ordered differently, or an expert selected or ordered differently within +#: the groups both kept -- passes only on a margin of at most this in the reference's own +#: group scores or selection scores. Routing on the same input is never excused: it is +#: compared exactly. +ROUTE_GAP = 3e-3 + +#: The forward's top-level op calls, in order: the composition this component claims to +#: run. The experts' calls nest inside BeamDispatchCombine. +EXPECTED_CALLS = ( + "FlashinferRmsnorm", + "CublasMm", + "CublasMm", + "CublasMm", + "CublasMm", + "FlashinferRmsnorm", + "FlashinferRmsnorm", + "BeamNeoxRope", + "ThopAttention", + "BeamOutputGate", + "CublasMm", + "FlashinferRmsnorm", + "BeamDepthResidual", + "FlashinferRmsnorm", + "BeamGroupedTopkRouter", + "BeamDispatchCombine", + "FlashinferRmsnorm", + "BeamDepthResidual", +) +#: What each selected expert runs, in order, inside BeamDispatchCombine. +EXPERT_CALLS = ("expert", "CublasMm", "CublasMm", "SwigluFp32", "CublasMm") +#: Every catalog entry the block's forward can call. +OPS = ( + flashinfer_rmsnorm, + cublas_mm, + beam_neox_rope, + thop_attention, + beam_output_gate, + beam_depth_residual, + beam_grouped_topk_router, + swiglu_fp32, + beam_dispatch_combine, +) +#: Keyword arguments of the entries' calls that identify the path. +_RECORDED_KWARGS = ( + "num_heads", + "num_kv_heads", + "head_size", + "attention_window_size", + "max_seq_len", + "tokens_per_block", + "mask_type", + "is_fused_qkv", + "num_contexts", + "num_ctx_tokens", + "use_paged_context_fmha", + "position_embedding_type", + "local_layer_idx", + "quant_mode", + "n_group", + "topk_group", + "top_k", + "eps", + "routed_scaling_factor", +) + + +def block_config(**overrides) -> types.SimpleNamespace: + fields = dict(BLOCK_CONFIG) + fields["layer_types"] = list(fields["layer_types"]) + fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) + fields["rope_parameters"] = { + k: (dict(v) if v else v) for k, v in fields["rope_parameters"].items() + } + fields.update(overrides) + return types.SimpleNamespace(**fields) + + +def replaced(items, index: int, value) -> list: + """A list copy of ``items`` with one entry replaced.""" + out = list(items) + out[index] = value + return out + + +# -------------------------------------------------- precision and environment + + +@pytest.fixture(scope="module", autouse=True) +def ieee_fp32(): + """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both + before the module's tests and after them.""" + yield from support.ieee_fp32_session(_write_evidence) + + +@pytest.fixture(scope="module", autouse=True) +def device_memory(): + """Hand the module's cached state and block back to the device when its tests are done.""" + yield + release_cache() + + +def _environment(ieee: dict) -> dict: + layer_mask = [i for i, kept in enumerate(LAYER_MASK) if kept] + return support.environment(ieee, EXECUTION, kv_cache=dict(KV_CACHE, layer_mask=layer_mask)) + + +_write_evidence = functools.partial(support.write_evidence, "BEAM_SPARSE_BLOCK_EVIDENCE") + + +# -------------------------------------------------- comparison + + +def compare(name: str, actual: torch.Tensor, expected: torch.Tensor, rule: str | None) -> dict: + """Score one bf16 tensor against one rule; `rule=None` measures without judging.""" + tolerance = None if rule is None else RULES[rule] + return support.compare(name, actual, expected, rule, tolerance, DTYPE) + + +def inversion_gap(candidate: list, ref: list, scores: list, members: set) -> float: + """Largest ``scores[y] - scores[x]`` over the pairs of ``members`` that the candidate + ranks x before y and the reference ranks y before x; 0.0 when there is none. A rank is + the position in the ordered selection, and a member a selection lacks ranks after all of + it (8 for experts, 4 for groups).""" + rank_candidate = {m: i for i, m in enumerate(candidate)} + rank_ref = {m: i for i, m in enumerate(ref)} + last_candidate, last_ref = len(candidate), len(ref) + gap = 0.0 + for x in members: + for y in members: + before = rank_candidate.get(x, last_candidate) < rank_candidate.get(y, last_candidate) + after = rank_ref.get(y, last_ref) < rank_ref.get(x, last_ref) + if before and after: + gap = max(gap, scores[y] - scores[x]) + return gap + + +def judge_routing(own: dict, ref: dict) -> tuple[dict, torch.Tensor]: + """The component's routing against the whole reference block's, token by token. + + A token routed identically passes. Otherwise each decision taken differently has to be + a near tie in the reference's own scores: groups kept or ordered differently by their + ``group_scores``, experts selected or ordered differently by their ``selection`` scores + among the experts of the groups both kept (an expert of a group only one side kept is + the group decision's consequence). Returns the judgement and the mask of the tokens + routed to the same expert set. + """ + own_ids, ref_ids = own["ids"].tolist(), ref["ids"].tolist() + own_groups, ref_groups = own["groups"].tolist(), ref["groups"].tolist() + selection = ref["selection"].tolist() + group_scores = ref["group_scores"].tolist() + per_group = moe_reference.PER_GROUP + differing = [] + for t, (ids, ref_t) in enumerate(zip(own_ids, ref_ids)): + if ids == ref_t and own_groups[t] == ref_groups[t]: + continue + common = set(own_groups[t]) & set(ref_groups[t]) + members = {e for e in set(ids) | set(ref_t) if e // per_group in common} + differing.append( + { + "token": t, + "ids": ids, + "reference_ids": ref_t, + "groups": own_groups[t], + "reference_groups": ref_groups[t], + "group_gap": inversion_gap( + own_groups[t], ref_groups[t], group_scores[t], set(range(len(group_scores[t]))) + ), + "expert_gap": inversion_gap(ids, ref_t, selection[t], members), + } + ) + worst = max([0.0] + [max(d["group_gap"], d["expert_gap"]) for d in differing]) + beyond = [d["token"] for d in differing if max(d["group_gap"], d["expert_gap"]) > ROUTE_GAP] + matched = torch.tensor( + [set(ids) == set(ref_t) for ids, ref_t in zip(own_ids, ref_ids)], + dtype=torch.bool, + device=own["ids"].device, + ) + problems = [] + if beyond: + problems.append(f"tokens {beyond[:8]} route differently by up to {worst:.3g}") + judgement = { + "tensor": "routing_vs_reference_block", + "rule": f"every differing decision within {ROUTE_GAP:g} of the reference's own scores", + "tokens": len(ref_ids), + "differing_tokens": differing, + "set_matched_tokens": int(matched.sum()), + "max_gap": worst, + "passed": not problems, + "problems": problems, + } + return judgement, matched + + +def output_contract(out, tokens: int) -> dict: + return support.output_contract(out, (1, tokens, 5120), DTYPE, SM) + + +# -------------------------------------------------- state, inputs, fresh context + +_STATE_CACHE: dict[int, dict] = {} + + +def release_cache() -> None: + """Drop the cached state and block and hand their memory back to the device.""" + _STATE_CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + + +def generated(seed: int) -> dict: + """The seed's 782 tensors and [1, 129, 5120] input, generated once on the GPU.""" + if seed not in _STATE_CACHE: + release_cache() # one seed's 18.75 GiB of expert weights, and its block, at a time + state = reference.generate_state(seed) + problems = reference.state_problems(state, seed) + assert not problems, problems + _STATE_CACHE[seed] = { + "state": state, + "moe_state": reference.moe_state(state), + "input": reference.generate_input(seed), + "block": None, + "loaded": False, + } + return _STATE_CACHE[seed] + + +def loaded_block(seed: int) -> BeamSparseBlockBf16Sm103Tp1: + """The seed's cached block, loaded with the seed's state.""" + entry = generated(seed) + if entry["block"] is None: + entry["block"] = BeamSparseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + if not entry["loaded"]: + entry["block"].load_weights(entry["state"]) + entry["loaded"] = True + return entry["block"] + + +def cheap_weights() -> "OrderedDict[str, torch.Tensor]": + """A valid 782-tensor mapping that allocates nothing (see support.cheap_weights).""" + return support.cheap_weights(reference.param_shapes()) + + +def fresh_context(num_tokens: int, cached_tokens: int = 0, num_requests: int = 1, **mgr_overrides): + """A KVCacheManagerV2 whose layer mask selects logical layer 3 alone, and prepared + TrtllmAttentionMetadata for one new request.""" + mgr_args = dict( + num_layers=1, + layer_mask=list(LAYER_MASK), + num_kv_heads=BLOCK_CONFIG["num_key_value_heads"], + head_dim=BLOCK_CONFIG["head_dim"], + tokens_per_block=KV_CACHE["tokens_per_block"], + max_seq_len=KV_CACHE["max_seq_len"], + max_batch_size=num_requests, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + dtype=DataType.BF16, + ) + mgr_args.update(mgr_overrides) + return support.fresh_context( + num_tokens, + cached_tokens, + num_requests, + max_kv_tokens=KV_CACHE["num_blocks"] * KV_CACHE["tokens_per_block"] * num_requests, + max_tokens=block_module.MAX_TOKENS, + request_id=REQUEST_ID, + mgr_args=mgr_args, + ) + + +def case_inputs(seed: int, length: int) -> tuple[torch.Tensor, torch.Tensor]: + x = generated(seed)["input"][:, :length].contiguous() + positions = torch.arange(length, device="cuda").view(1, length) + return x, positions + + +def reference_record(seed: int, x: torch.Tensor, positions: torch.Tensor) -> dict: + """The reference block of the seed's state on ``x``: its output and every named + intermediate, ``[T, ...]``, on the GPU.""" + record = {} + with torch.no_grad(): + out = reference.block(x, positions, generated(seed)["state"], record=record.__setitem__) + assert torch.equal(out[0], record["block_output"]) + return record + + +def run_block( + blk: BeamSparseBlockBf16Sm103Tp1, x: torch.Tensor, positions: torch.Tensor +) -> tuple[torch.Tensor, dict]: + """One forward on a fresh context: the output, the MoE's input (the pre-feed-forward + norm's output) and output from a forward hook, and the router op's mapping of the + call.""" + seen = [] + + def hook(module, args, output): + seen.append((args[0].clone(), output.clone())) + + handle = blk.mlp.register_forward_hook(hook) + try: + with fresh_context(x.shape[1]) as md, recording(beam_grouped_topk_router) as calls: + out = blk(x, positions, md) + torch.cuda.synchronize() + finally: + handle.remove() + assert len(seen) == 1 and len(calls) == 1, (len(seen), len(calls)) + ((pre_ff, moe_out),) = seen + return out, {"pre_ff": pre_ff, "moe_out": moe_out, "routing": calls[0]["output"]} + + +def block_results(seed: int, out: torch.Tensor, captured: dict, ref: dict) -> list[dict]: + """Every comparison of one forward against the reference block of the same input, + routing judged apart from numerical error.""" + moe_state = generated(seed)["moe_state"] + pre_ff, moe_out, routing = captured["pre_ff"], captured["moe_out"], captured["routing"] + tokens = pre_ff.shape[0] + results = [output_contract(out, tokens)] + # Same input, same routing: the reference routes the component's own MoE input exactly + # as the component did. + same_input = moe_reference.route( + pre_ff, moe_state["gate.weight"], moe_state["gate.e_score_correction_bias"] + ) + results.append(compare_exact("ids_on_own_moe_input", routing["ids"], same_input["ids"])) + results.append( + compare_exact("groups_on_own_moe_input", routing["groups"], same_input["groups"]) + ) + # Against the whole reference block: differing decisions are near ties. + judgement, matched = judge_routing(routing, ref["moe.routing"]) + results.append(judgement) + final = compare( + "block_output_on_set_matched_tokens", + out[0][matched], + ref["block_output"][matched], + "block_output", + ) + final["tokens"] = int(matched.sum()) + results.append(final) + results.append( + compare( + "moe_output_on_own_moe_input", + moe_out, + moe_reference.moe(pre_ff, moe_state), + "moe_output", + ) + ) + # Measured, not judged: how far the component's MoE input is from the reference's. + results.append( + compare("pre_feedforward_layernorm", pre_ff, ref["pre_feedforward_layernorm"], None) + ) + return results + + +# -------------------------------------------------- op-path instrumentation + + +def recording(*wrappers, experts: bool = False): + """Log every call of these entries, in order, with its nesting depth, the arguments that + identify the path (and a ``q`` keyword's shape and tensor) and its output; with + ``experts``, also every expert the MoE asks for and the rows it passes.""" + return support.recording( + *wrappers, + recorded_kwargs=_RECORDED_KWARGS, + experts=BeamMoeBf16Sm103Tp1 if experts else None, + qkv=True, + ) + + +def loggable(calls: list[dict]) -> list[dict]: + """The recorded calls without their tensors, for the evidence record.""" + keep = ("op", "depth", "shapes", "q_shape", "expert_id", *_RECORDED_KWARGS) + return support.loggable(calls, keep) + + +def builtin_beam_forbidden(): + """The built-in Beam modeling must not run while the component does.""" + if builtin_beam is None: + return support.builtin_forbidden([]) + return support.builtin_forbidden( + [ + (builtin_beam.BeamDecoderLayer, "forward"), + (builtin_beam.BeamAttention, "forward"), + (builtin_beam.BeamMoE, "forward"), + (builtin_beam.BeamMoE, "combine"), + (builtin_beam.BeamRouter, "forward"), + (builtin_beam.BeamRouter, "route"), + (builtin_beam.BeamSwiGLU, "forward"), + (builtin_beam.BeamRMSNorm, "forward"), + ] + ) + + +def no_dispatch(): + """Fail on any op call: rejected inputs must be rejected before a kernel runs.""" + return support.no_dispatch(OPS) + + +def cache_rows(md: TrtllmAttentionMetadata, length: int) -> tuple[torch.Tensor, torch.Tensor]: + """This request's K and V rows read back from the pool, [T, Hkv, D] each, under the + logical layer 3.""" + return support.cache_rows(md, length, LAYER_IDX, KV_CACHE["tokens_per_block"]) + + +# -------------------------------------------------- tests: configuration and construction + + +def test_configuration_is_the_fixed_scope() -> None: + """One configuration, no open choices: what the component accepts is what is tested.""" + assert CASES == ((17, 1), (17, 17), (17, 129), (41, 1), (41, 17), (41, 129)) + assert block_module.SUPPORTED_LAYER_IDX == LAYER_IDX + for field, value in block_module.SUPPORTED_CONFIG.items(): + assert BLOCK_CONFIG[field] == value, field + assert BLOCK_CONFIG["rope_parameters"]["sliding_attention"] == block_module.SUPPORTED_ROPE + assert BLOCK_CONFIG["layer_types"][LAYER_IDX] == "sliding_attention" + assert BLOCK_CONFIG["mlp_layer_types"][LAYER_IDX] == "sparse" + assert len(BLOCK_CONFIG["layer_types"]) == len(BLOCK_CONFIG["mlp_layer_types"]) == NUM_LAYERS + assert block_module.MAX_TOKENS == 1024 and max(LENGTHS) == reference.MAX_TOKENS + own = block_module.weight_shapes(block_config()) + assert list(own.items()) == list(reference.own_shapes().items()) + assert (reference.HEADS, reference.KV_HEADS, reference.HEAD_DIM) == (48, 8, 128) + assert reference.WINDOW == BLOCK_CONFIG["sliding_window"] == KV_CACHE["max_seq_len"] + assert reference.LAYER_IDX == LAYER_IDX and sum(LAYER_MASK) == 1 and LAYER_MASK[LAYER_IDX] + assert (reference.SEEDS, reference.LENGTHS) == (SEEDS, LENGTHS) + assert torch.cuda.get_device_capability() == SM, "this suite certifies sm_103 only" + + +def test_block_constructs_alone() -> None: + """The block is built directly from its configuration, without the model around it: its + twelve attention and norm tensors and the MoE component, no dense MLP, and block 3's + residual coefficients.""" + + def refuse(self, *args, **kwargs): + raise AssertionError(f"{type(self).__name__} constructed") + + with pytest.MonkeyPatch.context() as mp: + if builtin_beam is not None: + for cls in ( + builtin_beam.BeamForCausalLM, + builtin_beam.BeamModel, + builtin_beam.BeamDecoderLayer, + builtin_beam.BeamAttention, + builtin_beam.BeamMoE, + builtin_beam.BeamRouter, + builtin_beam.BeamSwiGLU, + ): + mp.setattr(cls, "__init__", refuse) + blk = BeamSparseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cuda") + assert isinstance(blk.mlp, BeamMoeBf16Sm103Tp1) + assert [name for name, _ in blk.named_children()] == ["mlp"] + tensors = blk.checkpoint_tensors() + names = list(tensors) + assert len(names) == 782 + assert names[:12] == list(reference.own_shapes()) + assert names[12:] == [reference.MLP_PREFIX + n for n in moe_reference.param_shapes()] + assert list(tensors) == list(reference.param_shapes()) + for dense in ("mlp.gate_proj.weight", "mlp.up_proj.weight", "mlp.down_proj.weight"): + assert dense not in tensors, dense + for name, (shape, dtype) in reference.param_shapes().items(): + t = tensors[name] + assert tuple(t.shape) == shape and t.dtype == dtype and t.is_cuda, name + assert tensors["mlp.gate.weight"] is blk.mlp.gate.weight + assert tensors["mlp.experts.255.down_proj.weight"] is blk.mlp.experts[255].down_proj + assert blk.attention_scales == (1 / math.sqrt(8), math.sqrt(7) / math.sqrt(8)) + assert blk.feed_forward_scales == (1 / 3, math.sqrt(8) / 3) + assert blk.attention_scales != block_module.depth_scales(0, True) + assert blk.feed_forward_scales != block_module.depth_scales(0, False) + assert "modeling_beam" not in Path(block_module.__file__).read_text() + + +# -------------------------------------------------- tests: declared limits + +_SLIDING_ROPE_1E6 = { + "sliding_attention": {"rope_type": "default", "rope_theta": 1e6}, + "full_attention": None, +} +_BAD_CONFIGS = [ + ( + "dense_block_3", + dict(mlp_layer_types=replaced(BLOCK_CONFIG["mlp_layer_types"], LAYER_IDX, "dense")), + "sparse", + ), + ( + "full_attention_block_3", + dict(layer_types=replaced(BLOCK_CONFIG["layer_types"], LAYER_IDX, "full_attention")), + "sliding_attention", + ), + ("rope_theta", dict(rope_parameters=_SLIDING_ROPE_1E6), "rope parameters"), + ("sliding_window", dict(sliding_window=1024), "unsupported sliding_window"), + ("head_dim", dict(head_dim=64), "unsupported head_dim"), +] + + +@pytest.mark.parametrize("case,overrides,match", _BAD_CONFIGS, ids=[c[0] for c in _BAD_CONFIGS]) +def test_rejects_unsupported_configuration(case, overrides, match) -> None: + """Anything outside the one configuration is refused at construction.""" + with pytest.raises(ValueError, match=match): + BeamSparseBlockBf16Sm103Tp1(block_config(**overrides), LAYER_IDX, device="cuda") + + +def test_rejects_other_blocks_and_devices() -> None: + for layer_idx in (0, 2, 6): + with pytest.raises(ValueError, match="only sparse blocks"): + BeamSparseBlockBf16Sm103Tp1(block_config(), layer_idx, device="cuda") + with pytest.raises(ValueError, match="sm_103"): + BeamSparseBlockBf16Sm103Tp1(block_config(), LAYER_IDX, device="cpu") + + +# -------------------------------------------------- tests: loading + + +def test_load_weights_maps_every_tensor() -> None: + """Exactly the 782 names, each copied exactly into its own tensor: the twelve attention + and norm tensors into the block, the ``mlp.`` ones into the MoE's router and into expert + e for every e.""" + seed = 17 + state = generated(seed)["state"] + blk = loaded_block(seed) + loaded = blk.checkpoint_tensors() + assert list(loaded) == list(state) == list(reference.param_shapes()) + wrong = [] + for name, (shape, dtype) in reference.param_shapes().items(): + t = loaded[name] + if tuple(t.shape) != shape or t.dtype != dtype or not t.is_cuda: + wrong.append(f"{name}: {tuple(t.shape)} {t.dtype} on {t.device}") + elif not torch.equal(t, state[name]): + wrong.append(f"{name}: values differ") + assert t.data_ptr() != state[name].data_ptr(), f"{name} aliases the given tensor" + assert not wrong, wrong[:8] + moe = blk.mlp + for expert_id, expert in enumerate(moe.experts): + for attr in ("gate_proj", "up_proj", "down_proj"): + given = state[f"mlp.experts.{expert_id}.{attr}.weight"] + assert torch.equal(getattr(expert, attr), given), (expert_id, attr) + assert moe.gate.weight.dtype == moe.gate.e_score_correction_bias.dtype == torch.float32 + assert torch.equal(moe.gate.weight, state["mlp.gate.weight"]) + assert torch.equal(moe.gate.e_score_correction_bias, state["mlp.gate.e_score_correction_bias"]) + + +_BAD_LOADS = [ + ("late_expert_shape", ValueError, r"experts\.255\.up_proj\.weight must be \[2560, 5120\]"), + ("q_norm_fp32", TypeError, r"self_attn\.q_norm\.weight must be torch\.bfloat16"), + ("router_weight_bf16", TypeError, r"gate\.weight must be torch\.float32"), +] + + +@pytest.mark.parametrize("case,error,match", _BAD_LOADS, ids=[c for c, _, _ in _BAD_LOADS]) +def test_rejected_load_changes_nothing(case, error, match) -> None: + """Every one of the 782 tensors is checked before any is copied: a mapping malformed + late is rejected without writing any parameter or buffer, every value stays, and the + block still runs.""" + seed = 17 + entry = generated(seed) + blk = loaded_block(seed) + bad = cheap_weights() + if case == "late_expert_shape": + bad["mlp.experts.255.up_proj.weight"] = bad["mlp.experts.255.up_proj.weight"].t() + elif case == "q_norm_fp32": + bad["self_attn.q_norm.weight"] = torch.zeros((), device="cuda").expand(128) + elif case == "router_weight_bf16": + bad["mlp.gate.weight"] = bad["mlp.gate.weight"].to(DTYPE) + names = ( + "self_attn.q_norm.weight", + "post_feedforward_layernorm.weight", + "mlp.gate.weight", + "mlp.gate.e_score_correction_bias", + "mlp.experts.255.up_proj.weight", + ) + snapshots = {name: blk.checkpoint_tensors()[name].clone() for name in names} + versions = tensor_versions(blk) + try: + with no_dispatch(), pytest.raises(error, match=match): + blk.load_weights(bad) + finally: + written = sorted(n for n, v in tensor_versions(blk).items() if v != versions[n]) + if written: + entry["loaded"] = False # reload the generated state before the block runs again + current = blk.checkpoint_tensors() + changed = [name for name, value in snapshots.items() if not torch.equal(current[name], value)] + assert not written, f"{len(written)} tensors written by a rejected load: {written[:8]}" + assert not changed, changed + x, positions = case_inputs(seed, 1) + with fresh_context(1) as md: + out = blk(x, positions, md) + assert out.shape == (1, 1, 5120) and bool(torch.isfinite(out).all()) + + +_BAD_INPUTS = [ + "fp16_hidden", + "fp32_hidden", + "hidden_5120x2", + "batch_2", + "too_long", + "empty", + "non_contiguous", + "positions_offset", + "float_positions", + "positions_shape", + "metadata_type", + "cached_prefix", + "two_requests", + "context_length_mismatch", + "kv_cache_heads", + "kv_cache_dtype", + "layer0_cache", +] + + +@pytest.mark.parametrize("case", _BAD_INPUTS) +def test_rejects_unsupported_inputs_before_dispatch(case) -> None: + """Each declared restriction raises ValueError on the loaded block, and no op runs + first -- including a cache manager that does not hold logical layer 3 (a default + one-layer manager holds layer 0).""" + blk = loaded_block(17) + x, positions, context, match = support.bad_block_input(case, block_module.MAX_TOKENS, DTYPE) + with fresh_context(**context) as md: + if case == "metadata_type": + md = object() + elif case == "layer0_cache": + assert dict(md.kv_cache_manager.layer_offsets) == {0: 0} + with no_dispatch(), pytest.raises(ValueError, match=match): + blk(x, positions, md) + + +def test_cache_slot_is_the_managers_local_index(ieee_fp32) -> None: + """Logical layer 3 lives in slot 0 of a manager whose layer mask selects it alone: the + attention op is called with local index 0, and the K and V it appended are read back + through the manager under the logical index 3.""" + seed, length = 17, 17 + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + with fresh_context(length) as md: + mgr = md.kv_cache_manager + layout = { + "manager": type(mgr).__name__, + "num_layers": mgr.num_layers, + "pp_layers": list(mgr.pp_layers), + "layer_offsets": dict(mgr.layer_offsets), + "num_local_layers": mgr.num_local_layers, + } + step = blk.step_args(md) + with recording(thop_attention) as calls: + out = blk(x, positions, md) + torch.cuda.synchronize() + keys, values = cache_rows(md, length) + _write_evidence( + f"cache-slot-seed{seed}-T{length}", + { + "test": "test_cache_slot_is_the_managers_local_index", + "layout": layout, + "step_local_layer_idx": step["local_layer_idx"], + "op_calls": loggable(calls), + "environment": _environment(ieee_fp32), + }, + ) + assert isinstance(mgr, KVCacheManagerV2) + assert layout["num_layers"] == NUM_LAYERS and layout["pp_layers"] == [LAYER_IDX] + assert layout["layer_offsets"] == {LAYER_IDX: 0} and layout["num_local_layers"] == 1 + assert step["local_layer_idx"] == 0 + (call,) = calls + assert { + k: call[k] + for k in ( + "local_layer_idx", + "num_heads", + "num_kv_heads", + "head_size", + "attention_window_size", + "mask_type", + "is_fused_qkv", + ) + } == { + "local_layer_idx": 0, + "num_heads": 48, + "num_kv_heads": 8, + "head_size": 128, + "attention_window_size": 512, + "mask_type": 1, + "is_fused_qkv": True, + } + assert call["q_shape"] == [length, (48 + 2 * 8) * 128] + assert out.shape == (1, length, 5120) and bool(torch.isfinite(out).all()) + # The op appended exactly this call's post-RoPE K and V to logical layer 3's slot. + qkv = call["qkv"] + k_in = qkv[:, 48 * 128 : 56 * 128].reshape(length, 8, 128) + v_in = qkv[:, 56 * 128 :].reshape(length, 8, 128) + assert torch.equal(keys, k_in) and torch.equal(values, v_in) + + +def test_cache_slot_in_a_manager_holding_two_layers(ieee_fp32) -> None: + """In a manager that holds logical layers 2 and 3, layer 3 is local slot 1: the attention + op is called with that slot, the K and V are appended there, and the block output is + bitwise the output on a manager holding layer 3 alone.""" + seed, length = 17, 17 + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + with fresh_context(length) as md: + alone = blk(x, positions, md) + with fresh_context( + length, num_layers=2, layer_mask=[i in (2, 3) for i in range(NUM_LAYERS)] + ) as md: + assert dict(md.kv_cache_manager.layer_offsets) == {2: 0, 3: 1} + assert blk.step_args(md)["local_layer_idx"] == 1 + with recording(thop_attention) as calls: + out = blk(x, positions, md) + torch.cuda.synchronize() + keys, values = cache_rows(md, length) + (call,) = calls + assert call["local_layer_idx"] == 1 + qkv = call["qkv"] + assert torch.equal(keys, qkv[:, 48 * 128 : 56 * 128].reshape(length, 8, 128)) + assert torch.equal(values, qkv[:, 56 * 128 :].reshape(length, 8, 128)) + assert bitwise_equal(out, alone) + + +# -------------------------------------------------- tests: the six cases + + +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_block_matches_reference(seed, length, ieee_fp32) -> None: + """One case, routing judged apart from numerical error: exact routing on the block's own + MoE input; against the whole reference block, routing differences only on near ties and + the output within its rule on the tokens routed alike; the MoE output within its rule + on the block's own MoE input; input and parameters untouched.""" + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + x_before = x.clone() + versions = tensor_versions(blk) + out, captured = run_block(blk, x, positions) + after = tensor_versions(blk) + ref = reference_record(seed, x, positions) + results = block_results(seed, out, captured, ref) + results.append(unchanged(x, x_before, versions, after)) + _write_evidence( + f"block-seed{seed}-T{length}", + { + "test": "test_block_matches_reference", + "case": {"seed": seed, "length": length}, + "rules": {k: list(v) for k, v in RULES.items()}, + "route_gap": ROUTE_GAP, + "results": results, + "input_sha256": tensor_sha256(x), + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(results) + + +# -------------------------------------------------- tests: the component path + + +def test_forward_runs_the_component_path(ieee_fp32) -> None: + """One case: the component's own composition ran, op by op, with the experts' calls + nested inside BeamDispatchCombine, through thop attention on this GPU, with no SDPA, no + built-in Beam module and every op's guard admitting its arguments.""" + seed, length = 17, 17 + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + with fresh_context(length) as md: + with builtin_beam_forbidden(), validating(*OPS), recording(*OPS, experts=True) as calls: + with torch.profiler.profile(activities=activities) as prof: + out = blk(x, positions, md) + torch.cuda.synchronize() + manager = md.kv_cache_manager + events = prof.events() + kernels = [e.name for e in events if e.device_type == torch.autograd.DeviceType.CUDA] + cpu_ops = sorted({e.name for e in events if e.device_type == torch.autograd.DeviceType.CPU}) + names = [c["op"] for c in calls] + record = { + "test": "test_forward_runs_the_component_path", + "case": {"seed": seed, "length": length}, + "block": f"{type(blk).__module__}.{type(blk).__qualname__}", + "op_calls": loggable(calls), + "cuda_kernels": kernels, + "attention_kernels": [k for k in kernels if support.ATTENTION_KERNEL.search(k)], + "sdpa_cpu_ops": [n for n in cpu_ops if support.SDPA_OPS.search(n)], + "kv_cache_manager": type(manager).__name__, + "environment": _environment(ieee_fp32), + } + _write_evidence(f"path-seed{seed}-T{length}", record) + + assert type(blk).__module__.endswith("sparse_block.block") + top = [c for c in calls if c["depth"] == 0] + assert tuple(c["op"] for c in top) == EXPECTED_CALLS, [c["op"] for c in top] + # Every other call ran inside BeamDispatchCombine: right after it, one level down. + position = names.index("BeamDispatchCombine") + inner = len(calls) - len(top) + depths = [c["depth"] for c in calls] + assert depths == [0] * (position + 1) + [1] * inner + [0] * (len(calls) - position - 1 - inner) + router = calls[names.index("BeamGroupedTopkRouter")] + ids = router["output"]["ids"] + selected = sorted(set(ids.reshape(-1).tolist())) + nested = calls[position + 1 : position + 1 + inner] + assert [c["op"] for c in nested] == list(EXPERT_CALLS) * len(selected) + assert [c["expert_id"] for c in nested if c["op"] == "expert"] == selected + assert sum(c["rows"].shape[0] for c in nested if c["op"] == "expert") == 8 * length + assert inner == len(EXPERT_CALLS) * len(selected) + (call,) = [c for c in calls if c["op"] == "ThopAttention"] + assert call["local_layer_idx"] == 0 and call["num_contexts"] == 1 + assert call["num_ctx_tokens"] == length and call["use_paged_context_fmha"] is False + assert call["quant_mode"] == 0 and call["position_embedding_type"] == 0 + assert isinstance(manager, KVCacheManagerV2) + assert record["attention_kernels"], "no attention kernel ran on the GPU" + assert not record["sdpa_cpu_ops"], record["sdpa_cpu_ops"] + assert torch.cuda.get_device_capability(x.device) == SM + assert out.shape == (1, length, 5120) and bool(torch.isfinite(out.float()).all()) + + +# -------------------------------------------------- tests: fixtures + + +def test_causal_prefix_independence(ieee_fp32) -> None: + """Two 129-token inputs sharing their first 17 rows: those rows reach the MoE bit for + bit alike and are routed alike, and their outputs agree within the block rule (each + expert's batch differs, so only the expert GEMMs' row counts change).""" + seed, length, prefix = 17, 129, 17 + blk = loaded_block(seed) + x_a, positions = case_inputs(seed, length) + suffix = reference.generate_input(41)[:, prefix:length] + x_b = torch.cat((x_a[:, :prefix], suffix), dim=1).contiguous() + assert not torch.equal(x_b[:, prefix:], x_a[:, prefix:]) + out_a, captured_a = run_block(blk, x_a, positions) + out_b, captured_b = run_block(blk, x_b, positions) + ids_a, ids_b = captured_a["routing"]["ids"], captured_b["routing"]["ids"] + results = [ + compare_exact("prefix_ids", ids_b[:prefix], ids_a[:prefix]), + { + "tensor": "prefix_moe_input", + "rule": "bitwise", + "passed": bitwise_equal(captured_b["pre_ff"][:prefix], captured_a["pre_ff"][:prefix]), + "problems": [], + }, + compare("prefix_block_output", out_b[0, :prefix], out_a[0, :prefix], "block_output"), + ] + if not results[1]["passed"]: + results[1]["problems"].append("the shared rows' MoE inputs differ") + _write_evidence( + f"prefix-seed{seed}-T{length}", + { + "test": "test_causal_prefix_independence", + "prefix": prefix, + "results": results, + "suffix_ids_differ": not torch.equal(ids_b[prefix:], ids_a[prefix:]), + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(results) + assert not torch.equal(out_b[0, prefix:], out_a[0, prefix:]), "the suffix changed nothing" + + +def test_residual_uses_block3_coefficients(ieee_fp32, monkeypatch) -> None: + """The residual entry with the block's coefficients is block 3's add bit for bit and + differs from block 0's; and the block comparison rejects a block run with block 0's + coefficients, and accepts the unchanged and the restored block.""" + seed, length = 17, 17 + blk = loaded_block(seed) + gen = torch.Generator(device="cuda") + gen.manual_seed(moe_reference.tensor_seed(seed, "residual.branch")) + branch = torch.randn(length, 5120, generator=gen, device="cuda").to(DTYPE) + gen.manual_seed(moe_reference.tensor_seed(seed, "residual.stream")) + stream = torch.randn(length, 5120, generator=gen, device="cuda").to(DTYPE) + assert blk.attention_scales == reference.ATTENTION_SCALES + assert blk.feed_forward_scales == reference.FEED_FORWARD_SCALES + for label, scales, is_attention in ( + ("attention", blk.attention_scales, True), + ("feed_forward", blk.feed_forward_scales, False), + ): + got = beam_depth_residual(branch, stream, *scales) + expected = reference.residual_add(branch, stream, reference.depth_coefs(3, is_attention)) + block0 = reference.residual_add(branch, stream, reference.depth_coefs(0, is_attention)) + assert bitwise_equal(got, expected), label + assert not torch.equal(got, block0), label + + x, positions = case_inputs(seed, length) + ref = reference_record(seed, x, positions) + runs = {} + for label in ("unchanged", "block0_coefficients", "restored"): + if label == "block0_coefficients": + monkeypatch.setattr(blk, "attention_scales", block_module.depth_scales(0, True)) + monkeypatch.setattr(blk, "feed_forward_scales", block_module.depth_scales(0, False)) + elif label == "restored": + monkeypatch.undo() + out, captured = run_block(blk, x, positions) + runs[label] = block_results(seed, out, captured, ref) + _write_evidence( + f"fault-residual-seed{seed}-T{length}", + { + "test": "test_residual_uses_block3_coefficients", + "fault": "block 0's residual coefficients in both adds, test-scoped", + "runs": runs, + "environment": _environment(ieee_fp32), + }, + ) + assert_all_pass(runs["unchanged"]) + failed = {r["tensor"] for r in runs["block0_coefficients"] if r["passed"] is False} + assert failed, "block 0's coefficients passed the block comparison" + assert failed <= {"routing_vs_reference_block", "block_output_on_set_matched_tokens"}, failed + assert blk.attention_scales == block_module.depth_scales(LAYER_IDX, True) + assert blk.feed_forward_scales == block_module.depth_scales(LAYER_IDX, False) + assert_all_pass(runs["restored"]) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py new file mode 100644 index 000000000000..7156f1efd367 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -0,0 +1,524 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generated-data tests for the Beam checkpoint layout component: bf16 / sm_103 / tp4. + +No model files. The layout tests use the architecture's configuration values only (no +tensors). The loader tests write small safetensors directories with this file's own writer +and compare every loaded byte with an expected value derived independently here: E4M3 +decoded from its bit fields, the product with the BF16 row scale formed exactly, and the +BF16 rounding done on the float32 bit pattern (round to nearest, ties to even). Every rank +of the four-rank layout is exercised, with tiles small enough to cross row seams. + +Four-rank execution (collectives, real checkpoint loading, KV cache) needs four GPUs and a +checkpoint and is not part of this file. + +Invocation:: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ + python -m pytest \\ + tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +""" + +from __future__ import annotations + +import json +import math +import os +import struct + +import pytest +import torch + +from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 + layout_bf16 as layout, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 + loader_bf16 as loader, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4.layout_bf16 import ( # noqa: E501 + BF16, + F32, + FP8, + TensorSpec, +) + +NUM_LAYERS = 52 +FULL = {5, 11, 17, 23, 29, 35, 41, 47, 51} + + +def model_config() -> dict: + """The architecture's configuration, as its config.json states it (values only).""" + cfg = dict(layout.SUPPORTED_CONFIG) + cfg["layer_types"] = [ + "full_attention" if i in FULL else "sliding_attention" for i in range(NUM_LAYERS) + ] + cfg["mlp_layer_types"] = ["dense"] * 3 + ["sparse"] * (NUM_LAYERS - 3) + cfg["rope_parameters"] = { + "full_attention": None, + "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}, + } + return cfg + + +# ------------------------------------------------------------------ independent expected values + + +def e4m3(code: int) -> float: + """E4M3 (fn) value of one byte from its bit fields: bias 7, subnormals, no infinities.""" + sign = -1.0 if code & 0x80 else 1.0 + exp, man = (code >> 3) & 0xF, code & 0x7 + if exp == 0xF and man == 0x7: + return math.nan + if exp == 0: + return sign * man / 8.0 * 2.0**-6 + return sign * (1.0 + man / 8.0) * 2.0 ** (exp - 7) + + +def bf16_bits_of_float(x: float) -> int: + """BF16 bits of the float32 value nearest ``x`` (exact here), rounded to nearest even.""" + (u,) = struct.unpack("> 16) & 1)) >> 16) & 0xFFFF + + +def bf16_value(bits: int) -> float: + return struct.unpack(" None: + """``shards``: file name -> list of (name, dtype, shape, raw bytes). Writes index + shards.""" + os.makedirs(path, exist_ok=True) + weight_map = {} + for filename, entries in shards.items(): + header, payload, offset = {}, b"", 0 + for name, dtype, shape, raw in entries: + header[name] = { + "dtype": dtype, + "shape": list(shape), + "data_offsets": [offset, offset + len(raw)], + } + payload += raw + offset += len(raw) + weight_map[name] = filename + blob = json.dumps(header).encode() + blob += b" " * (-len(blob) % 8) + with open(os.path.join(path, filename), "wb") as f: + f.write(struct.pack(" list: + """Codes cycling through every finite E4M3 byte (signs, zeros, subnormals, 448).""" + return [ + [FP8_CODES[(start + r * cols + c) % len(FP8_CODES)] for c in range(cols)] + for r in range(rows) + ] + + +def row_scales(rows: int, start: int) -> list: + """Distinct, non-uniform BF16 per-row scales (bits), as a quantizer's amax/448 would be.""" + return [ + bf16_bits_of_float(2.0 ** (-(r % 5) - 3) * (1.0 + (start + 3 * r) % 7 / 8.0)) + for r in range(rows) + ] + + +def fp8_entries(name: str, codes: list, scales: list) -> list: + rows, cols = len(codes), len(codes[0]) + raw = bytes(c for row in codes for c in row) + sraw = b"".join(struct.pack(" list: + """BF16 bits of bf16(fp32(e4m3(q)) * fp32(scale[row])), independently of torch.""" + return [ + [bf16_bits_of_float(e4m3(q) * bf16_value(scales[r])) for q in row] + for r, row in enumerate(codes) + ] + + +# A generated layout: row-split and column-split FP8, replicated BF16, row-split F32, on two shards. +SPECS = { + "a.rows.weight": TensorSpec("a.rows.weight", (8, 12), FP8, "gen", 0, None, 0), + "b.cols.weight": TensorSpec("b.cols.weight", (12, 16), FP8, "gen", 0, None, 1), + "c.norm.weight": TensorSpec("c.norm.weight", (10,), BF16, "gen", 0, None, None), + "d.router.weight": TensorSpec("d.router.weight", (8, 4), F32, "gen", 0, None, 0), +} +A_CODES, A_SCALES = fp8_matrix(8, 12, 0), row_scales(8, 0) +B_CODES, B_SCALES = fp8_matrix(12, 16, 97), row_scales(12, 5) +C_BITS = [ + bf16_bits_of_float(v) + for v in (-0.0, 1.5, -2.25, 3.0e-40, 448.0, 1.0 / 3, 7.0, -1e5, 0.0, 65504.0) +] +D_VALUES = [((-1) ** i) * (i + 0.5) / 3.0 for i in range(32)] + + +def generated_shards() -> dict: + return { + "model-00001-of-00002.safetensors": fp8_entries("a.rows.weight", A_CODES, A_SCALES) + + [("c.norm.weight", BF16, (10,), b"".join(struct.pack(" str: + path = str(tmp_path / "ckpt") + write_checkpoint(path, generated_shards()) + return path + + +def _bits16(t: torch.Tensor) -> list: + return t.contiguous().view(torch.int16).cpu().to(torch.int32).bitwise_and(0xFFFF).tolist() + + +# ------------------------------------------------------------------ layout + + +@pytest.mark.cpu_only +def test_layout_counts_intervals_and_banks() -> None: + specs = layout.tensor_specs(model_config()) + assert len(specs) == 3 + 3 * 15 + 49 * 782 + fp8 = [s for s in specs.values() if s.stored_dtype == FP8] + assert len(fp8) == 52 * 5 + 3 * 3 + 49 * 768 + layout.check_partition(specs) + want = { + "model.embed_tokens.weight": (0, 37984), + "lm_head.weight": (0, 37984), + "model.layers.0.self_attn.q_proj.weight": (0, 1536), + "model.layers.0.self_attn.gate_proj.weight": (0, 1536), + "model.layers.0.self_attn.k_proj.weight": (0, 256), + "model.layers.0.self_attn.v_proj.weight": (0, 256), + "model.layers.0.self_attn.o_proj.weight": (1, 1536), + "model.layers.0.self_attn.gate_bias": (0, 12), + "model.layers.0.mlp.gate_proj.weight": (0, 5120), + "model.layers.0.mlp.down_proj.weight": (1, 5120), + "model.layers.51.mlp.experts.255.up_proj.weight": (0, 640), + "model.layers.51.mlp.experts.255.down_proj.weight": (1, 640), + } + for name, (dim, size) in want.items(): + spec = specs[name] + got = [spec.interval(r) for r in range(4)] + assert got == [(dim, r * size, (r + 1) * size) for r in range(4)], name + for name in ( + "model.norm.weight", + "model.layers.3.mlp.gate.weight", + "model.layers.3.self_attn.q_norm.weight", + ): + assert all(specs[name].interval(r) is None for r in range(4)) + router = specs["model.layers.3.mlp.gate.e_score_correction_bias"] + assert router.stored_dtype == F32 and router.effective_dtype == F32 + expert = specs["model.layers.7.mlp.experts.200.down_proj.weight"] + assert expert.bank == "layers.7.experts.down" and expert.expert == 200 + assert expert.local_shape(2) == (5120, 640) and expert.scale_name.endswith( + "down_proj.weight_scale" + ) + assert ( + specs["lm_head.weight"].stored_dtype == BF16 and specs["lm_head.weight"].scale_name is None + ) + + +@pytest.mark.cpu_only +def test_gqa_pairing_and_rank_ownership() -> None: + specs = layout.tensor_specs(model_config(), layers=(5,)) + q, k = ( + specs["model.layers.5.self_attn.q_proj.weight"], + specs["model.layers.5.self_attn.k_proj.weight"], + ) + for rank in range(4): + _, q0, q1 = q.interval(rank) + _, k0, k1 = k.interval(rank) + q_heads = range(q0 // 128, q1 // 128) + kv_heads = set(range(k0 // 128, k1 // 128)) + assert {h // 6 for h in q_heads} == kv_heads == {2 * rank, 2 * rank + 1} + output = specs["model.layers.5.self_attn.o_proj.weight"] + assert [output.interval(rank) for rank in range(4)] == [ + (1, 1536 * rank, 1536 * (rank + 1)) for rank in range(4) + ] + expert = specs["model.layers.5.mlp.experts.9.gate_proj.weight"] + assert (expert.bank, expert.expert, expert.interval(3)) == ( + "layers.5.experts.gate", + 9, + (0, 1920, 2560), + ) + router = specs["model.layers.5.mlp.gate.weight"] + assert all(router.interval(rank) is None for rank in range(4)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "change", + [ + {"num_experts": 255}, + {"routed_scaling_factor": 2.0}, + {"tie_word_embeddings": True}, + {"mlp_layer_types": ["sparse"] + ["dense"] * 2 + ["sparse"] * 49}, + {"layer_types": ["sliding_attention"] * 51}, + { + "rope_parameters": { + "full_attention": None, + "sliding_attention": {"rope_theta": 1e6, "rope_type": "default"}, + } + }, + {"num_key_value_heads": 4}, + ], +) +def test_inconsistent_configuration_is_rejected(change) -> None: + cfg = model_config() + cfg.update(change) + with pytest.raises(ValueError): + layout.tensor_specs(cfg) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("rank, world", [(4, 4), (-1, 4), (0, 2), (0, 8), (True, 4), (1.0, 4)]) +def test_invalid_rank_or_world_is_rejected(rank, world) -> None: + with pytest.raises(ValueError): + layout.check_rank(rank, world) + spec = layout.tensor_specs(model_config(), layers=(0,))[ + "model.layers.0.self_attn.q_proj.weight" + ] + with pytest.raises(ValueError): + spec.interval(rank, world) + + +@pytest.mark.cpu_only +def test_unknown_layers_are_rejected() -> None: + for layers in ((52,), (-1,), (3, 3), ("3",)): + with pytest.raises(ValueError): + layout.tensor_specs(model_config(), layers=layers) + + +# ------------------------------------------------------------------ catalog validation + + +@pytest.mark.cpu_only +def test_catalog_accepts_the_exact_checkpoint(checkpoint) -> None: + catalog = loader.SafetensorsCatalog(checkpoint) + catalog.validate(SPECS, complete=True) + assert set(catalog.entries) == set(SPECS) | {"a.rows.weight_scale", "b.cols.weight_scale"} + + +def _mutated(entries_by_file: dict, how: str) -> dict: + shards = {f: list(e) for f, e in entries_by_file.items()} + first, second = list(shards) + if how == "missing_scale": + shards[first] = [e for e in shards[first] if e[0] != "a.rows.weight_scale"] + elif how == "extra_tensor": + shards[second].append(("e.unsupported.weight", BF16, (2,), b"\0" * 4)) + elif how == "scale_vector": + shards[first] = [ + (n, d, (8,) if n == "a.rows.weight_scale" else s, raw) for n, d, s, raw in shards[first] + ] + elif how == "wrong_dtype": + shards[second] = [ + (n, BF16 if n == "d.router.weight" else d, (8, 8) if n == "d.router.weight" else s, raw) + for n, d, s, raw in shards[second] + ] + elif how == "wrong_shape": + shards[first] = [ + (n, d, (12, 8) if n == "a.rows.weight" else s, raw) for n, d, s, raw in shards[first] + ] + elif how == "transposed_scale": + shards[second] = [ + (n, d, (1, 12) if n.endswith("_scale") else s, raw) for n, d, s, raw in shards[second] + ] + return shards + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "how", + [ + "missing_scale", + "extra_tensor", + "scale_vector", + "wrong_dtype", + "wrong_shape", + "transposed_scale", + ], +) +def test_catalog_rejects_malformed_metadata(tmp_path, how) -> None: + path = str(tmp_path / how) + write_checkpoint(path, _mutated(generated_shards(), how)) + with pytest.raises(loader.CheckpointFormatError): + loader.SafetensorsCatalog(path).validate(SPECS, complete=True) + + +@pytest.mark.cpu_only +def test_catalog_rejects_index_header_disagreement_and_truncation(checkpoint) -> None: + index_path = os.path.join(checkpoint, loader.INDEX_FILE) + index = json.load(open(index_path)) + index["weight_map"]["c.norm.weight"] = "model-00002-of-00002.safetensors" + json.dump(index, open(index_path, "w")) + with pytest.raises(loader.CheckpointFormatError): + loader.SafetensorsCatalog(checkpoint) + write_checkpoint(checkpoint, generated_shards()) + shard = os.path.join(checkpoint, "model-00002-of-00002.safetensors") + with open(shard, "rb+") as f: + f.truncate(os.path.getsize(shard) - 3) + with pytest.raises(loader.CheckpointFormatError): + loader.SafetensorsCatalog(checkpoint) + + +# ------------------------------------------------------------------ streaming reads on CUDA + +cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device") + + +@cuda +@pytest.mark.parametrize("tile_bytes", [16, 40, loader.TILE_BYTES]) +def test_fp8_slices_match_independent_bytes_on_every_rank(checkpoint, tile_bytes) -> None: + catalog = loader.SafetensorsCatalog(checkpoint) + catalog.validate(SPECS) + reader = loader.TileReader(tile_bytes=tile_bytes, pin=True) + want_a, want_b = expected_fp8(A_CODES, A_SCALES), expected_fp8(B_CODES, B_SCALES) + try: + for rank in range(4): + a = torch.empty( + SPECS["a.rows.weight"].local_shape(rank), dtype=torch.bfloat16, device="cuda" + ) + loader.read_slice_into(catalog, reader, SPECS["a.rows.weight"], rank, a) + assert _bits16(a) == want_a[2 * rank : 2 * rank + 2] + b = torch.empty( + SPECS["b.cols.weight"].local_shape(rank), dtype=torch.bfloat16, device="cuda" + ) + loader.read_slice_into(catalog, reader, SPECS["b.cols.weight"], rank, b) + assert _bits16(b) == [row[4 * rank : 4 * rank + 4] for row in want_b] + finally: + reader.close() + + +@cuda +def test_fp8_rule_is_a_row_scale_not_a_cast_or_transpose(checkpoint) -> None: + """The fixtures distinguish the contract from a plain cast, a reciprocal and a column scale.""" + want = expected_fp8(B_CODES, B_SCALES) + plain = [[bf16_bits_of_float(e4m3(q)) for q in row] for row in B_CODES] + recip = [ + [bf16_bits_of_float(e4m3(q) / bf16_value(B_SCALES[r])) for q in row] + for r, row in enumerate(B_CODES) + ] + assert want != plain and want != recip + catalog = loader.SafetensorsCatalog(checkpoint) + reader = loader.TileReader(tile_bytes=64) + try: + b = torch.empty(SPECS["b.cols.weight"].local_shape(3), dtype=torch.bfloat16, device="cuda") + loader.read_slice_into(catalog, reader, SPECS["b.cols.weight"], 3, b) + finally: + reader.close() + assert _bits16(b) == [row[12:16] for row in want] + + +@cuda +def test_bf16_and_f32_tensors_copy_byte_for_byte(checkpoint) -> None: + catalog = loader.SafetensorsCatalog(checkpoint) + reader = loader.TileReader(tile_bytes=24) + try: + for rank in range(4): + c = torch.empty(10, dtype=torch.bfloat16, device="cuda") + loader.read_slice_into(catalog, reader, SPECS["c.norm.weight"], rank, c) + assert _bits16(c) == C_BITS + d = torch.empty(2, 4, dtype=torch.float32, device="cuda") + loader.read_slice_into(catalog, reader, SPECS["d.router.weight"], rank, d) + want = torch.tensor(D_VALUES, dtype=torch.float32).view(8, 4)[2 * rank : 2 * rank + 2] + assert torch.equal(d.cpu().view(torch.int32), want.view(torch.int32)) + finally: + reader.close() + + +@cuda +def test_wrong_destination_and_short_read_raise(checkpoint) -> None: + catalog = loader.SafetensorsCatalog(checkpoint) + reader = loader.TileReader(tile_bytes=32) + try: + wrong = torch.empty(2, 12, dtype=torch.float32, device="cuda") + with pytest.raises(ValueError): + loader.read_slice_into(catalog, reader, SPECS["a.rows.weight"], 0, wrong) + shard = os.path.join(checkpoint, "model-00002-of-00002.safetensors") + with open(shard, "rb+") as f: + f.truncate(os.path.getsize(shard) - 64) + d = torch.empty(2, 4, dtype=torch.float32, device="cuda") + with pytest.raises(OSError): + loader.read_slice_into(catalog, reader, SPECS["d.router.weight"], 3, d) + finally: + reader.close() + + +# ------------------------------------------------------------------ component guards + + +def _tensor_state(tensor: torch.Tensor) -> tuple: + return ( + id(tensor), + tensor._version, + tensor.data_ptr(), + tuple(tensor.shape), + tuple(tensor.stride()), + tensor.dtype, + tensor.device, + ) + + +def _weight_state(model) -> tuple: + """Snapshot parameter identity, version and storage metadata for the test.""" + return tuple((name, _tensor_state(p)) for name, p in model.named_parameters()) + + +def _check_weight_state(model, state: tuple) -> None: + """Check that a loaded model still matches the test snapshot.""" + if not model.usable: + raise RuntimeError("the component is not usable: no complete load") + if _weight_state(model) != state: + raise RuntimeError("the weights changed after the test snapshot") + + +def _sm103() -> bool: + return torch.cuda.is_available() and torch.cuda.get_device_capability() == (10, 3) + + +@pytest.mark.skipif(not _sm103(), reason="the component is built for sm_103") +def test_mapping_rejections_and_weight_state_guard() -> None: + from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4.component_bf16 import ( # noqa: E501 + BeamCheckpointLayoutBf16Sm103Tp4, + ) + from tensorrt_llm.mapping import Mapping + + cfg = model_config() + for bad in ( + dict(world_size=2, rank=0, tp_size=2, moe_tp_size=2), + dict(world_size=4, rank=0, tp_size=4, moe_tp_size=1, moe_ep_size=4), + ): + with pytest.raises(ValueError): + BeamCheckpointLayoutBf16Sm103Tp4( + cfg, Mapping(gpus_per_node=4, pp_size=1, **bad), layers=(0,) + ) + mapping = Mapping( + world_size=4, rank=1, gpus_per_node=4, tp_size=4, pp_size=1, moe_tp_size=4, moe_ep_size=1 + ) + model = BeamCheckpointLayoutBf16Sm103Tp4(cfg, mapping, layers=(0,)) + assert model.vocab_interval == (37984, 75968) + assert tuple(model.tensor("model.layers.0.self_attn.k_proj.weight").shape) == (256, 5120) + state = _weight_state(model) + with pytest.raises(RuntimeError): # no complete load: unusable + _check_weight_state(model, state) + with pytest.raises(RuntimeError): + model.embed(torch.zeros(1, dtype=torch.int64, device="cuda")) + # White-box: mark usable to exercise the guard without a four-rank load. + model.usable = True + _check_weight_state(model, state) + model.tensor("model.layers.0.input_layernorm.weight").add_(0) # in place: version changes + with pytest.raises(RuntimeError): + _check_weight_state(model, state) + state = _weight_state(model) + name = "model__layers__0__self_attn__q_norm__weight" + old = getattr(model, name) + setattr( + model, name, torch.nn.Parameter(old.detach().clone(), requires_grad=False) + ) # same-shape replacement + with pytest.raises(RuntimeError): + _check_weight_state(model, state) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py new file mode 100644 index 000000000000..a67696f6cd97 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py @@ -0,0 +1,625 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generated-data tests for the Beam NVFP4-expert checkpoint layout: nvfp4 / sm_103 / tp4. + +No model files. The layout and recipe tests use the architecture's configuration values only. +The loader tests write small safetensors directories with this file's own writer and compare +every loaded byte with an independent slice of the generated bytes, on every rank of the +four-rank layout, with tiles small enough to cross row seams. The CUDA tests compare the +production block-scale relayout with this file's own 128x4 offset formula, and the NVFP4 expert +provider with this file's own decoder (E2M1 table, E4M3 bit fields, low nibble = even K index), +activation quantizer and FP64-accumulation GEMM on an asymmetric generated expert. + +Four-rank execution (collectives, real checkpoint loading, KV cache) needs four GPUs and a +checkpoint and is not part of this file. + +Invocation:: + + env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ + python -m pytest \\ + tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py +""" + +from __future__ import annotations + +import json +import math +import os +import struct + +import pytest +import torch + +from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 + layout_bf16 as bf16_layout, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + layout, + loader, +) + +NUM_LAYERS = 52 +FULL = {5, 11, 17, 23, 29, 35, 41, 47, 51} +SPARSE = range(3, NUM_LAYERS) + + +def model_config() -> dict: + """The architecture's configuration, as its config.json states it (values only).""" + cfg = dict(bf16_layout.SUPPORTED_CONFIG) + cfg["layer_types"] = [ + "full_attention" if i in FULL else "sliding_attention" for i in range(NUM_LAYERS) + ] + cfg["mlp_layer_types"] = ["dense"] * 3 + ["sparse"] * (NUM_LAYERS - 3) + cfg["rope_parameters"] = { + "full_attention": None, + "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}, + } + return cfg + + +def recipe() -> dict: + """A ModelOpt-style NVFP4 recipe excluding every higher-precision Linear module.""" + patterns = ["lm_head", "model.embed_tokens", "model.layers.0*", "model.layers.1.*"] + patterns.append("model.layers.2.*") + for layer in SPARSE: + patterns += [f"model.layers.{layer}.mlp.gate", f"model.layers.{layer}.self_attn*"] + return { + "producer": {"name": "modelopt", "version": "generated"}, + "quantization": { + "quant_algo": "NVFP4", + "kv_cache_quant_algo": None, + "group_size": 16, + "exclude_modules": patterns, + }, + } + + +# ------------------------------------------------------------------ independent format helpers + +_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0) + + +def e2m1(code: int) -> float: + """E2M1 value of one 4-bit code: bit 3 sign, bits 0-2 index the magnitude table.""" + return (-1.0 if code & 8 else 1.0) * _E2M1[code & 7] + + +def e4m3(code: int) -> float: + """E4M3 (fn) value of one byte from its bit fields: bias 7, subnormals, no infinities.""" + sign = -1.0 if code & 0x80 else 1.0 + exp, man = (code >> 3) & 0xF, code & 0x7 + if exp == 0xF and man == 0x7: + return math.nan + if exp == 0: + return sign * man / 8.0 * 2.0**-6 + return sign * (1.0 + man / 8.0) * 2.0 ** (exp - 7) + + +_E4M3_TABLE = torch.tensor([e4m3(b) for b in range(256)], dtype=torch.float64) +_E2M1_TABLE = torch.tensor([e2m1(c) for c in range(16)], dtype=torch.float64) + + +def unpack(packed: torch.Tensor) -> torch.Tensor: + """``[R, K/2]`` bytes -> ``[R, K]`` codes; the low nibble is the even K index.""" + p = packed.long() + return torch.stack((p & 0xF, p >> 4), dim=-1).reshape(p.shape[0], -1) + + +def decode(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """Packed codes x block scales, float64 ``[R, K]`` (global scale not applied).""" + codes = unpack(packed.cpu()) + vals = _E2M1_TABLE[codes] + sf = _E4M3_TABLE[scales.cpu().long()] + return (vals.view(vals.shape[0], -1, 16) * sf.unsqueeze(-1)).view(vals.shape) + + +def swizzle_offset(m: int, k: int, cols: int) -> int: + """Byte offset of block scale (row m, column k) in the 128x4 swizzled layout.""" + k_tiles = -(-cols // 4) + return ( + (m // 128) * k_tiles * 512 + (k // 4) * 512 + (m % 32) * 16 + ((m % 128) // 32) * 4 + k % 4 + ) + + +def unswizzle(flat: torch.Tensor, rows: int, cols: int) -> torch.Tensor: + idx = torch.tensor( + [swizzle_offset(m, k, cols) for m in range(rows) for k in range(cols)], dtype=torch.long + ) + return flat.reshape(-1).cpu()[idx].view(rows, cols) + + +# ------------------------------------------------------------------ layout and recipe + + +@pytest.mark.cpu_only +def test_layout_counts_dtypes_and_rank_intervals() -> None: + specs = layout.tensor_specs(model_config()) + quant = [s for s in specs.values() if isinstance(s, layout.QuantSpec)] + assert len(quant) == 49 * 256 * 3 + entries = layout.stored_entries(specs) + by_dtype = {} + for dtype, _ in entries.values(): + by_dtype[dtype] = by_dtype.get(dtype, 0) + 1 + # 636 higher-precision BF16 tensors, 98 FP32 router tensors, one quartet per projection + assert by_dtype == { + "BF16": 636, + "F32": 98 + 2 * len(quant), + "U8": len(quant), + "F8_E4M3": len(quant), + } + for name, spec in specs.items(): + if isinstance(spec, layout.QuantSpec): + continue + if ".mlp.gate." in name: + assert spec.stored_dtype == "F32" + else: + assert spec.stored_dtype == "BF16", name + gate = specs["model.layers.3.mlp.experts.7.gate_proj"] + down = specs["model.layers.51.mlp.experts.255.down_proj"] + stored = gate.stored() + assert stored["model.layers.3.mlp.experts.7.gate_proj.weight"] == ("U8", (2560, 2560)) + assert stored["model.layers.3.mlp.experts.7.gate_proj.weight_scale"] == ("F8_E4M3", (2560, 320)) + assert stored["model.layers.3.mlp.experts.7.gate_proj.weight_scale_2"] == ("F32", ()) + assert stored["model.layers.3.mlp.experts.7.gate_proj.input_scale"] == ("F32", ()) + for r in range(4): + assert gate.interval(r) == (0, 640 * r, 640 * (r + 1)) + assert gate.packed_interval(r) == (0, 640 * r, 640 * (r + 1)) + assert gate.scale_interval(r) == (0, 640 * r, 640 * (r + 1)) + assert gate.local_packed_shape(r) == (640, 2560) + assert gate.local_scale_shape(r) == (640, 320) + assert down.interval(r) == (1, 640 * r, 640 * (r + 1)) + assert down.packed_interval(r) == (1, 320 * r, 320 * (r + 1)) + assert down.scale_interval(r) == (1, 40 * r, 40 * (r + 1)) + assert down.local_packed_shape(r) == (5120, 320) + assert down.local_scale_shape(r) == (5120, 40) + assert gate.bank == "layers.3.experts.gate" and down.bank == "layers.51.experts.down" + layout.check_partition(specs) + assert layout.swizzled_numel(640, 320) == 640 * 320 + assert layout.swizzled_numel(17, 40) == 128 * 40 + + +@pytest.mark.cpu_only +def test_rank_slices_must_fall_on_whole_blocks() -> None: + # 2600 columns split four ways is 650, which is not a whole number of 16-element blocks. + spec = layout.QuantSpec("x.down_proj", (5120, 2600), 3, 0, "down_proj", 1) + with pytest.raises(ValueError, match="whole 16-element blocks"): + spec.interval(0) + + +@pytest.mark.cpu_only +def test_packed_and_scale_rank_ownership() -> None: + specs = layout.tensor_specs(model_config(), layers=(5,)) + expert = specs["model.layers.5.mlp.experts.200.down_proj"] + assert [expert.packed_interval(rank) for rank in range(4)] == [ + (1, 320 * rank, 320 * (rank + 1)) for rank in range(4) + ] + assert [expert.scale_interval(rank) for rank in range(4)] == [ + (1, 40 * rank, 40 * (rank + 1)) for rank in range(4) + ] + assert (expert.expert, expert.bank) == (200, "layers.5.experts.down") + stored = expert.stored() + assert stored[expert.name + ".weight_scale_2"] == (layout.F32, ()) + assert stored[expert.name + ".input_scale"] == (layout.F32, ()) + assert specs["model.layers.5.self_attn.q_proj.weight"].stored_dtype == "BF16" + + +@pytest.mark.cpu_only +def test_recipe_is_reconciled_against_module_names() -> None: + specs = layout.tensor_specs(model_config()) + summary = layout.check_recipe(recipe(), specs) + assert summary["quantized_modules"] == 49 * 256 * 3 + # 52 x 5 attention projections, 3 x 3 dense projections, 49 routers, the head + assert summary["excluded_linear_modules"] == 52 * 5 + 9 + 49 + 1 + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "mutate, match", + [ + (lambda q: q["quantization"].update(quant_algo="FP8"), "quant_algo"), + (lambda q: q["quantization"].update(group_size=32), "group_size"), + (lambda q: q["quantization"].update(kv_cache_quant_algo="FP8"), "kv_cache_quant_algo"), + ( + lambda q: q["quantization"]["exclude_modules"].append("model.layers.9.mlp.experts.3*"), + "excludes quantized experts", + ), + (lambda q: q["quantization"]["exclude_modules"].remove("lm_head"), "not excluded"), + ( + lambda q: q["quantization"]["exclude_modules"].remove("model.layers.7.mlp.gate"), + "not excluded", + ), + (lambda q: q["quantization"].update(exclude_modules="lm_head"), "list of strings"), + (lambda q: q.pop("quantization"), "quant_algo"), + ], +) +def test_recipe_rejections(mutate, match) -> None: + q = recipe() + mutate(q) + with pytest.raises(ValueError, match=match): + layout.check_recipe(q, layout.tensor_specs(model_config(), layers=(0, 7, 9))) + + +# ------------------------------------------------------------------ generated checkpoints + + +def write_checkpoint(path: str, shards: dict) -> None: + """``shards``: file -> {name: (dtype, shape, raw bytes)}; writes the shards and the index.""" + os.makedirs(path, exist_ok=True) + weight_map = {} + for filename, tensors in shards.items(): + header, payload, offset = {}, [], 0 + for name, (dtype, shape, raw) in tensors.items(): + header[name] = { + "dtype": dtype, + "shape": list(shape), + "data_offsets": [offset, offset + len(raw)], + } + payload.append(raw) + offset += len(raw) + weight_map[name] = filename + blob = json.dumps(header).encode() + with open(os.path.join(path, filename), "wb") as f: + f.write(struct.pack(" torch.Tensor: + g = torch.Generator().manual_seed(seed) + return torch.randint(0, 256, (rows, cols), dtype=torch.uint8, generator=g) + + +@pytest.fixture +def quartets(tmp_path) -> tuple: + """One expert's gate and down quartets (real geometry), split across two shards.""" + p = "model.layers.3.mlp.experts.7." + gate_w, gate_s = _bytes(2560, 2560, 1), _bytes(2560, 320, 2) & 0x7E + down_w, down_s = _bytes(5120, 1280, 3), _bytes(5120, 160, 4) & 0x7E + scalars = { + p + "gate_proj.weight_scale_2": 3.0517578125e-05, + p + "gate_proj.input_scale": 0.0123, + p + "down_proj.weight_scale_2": 1.5, + p + "down_proj.input_scale": 2.0**-7, + } + shards = { + "a.safetensors": { + p + "gate_proj.weight": ("U8", (2560, 2560), gate_w.numpy().tobytes()), + p + "gate_proj.weight_scale": ("F8_E4M3", (2560, 320), gate_s.numpy().tobytes()), + **{n: ("F32", (), struct.pack(" None: + path, expected, t, _ = quartets + catalog = loader.SafetensorsCatalog(path) + loader.validate(catalog, expected) + spec_g = layout.QuantSpec( + "model.layers.3.mlp.experts.7.gate_proj", (2560, 5120), 3, 7, "gate_proj", 0 + ) + spec_d = layout.QuantSpec( + "model.layers.3.mlp.experts.7.down_proj", (5120, 2560), 3, 7, "down_proj", 1 + ) + reader = loader.TileReader(tile_bytes, pin=False) + try: + for r in range(4): + _, a, b = spec_g.packed_interval(r) + dst = torch.empty(spec_g.local_packed_shape(r), dtype=torch.uint8) + loader.read_packed_into(catalog, reader, spec_g.name + ".weight", (a, b), None, dst) + assert torch.equal(dst, t["gate_w"][640 * r : 640 * (r + 1)]) + dst = torch.empty(spec_g.local_scale_shape(r), dtype=torch.uint8) + loader.read_packed_into( + catalog, reader, spec_g.name + ".weight_scale", (a, b), None, dst + ) + assert torch.equal(dst, t["gate_s"][640 * r : 640 * (r + 1)]) + _, a, b = spec_d.packed_interval(r) + dst = torch.empty(spec_d.local_packed_shape(r), dtype=torch.uint8) + loader.read_packed_into( + catalog, reader, spec_d.name + ".weight", (0, 5120), (a, b), dst + ) + assert torch.equal(dst, t["down_w"][:, 320 * r : 320 * (r + 1)]) + _, a, b = spec_d.scale_interval(r) + dst = torch.empty(spec_d.local_scale_shape(r), dtype=torch.uint8) + loader.read_packed_into( + catalog, reader, spec_d.name + ".weight_scale", (0, 5120), (a, b), dst + ) + assert torch.equal(dst, t["down_s"][:, 40 * r : 40 * (r + 1)]) + finally: + reader.close() + + +@pytest.mark.cpu_only +def test_scalars_read_exactly(quartets) -> None: + path, _, _, scalars = quartets + catalog = loader.SafetensorsCatalog(path) + reader = loader.TileReader(1 << 16, pin=False) + try: + got = loader.read_scalars(catalog, reader, list(scalars)) + finally: + reader.close() + assert {n: struct.pack(" None: + ok = torch.tensor([[0x00, 0x80, 0x01, 0x7E, 0x38]], dtype=torch.uint8) + loader.check_block_scales("ok", ok) + for bad in (0x7F, 0xFF, 0x81, 0xB8): + with pytest.raises(loader.CheckpointFormatError): + loader.check_block_scales("bad", torch.tensor([[0x38, bad]], dtype=torch.uint8)) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("how", ["missing", "extra", "dtype", "shape", "scalar_shape"]) +def test_validate_rejects_malformed_metadata(quartets, tmp_path, how) -> None: + path, expected, _, _ = quartets + catalog = loader.SafetensorsCatalog(path) + want = dict(expected) + name = "model.layers.3.mlp.experts.7.gate_proj.weight" + if how == "missing": + want["model.layers.3.mlp.experts.7.up_proj.weight"] = ("U8", (2560, 2560)) + elif how == "extra": + del want[name] + elif how == "dtype": + want[name] = ("I8", (2560, 2560)) + elif how == "shape": + want[name] = ("U8", (2560, 1280)) + else: + want["model.layers.3.mlp.experts.7.gate_proj.input_scale"] = ("F32", (1,)) + with pytest.raises(loader.CheckpointFormatError): + loader.validate(catalog, want) + + +@pytest.mark.cpu_only +def test_wrong_destination_raises(quartets) -> None: + path, _, _, _ = quartets + catalog = loader.SafetensorsCatalog(path) + reader = loader.TileReader(1 << 16, pin=False) + try: + with pytest.raises(ValueError): + loader.read_packed_into( + catalog, + reader, + "model.layers.3.mlp.experts.7.gate_proj.weight", + (0, 640), + None, + torch.empty(640, 2560, dtype=torch.int8), + ) + with pytest.raises(ValueError): + loader.read_packed_into( + catalog, + reader, + "model.layers.3.mlp.experts.7.gate_proj.weight", + (0, 640), + (0, 2561), + torch.empty(640, 2561, dtype=torch.uint8), + ) + finally: + reader.close() + + +# ------------------------------------------------------------------ CUDA: relayout and expert + + +def _sm103() -> bool: + return torch.cuda.is_available() and torch.cuda.get_device_capability() == (10, 3) + + +needs_sm103 = pytest.mark.skipif(not _sm103(), reason="the NVFP4 kernels are exercised on sm_103") + + +@needs_sm103 +@pytest.mark.parametrize("rows, cols", [(640, 320), (5120, 40), (256, 64)]) +def test_block_scale_relayout_matches_independent_offsets(rows, cols) -> None: + sf = _bytes(rows, cols, rows + cols).cuda() + out = torch.ops.trtllm.block_scale_interleave(sf).reshape(-1) + assert out.numel() == layout.swizzled_numel(rows, cols) + assert torch.equal(unswizzle(out, rows, cols), sf.cpu()) + + +def _e4m3_byte(x: torch.Tensor) -> torch.Tensor: + """FP32 -> E4M3 bytes, round to nearest even, saturating at 448 (torch's own cast).""" + return x.clamp(max=448.0).to(torch.float8_e4m3fn).view(torch.uint8) + + +def _e2m1_code(x: torch.Tensor) -> torch.Tensor: + """FP32 -> E2M1 codes, round to nearest even (ties to the even magnitude index), saturating.""" + mags = torch.tensor(_E2M1, dtype=torch.float32) + a = x.abs().clamp(max=6.0) + d = (a.unsqueeze(-1) - mags).abs() + best = d.min(-1, keepdim=True).values + ties = d == best + idx = torch.where(ties, torch.arange(8), 100) + even = torch.where(ties & (torch.arange(8) % 2 == 0), torch.arange(8), 100) + pick = torch.where(ties.sum(-1) > 1, even.min(-1).values, idx.min(-1).values) + return pick | ((x < 0) & (pick != 0)).long() * 8 + + +def _quantize(x: torch.Tensor, s: float) -> tuple: + """Documented static activation quantization, in the kernel's operation order: g = 1/s, + scale = e4m3(g * (amax * (1/6))), code = e2m1(x * (1 / (scale * (1/g)))). Returns codes + ``[M, K]``, block bytes ``[M, K/16]`` and the pre-rounding values.""" + one = torch.tensor(1.0, dtype=torch.float32) + g = one / torch.tensor(s, dtype=torch.float32) + blocks = x.float().cpu().view(x.shape[0], -1, 16) + sf = _e4m3_byte(g * (blocks.abs().amax(-1) * (one / 6.0))) + scale = _E4M3_TABLE[sf.long()].float() + out_scale = torch.where(scale > 0, one / (scale * (one / g)), torch.zeros_like(scale)) + scaled = (blocks * out_scale.unsqueeze(-1)).view(x.shape[0], -1) + return _e2m1_code(scaled), sf, scaled + + +#: Relative distance to an E2M1 midpoint inside which the kernel's approximate reciprocals may +#: round either way (the quantizer catalog entry's tie window). +_TIE = 2.0**-20 + + +def _near_tie(scaled: torch.Tensor) -> torch.Tensor: + mids = torch.tensor((0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)) + a = scaled.abs().unsqueeze(-1) + return ((a - mids).abs() <= _TIE * a).any(-1) + + +def _gemm(a_vals: torch.Tensor, w_vals: torch.Tensor, alpha: float) -> tuple: + acc = (a_vals @ w_vals.T).float() + out = (torch.tensor(alpha, dtype=torch.float32) * acc).to(torch.bfloat16) + abs_dot = (a_vals.abs() @ w_vals.abs().T).float() * abs(alpha) + return out, abs_dot + + +def _assert_gemm(out: torch.Tensor, a_codes, a_sf, w_packed, w_sf, alpha) -> None: + """Kernel GEMM output vs the exact product of the same quantized operands.""" + a_vals = _E2M1_TABLE[a_codes.cpu()].view(a_codes.shape[0], -1, 16) * _E4M3_TABLE[ + a_sf.cpu().long() + ].unsqueeze(-1) + ref, abs_dot = _gemm(a_vals.view(a_codes.shape[0], -1), decode(w_packed, w_sf), alpha) + r32 = ref.float() + ulp = torch.exp2(torch.frexp(r32.abs().clamp(min=2.0**-126))[1].float() - 8) + err = (out.cpu().float() - r32).abs() + assert bool((err <= ulp + 2.0**-20 * abs_dot).all()), float((err / ulp).max()) + + +@needs_sm103 +@pytest.mark.parametrize("shared_fc1", [True, False]) +def test_expert_provider_matches_independent_reference(shared_fc1, monkeypatch) -> None: + from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + expert as provider, + ) + + hidden, inter, rows, n_experts, expert = 1024, 256, 7, 3, 2 + gen = torch.Generator().manual_seed(7) + + def proj(n, k, w2, s, seed): + # every code point including signed zero; block scales spread over normal E4M3 values, + # one all-zero block scale per projection + packed = _bytes(n, k // 2, seed) + sf = torch.randint(0x28, 0x48, (n, k // 16), dtype=torch.uint8, generator=gen) + sf[0, 0] = 0 + # every other expert slot holds different codes and block scales + bank = { + "weight": torch.stack( + [_bytes(n, k // 2, seed + 100 * (i + 1)) for i in range(n_experts)] + ), + "weight_scale": None, + "linear_scale": torch.stack([(sf + 1 + i).to(torch.uint8) for i in range(n_experts)]), + "input_scale": torch.full((n_experts,), s, dtype=torch.float32), + "weight_scale_2": torch.full((n_experts,), w2, dtype=torch.float32), + } + bank["weight"][expert] = packed + bank["linear_scale"][expert] = sf + bank["g"] = torch.ones(n_experts, dtype=torch.float32) / bank["input_scale"] + bank["alpha"] = bank["input_scale"] * bank["weight_scale_2"] + bank["weight_scale"] = torch.stack( + [ + torch.ops.trtllm.block_scale_interleave(bank["linear_scale"][i].cuda()) + .reshape(-1) + .cpu() + for i in range(n_experts) + ] + ) + return {key: v.cuda() for key, v in bank.items()} + + s_up = 0.0123 if shared_fc1 else 0.0171 + bank = { + "gate": proj(inter, hidden, 2.0**-9, 0.0123, 11), + "up": proj(inter, hidden, 3.0e-3, s_up, 12), + "down": proj(hidden, inter, 1.25e-3, 0.0625, 13), + } + x = (torch.randn(rows, hidden, generator=gen) * 0.7).to(torch.bfloat16) + x[0, :16] = 0 # one all-zero activation block + # Observe real kernel operands/results in the test, without a runtime capture API. + call = {"fc1_up": None} + projections = {id(projection): name for name, projection in bank.items()} + quantize, gemm = provider._quantize, provider._gemm + + def observe_quantize(src, projection, expert_id): + assert expert_id == expert + packed, scales = quantize(src, projection, expert_id) + name = projections[id(projection)] + keys = {"gate": ("fc1", "sf1"), "up": ("fc1_up", "sf1_up"), "down": ("fc2", "sf2")} + code_key, scale_key = keys[name] + call[code_key], call[scale_key] = packed, scales + if name == "down": + call["act"] = src + return packed, scales + + def observe_gemm(packed, scales, projection, expert_id): + assert expert_id == expert + result = gemm(packed, scales, projection, expert_id) + call[projections[id(projection)]] = result + return result + + with monkeypatch.context() as patch: + patch.setattr(provider, "_quantize", observe_quantize) + patch.setattr(provider, "_gemm", observe_gemm) + y = provider.nvfp4_swiglu_expert(x.cuda(), bank, expert, shared_fc1) + assert y.dtype == torch.bfloat16 and y.shape == (rows, hidden) + assert (call["fc1_up"] is None) == shared_fc1 + + def check_quant(codes_packed, sf_flat, src, s): + k = src.shape[1] + codes = unpack(codes_packed.cpu()) + sf = unswizzle(sf_flat, src.shape[0], k // 16) + ref_codes, ref_sf, scaled = _quantize(src, s) + assert torch.equal(sf, ref_sf) + vals = _E2M1_TABLE[codes] * _E4M3_TABLE[sf.long()].repeat_interleave(16, 1) + ref_vals = _E2M1_TABLE[ref_codes] * _E4M3_TABLE[ref_sf.long()].repeat_interleave(16, 1) + differ = vals != ref_vals + assert not bool((differ & ~_near_tie(scaled)).any()) + assert int(differ.sum()) <= 2 + return codes, sf + + fc1_codes, fc1_sf = check_quant(call["fc1"], call["sf1"], x, 0.0123) + up_codes, up_sf = ( + (fc1_codes, fc1_sf) if shared_fc1 else check_quant(call["fc1_up"], call["sf1_up"], x, s_up) + ) + lin = {k: bank[k]["linear_scale"][expert] for k in bank} + _assert_gemm( + call["gate"], + fc1_codes, + fc1_sf, + bank["gate"]["weight"][expert], + lin["gate"], + 0.0123 * 2.0**-9, + ) + _assert_gemm( + call["up"], + up_codes, + up_sf, + bank["up"]["weight"][expert], + lin["up"], + float(torch.tensor(s_up, dtype=torch.float32) * torch.tensor(3.0e-3, dtype=torch.float32)), + ) + act = (torch.nn.functional.silu(call["gate"].float()) * call["up"].float()).to(torch.bfloat16) + assert torch.equal(act, call["act"]) + down_codes, down_sf = check_quant(call["fc2"], call["sf2"], call["act"].cpu(), 0.0625) + _assert_gemm( + y, + down_codes, + down_sf, + bank["down"]["weight"][expert], + lin["down"], + float( + torch.tensor(0.0625, dtype=torch.float32) * torch.tensor(1.25e-3, dtype=torch.float32) + ), + ) + # The provider used expert `expert`'s operands, not a neighbour's. + other = provider.nvfp4_swiglu_expert(x.cuda(), bank, (expert + 1) % n_experts, shared_fc1) + assert not torch.equal(other, y) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py new file mode 100644 index 000000000000..017db9a2a06d --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Diagnostic validation mode of the Beam checkpoint-layout components (CPU; no weights). + +Ordinary execution must not hash, copy back or agree on the MoE routing across ranks; the +explicit diagnostic mode (``TRTLLM_MODELING_V2_VALIDATE``) must, and every rank has to run the +same mode because the agreement is a collective. These checks stub the router, the expert +dispatch and the collectives, so they pin which work runs in which mode and in what order; the +real-checkpoint TP4 refactor-integrity runs exercise the collectives themselves. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch._experimental.modeling_v2._router_index import MODELING_V2_VALIDATE_ENV +from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 + component_bf16 as bf16, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + component as nvfp4, +) + +pytestmark = pytest.mark.cpu_only + +BF16_CLS = bf16.BeamCheckpointLayoutBf16Sm103Tp4 +NVFP4_CLS = nvfp4.BeamCheckpointLayoutNvfp4Sm103Tp4 +T, HIDDEN, EXPERTS, LAYER = 3, 8, 4, 3 + + +def _routing() -> dict: + keys = ( + "logits", + "scores", + "selection_scores", + "group_scores", + "groups", + "masked_selection_scores", + ) + routing = {key: torch.zeros(T, EXPERTS) for key in keys} + routing["ids"] = torch.tensor([[0, 1]] * T) + routing["weights"] = torch.full((T, 2), 0.5) + return routing + + +def _component(diagnostics: bool, agreements: list) -> SimpleNamespace: + component = SimpleNamespace(diagnostics=diagnostics, device=torch.device("cpu")) + component.route = lambda layer, h: _routing() + component.all_reduce = lambda t: t + component._bank = lambda layer, which: torch.zeros(EXPERTS, HIDDEN, HIDDEN) + component.expert_bank = lambda layer: {} + component._shared_fc1 = {LAYER: [True] * EXPERTS} + + def digests_agree(tensors): + agreements.append(len(tensors)) + return ["digest"] * len(tensors) + + component._digests_agree = digests_agree + return component + + +@pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) +@pytest.mark.parametrize("diagnostics", [False, True], ids=["off", "on"]) +def test_moe_router_agreement_runs_only_in_diagnostic_mode(monkeypatch, cls, diagnostics): + agreements = [] + component = _component(diagnostics, agreements) + + def dispatch(h, ids, weights, provider): + if cls is BF16_CLS: + provider(0, h[:1]) + return torch.ones_like(h) + + monkeypatch.setattr(bf16.dispatch_combine, "dispatch_and_combine", dispatch) + monkeypatch.setattr(bf16, "cublas_mm", lambda a, b: a.float() @ b.float()) + monkeypatch.setattr(bf16, "swiglu_fp32", lambda g, u: {"silu": g, "product": g, "act": g}) + h = torch.zeros(T, HIDDEN, dtype=torch.bfloat16) + + out = cls.moe(component, LAYER, h) + + assert torch.equal(out, torch.ones_like(h)) + # The router input, ids and weights are agreed on in diagnostic mode only. + assert agreements == ([3] if diagnostics else []) + + +class _Reduce: + """A stand-in for the four-rank all-reduce: returns the sum the peers would produce.""" + + def __init__(self, total: int): + self.total = total + self.calls = 0 + + def __call__(self, flag: torch.Tensor) -> torch.Tensor: + self.calls += 1 + return torch.tensor([float(self.total)]) + + +@pytest.mark.parametrize( + "mine,ranks_on,ok", + [(False, 0, True), (True, 4, True), (True, 3, False), (False, 1, False), (False, 2, False)], +) +def test_every_rank_runs_the_same_diagnostic_mode(mine, ranks_on, ok): + reduce = _Reduce(ranks_on) + component = SimpleNamespace( + diagnostics=mine, device=torch.device("cpu"), rank=0, _allreduce=reduce + ) + if ok: + BF16_CLS._check_diagnostics_agree(component) + else: + with pytest.raises(bf16.LoadError, match=f"on for {ranks_on} of 4 ranks"): + BF16_CLS._check_diagnostics_agree(component) + assert reduce.calls == 1 + + +@pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) +def test_load_settles_the_mode_before_any_other_collective(cls, tmp_path): + order = [] + component = SimpleNamespace() + + def mode_mismatch(): + order.append("mode") + raise bf16.LoadError("rank 0: diagnostic validation is on for 1 of 4 ranks") + + component._check_diagnostics_agree = mode_mismatch + component._agree = lambda ok: order.append("agree") or ok + with pytest.raises(bf16.LoadError, match="diagnostic validation"): + cls.load(component, str(tmp_path)) + assert order == ["mode"] + + +def test_router_disagreement_is_detected(monkeypatch): + component = SimpleNamespace(device=torch.device("cpu"), mapping=None) + tensors = ( + torch.ones(2, HIDDEN, dtype=torch.bfloat16), + torch.tensor([[0, 1], [2, 3]]), + torch.full((2, 2), 0.5), + ) + + def peers(differing_rank): + def allgather(local, mapping, dim=0): + rows = [local.clone() for _ in range(bf16.WORLD_SIZE)] + if differing_rank is not None: + rows[differing_rank][0] ^= 1 + return torch.cat(rows) + + return allgather + + monkeypatch.setattr(bf16, "allgather", peers(None)) + assert len(BF16_CLS._digests_agree(component, tensors)) == 3 + monkeypatch.setattr(bf16, "allgather", peers(2)) + with pytest.raises(RuntimeError, match="replicated MoE routing differs"): + BF16_CLS._digests_agree(component, tensors) + + +class _Stop(Exception): + pass + + +@pytest.mark.parametrize( + "call_checked,diagnostics,rechecks", + [(False, False, 1), (True, False, 0), (True, True, 1), (False, True, 1)], + ids=["standalone", "full-model-call", "full-model-call-diagnostic", "standalone-diagnostic"], +) +def test_attention_rechecks_the_step_per_layer_only_when_needed( + monkeypatch, call_checked, diagnostics, rechecks +): + calls = [] + + def stop(md, layer): + raise _Stop() + + monkeypatch.setattr(bf16._attention, "check_batch_metadata", lambda *a, **k: calls.append(a)) + monkeypatch.setattr(bf16._attention, "step_args", stop) + component = SimpleNamespace( + diagnostics=diagnostics, + hidden=HIDDEN, + device=torch.device("cpu"), + max_tokens=16, + local_kv_heads=2, + head_dim=4, + kv_cache_dtype=None, + _check_usable=lambda layer=None: None, + ) + h = torch.zeros(T, HIDDEN, dtype=torch.bfloat16) + with pytest.raises(_Stop): + BF16_CLS.attention(component, LAYER, h, torch.arange(T), None, call_checked=call_checked) + assert len(calls) == rechecks + + +def test_full_model_validates_once_and_marks_every_block_call_checked(monkeypatch): + from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + model as full, + ) + + blocks = [] + checks = [] + model = SimpleNamespace( + schedule=SimpleNamespace(is_sliding=lambda layer: True, is_sparse=lambda layer: True), + window=512, + _check_usable=lambda layer=None: None, + _select_rows=lambda logit_rows, spans: [T - 1], + embed=lambda ids: torch.zeros(T, HIDDEN), + final_norm=lambda x: x, + head=lambda x: x, + ) + + def check_call(input_ids, position_ids, md, **kwargs): + checks.append(kwargs) + return input_ids, position_ids, [(0, T)] + + def block(layer, x, positions, md, **kwargs): + blocks.append((layer, kwargs.get("call_checked"))) + return x + + model.check_call = check_call + model.block = block + md = SimpleNamespace(num_contexts=1) + full.BeamFullModelNvfp4Sm103Tp4.forward( + model, torch.zeros(T, dtype=torch.int64), torch.arange(T), md + ) + assert len(checks) == 1 + assert blocks == [(layer, True) for layer in range(full.NUM_LAYERS)] + + +@pytest.mark.parametrize("value,expected", [(None, False), ("0", False), ("1", True), ("on", True)]) +def test_diagnostic_mode_is_read_once_from_the_environment(monkeypatch, value, expected): + if value is None: + monkeypatch.delenv(MODELING_V2_VALIDATE_ENV, raising=False) + else: + monkeypatch.setenv(MODELING_V2_VALIDATE_ENV, value) + component = SimpleNamespace() + BF16_CLS._init_diagnostics(component) + assert component.diagnostics is expected + monkeypatch.setenv(MODELING_V2_VALIDATE_ENV, "0" if expected else "1") + assert component.diagnostics is expected diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py new file mode 100644 index 000000000000..8ac901fa41e1 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py @@ -0,0 +1,327 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU tests for the Beam MoE dispatch and weighted combination of block 3: bf16 / sm_103 / tp1. + +Configuration under test, fixed in ``_dispatch_combine_cases.py``: hidden 5120, 256 +experts, eight per token, FP32 mixing weights, routed scaling factor 2.5; one sm_103 GPU; +eager; controlled affine experts instead of expert matrices; seeds {17, 41} x tokens +{1, 17, 129}, and focused fixtures. + +What is compared. Every call goes through the public ``dispatch_and_combine``. Its output +must equal, bit for bit, the plain-torch golden of the source combination in +``_dispatch_combine_cases.py``: the steps are exact fp32 products, bf16 roundings and fp32 +additions in a fixed order, so any correct implementation reproduces them exactly. The +expert calls themselves -- which experts, in which order, with which rows -- are checked +against the routing table. + +Invocation:: + + python -m pytest tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py + +Dependencies: a TensorRT-LLM build for sm_103. No model files. +""" + +from __future__ import annotations + +import inspect + +import pytest +import torch + +from tensorrt_llm._torch._experimental.modeling_v2.models.beam import dispatch_combine # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.dispatch_combine import ( # noqa: E501 + dispatch_and_combine, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 + beam_dispatch_combine, +) + +__extra_import_path__ = ["."] +import _dispatch_combine_cases as cases # noqa: E402 +from _beam_component_support import TorchCallTrace # noqa: E402 + +assert torch.cuda.is_available(), "the Beam MoE dispatch/combine tests require a CUDA device" + +BF16, FP32, I64 = torch.bfloat16, torch.float32, torch.int64 + + +@pytest.fixture(scope="module") +def experts(): + return cases.affine_experts() + + +def call(x, ids, w, experts): + """One public call on fresh copies of the experts' log; returns (output, calls).""" + provider = experts.fresh() + out = dispatch_and_combine(x, ids, w, provider) + return out, provider.calls + + +def check_calls(calls, x, ids): + """Exactly the selected experts, ascending, once each, each with its tokens' rows in + ascending token order: every one of the T*8 assignments once and nothing else.""" + selected = sorted(set(ids.reshape(-1).tolist())) + assert [e for e, _, _ in calls] == selected + assignments = 0 + for e, rows, _ in calls: + tokens = (ids == e).any(dim=1).nonzero().squeeze(1) + assert rows.shape[0] == tokens.numel() >= 1, e + assert cases.bitwise_equal(rows, x[tokens]), f"expert {e} received the wrong rows" + assignments += rows.shape[0] + assert assignments == ids.numel() + + +def run_checked(x, ids, w, experts, zero=False): + """Public call on the GPU with every output and immutability check; returns + (output, calls, golden).""" + x, ids, w = x.cuda(), ids.cuda(), w.cuda() + before = (x.clone(), ids.clone(), w.clone()) + out, calls = call(x, ids, w, experts) + assert out.shape == (x.shape[0], 5120) and out.dtype == BF16 and out.device == x.device + assert bool(torch.isfinite(out).all()) + for name, now, then in zip( + ("hidden_states", "expert_ids", "mixing_weights"), (x, ids, w), before + ): + assert cases.bitwise_equal(now, then), f"{name} was modified" + check_calls(calls, x, ids) + gold = cases.golden(x, ids, w, experts.fresh()) + assert [e for e, _, _ in calls] == [e for e, _ in gold["calls"]] + return out, calls, gold + + +# ----------------------------------------------------------------------------- the interface + + +def test_public_callable_and_provider_interface(): + params = list(inspect.signature(dispatch_and_combine).parameters) + assert params == ["hidden_states", "expert_ids", "mixing_weights", "expert_provider"] + doc = " ".join(inspect.getdoc(dispatch_combine).split()) + for phrase in ( + "expert_provider(expert_id, rows)", + "owns dispatch and combination", + "owns the expert computation", + "[n, 5120]", + ): + assert phrase in doc, phrase + assert ( + dispatch_combine.HIDDEN_SIZE, + dispatch_combine.SUPPORTED_CONFIG["num_experts"], + dispatch_combine.EXPERTS_PER_TOKEN, + dispatch_combine.ROUTED_SCALING_FACTOR, + ) == (5120, 256, 8, 2.5) + + +def test_component_holds_no_parameters_and_allocates_nothing_persistent(experts): + """No router parameter, expert matrix, module or configuration object exists to build: + the component is a function, its entry holds no state, and after a call only the + returned tensor remains allocated.""" + assert inspect.isfunction(dispatch_and_combine) + assert vars(beam_dispatch_combine) == {} + assert not any( + isinstance(v, (torch.nn.Module, torch.Tensor)) for v in vars(dispatch_combine).values() + ) + x, ids, w = (t.cuda() for t in cases.matrix_case(17, 17)) + a, b = experts.a, experts.b + torch.cuda.synchronize() + before = torch.cuda.memory_stats()["requested_bytes.all.current"] + out = dispatch_and_combine(x, ids, w, lambda e, rows: (rows.float() * a[e] + b[e]).to(BF16)) + torch.cuda.synchronize() + after = torch.cuda.memory_stats()["requested_bytes.all.current"] + assert after - before == out.numel() * out.element_size() + + +_T = 17 + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("hidden_rank_1", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("hidden_rank_3", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("hidden_width_5119", ValueError, r"hidden_states must be \[tokens, 5120\]"), + ("zero_tokens", ValueError, "at least one token"), + ("hidden_fp32", TypeError, "hidden_states must be torch.bfloat16"), + ("hidden_fp16", TypeError, "hidden_states must be torch.bfloat16"), + ("hidden_not_a_tensor", TypeError, "hidden_states must be a torch.Tensor"), + ("hidden_on_host", ValueError, "sm_103 CUDA device"), + ("ids_rank_1", ValueError, r"expert_ids must be \[tokens, 8\]"), + ("ids_7_slots", ValueError, r"expert_ids must be \[tokens, 8\]"), + ("ids_extra_row", ValueError, r"expert_ids must be \[tokens, 8\]"), + ("ids_int32", TypeError, "expert_ids must be torch.int64"), + ("ids_float", TypeError, "expert_ids must be torch.int64"), + ("ids_on_host", ValueError, "expert_ids must be on hidden_states' device"), + ("weights_7_slots", ValueError, r"mixing_weights must be \[tokens, 8\]"), + ("weights_missing_row", ValueError, r"mixing_weights must be \[tokens, 8\]"), + ("weights_transposed", ValueError, r"mixing_weights must be \[tokens, 8\]"), + ("weights_bf16", TypeError, "mixing_weights must be torch.float32"), + ("weights_fp64", TypeError, "mixing_weights must be torch.float32"), + ("weights_on_host", ValueError, "mixing_weights must be on hidden_states' device"), + ("provider_not_callable", TypeError, "expert_provider must be callable"), + ], +) +def test_rejects_malformed_inputs_before_any_device_work(case, error, match, experts): + """Each argument is named in its error, nothing is computed and no provider is called; + the checks read metadata only, so they never synchronize with the device.""" + x, ids, w = (t.cuda() for t in cases.matrix_case(41, _T)) + provider = experts.fresh() + args = {"hidden_states": x, "expert_ids": ids, "mixing_weights": w, "expert_provider": provider} + change = { + "hidden_rank_1": ("hidden_states", x[0]), + "hidden_rank_3": ("hidden_states", x.unsqueeze(0)), + "hidden_width_5119": ("hidden_states", x[:, :5119]), + "zero_tokens": ("hidden_states", x[:0]), + "hidden_fp32": ("hidden_states", x.float()), + "hidden_fp16": ("hidden_states", x.half()), + "hidden_not_a_tensor": ("hidden_states", x.tolist()), + "hidden_on_host": ("hidden_states", x.cpu()), + "ids_rank_1": ("expert_ids", ids.reshape(-1)), + "ids_7_slots": ("expert_ids", ids[:, :7]), + "ids_extra_row": ("expert_ids", torch.cat([ids, ids[:1]])), + "ids_int32": ("expert_ids", ids.int()), + "ids_float": ("expert_ids", ids.float()), + "ids_on_host": ("expert_ids", ids.cpu()), + "weights_7_slots": ("mixing_weights", w[:, :7]), + "weights_missing_row": ("mixing_weights", w[:-1]), + "weights_transposed": ("mixing_weights", w.t()), + "weights_bf16": ("mixing_weights", w.to(BF16)), + "weights_fp64": ("mixing_weights", w.double()), + "weights_on_host": ("mixing_weights", w.cpu()), + "provider_not_callable": ("expert_provider", None), + }[case] + args[change[0]] = change[1] + torch.cuda.synchronize() + torch.cuda.set_sync_debug_mode("error") + try: + with pytest.raises(error, match=match): + dispatch_and_combine(**args) + finally: + torch.cuda.set_sync_debug_mode("default") + assert provider.calls == [] + + +@pytest.mark.parametrize( + "case,error,match", + [ + ("fp32", TypeError, r"expert_provider\(\d+\) must return torch.bfloat16"), + ("narrow", ValueError, r"expert_provider\(\d+\) must return \[\d+, 5120\]"), + ("extra_row", ValueError, r"expert_provider\(\d+\) must return \[\d+, 5120\]"), + ("host", ValueError, r"expert_provider\(\d+\) must return a dense tensor on cuda"), + ("not_a_tensor", TypeError, r"expert_provider\(\d+\) must return a torch.Tensor"), + ], +) +def test_rejects_provider_outputs_outside_the_contract(case, error, match, experts): + x, ids, w = (t.cuda() for t in cases.matrix_case(17, _T)) + good = experts.fresh() + bad = { + "fp32": lambda e, rows: good(e, rows).float(), + "narrow": lambda e, rows: good(e, rows)[:, :5119], + "extra_row": lambda e, rows: torch.cat([good(e, rows), good(e, rows[:1])]), + "host": lambda e, rows: good(e, rows).cpu(), + "not_a_tensor": lambda e, rows: good(e, rows).tolist(), + }[case] + with pytest.raises(error, match=match): + dispatch_and_combine(x, ids, w, bad) + assert len(good.calls) in (1, 2) # refused at the first expert's output + + +# ----------------------------------------------------------------------------- the matrix + + +@pytest.mark.parametrize("tokens", cases.TOKENS) +@pytest.mark.parametrize("seed", cases.SEEDS) +def test_matrix_matches_the_source_combination(seed, tokens, experts): + out, calls, gold = run_checked(*cases.matrix_case(seed, tokens), experts) + assert cases.bitwise_equal(out, gold["output"]), f"seed {seed}, {tokens} tokens" + # Each expert's output went back to its own tokens: the scattered golden is different. + if tokens > 1: + wrong = cases.golden( + *(t.cuda() for t in cases.matrix_case(seed, tokens)), + experts.fresh(), + fault="wrong_scatter_mapping", + ) + assert not torch.equal(wrong["output"], out) + + +# ----------------------------------------------------------------------------- fixtures + + +def test_shared_extreme_experts_keep_tokens_and_experts_apart(experts): + x, ids, w, _ = cases.fixtures()["shared_extremes"] + out, calls, gold = run_checked(x, ids, w, experts) + assert cases.bitwise_equal(out, gold["output"]) + by_expert = {e: rows for e, rows, _ in calls} + for e in (0, 255): # every token selects both ends of the id range, in different slots + assert cases.bitwise_equal(by_expert[e], x.cuda()) + assert calls[0][0] == 0 and calls[-1][0] == 255 + # Leakage across tokens, across slots and a re-normalization each change the output. + for fault in ("wrong_scatter_mapping", "weights_slot_rotated", "renormalized_weights"): + wrong = cases.golden(x.cuda(), ids.cuda(), w.cuda(), experts.fresh(), fault=fault) + assert not torch.equal(wrong["output"], out), fault + + +def test_cancellation_witness_needs_every_rounding_step_in_place(experts): + """Signed routes of unequal size: the rounded route, the fp32 accumulation and the single + final scale are each visible in the output.""" + x, ids, w, _ = cases.fixtures()["cancellation_witness"] + out, calls, gold = run_checked(x, ids, w, experts) + assert cases.bitwise_equal(out, gold["output"]) + route = gold["route"].float() + assert int((route > 0).sum()) > 0 and int((route < 0).sum()) > 0 + assert bool((w.cuda() != w.cuda().to(BF16).float()).all()) # no weight is a bf16 value + for fault in cases.FAULTS: + if fault == "renormalized_weights": + continue # these weights are already normalized + wrong = cases.golden(x.cuda(), ids.cuda(), w.cuda(), experts.fresh(), fault=fault) + assert not torch.equal(wrong["output"], out), fault + + +def test_zero_weight_routes_contribute_exactly_zero(experts): + x, ids, w, _ = cases.fixtures()["zero_weight_route"] + out, calls, gold = run_checked(x, ids, w, experts) + assert cases.bitwise_equal(out, gold["output"]) + # A zero-weight route still calls its expert, and removing it changes nothing. + called = [e for e, _, _ in calls] + for t, j in enumerate(cases.ZERO_ROUTE_SLOT): + assert int(ids[t, j]) in called + without = cases.golden( + x.cuda(), ids.cuda(), w.cuda(), experts.fresh(), skip_slots=cases.ZERO_ROUTE_SLOT + ) + assert cases.bitwise_equal(out, without["output"]) + assert bool((out != 0).any(dim=1).all()) + + +@pytest.mark.parametrize("name", ["all_zero_weights", "all_zero_outputs"]) +def test_all_zero_weights_or_outputs_give_exact_zero(name, experts): + x, ids, w, zero = cases.fixtures()[name] + provider = cases.affine_experts(zero=True) if zero else experts + out, _, _ = run_checked(x, ids, w, provider) + assert bool((out == 0).all()) + # The hidden states are nonzero, so no residual or bias term reaches the output. + assert bool((x != 0).any()) + + +@pytest.mark.parametrize("seed", cases.SEEDS) +@pytest.mark.parametrize("tokens", [t for t in cases.TOKENS if t > 1]) +def test_row_permutation_keeps_each_token_its_output(seed, tokens, experts): + x, ids, w = (t.cuda() for t in cases.matrix_case(seed, tokens)) + perm = cases.permutation(seed, tokens).cuda() + base, _ = call(x, ids, w, experts) + permuted, _ = call(x[perm], ids[perm], w[perm], experts) + assert cases.bitwise_equal(permuted[torch.argsort(perm)], base) + + +# ----------------------------------------------------------------------------- execution + + +def test_token_arithmetic_runs_on_the_gpu(experts): + """Gathers, products, roundings, the accumulation and the scale all produce tensors on + the input's device; only the expert list itself comes to the host (``tolist``).""" + x, ids, w = (t.cuda() for t in cases.matrix_case(41, 129)) + with TorchCallTrace() as trace: + dispatch_and_combine(x, ids, w, experts.fresh()) + names = [name for name, _ in trace.calls] + short = {name.rsplit(".", 1)[-1] for name in names} + assert {"__getitem__", "to", "index_add_"} <= short and {"__mul__", "mul"} & short, short + produced = [device for _, devices in trace.calls for device in devices] + assert produced and all(device == str(x.device) for device in produced), trace.calls + assert not any("reduce" in n or "all_gather" in n or "distributed" in n for n in names) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py new file mode 100644 index 000000000000..809e4955e98c --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Manual four-GPU load-error qualification with one 32-byte generated tensor. + +Run in a matching installed/native TRT-LLM environment on one node:: + + env -u TLLM_DISABLE_MPI timeout 120s mpirun -n 4 python -m pytest -q \\ + tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py + +This uses real MPI error exchange, NCCL diagnostic agreement and CUDA copies. +Only model geometry/quantization metadata are reduced; no checkpoint, inference, +NVFP4 kernel or full-model qualification is involved. Not scheduled in L0. +The outer timeout bounds a broken collective, including pytest/process startup. +""" + +import json +from types import SimpleNamespace + +import pytest +import torch + + +@pytest.mark.parametrize("packed", [False, True], ids=["bf16", "nvfp4"]) +def test_distributed_loading(tmp_path, monkeypatch, packed): + from tensorrt_llm import mpi_rank, mpi_world_size + from tensorrt_llm._utils import local_mpi_rank, mpi_disabled + + if mpi_disabled() or mpi_world_size() != 4 or torch.cuda.device_count() < 4: + pytest.skip("requires MPI with four ranks and four visible GPUs on one node") + + from safetensors.torch import save_file + + from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( + component_bf16 as bf16, + ) + from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import layout_bf16 as layout + from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import loader_bf16 as loader + from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( + component as nvfp4, + ) + from tensorrt_llm._torch.distributed import AllReduce + from tensorrt_llm._torch.distributed.communicator import Distributed + from tensorrt_llm.functional import AllReduceStrategy + from tensorrt_llm.mapping import Mapping + + rank = mpi_rank() + torch.cuda.set_device(local_mpi_rank()) + device = torch.device("cuda", local_mpi_rank()) + mapping = Mapping(world_size=4, rank=rank, tp_size=4) + cls = ( + nvfp4.BeamCheckpointLayoutNvfp4Sm103Tp4 if packed else bf16.BeamCheckpointLayoutBf16Sm103Tp4 + ) + obj = cls.__new__(cls) + torch.nn.Module.__init__(obj) + source = torch.arange(8, dtype=torch.float32).reshape(4, 2) + spec = layout.TensorSpec("weight", (4, 2), layout.F32, "dense", None, None, None) + obj.config, obj.specs = {"test": True}, {spec.name: spec} + obj.mapping, obj.rank, obj.device = mapping, rank, device + obj.layers, obj.expert_names, obj._shared_fc1 = (), [], {} + obj.schedule = SimpleNamespace(is_sparse=lambda layer: False) + obj.usable, obj.load_record, obj.diagnostics = False, None, False + obj._allreduce = AllReduce(mapping=mapping, strategy=AllReduceStrategy.NCCL) + obj.register_parameter( + bf16._attr(spec.name), + torch.nn.Parameter(torch.zeros_like(source, device=device), requires_grad=False), + ) + monkeypatch.setattr(cls, "_tensor_specs", staticmethod(lambda *args: obj.specs)) + if packed: + monkeypatch.setattr(nvfp4, "tensor_specs", lambda *args: obj.specs) + monkeypatch.setattr(nvfp4, "read_quant_config", lambda *args: {}) + monkeypatch.setattr(nvfp4, "check_recipe", lambda *args: {}) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(obj.config)) + save_file({"weight": source}, str(tmp_path / "weights.safetensors")) + (tmp_path / loader.INDEX_FILE).write_text( + json.dumps({"weight_map": {"weight": "weights.safetensors"}}) + ) + # Keep staging tiny as well as the generated tensor; retain real CUDA copies. + reader_type = loader.TileReader + monkeypatch.setattr(bf16, "TileReader", lambda: reader_type(tile_bytes=16)) + original_read = bf16.read_slice_into + + def fail_read(*args, **kwargs): + if rank == 2: + raise OSError("injected rank-two read failure") + return original_read(*args, **kwargs) + + obj.load(str(tmp_path), record_digests=False) + assert obj.usable and torch.equal(obj.tensor("weight").cpu(), source) + previous = obj.load_record + if rank == 2: + config_path.write_text("{}") + with pytest.raises(bf16.LoadError, match="rank 2: CheckpointFormatError"): + obj.load(str(tmp_path), record_digests=False) + assert obj.usable and obj.load_record is previous # No writes during validation. + config_path.write_text(json.dumps(obj.config)) + + with monkeypatch.context() as patch: + patch.setattr(bf16, "read_slice_into", fail_read) + with pytest.raises(bf16.LoadError, match="rank 2: OSError.*injected rank-two read failure"): + obj.load(str(tmp_path), record_digests=False) + assert not obj.usable and obj.load_record is None + if packed: + assert obj._shared_fc1 == {} + + obj.load(str(tmp_path), record_digests=False) + assert obj.usable and torch.equal(obj.tensor("weight").cpu(), source) + assert Distributed.get(mapping).allgather(obj.usable) == [True] * 4 From 0d1d796221665871596dbc0b33d5c81c0abd307d Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Thu, 12 Nov 2026 13:29:16 -0800 Subject: [PATCH 02/33] test: group Beam CPU checks with ModelingV2 Signed-off-by: Maxim Gluhovskoi --- tests/integration/test_lists/test-db/l0_cpu.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index d922ed058336..93ed683580dc 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -123,13 +123,12 @@ l0_cpu: - unittest/_torch/peft - unittest/_torch/memory - unittest/_torch/modeling - # modeling_v2 routing and routing-table checks: config shapes and a monkeypatched - # compute capability, no device. The default targets' shard tests: small CPU - # tensors and the CPU block_scale_interleave. The rest of unittest/_torch/modeling_v2 - # needs a GPU. + # ModelingV2 routing, sharding and Beam component, diagnostic and checkpoint-layout + # checks run on CPU. Mixed CPU/GPU modules select only cpu_only-marked cases here. - unittest/_torch/modeling_v2/models/beam/test_component_contracts.py - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py + - unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py - unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py - unittest/_torch/modeling_v2/test_modeling_v2_contracts.py - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -456,6 +455,3 @@ l0_cpu: - unittest/utils/test_confidential_compute.py - unittest/utils/test_llm_data.py - unittest/visual_gen/test_output.py - - # Beam baseline diagnostic checks (no checkpoints or GPU). - - unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py From a093e311b86903d11b1658def4db7240b6794204 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Thu, 12 Nov 2026 13:38:36 -0800 Subject: [PATCH 03/33] test: annotate Beam component helpers and test interfaces Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_beam_component_support.py | 70 +++++++++++++------ .../beam/_components/dense_block/block.py | 10 ++- .../_components/moe_expert/test_expert.py | 43 +++++++----- .../models/beam/test_diagnostic_mode.py | 58 ++++++++++----- .../models/beam/test_dispatch_combine.py | 54 ++++++++++---- 5 files changed, 163 insertions(+), 72 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index ff7bbbdeedec..cec63ebadc30 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -29,8 +29,9 @@ import types import zlib from collections import OrderedDict +from collections.abc import Callable, Iterator, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NoReturn import torch from torch.overrides import TorchFunctionMode, resolve_name @@ -38,6 +39,8 @@ import tensorrt_llm if TYPE_CHECKING: + from _components.moe.moe import BeamMoeBf16Sm103Tp1 + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata #: CPU op names of PyTorch's SDPA paths, and CUDA kernel names of attention kernels. @@ -66,7 +69,7 @@ def math_state() -> dict: } -def ieee_fp32_session(write_evidence): +def ieee_fp32_session(write_evidence: Callable[[str, dict], None]) -> Iterator[dict]: """Enforce IEEE fp32 and verify it before and after the module's tests.""" matmul = torch.backends.cuda.matmul saved = ( @@ -113,7 +116,7 @@ def git_revision() -> str | None: return out.stdout.strip() or None -def environment(ieee: dict, execution, **extra) -> dict: +def environment(ieee: dict, execution: Mapping[str, object], **extra: object) -> dict: """The device and software a case ran on; ``extra`` entries follow ``execution``.""" device = torch.cuda.current_device() return { @@ -241,7 +244,9 @@ def compare_exact(name: str, actual: torch.Tensor, expected: torch.Tensor) -> di } -def output_contract(out, shape: tuple[int, ...], dtype: torch.dtype, sm: tuple[int, int]) -> dict: +def output_contract( + out: object, shape: tuple[int, ...], dtype: torch.dtype, sm: tuple[int, int] +) -> dict: """The forward returned a finite ``shape`` bf16 tensor on the sm_103 device.""" problems = [] if not isinstance(out, torch.Tensor): @@ -302,7 +307,9 @@ def assert_all_pass(results: list[dict]) -> None: # -------------------------------------------------- weights -def cheap_weights(param_shapes) -> "OrderedDict[str, torch.Tensor]": +def cheap_weights( + param_shapes: Mapping[str, tuple[tuple[int, ...], torch.dtype]], +) -> "OrderedDict[str, torch.Tensor]": """A valid mapping for ``param_shapes`` (name -> (shape, dtype)) that allocates nothing: one expanded constant per dtype, so a load that copied any of it would change the generated values it replaced.""" @@ -320,12 +327,12 @@ def cheap_weights(param_shapes) -> "OrderedDict[str, torch.Tensor]": @contextlib.contextmanager def recording( - *wrappers, + *wrappers: Callable[..., object], recorded_kwargs: tuple[str, ...], - experts=None, + experts: type[BeamMoeBf16Sm103Tp1] | None = None, data_ptrs: bool = False, qkv: bool = False, -): +) -> Iterator[list[dict]]: """Log every call of these entries, in order, with its nesting depth, the arguments that identify the path and its output. ``experts``: an MoE class whose ``_expert`` calls are also logged, with the expert asked for and the rows it is passed. ``data_ptrs``: the @@ -338,7 +345,13 @@ def recording( original = cls.__call__ saved.append((cls, "__call__", original)) - def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): + def logged( + self, + *args: object, + _original: Callable[..., object] = original, + _name: str = cls.__name__, + **kwargs: object, + ) -> object: tensors = [a for a in args if isinstance(a, torch.Tensor)] entry = { "op": _name, @@ -365,7 +378,14 @@ def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): original_expert = experts._expert saved.append((experts, "_expert", original_expert)) - def asked(self, expert_id, rows, _original=original_expert): + def asked( + self, + expert_id: int, + rows: torch.Tensor, + _original: Callable[ + [BeamMoeBf16Sm103Tp1, int, torch.Tensor], torch.Tensor + ] = original_expert, + ) -> torch.Tensor: calls.append({"op": "expert", "depth": depth[0], "expert_id": expert_id, "rows": rows}) return _original(self, expert_id, rows) @@ -389,12 +409,12 @@ def loggable(calls: list[dict], keep: tuple[str, ...]) -> list[dict]: @contextlib.contextmanager -def builtin_forbidden(targets): +def builtin_forbidden(targets: Sequence[tuple[type, str]]) -> Iterator[None]: """The built-in Beam modeling must not run while the component does: every ``(class, method)`` in ``targets`` raises for the duration of the block.""" saved = [(cls, name, getattr(cls, name)) for cls, name in targets] - def refuse(self, *args, **kwargs): + def refuse(self, *args: object, **kwargs: object) -> NoReturn: raise AssertionError(f"the built-in {type(self).__name__} ran inside the component") for cls, name, _ in saved: @@ -407,14 +427,14 @@ def refuse(self, *args, **kwargs): @contextlib.contextmanager -def no_dispatch(ops): +def no_dispatch(ops: Sequence[Callable[..., object]]) -> Iterator[None]: """Fail on any call of ``ops``: rejected inputs must be rejected before a kernel runs.""" saved = [] for w in ops: cls = type(w) saved.append((cls, cls.__call__)) - def refuse(self, *args, _name=cls.__name__, **kwargs): + def refuse(self, *args: object, _name: str = cls.__name__, **kwargs: object) -> NoReturn: raise AssertionError(f"{_name} dispatched before the input was rejected") cls.__call__ = refuse @@ -428,11 +448,17 @@ def refuse(self, *args, _name=cls.__name__, **kwargs): class TorchCallTrace(TorchFunctionMode): """The name of every torch call made inside the block and the devices of its outputs.""" - def __init__(self): + def __init__(self) -> None: super().__init__() self.calls: list[tuple[str, list[str]]] = [] - def __torch_function__(self, func, types, args=(), kwargs=None): + def __torch_function__( + self, + func: Callable[..., object], + types: tuple[type, ...], + args: tuple[object, ...] = (), + kwargs: dict[str, object] | None = None, + ) -> object: out = func(*args, **(kwargs or {})) outs = out if isinstance(out, (tuple, list)) else [out] devices = [str(t.device) for t in outs if isinstance(t, torch.Tensor)] @@ -453,7 +479,7 @@ def fresh_context( max_tokens: int, request_id: int, mgr_args: dict, -): +) -> Iterator[TrtllmAttentionMetadata]: """A KVCacheManagerV2 built from ``mgr_args`` and prepared TrtllmAttentionMetadata for ``num_requests`` new requests of ``num_tokens`` after ``cached_tokens`` cached ones.""" from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata @@ -523,7 +549,9 @@ def cache_rows( } -def bad_block_input(case: str, max_tokens: int, dtype: torch.dtype): +def bad_block_input( + case: str, max_tokens: int, dtype: torch.dtype +) -> tuple[torch.Tensor, torch.Tensor, dict, str | None]: """One unsupported call of a block at 17 tokens: ``(hidden_states, position_ids, fresh_context kwargs, ValueError message pattern or None)``. ``metadata_type`` keeps the valid inputs; the caller replaces the metadata itself.""" @@ -609,7 +637,7 @@ def bad_block_input(case: str, max_tokens: int, dtype: torch.dtype): ) -def small_model_config(**overrides) -> types.SimpleNamespace: +def small_model_config(**overrides: object) -> types.SimpleNamespace: fields = dict(SMALL_MODEL_CONFIG) fields["layer_types"] = list(fields["layer_types"]) fields["mlp_layer_types"] = list(fields["mlp_layer_types"]) @@ -634,7 +662,9 @@ def _small_model_half_width(name: str, shape: tuple, embed_name: str) -> tuple[f return math.sqrt(3.0 / shape[-1]), 0.0 -def generate_small_model_state(seed: int, model_module) -> "OrderedDict[str, torch.Tensor]": +def generate_small_model_state( + seed: int, model_module: types.ModuleType +) -> "OrderedDict[str, torch.Tensor]": """All 2394 tensors of ``model_module.weight_schema``, each drawn from its own generator keyed by seed and name, fp32 then rounded once to the stored dtype.""" state = OrderedDict() diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py index 5215e561fb31..164a5acf80c7 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py @@ -33,7 +33,8 @@ from __future__ import annotations from collections import OrderedDict -from typing import Mapping +from types import SimpleNamespace +from typing import TYPE_CHECKING, Mapping import torch from torch import nn @@ -51,6 +52,9 @@ depth_scales, ) +if TYPE_CHECKING: + from transformers import PretrainedConfig + #: The configuration this component computes, as the checkpoint's config.json states it. #: Anything else is rejected at construction rather than computed approximately. SUPPORTED_CONFIG = { @@ -75,7 +79,9 @@ MAX_TOKENS = 1024 -def weight_shapes(config) -> "OrderedDict[str, tuple[int, ...]]": +def weight_shapes( + config: PretrainedConfig | SimpleNamespace, +) -> "OrderedDict[str, tuple[int, ...]]": """The block's fifteen tensors by checkpoint name, in checkpoint order, [out, in] for matrices.""" hidden, inter = config.hidden_size, config.intermediate_size q_width = config.num_attention_heads * config.head_dim diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py index 0e951239b020..a2b293d7b252 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -24,6 +24,7 @@ import contextlib import gc +from collections.abc import Iterator import pytest import torch @@ -48,7 +49,7 @@ @pytest.fixture(scope="module", autouse=True) -def controls(): +def controls() -> Iterator[None]: """IEEE fp32, no reduced-precision bf16 reduction and no autocast for the whole module.""" with cases.numerical_controls(): yield @@ -58,17 +59,19 @@ def controls(): @contextlib.contextmanager -def observed_calls(): +def observed_calls() -> Iterator[ + dict[str, list[tuple[tuple[torch.Tensor, ...], torch.Tensor | dict[str, torch.Tensor]]]] +]: """Every cublas_mm result and activation mapping made inside the block, unchanged.""" seen = {"mm": [], "act": []} mm_call, act_call = CublasMm.__call__, SwigluFp32.__call__ - def mm(self, *args, **kwargs): + def mm(self, *args: torch.Tensor, **kwargs: object) -> torch.Tensor: out = mm_call(self, *args, **kwargs) seen["mm"].append((args, out)) return out - def act(self, *args, **kwargs): + def act(self, *args: torch.Tensor, **kwargs: object) -> dict[str, torch.Tensor]: out = act_call(self, *args, **kwargs) seen["act"].append((args, out)) return out @@ -122,7 +125,7 @@ def _requested_cuda_bytes() -> int: return torch.cuda.memory_stats()["requested_bytes.all.current"] -def test_construction_holds_one_expert_and_nothing_else(): +def test_construction_holds_one_expert_and_nothing_else() -> None: gc.collect() torch.cuda.synchronize() before = _requested_cuda_bytes() @@ -143,13 +146,13 @@ def test_construction_holds_one_expert_and_nothing_else(): assert list(expert.checkpoint_tensors()) == list(WEIGHT_SHAPES) -def test_construction_rejects_a_host_device(): +def test_construction_rejects_a_host_device() -> None: with pytest.raises(ValueError, match="sm_103 CUDA devices"): BeamMoeExpertBf16Sm103Tp1(device="cpu") @pytest.mark.parametrize("source_device", ["cpu", "cuda"]) -def test_load_copies_exact_values(source_device): +def test_load_copies_exact_values(source_device: str) -> None: weights = {k: v.to(source_device) for k, v in cases.matrix_case(17)[0].items()} expert = loaded(weights) for name, tensor in expert.checkpoint_tensors().items(): @@ -173,7 +176,9 @@ def test_load_copies_exact_values(source_device): ("not_a_tensor", TypeError, "must be a torch.Tensor"), ], ) -def test_rejects_unsupported_inputs_before_any_kernel(case, error, match): +def test_rejects_unsupported_inputs_before_any_kernel( + case: str, error: type[Exception], match: str +) -> None: weights, x_all = cases.matrix_case(17) expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") if case != "not_loaded": @@ -209,7 +214,9 @@ def test_rejects_unsupported_inputs_before_any_kernel(case, error, match): ("not_a_mapping", TypeError, "weights must be a mapping"), ], ) -def test_rejected_load_leaves_the_expert_unchanged(case, error, match): +def test_rejected_load_leaves_the_expert_unchanged( + case: str, error: type[Exception], match: str +) -> None: """Every other tensor is a valid replacement; a bad down tensor is the last one checked.""" weights, x_all = cases.matrix_case(17) expert = loaded(weights) @@ -236,7 +243,7 @@ def test_rejected_load_leaves_the_expert_unchanged(case, error, match): assert bitwise_equal(expert(x), y_before) -def test_rejected_first_load_leaves_the_expert_unloaded(): +def test_rejected_first_load_leaves_the_expert_unloaded() -> None: weights = dict(cases.matrix_case(17)[0]) weights["down_proj.weight"] = weights["down_proj.weight"].float() expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") @@ -251,7 +258,7 @@ def test_rejected_first_load_leaves_the_expert_unloaded(): @pytest.mark.parametrize("tokens", cases.TOKENS) @pytest.mark.parametrize("seed", cases.SEEDS) -def test_matrix_every_step_matches_the_source_expert(seed, tokens): +def test_matrix_every_step_matches_the_source_expert(seed: int, tokens: int) -> None: weights, x_all = cases.matrix_case(seed) expert = loaded(weights) x = x_all[:tokens].cuda() @@ -275,7 +282,7 @@ def test_matrix_every_step_matches_the_source_expert(seed, tokens): # ----------------------------------------------------------------------------- fixtures -def test_zero_input_gives_exact_zero(): +def test_zero_input_gives_exact_zero() -> None: expert = loaded(cases.matrix_case(17)[0]) steps = run(expert, torch.zeros(17, 5120, dtype=BF16, device="cuda")) for stage, tensor in steps.items(): @@ -283,7 +290,7 @@ def test_zero_input_gives_exact_zero(): assert bool((tensor == 0).all()), stage -def test_signed_gates_exercise_silu_exactly(): +def test_signed_gates_exercise_silu_exactly() -> None: weights, x, expected = cases.signed_gate_fixture() gates = expected["gate"].double() assert int((gates < 0).sum()) >= 2000 and int((gates > 0).sum()) >= 2000 @@ -295,7 +302,7 @@ def test_signed_gates_exercise_silu_exactly(): assert not torch.equal(silu, gates) and not torch.equal(silu, gates / 2) -def test_precision_fixture_tells_fp32_activation_from_bf16_steps(): +def test_precision_fixture_tells_fp32_activation_from_bf16_steps() -> None: weights, x, expected, premature_act = cases.precision_fixture() steps = run(loaded(weights), x.cuda()) cases.assert_stages(steps, expected, cases.EXACT_RULES, "precision fixture") @@ -303,7 +310,7 @@ def test_precision_fixture_tells_fp32_activation_from_bf16_steps(): assert bool((steps["act"].cpu().double() != premature_act.double()).all()) -def test_sparse_mapping_fixes_orientation_and_feature_maps(): +def test_sparse_mapping_fixes_orientation_and_feature_maps() -> None: weights, x, expected = cases.sparse_mapping_fixture() steps = run(loaded(weights), x.cuda()) cases.assert_stages(steps, expected, cases.EXACT_RULES, "sparse mapping") @@ -319,7 +326,7 @@ def test_sparse_mapping_fixes_orientation_and_feature_maps(): assert not torch.equal(gold["y"].cpu(), expected["y"]), name -def test_row_permutation_keeps_tokens_apart(): +def test_row_permutation_keeps_tokens_apart() -> None: weights, x_all = cases.matrix_case(17) expert = loaded(weights) x = x_all.cuda() @@ -334,7 +341,7 @@ def test_row_permutation_keeps_tokens_apart(): assert cases.violations(mixed, base["y"], *cases.MATRIX_RULES["y"])["failing"] > 0 -def test_no_residual_weighting_or_collective(): +def test_no_residual_weighting_or_collective() -> None: """Zero down weights with a nonzero input: an added residual, bias or any other term would leave the output nonzero. Exact outputs on the fixtures above rule out a mixing weight or the routed scaling factor.""" @@ -350,7 +357,7 @@ def test_no_residual_weighting_or_collective(): assert not any("reduce" in n or "all_gather" in n or "distributed" in n for n in names), names -def test_forward_runs_on_the_gpu(): +def test_forward_runs_on_the_gpu() -> None: weights, x_all = cases.matrix_case(41) expert = loaded(weights) x = x_all[:129].cuda() diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py index 017db9a2a06d..1b7419e9bea4 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py @@ -9,7 +9,12 @@ real-checkpoint TP4 refactor-integrity runs exercise the collectives themselves. """ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from pathlib import Path from types import SimpleNamespace +from typing import NoReturn import pytest import torch @@ -52,7 +57,7 @@ def _component(diagnostics: bool, agreements: list) -> SimpleNamespace: component.expert_bank = lambda layer: {} component._shared_fc1 = {LAYER: [True] * EXPERTS} - def digests_agree(tensors): + def digests_agree(tensors: Sequence[torch.Tensor]) -> list[str]: agreements.append(len(tensors)) return ["digest"] * len(tensors) @@ -62,11 +67,20 @@ def digests_agree(tensors): @pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) @pytest.mark.parametrize("diagnostics", [False, True], ids=["off", "on"]) -def test_moe_router_agreement_runs_only_in_diagnostic_mode(monkeypatch, cls, diagnostics): +def test_moe_router_agreement_runs_only_in_diagnostic_mode( + monkeypatch: pytest.MonkeyPatch, + cls: type[bf16.BeamCheckpointLayoutBf16Sm103Tp4], + diagnostics: bool, +) -> None: agreements = [] component = _component(diagnostics, agreements) - def dispatch(h, ids, weights, provider): + def dispatch( + h: torch.Tensor, + ids: torch.Tensor, + weights: torch.Tensor, + provider: Callable[[int, torch.Tensor], torch.Tensor], + ) -> torch.Tensor: if cls is BF16_CLS: provider(0, h[:1]) return torch.ones_like(h) @@ -86,7 +100,7 @@ def dispatch(h, ids, weights, provider): class _Reduce: """A stand-in for the four-rank all-reduce: returns the sum the peers would produce.""" - def __init__(self, total: int): + def __init__(self, total: int) -> None: self.total = total self.calls = 0 @@ -99,7 +113,7 @@ def __call__(self, flag: torch.Tensor) -> torch.Tensor: "mine,ranks_on,ok", [(False, 0, True), (True, 4, True), (True, 3, False), (False, 1, False), (False, 2, False)], ) -def test_every_rank_runs_the_same_diagnostic_mode(mine, ranks_on, ok): +def test_every_rank_runs_the_same_diagnostic_mode(mine: bool, ranks_on: int, ok: bool) -> None: reduce = _Reduce(ranks_on) component = SimpleNamespace( diagnostics=mine, device=torch.device("cpu"), rank=0, _allreduce=reduce @@ -113,11 +127,13 @@ def test_every_rank_runs_the_same_diagnostic_mode(mine, ranks_on, ok): @pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) -def test_load_settles_the_mode_before_any_other_collective(cls, tmp_path): +def test_load_settles_the_mode_before_any_other_collective( + cls: type[bf16.BeamCheckpointLayoutBf16Sm103Tp4], tmp_path: Path +) -> None: order = [] component = SimpleNamespace() - def mode_mismatch(): + def mode_mismatch() -> NoReturn: order.append("mode") raise bf16.LoadError("rank 0: diagnostic validation is on for 1 of 4 ranks") @@ -128,7 +144,7 @@ def mode_mismatch(): assert order == ["mode"] -def test_router_disagreement_is_detected(monkeypatch): +def test_router_disagreement_is_detected(monkeypatch: pytest.MonkeyPatch) -> None: component = SimpleNamespace(device=torch.device("cpu"), mapping=None) tensors = ( torch.ones(2, HIDDEN, dtype=torch.bfloat16), @@ -136,8 +152,8 @@ def test_router_disagreement_is_detected(monkeypatch): torch.full((2, 2), 0.5), ) - def peers(differing_rank): - def allgather(local, mapping, dim=0): + def peers(differing_rank: int | None) -> Callable[..., torch.Tensor]: + def allgather(local: torch.Tensor, mapping: object, dim: int = 0) -> torch.Tensor: rows = [local.clone() for _ in range(bf16.WORLD_SIZE)] if differing_rank is not None: rows[differing_rank][0] ^= 1 @@ -162,11 +178,11 @@ class _Stop(Exception): ids=["standalone", "full-model-call", "full-model-call-diagnostic", "standalone-diagnostic"], ) def test_attention_rechecks_the_step_per_layer_only_when_needed( - monkeypatch, call_checked, diagnostics, rechecks -): + monkeypatch: pytest.MonkeyPatch, call_checked: bool, diagnostics: bool, rechecks: int +) -> None: calls = [] - def stop(md, layer): + def stop(md: object, layer: int) -> NoReturn: raise _Stop() monkeypatch.setattr(bf16._attention, "check_batch_metadata", lambda *a, **k: calls.append(a)) @@ -187,7 +203,9 @@ def stop(md, layer): assert len(calls) == rechecks -def test_full_model_validates_once_and_marks_every_block_call_checked(monkeypatch): +def test_full_model_validates_once_and_marks_every_block_call_checked( + monkeypatch: pytest.MonkeyPatch, +) -> None: from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 model as full, ) @@ -204,11 +222,15 @@ def test_full_model_validates_once_and_marks_every_block_call_checked(monkeypatc head=lambda x: x, ) - def check_call(input_ids, position_ids, md, **kwargs): + def check_call( + input_ids: torch.Tensor, position_ids: torch.Tensor, md: object, **kwargs: object + ) -> tuple[torch.Tensor, torch.Tensor, list[tuple[int, int]]]: checks.append(kwargs) return input_ids, position_ids, [(0, T)] - def block(layer, x, positions, md, **kwargs): + def block( + layer: int, x: torch.Tensor, positions: torch.Tensor, md: object, **kwargs: object + ) -> torch.Tensor: blocks.append((layer, kwargs.get("call_checked"))) return x @@ -223,7 +245,9 @@ def block(layer, x, positions, md, **kwargs): @pytest.mark.parametrize("value,expected", [(None, False), ("0", False), ("1", True), ("on", True)]) -def test_diagnostic_mode_is_read_once_from_the_environment(monkeypatch, value, expected): +def test_diagnostic_mode_is_read_once_from_the_environment( + monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool +) -> None: if value is None: monkeypatch.delenv(MODELING_V2_VALIDATE_ENV, raising=False) else: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py index 8ac901fa41e1..bab49aff1d60 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py @@ -46,18 +46,22 @@ @pytest.fixture(scope="module") -def experts(): +def experts() -> cases.AffineExperts: return cases.affine_experts() -def call(x, ids, w, experts): +def call( + x: torch.Tensor, ids: torch.Tensor, w: torch.Tensor, experts: cases.AffineExperts +) -> tuple[torch.Tensor, list[tuple[int, torch.Tensor, torch.Tensor]]]: """One public call on fresh copies of the experts' log; returns (output, calls).""" provider = experts.fresh() out = dispatch_and_combine(x, ids, w, provider) return out, provider.calls -def check_calls(calls, x, ids): +def check_calls( + calls: list[tuple[int, torch.Tensor, torch.Tensor]], x: torch.Tensor, ids: torch.Tensor +) -> None: """Exactly the selected experts, ascending, once each, each with its tokens' rows in ascending token order: every one of the T*8 assignments once and nothing else.""" selected = sorted(set(ids.reshape(-1).tolist())) @@ -71,7 +75,13 @@ def check_calls(calls, x, ids): assert assignments == ids.numel() -def run_checked(x, ids, w, experts, zero=False): +def run_checked( + x: torch.Tensor, + ids: torch.Tensor, + w: torch.Tensor, + experts: cases.AffineExperts, + zero: bool = False, +) -> tuple[torch.Tensor, list[tuple[int, torch.Tensor, torch.Tensor]], dict]: """Public call on the GPU with every output and immutability check; returns (output, calls, golden).""" x, ids, w = x.cuda(), ids.cuda(), w.cuda() @@ -92,7 +102,7 @@ def run_checked(x, ids, w, experts, zero=False): # ----------------------------------------------------------------------------- the interface -def test_public_callable_and_provider_interface(): +def test_public_callable_and_provider_interface() -> None: params = list(inspect.signature(dispatch_and_combine).parameters) assert params == ["hidden_states", "expert_ids", "mixing_weights", "expert_provider"] doc = " ".join(inspect.getdoc(dispatch_combine).split()) @@ -111,7 +121,9 @@ def test_public_callable_and_provider_interface(): ) == (5120, 256, 8, 2.5) -def test_component_holds_no_parameters_and_allocates_nothing_persistent(experts): +def test_component_holds_no_parameters_and_allocates_nothing_persistent( + experts: cases.AffineExperts, +) -> None: """No router parameter, expert matrix, module or configuration object exists to build: the component is a function, its entry holds no state, and after a call only the returned tensor remains allocated.""" @@ -159,7 +171,9 @@ def test_component_holds_no_parameters_and_allocates_nothing_persistent(experts) ("provider_not_callable", TypeError, "expert_provider must be callable"), ], ) -def test_rejects_malformed_inputs_before_any_device_work(case, error, match, experts): +def test_rejects_malformed_inputs_before_any_device_work( + case: str, error: type[Exception], match: str, experts: cases.AffineExperts +) -> None: """Each argument is named in its error, nothing is computed and no provider is called; the checks read metadata only, so they never synchronize with the device.""" x, ids, w = (t.cuda() for t in cases.matrix_case(41, _T)) @@ -209,7 +223,9 @@ def test_rejects_malformed_inputs_before_any_device_work(case, error, match, exp ("not_a_tensor", TypeError, r"expert_provider\(\d+\) must return a torch.Tensor"), ], ) -def test_rejects_provider_outputs_outside_the_contract(case, error, match, experts): +def test_rejects_provider_outputs_outside_the_contract( + case: str, error: type[Exception], match: str, experts: cases.AffineExperts +) -> None: x, ids, w = (t.cuda() for t in cases.matrix_case(17, _T)) good = experts.fresh() bad = { @@ -229,7 +245,9 @@ def test_rejects_provider_outputs_outside_the_contract(case, error, match, exper @pytest.mark.parametrize("tokens", cases.TOKENS) @pytest.mark.parametrize("seed", cases.SEEDS) -def test_matrix_matches_the_source_combination(seed, tokens, experts): +def test_matrix_matches_the_source_combination( + seed: int, tokens: int, experts: cases.AffineExperts +) -> None: out, calls, gold = run_checked(*cases.matrix_case(seed, tokens), experts) assert cases.bitwise_equal(out, gold["output"]), f"seed {seed}, {tokens} tokens" # Each expert's output went back to its own tokens: the scattered golden is different. @@ -245,7 +263,7 @@ def test_matrix_matches_the_source_combination(seed, tokens, experts): # ----------------------------------------------------------------------------- fixtures -def test_shared_extreme_experts_keep_tokens_and_experts_apart(experts): +def test_shared_extreme_experts_keep_tokens_and_experts_apart(experts: cases.AffineExperts) -> None: x, ids, w, _ = cases.fixtures()["shared_extremes"] out, calls, gold = run_checked(x, ids, w, experts) assert cases.bitwise_equal(out, gold["output"]) @@ -259,7 +277,9 @@ def test_shared_extreme_experts_keep_tokens_and_experts_apart(experts): assert not torch.equal(wrong["output"], out), fault -def test_cancellation_witness_needs_every_rounding_step_in_place(experts): +def test_cancellation_witness_needs_every_rounding_step_in_place( + experts: cases.AffineExperts, +) -> None: """Signed routes of unequal size: the rounded route, the fp32 accumulation and the single final scale are each visible in the output.""" x, ids, w, _ = cases.fixtures()["cancellation_witness"] @@ -275,7 +295,7 @@ def test_cancellation_witness_needs_every_rounding_step_in_place(experts): assert not torch.equal(wrong["output"], out), fault -def test_zero_weight_routes_contribute_exactly_zero(experts): +def test_zero_weight_routes_contribute_exactly_zero(experts: cases.AffineExperts) -> None: x, ids, w, _ = cases.fixtures()["zero_weight_route"] out, calls, gold = run_checked(x, ids, w, experts) assert cases.bitwise_equal(out, gold["output"]) @@ -291,7 +311,9 @@ def test_zero_weight_routes_contribute_exactly_zero(experts): @pytest.mark.parametrize("name", ["all_zero_weights", "all_zero_outputs"]) -def test_all_zero_weights_or_outputs_give_exact_zero(name, experts): +def test_all_zero_weights_or_outputs_give_exact_zero( + name: str, experts: cases.AffineExperts +) -> None: x, ids, w, zero = cases.fixtures()[name] provider = cases.affine_experts(zero=True) if zero else experts out, _, _ = run_checked(x, ids, w, provider) @@ -302,7 +324,9 @@ def test_all_zero_weights_or_outputs_give_exact_zero(name, experts): @pytest.mark.parametrize("seed", cases.SEEDS) @pytest.mark.parametrize("tokens", [t for t in cases.TOKENS if t > 1]) -def test_row_permutation_keeps_each_token_its_output(seed, tokens, experts): +def test_row_permutation_keeps_each_token_its_output( + seed: int, tokens: int, experts: cases.AffineExperts +) -> None: x, ids, w = (t.cuda() for t in cases.matrix_case(seed, tokens)) perm = cases.permutation(seed, tokens).cuda() base, _ = call(x, ids, w, experts) @@ -313,7 +337,7 @@ def test_row_permutation_keeps_each_token_its_output(seed, tokens, experts): # ----------------------------------------------------------------------------- execution -def test_token_arithmetic_runs_on_the_gpu(experts): +def test_token_arithmetic_runs_on_the_gpu(experts: cases.AffineExperts) -> None: """Gathers, products, roundings, the accumulation and the scale all produce tensors on the input's device; only the expert list itself comes to the host (``tolist``).""" x, ids, w = (t.cuda() for t in cases.matrix_case(41, 129)) From 98065d67abe0e9d66fec21c81ab5c29ec377faa1 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Thu, 12 Nov 2026 14:31:34 -0800 Subject: [PATCH 04/33] fix: accept sufficient Beam sliding-cache retention Signed-off-by: Maxim Gluhovskoi --- .../beam/full_model_nvfp4__sm_103__tp4/model.py | 7 +++---- .../models/beam/test_component_contracts.py | 12 +++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py index daa7df387167..a47b1c26d243 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py @@ -124,10 +124,9 @@ def check_cache_manager(self, manager: KVCacheManagerV2) -> None: raise ValueError(f"the KV cache manager states {len(windows)} attention windows") for layer, window in enumerate(windows): if self.schedule.is_sliding(layer): - # The estimator uses a full-context pool, and short sequences - # normalize the sliding window to full context. The attention - # operator still applies self.window independently of retention. - ok = window is None or int(window) in (self.window, manager.max_seq_len) + # Retention may exceed the attention window, or cover a shorter + # whole sequence. The attention operator still applies self.window. + ok = window is None or int(window) >= min(self.window, manager.max_seq_len) else: ok = window is None or int(window) >= manager.max_seq_len if not ok: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py index 26137dd9af34..25fe04757c4d 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py @@ -33,12 +33,12 @@ def config(): @pytest.mark.parametrize("length", [128, 512, 8192]) -@pytest.mark.parametrize("retention", ["sliding", "full", "normalized"]) +@pytest.mark.parametrize("retention", ["sliding", "full", "normalized", "extended"]) def test_cache_allows_estimator_and_short_sequence_geometry(length, retention): component = NS( head_dim=128, local_kv_heads=2, window=512, schedule=layout.check_config(config()) ) - window = {"sliding": 512, "full": length, "normalized": None}[retention] + window = {"sliding": 512, "full": length, "normalized": None, "extended": 1024}[retention] manager = NS( layer_offsets=dict(enumerate(range(52))), dtype=DataType.BF16, @@ -52,8 +52,10 @@ def test_cache_allows_estimator_and_short_sequence_geometry(length, retention): assert Component.attention_windows(component, length) == [512] * 51 + [length] -@pytest.mark.parametrize("layer,window", [(0, 256), (0, 1024), (51, 512)]) -def test_cache_rejects_incompatible_retention(layer, window): +@pytest.mark.parametrize( + "length,layer,window", [(8192, 0, 256), (8192, 0, 511), (128, 0, 127), (8192, 51, 512)] +) +def test_cache_rejects_incompatible_retention(length, layer, window): component = NS( head_dim=128, local_kv_heads=2, window=512, schedule=layout.check_config(config()) ) @@ -63,7 +65,7 @@ def test_cache_rejects_incompatible_retention(layer, window): head_dim=128, num_kv_heads_per_layer=[2] * 52, max_attention_window_vec=[512] * 51 + [None], - max_seq_len=8192, + max_seq_len=length, ) manager.max_attention_window_vec[layer] = window with pytest.raises(ValueError, match="cache keeps a window"): From ec73c2f45cc08ca6468c870b9329c4d16adde402 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Fri, 13 Nov 2026 15:17:23 -0800 Subject: [PATCH 05/33] fix: reject unsupported Beam RoPE configuration Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_tp4/layout_bf16.py | 2 ++ .../test_checkpoint_layout.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/layout_bf16.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/layout_bf16.py index 228c57da173e..737c85dca79a 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/layout_bf16.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/layout_bf16.py @@ -170,6 +170,8 @@ def check_config(config: Any) -> Schedule: and mlp.index("sparse") < len(mlp) - mlp[::-1].index("dense") ): raise ValueError("dense feed-forward layers must precede the sparse ones") + if _get(config, "rope_scaling") is not None: + raise ValueError("rope_scaling is unsupported; sliding layers use unscaled RoPE") rope = _get(config, "rope_parameters") sliding = (rope or {}).get("sliding_attention") if isinstance(rope, Mapping) else None if ( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index 7156f1efd367..60043244244b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -25,6 +25,7 @@ import math import os import struct +from types import SimpleNamespace import pytest import torch @@ -275,6 +276,26 @@ def test_inconsistent_configuration_is_rejected(change) -> None: layout.tensor_specs(cfg) +@pytest.mark.cpu_only +@pytest.mark.parametrize("as_object", [False, True]) +@pytest.mark.parametrize("scaling", [{}, {"rope_type": "linear", "factor": 2.0}, False]) +def test_top_level_rope_scaling_is_rejected(as_object: bool, scaling: object) -> None: + cfg = model_config() + cfg["rope_scaling"] = scaling + with pytest.raises(ValueError, match="rope_scaling"): + layout.check_config(SimpleNamespace(**cfg) if as_object else cfg) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("partial", [None, 1.0]) +def test_supported_rope_with_null_scaling(partial: float | None) -> None: + cfg = model_config() + cfg["rope_scaling"] = None + if partial is not None: + cfg["rope_parameters"]["sliding_attention"]["partial_rotary_factor"] = partial + assert layout.check_config(cfg).num_layers == NUM_LAYERS + + @pytest.mark.cpu_only @pytest.mark.parametrize("rank, world", [(4, 4), (-1, 4), (0, 2), (0, 8), (True, 4), (1.0, 4)]) def test_invalid_rank_or_world_is_rejected(rank, world) -> None: From 4586bf2c72f9c24aacf6a33ac4538ab89d3e71a8 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Sun, 15 Nov 2026 21:04:09 -0800 Subject: [PATCH 06/33] fix: respect pinned-memory policy in Beam tile reader Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_tp4/loader_bf16.py | 9 ++++-- .../test_checkpoint_layout.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py index 09fc75346ecf..87b8e2f15b9c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py @@ -31,6 +31,8 @@ import torch +from tensorrt_llm._utils import prefer_pinned + from .layout_bf16 import BF16, F32, FP8, TensorSpec INDEX_FILE = "model.safetensors.index.json" @@ -179,16 +181,19 @@ def validate_entries( class TileReader: - """Bounded byte-range reads: at most two pinned staging buffers of ``tile_bytes``. + """Bounded byte-range reads: at most two staging buffers of ``tile_bytes``. ``tiles(entry, rows)`` yields ``(row_start, row_stop, host_view)`` for consecutive row ranges; the next tile is read on a background thread while the caller consumes the current one, and a buffer is refilled only after the caller's device copy of it completed. Counts the bytes read and the peak staging footprint. + Pinning follows ``prefer_pinned()`` unless explicitly overridden. """ - def __init__(self, tile_bytes: int = TILE_BYTES, pin: bool = True) -> None: + def __init__(self, tile_bytes: int = TILE_BYTES, pin: bool | None = None) -> None: self.tile_bytes = tile_bytes + if pin is None: + pin = prefer_pinned() self.buffers = [ torch.empty(tile_bytes, dtype=torch.uint8, pin_memory=pin) for _ in range(2) ] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index 60043244244b..398225b91d9f 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -387,6 +387,35 @@ def test_catalog_rejects_index_header_disagreement_and_truncation(checkpoint) -> loader.SafetensorsCatalog(checkpoint) +@pytest.mark.cpu_only +@pytest.mark.parametrize("preferred", [False, True]) +@pytest.mark.parametrize("pin", [None, False, True]) +def test_tile_reader_pinning_policy( + monkeypatch: pytest.MonkeyPatch, preferred: bool, pin: bool | None +) -> None: + policy_calls = [] + allocations = [] + empty = torch.empty + + def policy() -> bool: + policy_calls.append(True) + return preferred + + def allocate(size: int, *, dtype: torch.dtype, pin_memory: bool) -> torch.Tensor: + allocations.append(pin_memory) + return empty(size, dtype=dtype) + + monkeypatch.setattr(loader, "prefer_pinned", policy) + monkeypatch.setattr(loader.torch, "empty", allocate) + reader = loader.TileReader(tile_bytes=16) if pin is None else loader.TileReader(16, pin=pin) + try: + assert allocations == [preferred if pin is None else pin] * 2 + assert len(policy_calls) == (1 if pin is None else 0) + assert all(buffer.numel() == 16 for buffer in reader.buffers) + finally: + reader.close() + + # ------------------------------------------------------------------ streaming reads on CUDA cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device") From 82b4dbce6a7a58ad80348f0773a15c0c5fd9df72 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 10:14:45 -0800 Subject: [PATCH 07/33] fix: reuse catalog collectives and checkpoint metadata for Beam Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_tp4/component_bf16.py | 44 +++++++-- .../models/beam/_tp4/loader_bf16.py | 91 +++++++------------ .../test_checkpoint_layout.py | 27 ++++++ .../models/beam/test_component_contracts.py | 22 +---- .../models/beam/test_diagnostic_mode.py | 47 +++++++++- .../models/beam/test_distributed_loading.py | 69 +++++++++++++- 6 files changed, 208 insertions(+), 92 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py index 0ed28d9ccb95..bab6f5682562 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py @@ -16,7 +16,7 @@ Per call, rank r computes its 12 query heads / 2 kv heads, its dense or expert intermediate slice and its vocabulary rows; the enclosing method applies exactly one collective at each -boundary, on the TRT-LLM NCCL all-reduce or all-gather:: +boundary, through the ModelingV2 catalog all-reduce or all-gather:: embed masked local lookup, one all-reduce (sum) attention local q/k/v/gate, q/k norms, RoPE or NoPE scale, TRTLLM attention on the @@ -37,6 +37,7 @@ import json import os from collections import OrderedDict +from functools import partial from typing import Any, Mapping, Optional, Sequence import torch @@ -48,8 +49,15 @@ MODELING_V2_VALIDATE_ENV, step_contract_enabled, ) +from tensorrt_llm._torch.modeling_v2.catalog.comm.allgather import allgather +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import ( + allreduce, + allreduce_group_args, +) from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( beam_logit_softcap, ) @@ -62,7 +70,8 @@ beam_depth_residual, depth_scales, ) -from tensorrt_llm.functional import AllReduceStrategy +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm.functional import AllReduceFusionOp from tensorrt_llm.mapping import Mapping as ParallelMapping from .. import _attention, dispatch_combine @@ -161,11 +170,28 @@ def __init__( self.softcap = config["final_logit_softcapping"] self.vocab_interval = self.specs[EMBED].interval(self.rank)[1:] self._allocate_storage() - self._allreduce = AllReduce(mapping=mapping, strategy=AllReduceStrategy.NCCL) + self._init_collectives() self.usable = False self.load_record: Optional[dict] = None self._init_diagnostics() + def _init_collectives(self) -> None: + args = allreduce_group_args(self.mapping) + self._allreduce = partial( + allreduce, + residual=None, + norm_weight=None, + scale=None, + bias=None, + op=int(AllReduceFusionOp.NONE), + eps=0.0, + trigger_completion_at_end=False, + **args, + ) + self._allgather = partial( + allgather, sizes=None, group=args["group"], process_group=args["process_group"] + ) + def _allocate_storage(self) -> None: #: Storage is always resolved from the currently registered parameters, never from #: references taken here, so a replaced parameter or bank is what every call uses. @@ -206,8 +232,8 @@ def _init_diagnostics(self) -> None: def _check_diagnostics_agree(self) -> None: """Raise on every rank unless all four ranks run the same diagnostic mode (collective).""" - flag = torch.tensor([1.0 if self.diagnostics else 0.0], device=self.device) - on = int(self._allreduce(flag).item()) + flag = torch.tensor([int(self.diagnostics)], dtype=torch.int32, device=self.device) + on = int(self._allgather(flag).sum().item()) if on not in (0, WORLD_SIZE): raise LoadError( f"rank {self.rank}: diagnostic validation is on for {on} of {WORLD_SIZE} ranks; " @@ -361,7 +387,7 @@ def _bank(self, layer: int, which: str) -> torch.Tensor: return self.bank(f"layers.{layer}.experts.{which}") def all_reduce(self, tensor: torch.Tensor) -> torch.Tensor: - """The one sum over the four ranks at a partial-result boundary (NCCL).""" + """The one sum over the four ranks at a partial-result boundary (catalog AUTO).""" return self._allreduce(tensor) def _digests_agree(self, tensors: Sequence[torch.Tensor]) -> list: @@ -376,7 +402,7 @@ def _digests_agree(self, tensors: Sequence[torch.Tensor]) -> list: h.update(memoryview(t.contiguous().cpu().view(torch.uint8).numpy().reshape(-1))) digests.append(h.digest()) local = torch.frombuffer(bytearray(b"".join(digests)), dtype=torch.uint8).to(self.device) - gathered = allgather(local, self.mapping, dim=0).view(WORLD_SIZE, -1).cpu() + gathered = self._allgather(local).view(WORLD_SIZE, -1).cpu() if not all(torch.equal(gathered[0], gathered[r]) for r in range(1, WORLD_SIZE)): raise RuntimeError( "replicated MoE routing differs between ranks (input, ids or weights)" @@ -562,7 +588,7 @@ def head(self, n: torch.Tensor) -> torch.Tensor: self._check_usable() _check_hidden(n, self.hidden, self.device, self.max_tokens) local = cublas_mm(n, self.tensor(HEAD).t()) - gathered = allgather(local, self.mapping, dim=-1) + gathered = self._allgather(local.t().contiguous()).t().contiguous() logits = beam_logit_softcap(gathered, self.softcap) return logits diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py index 87b8e2f15b9c..c393ea782a6b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/loader_bf16.py @@ -24,7 +24,6 @@ import json import math import os -import struct import threading from collections import OrderedDict from typing import Generator, Mapping, Optional, Sequence @@ -39,7 +38,6 @@ TILE_BYTES = 16 << 20 _ITEMSIZE = {"U8": 1, FP8: 1, BF16: 2, F32: 4} _TORCH_DTYPE = {FP8: torch.float8_e4m3fn, BF16: torch.bfloat16, F32: torch.float32} -_MAX_HEADER = 256 << 20 class CheckpointFormatError(ValueError): @@ -65,26 +63,6 @@ def row_bytes(self) -> int: return cols * _ITEMSIZE[self.dtype] -def _read_header(path: str) -> tuple: - with open(path, "rb") as f: - raw = f.read(8) - if len(raw) != 8: - raise CheckpointFormatError(f"{path}: truncated safetensors header length") - (length,) = struct.unpack(" None: weight_map = index.get("weight_map") if isinstance(index, dict) else None if not isinstance(weight_map, dict) or not weight_map: raise CheckpointFormatError(f"{INDEX_FILE} has no weight_map") - self.entries: "OrderedDict[str, Entry]" = OrderedDict() - for filename in sorted(set(weight_map.values())): - if not isinstance(filename, str) or os.path.basename(filename) != filename: + # Checkpoint adapters are imported when loading, after model registration. + from tensorrt_llm._torch.models.checkpoints.hf.checkpoint_catalog import ( + build_safetensors_checkpoint_catalog, + ) + + for filename in weight_map.values(): + if ( + not isinstance(filename, str) + or not filename + or os.path.basename(filename) != filename + or filename in (".", "..") + ): raise CheckpointFormatError(f"{INDEX_FILE} names an unsafe shard path {filename!r}") - path = os.path.join(self.checkpoint_dir, filename) - table, data_start = _read_header(path) - size = os.path.getsize(path) - spans = [] - for name, info in table.items(): - if name == "__metadata__": - continue - try: - dtype, shape = info["dtype"], info["shape"] - begin, end = info["data_offsets"] - except (KeyError, TypeError, ValueError): - raise CheckpointFormatError( - f"{filename}: malformed header entry {name!r}" - ) from None - if weight_map.get(name) != filename: - raise CheckpointFormatError( - f"{name} is stored in {filename} but the index maps it to {weight_map.get(name)!r}" - ) - if type(begin) is not int or type(end) is not int: - raise CheckpointFormatError(f"{filename}: {name} offsets must be integers") - if not 0 <= begin <= end or data_start + end > size: - raise CheckpointFormatError( - f"{filename}: {name} spans [{begin}, {end}) outside the file" - ) - spans.append((begin, end, name)) - self.entries[name] = Entry( - name, path, dtype, shape, data_start + begin, data_start + end + paths = [ + os.path.join(self.checkpoint_dir, name) for name in sorted(set(weight_map.values())) + ] + try: + catalog = build_safetensors_checkpoint_catalog(paths) + except ValueError as exc: + raise CheckpointFormatError(str(exc)) from exc + self.entries: "OrderedDict[str, Entry]" = OrderedDict() + for tensor in catalog.tensors: + (extent,) = tensor.extents + filename = extent.object_id + if weight_map.get(tensor.name) != filename: + raise CheckpointFormatError( + f"{tensor.name} is stored in {filename} but the index maps it to " + f"{weight_map.get(tensor.name)!r}" ) - spans.sort() - for (b0, e0, n0), (b1, _, n1) in zip(spans, spans[1:]): - if b1 < e0: - raise CheckpointFormatError(f"{filename}: {n0} and {n1} overlap") + self.entries[tensor.name] = Entry( + tensor.name, + os.path.join(self.checkpoint_dir, filename), + tensor.dtype, + tensor.shape, + extent.offset_bytes, + extent.end_offset_bytes, + ) missing = sorted(set(weight_map) - set(self.entries)) if missing: raise CheckpointFormatError( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index 398225b91d9f..909cb788d4e7 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -572,3 +572,30 @@ def test_mapping_rejections_and_weight_state_guard() -> None: ) # same-shape replacement with pytest.raises(RuntimeError): _check_weight_state(model, state) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "field,value", + [ + ("data_offsets", [0.0, 2.0]), + ("data_offsets", ["0", 2]), + ("data_offsets", [False, 2]), + ("data_offsets", [0, 3]), + ("shape", [1.0]), + ("shape", [True]), + ("shape", [-1]), + ], +) +def test_catalog_uses_shared_metadata_validation(tmp_path, field, value) -> None: + info = {"dtype": "BF16", "shape": [1], "data_offsets": [0, 2]} + info[field] = value + header = json.dumps({"weight": info}).encode() + (tmp_path / "weights.safetensors").write_bytes( + struct.pack(" ) spec = layout.TensorSpec(layout.EMBED, (8, 2), layout.BF16, "embedding", None, None, 0) monkeypatch.setattr(cls, "_tensor_specs", staticmethod(lambda *args: {spec.name: spec})) - monkeypatch.setattr(bf16, "AllReduce", lambda **kwargs: NS()) + monkeypatch.setattr( + bf16, "allreduce_group_args", lambda mapping: dict(group=[0, 1, 2, 3], process_group=None) + ) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (10, 3)) real_empty = torch.empty diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py index 1b7419e9bea4..c5afed66407b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py @@ -116,7 +116,7 @@ def __call__(self, flag: torch.Tensor) -> torch.Tensor: def test_every_rank_runs_the_same_diagnostic_mode(mine: bool, ranks_on: int, ok: bool) -> None: reduce = _Reduce(ranks_on) component = SimpleNamespace( - diagnostics=mine, device=torch.device("cpu"), rank=0, _allreduce=reduce + diagnostics=mine, device=torch.device("cpu"), rank=0, _allgather=reduce ) if ok: BF16_CLS._check_diagnostics_agree(component) @@ -153,7 +153,7 @@ def test_router_disagreement_is_detected(monkeypatch: pytest.MonkeyPatch) -> Non ) def peers(differing_rank: int | None) -> Callable[..., torch.Tensor]: - def allgather(local: torch.Tensor, mapping: object, dim: int = 0) -> torch.Tensor: + def allgather(local: torch.Tensor) -> torch.Tensor: rows = [local.clone() for _ in range(bf16.WORLD_SIZE)] if differing_rank is not None: rows[differing_rank][0] ^= 1 @@ -161,9 +161,9 @@ def allgather(local: torch.Tensor, mapping: object, dim: int = 0) -> torch.Tenso return allgather - monkeypatch.setattr(bf16, "allgather", peers(None)) + component._allgather = peers(None) assert len(BF16_CLS._digests_agree(component, tensors)) == 3 - monkeypatch.setattr(bf16, "allgather", peers(2)) + component._allgather = peers(2) with pytest.raises(RuntimeError, match="replicated MoE routing differs"): BF16_CLS._digests_agree(component, tensors) @@ -257,3 +257,42 @@ def test_diagnostic_mode_is_read_once_from_the_environment( assert component.diagnostics is expected monkeypatch.setenv(MODELING_V2_VALIDATE_ENV, "0" if expected else "1") assert component.diagnostics is expected + + +@pytest.mark.parametrize("process_group", [None, object()], ids=["mpi", "c10d"]) +def test_catalog_collectives_preserve_vocabulary_order(monkeypatch, process_group) -> None: + group = [0, 1, 2, 3] + args = dict(group=group, process_group=process_group, strategy=0, workspace=None, rank=0) + monkeypatch.setattr(bf16, "allreduce_group_args", lambda mapping: args) + local = torch.arange(6, dtype=torch.bfloat16).reshape(2, 3) + peers = [local + rank * 10 for rank in group] + + def gather(input, *, sizes, group, process_group): + assert sizes is None and group == args["group"] + assert process_group is args["process_group"] + assert input.is_contiguous() and torch.equal(input, local.t()) + return torch.cat([peer.t() for peer in peers], dim=0) + + def reduce(input, **kwargs): + assert all(kwargs[key] is value for key, value in args.items()) + assert kwargs["op"] == 0 and kwargs["residual"] is None + return input * 4 + + monkeypatch.setattr(bf16, "allgather", gather) + monkeypatch.setattr(bf16, "allreduce", reduce) + monkeypatch.setattr(bf16, "cublas_mm", lambda x, w: local) + monkeypatch.setattr(bf16, "beam_logit_softcap", lambda x, cap: x) + component = SimpleNamespace( + mapping=object(), + hidden=3, + device=torch.device("cpu"), + max_tokens=2, + softcap=30.0, + tensor=lambda name: torch.zeros(3, 3), + _check_usable=lambda: None, + ) + BF16_CLS._init_collectives(component) + assert torch.equal(BF16_CLS.all_reduce(component, local), local * 4) + output = BF16_CLS.head(component, local) + assert output.is_contiguous() + assert torch.equal(output, torch.cat(peers, dim=-1)) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py index 809e4955e98c..0a554c6fd1fa 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py @@ -7,7 +7,7 @@ env -u TLLM_DISABLE_MPI timeout 120s mpirun -n 4 python -m pytest -q \\ tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py -This uses real MPI error exchange, NCCL diagnostic agreement and CUDA copies. +This uses real MPI error exchange, catalog diagnostic agreement and CUDA copies. Only model geometry/quantization metadata are reduced; no checkpoint, inference, NVFP4 kernel or full-model qualification is involved. Not scheduled in L0. The outer timeout bounds a broken collective, including pytest/process startup. @@ -38,9 +38,7 @@ def test_distributed_loading(tmp_path, monkeypatch, packed): from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( component as nvfp4, ) - from tensorrt_llm._torch.distributed import AllReduce from tensorrt_llm._torch.distributed.communicator import Distributed - from tensorrt_llm.functional import AllReduceStrategy from tensorrt_llm.mapping import Mapping rank = mpi_rank() @@ -59,7 +57,7 @@ def test_distributed_loading(tmp_path, monkeypatch, packed): obj.layers, obj.expert_names, obj._shared_fc1 = (), [], {} obj.schedule = SimpleNamespace(is_sparse=lambda layer: False) obj.usable, obj.load_record, obj.diagnostics = False, None, False - obj._allreduce = AllReduce(mapping=mapping, strategy=AllReduceStrategy.NCCL) + obj._init_collectives() obj.register_parameter( bf16._attr(spec.name), torch.nn.Parameter(torch.zeros_like(source, device=device), requires_grad=False), @@ -106,3 +104,66 @@ def fail_read(*args, **kwargs): obj.load(str(tmp_path), record_digests=False) assert obj.usable and torch.equal(obj.tensor("weight").cpu(), source) assert Distributed.get(mapping).allgather(obj.usable) == [True] * 4 + + +def test_catalog_collectives() -> None: + """Manual TP4 Beam shapes, no checkpoint; exact sums and rank-ordered vocabulary.""" + from tensorrt_llm import mpi_rank, mpi_world_size + from tensorrt_llm._utils import local_mpi_rank, mpi_disabled + + if mpi_disabled() or mpi_world_size() != 4 or torch.cuda.device_count() < 4: + pytest.skip("requires MPI with four ranks and four visible GPUs on one node") + + from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( + component_bf16 as bf16, + ) + from tensorrt_llm.mapping import Mapping + + rank = mpi_rank() + torch.cuda.set_device(local_mpi_rank()) + device = torch.device("cuda", local_mpi_rank()) + cls = bf16.BeamCheckpointLayoutBf16Sm103Tp4 + obj = cls.__new__(cls) + torch.nn.Module.__init__(obj) + obj.mapping = Mapping(world_size=4, rank=rank, tp_size=4) + obj.rank, obj.device = rank, device + obj._init_collectives() + # Dyadic values keep the four-rank sum exactly representable in both dtypes. + for dtype in (torch.bfloat16, torch.float32): + for rows in (1, 17, 129, 512): + base = (torch.arange(rows * 5120, device=device) % 31).reshape(rows, 5120) + local = (base.float() / 32 + rank / 8).to(dtype) + result = obj.all_reduce(local) + expected = (base.float() / 8 + 0.75).to(dtype) + torch.testing.assert_close(result, expected, rtol=0, atol=0) + torch.testing.assert_close( + local, (base.float() / 32 + rank / 8).to(dtype), rtol=0, atol=0 + ) + for rows in (1, 17, 129): + base = (torch.arange(rows * 37984, device=device) % 31).reshape(rows, 37984) + local = (base.float() / 32 + rank / 8).to(torch.bfloat16) + gathered = obj._allgather(local.t().contiguous()).t().contiguous() + expected = torch.cat( + [(base.float() / 32 + peer / 8).to(local.dtype) for peer in range(4)], dim=1 + ) + torch.testing.assert_close(gathered, expected, rtol=0, atol=0) + # Capture after warmup, then replay with changed inputs and an intervening eager call. + static = torch.full((1, 5120), rank / 8, dtype=torch.bfloat16, device=device) + obj.all_reduce(static) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = obj.all_reduce(static) + static.fill_(rank / 8 + 0.5) + obj.all_reduce(torch.zeros(17, 5120, dtype=static.dtype, device=device)) + graph.replay() + torch.testing.assert_close(captured, torch.full_like(captured, 2.75), rtol=0, atol=0) + # Diagnostic collectives use integer flags and byte digests, outside the hot path. + obj.diagnostics = False + obj._check_diagnostics_agree() + assert len(obj._digests_agree([torch.ones(3, device=device)])) == 1 + obj.diagnostics = rank == 2 + with pytest.raises(bf16.LoadError, match="on for 1 of 4 ranks"): + obj._check_diagnostics_agree() + with pytest.raises(RuntimeError, match="replicated MoE routing differs"): + obj._digests_agree([torch.tensor([rank], device=device)]) From db9e379a0e3ad0c6b1be5925c9bbe77a19035e3f Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 10:29:57 -0800 Subject: [PATCH 08/33] docs: align Beam README with available components and serving features Signed-off-by: Maxim Gluhovskoi --- .../_torch/modeling_v2/models/beam/README.md | 88 +++++++++++++------ 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/README.md b/tensorrt_llm/_torch/modeling_v2/models/beam/README.md index 9c04ef9a75df..877b91e61298 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/README.md +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/README.md @@ -1,36 +1,74 @@ -# Beam StairCase components +# Beam components with ModelingV2 -This directory implements Beam components using the experimental ModelingV2/StairCase -architecture: family-scoped operators and a 52-layer NVFP4-expert component for attention TP4 and MoE TP4/EP1 on four GB300 GPUs. +This directory contains Beam family operators and a directly constructed 52-layer +NVFP4-expert component. It does not register an engine target: `LLM()` and +`trtllm-serve` cannot select these components through ModelingV2. Component users +construct `BeamFullModelNvfp4Sm103Tp4` from `full_model_nvfp4__sm_103__tp4/model.py` +and provide the checkpoint configuration, rank mapping and device; the component's +`load()` method loads the caller-provided checkpoint before execution. The tests +below exercise the component APIs and their input contracts. -The full component implements sliding-window and full attention with a BF16 KV cache. -Its loader streams caller-provided weights into rank-local storage; no checkpoint is -included. Router projection and selection use FP32, and packed NVFP4 experts use the -native GEMM path. +## Configuration and limits -These are directly constructed components, not a registered `LLM()` or `trtllm-serve` -target. Engine routing, the engine adapter, checkpoint-loader registration and a named -whole-model accuracy gate belong to the serving integration. Operator certification and -CPU component-contract tests accompany these components; broader component tests are -reviewed separately. Speculative decoding, FP8 KV cache, disaggregated serving, CUDA -graphs and overlap scheduling are outside this component baseline. +| Setting | Component configuration | +|---|---| +| Hardware and layout | Four GB300 GPUs (SM103), attention TP4, MoE TP4/EP1, PP1 | +| Parameters | Packed NVFP4 experts; BF16 dense tensors; FP32 router projection and selection | +| Attention | Checkpoint-defined sliding-window and full attention, BF16 KV cache | +| Execution | Eager; CUDA graphs and overlap scheduling disabled | -Beam-only operators live under `ops/`, with tests in the matching Beam test tree. -Component-specific cases for shared catalog operators reuse the catalog test drivers, -including TP1 shapes and the per-rank TP4 projection, attention and expert shapes. +Checkpoint weights are supplied separately and streamed into rank-local storage. +No checkpoint, reference captures or private evaluation artifacts are included. +Other GPU architectures and parallel layouts, speculative decoding, FP8 KV cache, +disaggregated serving, CUDA graphs and overlap scheduling are outside this baseline. -## Checkout-only bring-up helpers +## File organization -The six TP1 bring-up components and their generation controller live with their tests in -`tests/unittest/_torch/modeling_v2/models/beam/_components/`. They are not installed in -the TensorRT-LLM wheel. The runtime does not expose intermediate-tensor capture hooks. -Normal runtime validation and rank-consistency diagnostics remain available. +- `ops/` contains Beam-only operators. Their tests mirror that directory under + `tests/unittest/_torch/modeling_v2/models/beam/ops/`. +- `_tp4/` contains shared BF16 layout, loading and component base modules. +- `full_model_nvfp4__sm_103__tp4/` adds packed expert storage and full-depth execution. +- `tests/unittest/_torch/modeling_v2/models/beam/_components/` contains checkout-only + TP1 bring-up helpers, including the six-layer model and its generation controller. + They are not installed in the wheel. Intermediate capture instrumentation lives + in tests; runtime validation and rank-consistency diagnostics remain in the library. -The TP4 checkpoint layout, expert provider and full-model execution remain in this -library. Public serving integration uses `LLM()` and `trtllm-serve`. +## Run component tests -Shared BF16 TP4 layout, loading and component base modules live in `_tp4/`; the -`full_model_nvfp4__sm_103__tp4/` package adds NVFP4 expert storage and execution. +Run from the repository root with a matching TensorRT-LLM installation and its +native dependencies. `cpu_only` selects tests that do not execute GPU operations; +it does not make the TensorRT-LLM package independent of its native dependencies. + +```bash +python -m pytest -m cpu_only \ + tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py \ + tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py \ + tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +``` + +For generated-input operator coverage on one compatible GPU: + +```bash +python -m pytest tests/unittest/_torch/modeling_v2/models/beam/ops +``` + +For the small four-rank load-error and collective checks, with four GPUs visible +and working MPI peer communication: + +```bash +env -u TLLM_DISABLE_MPI mpirun -n 4 python -m pytest -q \ + tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py +``` + +These checks use generated data, not a checkpoint. The collective test includes +capture/replay of the communication operation; this does not qualify whole-model +CUDA graphs. Component checks do not establish whole-model accuracy. + +CPU and bounded single-GPU CI coverage is selected explicitly in +the test lists under `tests/integration/test_lists/test-db/`, including +`l0_b300.yml` for GPU operators. +Large component and four-rank checks are manual; do not run the entire Beam test +directory as a lightweight smoke test. Full GSM8K is not part of CI. From 95ddc83e5ba87f9d76232617e8c300c8432c2771 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 10:35:02 -0800 Subject: [PATCH 09/33] fix: allowlist the publicly documented Beam architecture Signed-off-by: Maxim Gluhovskoi --- docs/source/models/supported-models.md | 2 +- tensorrt_llm/usage/architecture_allowlist.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index b9b9b1392318..468e6021d3a7 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -80,7 +80,7 @@ following its [Docker guide](https://github.com/NVIDIA/TensorRT-LLM/blob/main/do | ----- | ----------- | ---------------- | ----------------- | | MiniMax-M3.1-Flash | [`feat/minimax-m3.1-flash`](https://github.com/NVIDIA/TensorRT-LLM/tree/feat/minimax-m3.1-flash) | [Serving MiniMax-M3.1-Flash](https://github.com/NVIDIA/TensorRT-LLM/blob/feat/minimax-m3.1-flash/docs/source/deployment-guide/deployment-guide-for-minimax-m3-on-trtllm.md#serving-minimax-m31-flash) in the MiniMax-M3 deployment guide | `nvcr.io/nvidia/tensorrt-llm/release:feat-minimax-m3.1-flash-1.4.0a0.dev202611050719` | | Kimi K3.1 Pro | [`feat/kimi-k3.1-pro`](https://github.com/NVIDIA/TensorRT-LLM/tree/feat/kimi-k3.1-pro) | [Kimi K3.1 Pro deployment guide](https://github.com/NVIDIA/TensorRT-LLM/blob/feat/kimi-k3.1-pro/docs/source/deployment-guide/deployment-guide-for-kimi-k3.1-pro-on-trtllm.md) | `nvcr.io/nvidia/tensorrt-llm/release:feat-kimi-k3.1-pro-1.4.0a0.dev202611050000` | -| Beam (Reflection AI) | [`feat/reflection-beam`](https://github.com/NVIDIA/TensorRT-LLM/tree/feat/reflection-beam) | [Beam deployment guide](https://github.com/NVIDIA/TensorRT-LLM/blob/feat/reflection-beam/docs/source/deployment-guide/deployment-guide-for-beam-on-trtllm.md) | No container published yet — build from the branch sources per the guide | +| Beam (Reflection AI), `BeamForCausalLM` ([public NVFP4 checkpoint](https://huggingface.co/reflection/Beam-1.0-501B-A23B-NVFP4)) | [`feat/reflection-beam`](https://github.com/NVIDIA/TensorRT-LLM/tree/feat/reflection-beam) | [Beam deployment guide](https://github.com/NVIDIA/TensorRT-LLM/blob/feat/reflection-beam/docs/source/deployment-guide/deployment-guide-for-beam-on-trtllm.md) | No container published yet — build from the branch sources per the guide | ## Model-Feature Support Matrix (Key Models) diff --git a/tensorrt_llm/usage/architecture_allowlist.py b/tensorrt_llm/usage/architecture_allowlist.py index cd6e2e30a0aa..3f0cfd8da5db 100644 --- a/tensorrt_llm/usage/architecture_allowlist.py +++ b/tensorrt_llm/usage/architecture_allowlist.py @@ -27,6 +27,7 @@ "Apertus1p5ForConditionalGeneration", "ApertusForCausalLM", "BartForConditionalGeneration", + "BeamForCausalLM", "BertForSequenceClassification", "CLIPVisionModel", "Cohere2ForCausalLM", From e40526b4716c75718c9394d583ba8d0a4c6b190c Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 10:47:23 -0800 Subject: [PATCH 10/33] fix: reject nested Beam checkpoint KV-cache schemes Signed-off-by: Maxim Gluhovskoi --- .../full_model_nvfp4__sm_103__tp4/layout.py | 2 ++ .../test_checkpoint_layout_nvfp4.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/layout.py b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/layout.py index 4e9c8aeaf320..8dc9dc4fa5a8 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/layout.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/layout.py @@ -262,6 +262,8 @@ def check_recipe(quant_config: Any, specs: Mapping[str, Any]) -> dict: if quant_config.get("kv_cache_scheme") is not None: raise ValueError("kv_cache_scheme is not supported; this target requires BF16 KV cache") q = read_modelopt_quant_config(dict(quant_config)) + if q.get("kv_cache_scheme") is not None: + raise ValueError("kv_cache_scheme is not supported; this target requires BF16 KV cache") if q.get("quant_algo") != QUANT_ALGO: raise ValueError(f"unsupported quant_algo {q.get('quant_algo')!r}; expected {QUANT_ALGO}") if q.get("group_size") != GROUP or type(q.get("group_size")) is not int: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py index a67696f6cd97..996b65b744a5 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py @@ -239,6 +239,27 @@ def test_recipe_rejections(mutate, match) -> None: layout.check_recipe(q, layout.tensor_specs(model_config(), layers=(0, 7, 9))) +@pytest.mark.cpu_only +@pytest.mark.parametrize("nested", [False, True], ids=["top-level", "nested"]) +@pytest.mark.parametrize("scheme", [None, "FP8", "unknown", {}, {"type": "float", "num_bits": 8}]) +def test_checkpoint_kv_cache_scheme(tmp_path, monkeypatch, nested, scheme) -> None: + q = recipe() + (q["quantization"] if nested else q)["kv_cache_scheme"] = scheme + cfg = model_config() + specs = layout.tensor_specs(cfg, layers=(0, 7, 9)) + monkeypatch.setattr(layout, "tensor_specs", lambda config: specs) + (tmp_path / "config.json").write_text(json.dumps(cfg)) + (tmp_path / "hf_quant_config.json").write_text(json.dumps(q)) + if scheme is None: + assert layout.check_recipe(q, specs)["kv_cache_quant_algo"] is None + assert layout.check_checkpoint(tmp_path) == cfg + else: + with pytest.raises(ValueError, match="kv_cache_scheme"): + layout.check_recipe(q, specs) + with pytest.raises(ValueError, match="kv_cache_scheme"): + layout.check_checkpoint(tmp_path) + + # ------------------------------------------------------------------ generated checkpoints From d9f5b558af14f74c2cb905fe6fd2b7cd25caaa48 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 10:56:02 -0800 Subject: [PATCH 11/33] test: verify Beam diagnostic flags and per-instance settings Signed-off-by: Maxim Gluhovskoi --- .../models/beam/test_diagnostic_mode.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py index c5afed66407b..b75120e6d56b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py @@ -97,16 +97,16 @@ def dispatch( assert agreements == ([3] if diagnostics else []) -class _Reduce: - """A stand-in for the four-rank all-reduce: returns the sum the peers would produce.""" +class _Gather: + """Gather the supplied local flag with three independently configured peer flags.""" - def __init__(self, total: int) -> None: - self.total = total - self.calls = 0 + def __init__(self, peers_on: int) -> None: + self.peers = [1] * peers_on + [0] * (3 - peers_on) + self.inputs: list[torch.Tensor] = [] def __call__(self, flag: torch.Tensor) -> torch.Tensor: - self.calls += 1 - return torch.tensor([float(self.total)]) + self.inputs.append(flag.clone()) + return torch.cat((flag, flag.new_tensor(self.peers))) @pytest.mark.parametrize( @@ -114,16 +114,17 @@ def __call__(self, flag: torch.Tensor) -> torch.Tensor: [(False, 0, True), (True, 4, True), (True, 3, False), (False, 1, False), (False, 2, False)], ) def test_every_rank_runs_the_same_diagnostic_mode(mine: bool, ranks_on: int, ok: bool) -> None: - reduce = _Reduce(ranks_on) + gather = _Gather(ranks_on - int(mine)) component = SimpleNamespace( - diagnostics=mine, device=torch.device("cpu"), rank=0, _allgather=reduce + diagnostics=mine, device=torch.device("cpu"), rank=0, _allgather=gather ) if ok: BF16_CLS._check_diagnostics_agree(component) else: with pytest.raises(bf16.LoadError, match=f"on for {ranks_on} of 4 ranks"): BF16_CLS._check_diagnostics_agree(component) - assert reduce.calls == 1 + assert len(gather.inputs) == 1 + assert torch.equal(gather.inputs[0], torch.tensor([int(mine)], dtype=torch.int32)) @pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) @@ -244,18 +245,28 @@ def block( assert blocks == [(layer, True) for layer in range(full.NUM_LAYERS)] +@pytest.mark.parametrize("cls", [BF16_CLS, NVFP4_CLS], ids=["bf16", "nvfp4"]) @pytest.mark.parametrize("value,expected", [(None, False), ("0", False), ("1", True), ("on", True)]) def test_diagnostic_mode_is_read_once_from_the_environment( - monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool + monkeypatch: pytest.MonkeyPatch, + cls: type[bf16.BeamCheckpointLayoutBf16Sm103Tp4], + value: str | None, + expected: bool, ) -> None: if value is None: monkeypatch.delenv(MODELING_V2_VALIDATE_ENV, raising=False) else: monkeypatch.setenv(MODELING_V2_VALIDATE_ENV, value) - component = SimpleNamespace() - BF16_CLS._init_diagnostics(component) + # Exercise initialization on real instances without allocating checkpoint storage. + component = cls.__new__(cls) + torch.nn.Module.__init__(component) + component._init_diagnostics() assert component.diagnostics is expected monkeypatch.setenv(MODELING_V2_VALIDATE_ENV, "0" if expected else "1") + second = cls.__new__(cls) + torch.nn.Module.__init__(second) + second._init_diagnostics() + assert second.diagnostics is not expected assert component.diagnostics is expected From 5b91a92b0c1a35d03bef8587a9df18e06941016b Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 11:28:48 -0800 Subject: [PATCH 12/33] test: correct Beam cache readback for nonzero pool slots Signed-off-by: Maxim Gluhovskoi --- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../models/beam/_beam_component_support.py | 1 - .../models/beam/test_component_support.py | 45 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 93ed683580dc..bacf50eee6cf 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -129,6 +129,7 @@ l0_cpu: - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py - unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py - unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py + - unittest/_torch/modeling_v2/models/beam/test_component_support.py - unittest/_torch/modeling_v2/test_modeling_v2_ci_selection.py - unittest/_torch/modeling_v2/test_modeling_v2_contracts.py - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index cec63ebadc30..2650ac21a1ea 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -521,7 +521,6 @@ def cache_rows( APIs take the logical layer and translate it to the local slot themselves.""" mgr = md.kv_cache_manager view = mgr.get_buffers(layer_idx, kv_layout="HND") - view = view[:: mgr.get_layer_page_index_scale(layer_idx) // 2] keys, values = [], [] for page in range((length + tokens_per_block - 1) // tokens_per_block): k_off = int(md.kv_cache_block_offsets[0, 0, 0, page]) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py new file mode 100644 index 000000000000..634a6c0dd644 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU checks for the test harness's cache readback indexing.""" + +from types import SimpleNamespace + +import pytest +import torch + +__extra_import_path__ = ["."] + +from _beam_component_support import cache_rows # noqa: E402 + +pytestmark = pytest.mark.cpu_only + + +@pytest.mark.parametrize("num_layers,local_layer", [(1, 0), (2, 1), (3, 2)]) +def test_cache_rows_reads_nonzero_slots(num_layers: int, local_layer: int) -> None: + slots, heads, block, dim = 4, 2, 4, 3 + pool = torch.arange(slots * num_layers * 2 * heads * block * dim).reshape( + slots, num_layers, 2, heads, block, dim + ) + # get_buffers starts at the requested layer's K page; rows are K/V pairs. + view = pool.flatten(0, 1)[local_layer:] + + def get_buffers(layer_idx: int, kv_layout: str) -> torch.Tensor: + assert layer_idx == 7 and kv_layout == "HND" + return view + + selected = [2, 0, 1] + scale = 2 * num_layers + offsets = torch.tensor(selected) * scale + manager = SimpleNamespace( + get_buffers=get_buffers, get_layer_page_index_scale=lambda layer: scale + ) + md = SimpleNamespace( + kv_cache_manager=manager, kv_cache_block_offsets=offsets.reshape(1, 1, 1, -1) + ) + length = 2 * block + 2 + keys, values = cache_rows(md, length, 7, block) + for actual, role in ((keys, 0), (values, 1)): + expected = torch.cat([pool[slot, local_layer, role].permute(1, 0, 2) for slot in selected])[ + :length + ] + torch.testing.assert_close(actual, expected, rtol=0, atol=0) From 0cdf10ad4f937a40d7ef9e0edc85bfdb0e5bd453 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 11:41:50 -0800 Subject: [PATCH 13/33] test: release Beam dense state and isolate reference RNG Signed-off-by: Maxim Gluhovskoi --- .../dense_block/_dense_block_reference.py | 8 ++++---- .../dense_block/test_dense_block.py | 10 ++++++++++ .../models/beam/test_component_support.py | 19 +++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py index 5578100d6cdb..642f008cb9b3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/_dense_block_reference.py @@ -47,8 +47,8 @@ class Dims: LENGTHS = (1, 17, 129) STATE_RECIPE = { - "construction": "torch.manual_seed(1300 + seed + layer_id); fp32 kaiming_uniform_(a=sqrt(5)) on CPU for the " - "eight projection matrices in module construction order (q, k, v, gate, o, mlp gate, up, down)", + "construction": "torch.Generator().manual_seed(1300 + seed + layer_id); fp32 kaiming_uniform_(a=sqrt(5)) " + "on CPU for the eight projection matrices in module construction order (q, k, v, gate, o, mlp gate, up, down)", "overrides": "torch.Generator().manual_seed(10000 + seed) in state_dict order: *norm.weight = 1 + 0.25 * N(0, 1); " "self_attn.gate_bias = 0.75 * N(0, 1)", "dtype": "every tensor cast once to bfloat16", @@ -103,11 +103,11 @@ def param_shapes(d: Dims = BLOCK0) -> "OrderedDict[str, tuple[int, ...]]": def generate_state(seed: int, d: Dims = BLOCK0) -> "OrderedDict[str, torch.Tensor]": """Deterministic bf16 block state on CPU (see STATE_RECIPE).""" shapes = param_shapes(d) - torch.manual_seed(1300 + seed + d.layer_id) + projection_gen = torch.Generator().manual_seed(1300 + seed + d.layer_id) matrices = {} for name in INIT_ORDER: weight = torch.empty(shapes[name], dtype=torch.float32) - torch.nn.init.kaiming_uniform_(weight, a=math.sqrt(5)) + torch.nn.init.kaiming_uniform_(weight, a=math.sqrt(5), generator=projection_gen) matrices[name] = weight gen = torch.Generator().manual_seed(10000 + seed) state: "OrderedDict[str, torch.Tensor]" = OrderedDict() diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 9cc63a031431..41b9ee95f7e5 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -41,6 +41,7 @@ import math import os import types +from collections.abc import Iterator from pathlib import Path import pytest @@ -279,6 +280,15 @@ def compare(name: str, actual: torch.Tensor, expected: torch.Tensor, rule: str | _STATE_CACHE: dict[int, dict] = {} +@pytest.fixture(scope="module", autouse=True) +def release_generated_state() -> Iterator[None]: + try: + yield + finally: + _STATE_CACHE.clear() + torch.cuda.empty_cache() + + def generated(seed: int) -> dict: """The seed's bf16 state and [1, 129, hidden] input, generated once on the CPU.""" if seed not in _STATE_CACHE: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py index 634a6c0dd644..5152ff0b7f8c 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py @@ -1,14 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""CPU checks for the test harness's cache readback indexing.""" +"""CPU checks for cache readback and reference-state isolation.""" from types import SimpleNamespace import pytest import torch -__extra_import_path__ = ["."] +__extra_import_path__ = [".", "_components/dense_block"] +import _dense_block_reference as reference # noqa: E402 from _beam_component_support import cache_rows # noqa: E402 pytestmark = pytest.mark.cpu_only @@ -43,3 +44,17 @@ def get_buffers(layer_idx: int, kv_layout: str) -> torch.Tensor: :length ] torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("seed", [17, 41]) +def test_dense_reference_preserves_rng(seed: int) -> None: + dims = reference.Dims(hidden=8, intermediate=16, heads=2, kv_heads=1, head_dim=4) + with torch.random.fork_rng(devices=[]): + before = torch.get_rng_state().clone() + first = reference.generate_state(seed, dims) + assert torch.equal(before, torch.get_rng_state()) + second = reference.generate_state(seed, dims) + assert torch.equal(before, torch.get_rng_state()) + assert first.keys() == second.keys() + for name in first: + torch.testing.assert_close(first[name], second[name], rtol=0, atol=0) From 18b26d43f3614fa01a58316898fd640b841f6e89 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 11:51:00 -0800 Subject: [PATCH 14/33] test: check Beam behavior and native router tie order Signed-off-by: Maxim Gluhovskoi --- .../dense_block/test_dense_block.py | 1 - .../models/beam/_components/moe/test_moe.py | 3 - .../_components/moe_router/_router_cases.py | 22 ++--- .../_components/moe_router/test_router.py | 26 ++++-- .../test_small_model_prefill.py | 93 ++++++++++++------- .../sparse_block/test_sparse_block.py | 2 - .../models/beam/test_dispatch_combine.py | 8 -- 7 files changed, 84 insertions(+), 71 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 41b9ee95f7e5..2cc9cd6101f1 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -703,7 +703,6 @@ def refuse(self, *args, **kwargs): assert blk.checkpoint_tensors()["self_attn.v_proj.weight"].shape == (1024, 5120) assert blk.attention_scales == (1 / math.sqrt(2), 1 / math.sqrt(2)) assert blk.feed_forward_scales == (1 / math.sqrt(3), math.sqrt(2) / math.sqrt(3)) - assert "modeling_beam" not in Path(block_module.__file__).read_text() # ----------------------------------------------------------------------------- tests: declared limits diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py index 646ebc604639..90b8a2f3c75d 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -38,7 +38,6 @@ import gc import types from collections import OrderedDict -from pathlib import Path import pytest import torch @@ -63,7 +62,6 @@ ) from _validating import validating # noqa: E402 -from _components.moe import moe as moe_module from _components.moe.moe import BeamMoeBf16Sm103Tp1 from _components.moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 # noqa: E501 from _components.moe_router.router import BeamMoeRouterBf16Sm103Tp1 # noqa: E501 @@ -440,7 +438,6 @@ def refuse(self, *args, **kwargs): assert tensors[f"experts.{expert_id}.{attr}.weight"] is getattr(expert, attr) pointers = [t.data_ptr() for name, t in tensors.items() if name.startswith("experts.")] assert len(pointers) == 768 and len(set(pointers)) == 768 - assert "modeling_beam" not in Path(moe_module.__file__).read_text() # -------------------------------------------------- tests: declared limits diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/_router_cases.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/_router_cases.py index 6ef460f0d26e..56818217f47a 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/_router_cases.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/_router_cases.py @@ -273,18 +273,16 @@ def top7() -> list: ) -#: The order `torch.topk(..., sorted=True)` returns for each tie row -- the source router's -#: order, measured on sm_103. A tie at a selection boundary keeps the lower index; tied -#: values inside the selection are not in index order. -TIE_ROUTES = { - "expert_boundary_tie_same_group_39_57": ([0, 1, 2, 3], [0, 1, 32, 33, 64, 65, 96, 39]), - "expert_boundary_tie_distant_3_126": ([0, 1, 2, 3], [0, 1, 32, 33, 64, 65, 96, 3]), - "expert_order_tie_33_62": ([0, 1, 2, 3], [0, 1, 33, 62, 64, 65, 96, 97]), - "group_boundary_tie_3_6": ([0, 1, 2, 3], [0, 1, 32, 33, 64, 65, 96, 97]), - "group_order_tie_1_4": ([0, 4, 1, 2], [0, 1, 128, 32, 33, 129, 64, 65]), - "all_equal": ([1, 0, 2, 3], [7, 6, 4, 5, 1, 0, 2, 3]), - "expert_boundary_tie_10way": ([0, 1, 2, 3], [0, 1, 32, 33, 64, 65, 96, 5]), -} +def assert_topk_membership(scores: torch.Tensor, indices: torch.Tensor, k: int) -> None: + """Selected IDs include every score above the boundary and only valid ties at it.""" + assert scores.ndim == indices.ndim == 1 + assert indices.numel() == k and indices.unique().numel() == k + assert bool(((indices >= 0) & (indices < scores.numel())).all()) + boundary = scores.sort(descending=True).values[k - 1] + assert bool((scores[indices] >= boundary).all()) + required = torch.nonzero(scores > boundary).flatten() + assert bool(torch.isin(required, indices).all()) + EPSILON_IDS = [0, 1, 32, 33, 64, 65, 96, 97] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index b59ac880b367..d7bd544ed27d 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -841,33 +841,39 @@ def test_expert_boundary_moves_one_expert() -> None: ] -def test_ties_follow_the_pinned_topk_order() -> None: - """Exactly tied scores are routed in the order `torch.topk(..., sorted=True)` returns - -- the source's order, stated in `_router_cases.TIE_ROUTES` -- whether a row is routed - in a batch or alone. Ids are compared in order, never sorted.""" +def test_ties_follow_native_topk_order() -> None: + """Match the native reference in each batch shape, including valid boundary ties.""" route, _, results = route_fixture("ties") spec = cases.fixtures()["ties"] weight, bias, hidden = cases.fixture_tensors(spec) router = loaded_router(weight, bias) alone = {} for i, name in enumerate(spec["rows"]): - _, _, single = routed(router, hidden[i : i + 1].cuda()) + single_input = hidden[i : i + 1].cuda() + _, _, single = routed(router, single_input) + expected = reference_route(router, single_input) + results += cases.compare_routes(single, expected, f"reference:{name}") alone[name] = (single["groups"][0].tolist(), single["ids"][0].tolist()) + for observed, row in ((route, i), (single, 0)): + groups, ids = observed["groups"][row], observed["ids"][row] + cases.assert_topk_membership(observed["group_scores"][row], groups, cases.TOPK_GROUP) + allowed = torch.isin( + torch.arange(cases.EXPERTS, device=ids.device) // cases.PER_GROUP, groups + ) + scores = observed["selection_scores"][row].masked_fill(~allowed, -float("inf")) + cases.assert_topk_membership(scores, ids, cases.TOP_K) batched = { name: (route["groups"][i].tolist(), route["ids"][i].tolist()) for i, name in enumerate(spec["rows"]) } _record_fixture( "ties", - "test_ties_follow_the_pinned_topk_order", + "test_ties_follow_native_topk_order", route, results, - {"batched": batched, "alone": alone, "pinned": cases.TIE_ROUTES}, + {"batched": batched, "alone": alone}, ) cases.assert_all_pass(results) - for name, pinned in cases.TIE_ROUTES.items(): - assert batched[name] == (list(pinned[0]), list(pinned[1])), (name, batched[name]) - assert alone[name] == batched[name], (name, alone[name]) def test_epsilon_is_the_source_denominator() -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index e945889e7f7e..6d548727ca24 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -413,7 +413,18 @@ def test_rejected_load_changes_nothing(case) -> None: bad["model.layers.5.mlp.gate.weight"] = state["model.layers.5.mlp.gate.weight"].bfloat16() else: bad[model_module.HEAD] = state[model_module.HEAD].tolist - with pytest.raises((ValueError, TypeError)): + error, match = { + "missing_head": (ValueError, r"weights mismatch: missing .*lm_head\.weight"), + "unexpected_name": ( + ValueError, + r"weights mismatch: missing \[\], unexpected .*model\.layers\.6", + ), + "late_expert_fp32": (TypeError, r"experts\.255\.down_proj\.weight must be torch.bfloat16"), + "late_expert_transposed": (ValueError, r"experts\.255\.down_proj\.weight must be \["), + "router_bf16": (TypeError, r"mlp\.gate\.weight must be torch.float32"), + "head_not_a_tensor": (TypeError, r"lm_head\.weight must be a torch.Tensor"), + }[case] + with pytest.raises(error, match=match): model.load_weights(bad) after = {n: t._version for n, t in model.checkpoint_tensors().items()} assert after == versions @@ -439,35 +450,6 @@ def broken(weights): # -------------------------------------------------- tests: input and cache contracts -_BAD_INPUTS = [ - "float_ids", - "bool_ids", - "int16_ids", - "negative_id", - "id_vocab_size", - "batch_2", - "flat_ids", - "empty", - "too_long", - "host_ids", - "non_contiguous_ids", - "positions_offset", - "float_positions", - "positions_shape", - "host_positions", - "metadata_type", - "cached_prefix", - "context_length_mismatch", - "one_layer_cache", - "seven_layer_cache", - "duplicate_slots", - "slot_out_of_range", - "missing_layer", - "all_sliding_windows", - "no_sliding_windows", - "fp16_cache", - "four_kv_heads", -] #: Host pool tables replaced after the manager built them: rows sharing a slot's storage or #: swapped between slots, an offset outside its pool, a fractional, int64 or misshapen #: mapping, and pointer rows other than the manager's. @@ -481,11 +463,51 @@ def broken(weights): "aliased_pool_pointers", "secondary_pool_pointer", ] -_BAD_INPUTS += _BAD_POOL_TABLES + ["swa_scratch_cache"] - -@pytest.mark.parametrize("case", _BAD_INPUTS) -def test_rejects_unsupported_inputs_before_dispatch(case, monkeypatch) -> None: +_BAD_INPUTS = { + "float_ids": "input_ids must be a dense torch.int32 or torch.int64 tensor", + "bool_ids": "input_ids must be a dense torch.int32 or torch.int64 tensor", + "int16_ids": "input_ids must be a dense torch.int32 or torch.int64 tensor", + "negative_id": "token ids must be in", + "id_vocab_size": "token ids must be in", + "batch_2": "input_ids must be \\[1, T\\]", + "flat_ids": "input_ids must be \\[1, T\\]", + "empty": "supported token counts per call", + "too_long": "supported token counts per call", + "host_ids": "input_ids must be contiguous on the model", + "non_contiguous_ids": "input_ids must be contiguous on the model", + "positions_offset": "positions must be exactly those", + "float_positions": "position_ids must be a contiguous integer", + "positions_shape": "position_ids must be a contiguous integer", + "host_positions": "position_ids must be a contiguous integer", + "metadata_type": "attn_metadata must be TrtllmAttentionMetadata", + "cached_prefix": "paged-context FMHA is required", + "context_length_mismatch": "new tokens .* do not fill the call's 17 tokens", + "one_layer_cache": "does not hold logical layer 1", + "seven_layer_cache": "must hold exactly logical layers 0-5", + "duplicate_slots": "must map one-to-one onto slots 0-5", + "slot_out_of_range": "maps logical layer 5 to slot 6", + "missing_layer": "maps logical layer 0 to slot 0.*holding 5 logical layers", + "all_sliding_windows": "full-attention layer 5 must keep the whole context", + "no_sliding_windows": "sliding layer 0 must keep a 512-token window", + "fp16_cache": "KV cache must be BF16 with 8 kv heads of 128", + "four_kv_heads": "KV cache must be BF16 with 8 kv heads of 128", + "aliased_pool_rows": "layer 4's pool row .* addresses .*manager keeps slot 4's keys", + "swapped_pool_offsets": "layer 0's pool row .* addresses .*manager keeps slot 0's keys", + "pool_offset_out_of_range": "layer 5 has pool row", + "fractional_pool_offsets": "pool mapping must be a host torch.int32", + "int64_pool_mapping": "pool mapping must be a host torch.int32", + "extra_pool_mapping_row": "pool mapping must be a host torch.int32", + "aliased_pool_pointers": "pool row .* addresses .*manager keeps slot", + "secondary_pool_pointer": "layer 0's pool row .* addresses .*manager keeps slot 0's keys", + "swa_scratch_cache": "without sliding-window scratch reuse", +} + + +@pytest.mark.parametrize("case,match", _BAD_INPUTS.items()) +def test_rejects_unsupported_inputs_before_dispatch( + case: str, match: str, monkeypatch: pytest.MonkeyPatch +) -> None: """Each declared restriction raises ValueError before any op or the embedding runs.""" model, _ = loaded(17) length = 17 @@ -549,6 +571,7 @@ def test_rejects_unsupported_inputs_before_dispatch(case, monkeypatch) -> None: md = object() elif case == "cached_prefix": md.kv_lens_runtime[0] = length + 8 + md.use_paged_context_fmha = False elif case in ("duplicate_slots", "slot_out_of_range", "missing_layer"): offsets = dict(md.kv_cache_manager.layer_offsets) if case == "duplicate_slots": @@ -583,7 +606,7 @@ def test_rejects_unsupported_inputs_before_dispatch(case, monkeypatch) -> None: monkeypatch.setattr(mgr, "kv_cache_pool_pointers", pointers) elif case == "swa_scratch_cache": monkeypatch.setattr(md.kv_cache_manager, "enable_swa_scratch_reuse", True) - with no_dispatch(), pytest.raises(ValueError): + with no_dispatch(), pytest.raises(ValueError, match=match): model(ids, positions, md) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 03d15147fc46..aee242adc2dd 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -45,7 +45,6 @@ import math import types from collections import OrderedDict -from pathlib import Path import pytest import torch @@ -636,7 +635,6 @@ def refuse(self, *args, **kwargs): assert blk.feed_forward_scales == (1 / 3, math.sqrt(8) / 3) assert blk.attention_scales != block_module.depth_scales(0, True) assert blk.feed_forward_scales != block_module.depth_scales(0, False) - assert "modeling_beam" not in Path(block_module.__file__).read_text() # -------------------------------------------------- tests: declared limits diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py index bab49aff1d60..b0b4f790cfeb 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py @@ -105,14 +105,6 @@ def run_checked( def test_public_callable_and_provider_interface() -> None: params = list(inspect.signature(dispatch_and_combine).parameters) assert params == ["hidden_states", "expert_ids", "mixing_weights", "expert_provider"] - doc = " ".join(inspect.getdoc(dispatch_combine).split()) - for phrase in ( - "expert_provider(expert_id, rows)", - "owns dispatch and combination", - "owns the expert computation", - "[n, 5120]", - ): - assert phrase in doc, phrase assert ( dispatch_combine.HIDDEN_SIZE, dispatch_combine.SUPPORTED_CONFIG["num_experts"], From deb9458a7c75c041e1ad77167efd6ec97f008b36 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 13:04:25 -0800 Subject: [PATCH 15/33] test: cover Beam paged prefixes beyond sliding window Signed-off-by: Maxim Gluhovskoi --- .../beam/ops/attention/test_modeling_v2_beam_thop_attention.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_thop_attention.py b/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_thop_attention.py index 8891540fc87d..caa7e6754e58 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_thop_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_thop_attention.py @@ -60,6 +60,7 @@ ("fresh decode-sized prefill", (0,), (1,), 1), ("fresh prefill", (0,), (129,), 1), ("prefix reuse", (64,), (17,), 1), + ("prefix reuse beyond window", (576,), (17,), 1), ("prefill beyond window", (0,), (513,), 1), ("prefix at window", (512,), (1,), 1), ("decode across window", (511, 512, 513), (1, 1, 1), 0), From 98b291eeb310a063942ff595943072ed941b05bb Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 13:33:32 -0800 Subject: [PATCH 16/33] test: consolidate Beam component helpers and block reference math Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_beam_component_support.py | 59 +++++++-- .../dense_block/test_dense_block.py | 116 ++++-------------- .../models/beam/_components/moe/test_moe.py | 2 +- .../_components/moe_expert/test_expert.py | 10 +- .../_components/moe_router/test_router.py | 46 +------ .../test_small_model_prefill.py | 2 +- .../sparse_block/_sparse_block_reference.py | 46 ++----- .../sparse_block/test_sparse_block.py | 2 +- .../models/beam/_dispatch_combine_cases.py | 13 +- .../models/beam/test_component_support.py | 29 ++++- 10 files changed, 127 insertions(+), 198 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index 2650ac21a1ea..59e67ad08596 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -36,8 +36,6 @@ import torch from torch.overrides import TorchFunctionMode, resolve_name -import tensorrt_llm - if TYPE_CHECKING: from _components.moe.moe import BeamMoeBf16Sm103Tp1 @@ -103,6 +101,8 @@ def ieee_fp32_session(write_evidence: Callable[[str, dict], None]) -> Iterator[d def git_revision() -> str | None: + import tensorrt_llm + root = Path(tensorrt_llm.__file__).resolve().parents[1] try: out = subprocess.run( @@ -118,6 +118,8 @@ def git_revision() -> str | None: def environment(ieee: dict, execution: Mapping[str, object], **extra: object) -> dict: """The device and software a case ran on; ``extra`` entries follow ``execution``.""" + import tensorrt_llm + device = torch.cuda.current_device() return { "device_name": torch.cuda.get_device_name(device), @@ -167,12 +169,15 @@ def compare( rule: str | None, tolerance: tuple[float, float] | None, dtype: torch.dtype, + *, + max_unequal: float | None = None, ) -> dict: """Score one tensor against one rule; ``rule=None`` measures without judging. ``tolerance`` is the rule's ``(atol, rtol)`` from the calling test file and ``dtype`` the dtype both tensors must have. An element passes when |actual - expected| <= atol + rtol * |expected|, both widened to fp64; no NaN or Inf is accepted. + ``max_unequal`` optionally limits the fraction of numerically unequal elements. """ result = { "tensor": name, @@ -215,6 +220,11 @@ def compare( f"{violations} of {numel} elements outside atol {atol} + rtol {rtol} * |ref| " f"(max abs {result['max_abs']:.3g})" ) + if max_unequal is not None: + allowed = math.floor(max_unequal * numel) + result["max_unequal"] = allowed + if result["unequal"] > allowed: + problems.append(f"{result['unequal']} unequal elements, at most {allowed} allowed") result.update(passed=not problems, problems=problems) return result @@ -291,6 +301,7 @@ def bitwise_equal(a: torch.Tensor, b: torch.Tensor) -> bool: """Same dtype, shape and bits (so -0.0 differs from +0.0).""" if a.dtype != b.dtype or a.shape != b.shape: return False + b = b.to(a.device) view = {torch.bfloat16: torch.int16, torch.float32: torch.int32}.get(a.dtype) if view is None: return bool(torch.equal(a, b)) @@ -445,12 +456,26 @@ def refuse(self, *args: object, _name: str = cls.__name__, **kwargs: object) -> cls.__call__ = original +def _tensors_of(value: object) -> list[torch.Tensor]: + if isinstance(value, torch.Tensor): + return [value] + if isinstance(value, (tuple, list)): + return [t for v in value for t in _tensors_of(v)] + if isinstance(value, dict): + return [t for v in value.values() for t in _tensors_of(v)] + return [] + + class TorchCallTrace(TorchFunctionMode): - """The name of every torch call made inside the block and the devices of its outputs.""" + """Trace torch calls; a projection enables detailed router precision/output records.""" - def __init__(self) -> None: + def __init__( + self, *, projection: Callable[..., object] | None = None, include_tf32: bool = False + ) -> None: super().__init__() - self.calls: list[tuple[str, list[str]]] = [] + self.projection = projection + self.include_tf32 = include_tf32 + self.calls: list[tuple[str, list[str]] | dict] = [] def __torch_function__( self, @@ -459,10 +484,28 @@ def __torch_function__( args: tuple[object, ...] = (), kwargs: dict[str, object] | None = None, ) -> object: + name = resolve_name(func) or getattr(func, "__qualname__", None) or repr(func) + entry = {"func": name} + if func is self.projection: + entry["math"] = { + "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, + "autocast_cuda": torch.is_autocast_enabled("cuda"), + } + if self.include_tf32: + entry["math"]["cuda_matmul_allow_tf32"] = bool( + torch.backends.cuda.matmul.allow_tf32 + ) + entry["operand_dtypes"] = [str(t.dtype) for t in _tensors_of(args)] out = func(*args, **(kwargs or {})) - outs = out if isinstance(out, (tuple, list)) else [out] - devices = [str(t.device) for t in outs if isinstance(t, torch.Tensor)] - self.calls.append((resolve_name(func) or repr(func), devices)) + if self.projection is None: + outs = out if isinstance(out, (tuple, list)) else [out] + devices = [str(t.device) for t in outs if isinstance(t, torch.Tensor)] + self.calls.append((resolve_name(func) or repr(func), devices)) + else: + produced = _tensors_of(out) + if produced: + entry["outputs"] = [[str(t.device), str(t.dtype), list(t.shape)] for t in produced] + self.calls.append(entry) return out diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 2cc9cd6101f1..79bfa2ffc789 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the Beam dense block 0 component: bf16 / sm_103 / tp1. -Configuration under test, fixed below and nowhere else: block 0 (sliding attention, +Configuration under test: block 0 (sliding attention, dense MLP) at hidden 5120, intermediate 20480 and 48 q / 8 kv heads of 128; bf16 weights and activations with the source model's fp32 steps; one sm_103 GPU; eager, batch 1, one fresh context per call; generated weights; seeds {17, 41} x lengths @@ -36,7 +36,6 @@ from __future__ import annotations -import contextlib import functools import math import os @@ -225,54 +224,10 @@ def _environment(ieee: dict) -> dict: def compare(name: str, actual: torch.Tensor, expected: torch.Tensor, rule: str | None) -> dict: - """Score one tensor against one rule; `rule=None` measures without judging.""" - result = { - "tensor": name, - "rule": rule, - "shape": list(expected.shape), - "dtype": str(actual.dtype), - } - problems = [] - if tuple(actual.shape) != tuple(expected.shape): - problems.append(f"shape {tuple(actual.shape)} != {tuple(expected.shape)}") - if actual.dtype != DTYPE or expected.dtype != DTYPE: - problems.append(f"dtypes {actual.dtype}/{expected.dtype}, need {DTYPE}") - if problems: - result.update(passed=False, problems=problems) - return result - a, r = actual.double(), expected.double() - if not bool(torch.isfinite(a).all()) or not bool(torch.isfinite(r).all()): - result.update(passed=False, problems=["non-finite values"]) - return result - diff = (a - r).abs() - numel = diff.numel() - worst = int(diff.argmax()) - result.update( - numel=numel, - max_abs=float(diff.max()), - mean_abs=float(diff.mean()), - worst_index=[int(i) for i in torch.unravel_index(torch.tensor(worst), diff.shape)], - worst_actual=float(a.reshape(-1)[worst]), - worst_expected=float(r.reshape(-1)[worst]), - unequal=int((a != r).sum()), + atol, rtol, max_unequal = RULES[rule] if rule is not None else (None, None, None) + return support.compare( + name, actual, expected, rule, (atol, rtol), DTYPE, max_unequal=max_unequal ) - if rule is None: - result["passed"] = None - return result - atol, rtol, max_unequal = RULES[rule] - violations = int((diff > atol + rtol * r.abs()).sum()) - result.update(atol=atol, rtol=rtol, violations=violations) - if violations: - problems.append( - f"{violations} of {numel} elements outside atol {atol} + rtol {rtol} * |ref|" - ) - if max_unequal is not None: - allowed = math.floor(max_unequal * numel) - result["max_unequal"] = allowed - if result["unequal"] > allowed: - problems.append(f"{result['unequal']} unequal elements, at most {allowed} allowed") - result.update(passed=not problems, problems=problems) - return result # ----------------------------------------------------------------------------- state, inputs, fresh context @@ -577,50 +532,27 @@ def accounting(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> dict: # ----------------------------------------------------------------------------- op-path instrumentation -@contextlib.contextmanager def recording(*wrappers): - """Log every call of these entries, in order, with the arguments that identify the path.""" - calls: list[dict] = [] - saved = [] - for w in wrappers: - cls = type(w) - original = cls.__call__ - saved.append((cls, original)) - - def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): - entry = {"op": _name} - tensors = [a for a in args if isinstance(a, torch.Tensor)] - entry["shapes"] = [list(t.shape) for t in tensors] - for key in ( - "num_heads", - "num_kv_heads", - "head_size", - "attention_window_size", - "max_seq_len", - "tokens_per_block", - "mask_type", - "is_fused_qkv", - "num_contexts", - "num_ctx_tokens", - "use_paged_context_fmha", - "position_embedding_type", - "local_layer_idx", - "quant_mode", - ): - if key in kwargs: - entry[key] = kwargs[key] - if "q" in kwargs: - entry["q_shape"] = list(kwargs["q"].shape) - entry["qkv"] = kwargs["q"] - calls.append(entry) - return _original(self, *args, **kwargs) - - cls.__call__ = logged - try: - yield calls - finally: - for cls, original in reversed(saved): - cls.__call__ = original + return support.recording( + *wrappers, + recorded_kwargs=( + "num_heads", + "num_kv_heads", + "head_size", + "attention_window_size", + "max_seq_len", + "tokens_per_block", + "mask_type", + "is_fused_qkv", + "num_contexts", + "num_ctx_tokens", + "use_paged_context_fmha", + "position_embedding_type", + "local_layer_idx", + "quant_mode", + ), + qkv=True, + ) def builtin_beam_forbidden(): diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py index 90b8a2f3c75d..73d177714ecf 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the Beam MoE component of decoder block 3: bf16 / sm_103 / tp1. -Configuration under test, fixed below and nowhere else: block 3's MoE at hidden 5120 -- +Configuration under test: block 3's MoE at hidden 5120 -- 256 experts of intermediate 2560 in 8 groups of 32, 4 groups and 8 experts per token, routed scaling factor 2.5; an fp32 router and bf16 experts with the source model's fp32 steps; one sm_103 GPU; eager, one call per case on already-normalized bf16 hidden diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py index a2b293d7b252..0b1c1e9c2824 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -32,7 +32,7 @@ __extra_import_path__ = [".", "../..", "../../../.."] import _expert_cases as cases # noqa: E402 -from _beam_component_support import TorchCallTrace # noqa: E402 +from _beam_component_support import TorchCallTrace, bitwise_equal # noqa: E402 from _validating import validating # noqa: E402 from _components.moe_expert.expert import WEIGHT_SHAPES, BeamMoeExpertBf16Sm103Tp1 # noqa: E501 @@ -109,14 +109,6 @@ def loaded(weights: dict) -> BeamMoeExpertBf16Sm103Tp1: return expert -def bitwise_equal(a: torch.Tensor, b: torch.Tensor) -> bool: - return ( - a.dtype == b.dtype - and a.shape == b.shape - and torch.equal(a.view(torch.int16), b.view(torch.int16)) - ) - - # ----------------------------------------------------------------------------- state diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index d7bd544ed27d..e4c2494b5e8e 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the Beam MoE router component of block 3: bf16 / sm_103 / tp1. -Configuration under test, fixed in ``_router_cases.py`` and nowhere else: 256 experts in +Configuration under test, fixed in ``_router_cases.py``: 256 experts in 8 groups of 32, 4 groups kept, 8 experts per token, hidden 5120; FP32 router weight and correction bias, BF16 already-normalized input; one sm_103 GPU; eager; generated parameters; seeds {17, 41} x tokens {1, 17, 129}, and the controlled fixtures there. @@ -46,7 +46,6 @@ import pytest import torch -from torch.overrides import TorchFunctionMode, resolve_name import tensorrt_llm @@ -268,42 +267,6 @@ def refuse(*args, **kwargs): setattr(builtin_beam.BeamRouter, name, fn) -def _tensors_of(value): - if isinstance(value, torch.Tensor): - return [value] - if isinstance(value, (tuple, list)): - return [t for v in value for t in _tensors_of(v)] - if isinstance(value, dict): - return [t for v in value.values() for t in _tensors_of(v)] - return [] - - -class TorchCallTrace(TorchFunctionMode): - """Every torch call made inside the block, with the device and dtype of what it returned, - and the fp32 matmul and autocast state at each projection.""" - - def __init__(self): - super().__init__() - self.calls: list[dict] = [] - - def __torch_function__(self, func, types, args=(), kwargs=None): - kwargs = kwargs or {} - name = resolve_name(func) or getattr(func, "__qualname__", None) or repr(func) - entry = {"func": name} - if func is torch.ops.trtllm.cublas_mm: - entry["math"] = { - "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, - "autocast_cuda": torch.is_autocast_enabled("cuda"), - } - entry["operand_dtypes"] = [str(t.dtype) for t in _tensors_of(args)] - out = func(*args, **kwargs) - produced = _tensors_of(out) - if produced: - entry["outputs"] = [[str(t.device), str(t.dtype), list(t.shape)] for t in produced] - self.calls.append(entry) - return out - - def loaded_router(weight: torch.Tensor, bias: torch.Tensor) -> BeamMoeRouterBf16Sm103Tp1: router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") router.load_weights(weight.cuda(), bias.cuda()) @@ -568,7 +531,10 @@ def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: torch.backends.cuda.matmul.fp32_precision = "tf32" try: ambient = torch.backends.cuda.matmul.fp32_precision - with torch.autocast("cuda", dtype=torch.bfloat16), TorchCallTrace() as trace: + with ( + torch.autocast("cuda", dtype=torch.bfloat16), + support.TorchCallTrace(projection=torch.ops.trtllm.cublas_mm) as trace, + ): weights, ids, route = routed(router, hidden) after = torch.backends.cuda.matmul.fp32_precision finally: @@ -626,7 +592,7 @@ def test_matrix(seed, tokens) -> None: activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] with torch.profiler.profile(activities=activities) as prof: - with TorchCallTrace() as trace: + with support.TorchCallTrace(projection=torch.ops.trtllm.cublas_mm) as trace: previous = torch.cuda.get_sync_debug_mode() torch.cuda.set_sync_debug_mode("error") try: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index 6d548727ca24..45f6184c988c 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the Beam six-block small model: bf16 / sm_103 / tp1. -Configuration under test, fixed below and nowhere else: the first six logical layers of the +Configuration under test: the first six logical layers of the checkpoint's schedule -- dense sliding-attention blocks 0-2, sparse sliding-attention blocks 3-4 and the sparse full-attention block 5 without position embedding -- at hidden 5120 with 48 q / 8 kv heads of 128, 256 experts of 2560 per sparse block, vocabulary 151936, untied diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py index 8a93c5c0adc3..044e0b35bec7 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py @@ -11,7 +11,7 @@ (1/sqrt(8), sqrt(7)/sqrt(8)) after attention and (1/3, sqrt(8)/3) after the feed-forward, each scaled term and each sum rounded to bf16. -Only torch and that MoE reference are imported. The attention core is torch's SDPA, which +Only torch and the independent dense/MoE references are imported. The attention core is torch's SDPA, which the component never runs. State and input are generated on the GPU as the MoE reference generates its own, one seeded generator per tensor; the block's 770 ``mlp.`` tensors are exactly the MoE reference's state of the same seed. @@ -25,7 +25,8 @@ import torch import torch.nn.functional as F -__extra_import_path__ = ["../moe"] +__extra_import_path__ = ["../moe", "../dense_block"] +import _dense_block_reference as block_reference # noqa: E402 import _moe_reference as moe_reference # noqa: E402 HIDDEN = 5120 @@ -54,12 +55,7 @@ } -def depth_coefs(layer_idx: int, is_attention: bool) -> tuple[float, float]: - """(branch, stream) scales of the n-th residual add, 1 / sqrt(n) and sqrt(n - 1) / - sqrt(n); block i's attention add is the 2(i + 1)-th, its feed-forward add the next.""" - num_adds = 2 * (layer_idx + 1) + (0 if is_attention else 1) - denom = math.sqrt(num_adds) - return 1.0 / denom, math.sqrt(num_adds - 1) / denom +depth_coefs = block_reference.depth_coefs ATTENTION_SCALES = depth_coefs(LAYER_IDX, True) @@ -178,10 +174,7 @@ def state_problems(state: "dict[str, torch.Tensor]", seed: int) -> list[str]: def rms_norm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """Over the last dim: fp32 statistics, normalization and affine, rounded once.""" - x32 = x.float() - normed = x32 * torch.rsqrt(x32.pow(2).mean(dim=-1, keepdim=True) + EPS) - return (normed * weight.float()).to(x.dtype) + return block_reference.rms_norm(x, weight, EPS) def rope( @@ -206,32 +199,15 @@ def rotate(x: torch.Tensor) -> torch.Tensor: def attention_core(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: - """Causal windowed GQA over one fresh context: ``q [T, 48, 128]``, ``k`` and ``v`` - ``[T, 8, 128]`` bf16, q head h reading kv head h // 6; returns ``[T, 48, 128]`` bf16.""" - tokens = q.shape[0] - repeat = HEADS // KV_HEADS - k = k.repeat_interleave(repeat, dim=1) - v = v.repeat_interleave(repeat, dim=1) - position = torch.arange(tokens, device=q.device) - distance = position.unsqueeze(1) - position.unsqueeze(0) - keep = (distance >= 0) & (distance < WINDOW) - out = F.scaled_dot_product_attention( - q.transpose(0, 1).unsqueeze(0), - k.transpose(0, 1).unsqueeze(0), - v.transpose(0, 1).unsqueeze(0), - attn_mask=keep, - dropout_p=0.0, - scale=HEAD_DIM**-0.5, - ) - return out.squeeze(0).transpose(0, 1) + """Adapt the shared reference's batched GQA to this block's single context.""" + dims = block_reference.Dims(heads=HEADS, kv_heads=KV_HEADS, head_dim=HEAD_DIM, window=WINDOW) + return block_reference.attention_core( + q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), dims + ).squeeze(0) def output_gate(core: torch.Tensor, gate: torch.Tensor, gate_bias: torch.Tensor) -> torch.Tensor: - """``core * sigmoid(gate + gate_bias)`` per head, ``[T, 48, 128]``: the add, the - sigmoid and the product each rounded to bf16.""" - biased = (gate.float() + gate_bias.float().view(1, HEADS, 1)).to(torch.bfloat16) - weight = torch.sigmoid(biased.float()).to(torch.bfloat16) - return (core.float() * weight.float()).to(torch.bfloat16) + return block_reference.output_gate(core.unsqueeze(0), gate.unsqueeze(0), gate_bias).squeeze(0) def attention( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index aee242adc2dd..6200cd77468f 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GPU tests for the Beam sparse decoder block 3 component: bf16 / sm_103 / tp1. -Configuration under test, fixed below and nowhere else: block 3 of the 52-block model +Configuration under test: block 3 of the 52-block model (sliding attention, MoE feed-forward) at hidden 5120 with 48 q / 8 kv heads of 128, and block 3's MoE: 256 experts of intermediate 2560, 8 per token from 4 of 8 groups, routed scaling factor 2.5; bf16 weights and activations with the source model's fp32 steps and diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_dispatch_combine_cases.py b/tests/unittest/_torch/modeling_v2/models/beam/_dispatch_combine_cases.py index f2328f547130..27e50cfc9825 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_dispatch_combine_cases.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_dispatch_combine_cases.py @@ -14,6 +14,9 @@ import torch +__extra_import_path__ = ["."] +from _beam_component_support import bitwise_equal as bitwise_equal # noqa: E402 + HIDDEN, EXPERTS, TOP_K, SCALE = 5120, 256, 8, 2.5 SEEDS = (17, 41) TOKENS = (1, 17, 129) @@ -214,13 +217,3 @@ def golden(x, ids, w, experts, fault: str | None = None, skip_slots=None) -> dic stages["product"] = torch.cat(stages["product"]) stages["route"] = torch.cat(stages["route"]) return stages - - -def bitwise_equal(a: torch.Tensor, b: torch.Tensor) -> bool: - """Same dtype, shape and bits (so -0.0 differs from +0.0).""" - if a.dtype != b.dtype or a.shape != b.shape: - return False - view = {BF16: torch.int16, FP32: torch.int32}.get(a.dtype) - if view is None: - return bool(torch.equal(a, b.to(a.device))) - return bool(torch.equal(a.contiguous().view(view), b.to(a.device).contiguous().view(view))) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py index 5152ff0b7f8c..22e221815f7c 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_support.py @@ -10,7 +10,7 @@ __extra_import_path__ = [".", "_components/dense_block"] import _dense_block_reference as reference # noqa: E402 -from _beam_component_support import cache_rows # noqa: E402 +from _beam_component_support import TorchCallTrace, cache_rows, compare # noqa: E402 pytestmark = pytest.mark.cpu_only @@ -58,3 +58,30 @@ def test_dense_reference_preserves_rng(seed: int) -> None: assert first.keys() == second.keys() for name in first: torch.testing.assert_close(first[name], second[name], rtol=0, atol=0) + + +@pytest.mark.parametrize( + "unequal,fraction,passed", [(1, 0.25, True), (2, 0.25, False), (1, 0.24, False)] +) +def test_compare_unequal_limit(unequal: int, fraction: float, passed: bool) -> None: + expected = torch.zeros(4) + actual = expected.clone() + actual[:unequal] = 0.01 # Within absolute tolerance, but still unequal. + result = compare( + "fraction", actual, expected, "test", (1.0, 0.0), actual.dtype, max_unequal=fraction + ) + assert result["passed"] is passed + assert result["violations"] == 0 + + +def test_projection_trace_retains_precision_and_outputs() -> None: + x, w = torch.ones(1, 2), torch.ones(3, 2) + with TorchCallTrace(projection=torch.nn.functional.linear) as trace: + output = torch.nn.functional.linear(x, w) + projection = next(call for call in trace.calls if "math" in call) + assert projection["math"] == { + "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, + "autocast_cuda": torch.is_autocast_enabled("cuda"), + } + assert projection["operand_dtypes"] == [str(x.dtype), str(w.dtype)] + assert projection["outputs"] == [["cpu", str(output.dtype), [1, 3]]] From db330e08da165c794b5ab864d7f8ea27643543a1 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 13:42:58 -0800 Subject: [PATCH 17/33] test: preserve metadata-only dense operation records Signed-off-by: Maxim Gluhovskoi --- .../modeling_v2/models/beam/_beam_component_support.py | 7 +++++-- .../beam/_components/dense_block/test_dense_block.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index 59e67ad08596..98ff9e2f6c1b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -343,11 +343,13 @@ def recording( experts: type[BeamMoeBf16Sm103Tp1] | None = None, data_ptrs: bool = False, qkv: bool = False, + record_output: bool = True, ) -> Iterator[list[dict]]: """Log every call of these entries, in order, with its nesting depth, the arguments that identify the path and its output. ``experts``: an MoE class whose ``_expert`` calls are also logged, with the expert asked for and the rows it is passed. ``data_ptrs``: the - positional tensors' data pointers. ``qkv``: the shape and tensor of a ``q`` keyword.""" + positional tensors' data pointers. ``qkv``: the shape and tensor of a ``q`` keyword. + Set ``record_output=False`` for metadata-only records that must not retain outputs.""" calls: list[dict] = [] depth = [0] saved = [] @@ -381,7 +383,8 @@ def logged( out = _original(self, *args, **kwargs) finally: depth[0] -= 1 - entry["output"] = out + if record_output: + entry["output"] = out return out cls.__call__ = logged diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 79bfa2ffc789..5a7b45f8cf35 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -552,6 +552,7 @@ def recording(*wrappers): "quant_mode", ), qkv=True, + record_output=False, ) From 5f6cdb806b3c815862158616ce047f09f9fd0edb Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 14:54:44 -0800 Subject: [PATCH 18/33] Strengthen Beam component test assertions Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_beam_component_support.py | 7 ++++++- .../six_layer_model/test_small_model_prefill.py | 8 +++++++- .../_components/sparse_block/test_sparse_block.py | 3 +++ .../test_checkpoint_layout_nvfp4.py | 11 +++++++++-- .../test_modeling_v2_beam_nope_query_scale.py | 8 ++++++-- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index 98ff9e2f6c1b..9bb1dc7db9c9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -579,7 +579,7 @@ def cache_rows( # -------------------------------------------------- unsupported block inputs #: The message each hidden-state or position refusal of ``_attention.check_step`` raises. -#: Metadata refusals have no pattern here: they are checked by type alone. +#: Include metadata messages so unrelated validation failures cannot satisfy those cases. _BAD_INPUT_MATCH = { "fp16_hidden": "hidden_states must be a torch.bfloat16 tensor", "fp32_hidden": "hidden_states must be a torch.bfloat16 tensor", @@ -589,6 +589,9 @@ def cache_rows( "empty": "supported token counts per call are", "non_contiguous": "hidden_states must be contiguous", "positions_offset": "its positions must be exactly those", + "cached_prefix": "paged-context FMHA is required", + "two_requests": "one request per call is supported; metadata has 2", + "context_length_mismatch": "do not fill the call's 17 tokens", "float_positions": r"position_ids must be an int32/int64 tensor of shape \(1, 17\) on cuda", "positions_shape": r"position_ids must be an int32/int64 tensor of shape \(1, 17\) on cuda", } @@ -634,6 +637,8 @@ def bad_block_input( context = dict(num_tokens=length, cached_tokens=8) elif case == "two_requests": context = dict(num_tokens=length, num_requests=2) + x = torch.cat((x, x), dim=1) + positions = torch.arange(2 * length, device="cuda").view(1, -1) elif case == "context_length_mismatch": context = dict(num_tokens=length - 1) elif case == "kv_cache_heads": diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index 45f6184c988c..2de4fdd7d005 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -196,7 +196,7 @@ def recording(model): saved.append((cls, original)) def logged(self, *args, _original=original, _name=cls.__name__, **kwargs): - keep = weight_args_from.get(_name, len(args)) + keep = len(args) if current[0] is None else weight_args_from.get(_name, len(args)) entry = { "op": _name, "block": current[0], @@ -687,6 +687,12 @@ def test_forward_composes_the_six_blocks(seed, length) -> None: assert [c["op"] for c in top] == ["FlashinferRmsnorm", "CublasMm", "BeamLogitSoftcap"] norm, head, softcap = top assert bitwise_equal(norm["args"][0], events[-1][2].view(length, HIDDEN)) + assert norm["args"][1] is model.norm + assert norm["args"][2] == model.eps == 1e-6 + head_weight = head["args"][1] + assert head_weight.data_ptr() == model.lm_head.data_ptr() + assert head_weight.shape == model.lm_head.t().shape + assert head_weight.stride() == model.lm_head.t().stride() assert bitwise_equal(head["args"][0], norm["output"]) assert head["output"].dtype == torch.bfloat16 and head["output"].shape == (length, VOCAB) # The softcap, exactly once: the returned logits are this expression of the head output. diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 6200cd77468f..5aca151c610e 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -506,6 +506,9 @@ def block_results(seed: int, out: torch.Tensor, captured: dict, ref: dict) -> li "block_output", ) final["tokens"] = int(matched.sum()) + if final["tokens"] == 0: + final["passed"] = False + final["problems"].append("no set-matched tokens: block output was not compared") results.append(final) results.append( compare( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py index 996b65b744a5..102529871c1b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py @@ -540,9 +540,16 @@ def proj(n, k, w2, s, seed): ), "weight_scale": None, "linear_scale": torch.stack([(sf + 1 + i).to(torch.uint8) for i in range(n_experts)]), - "input_scale": torch.full((n_experts,), s, dtype=torch.float32), - "weight_scale_2": torch.full((n_experts,), w2, dtype=torch.float32), + "input_scale": torch.tensor( + [s * (i + 2) for i in range(n_experts)], dtype=torch.float32 + ), + "weight_scale_2": torch.tensor( + [w2 / (i + 2) ** 2 for i in range(n_experts)], dtype=torch.float32 + ), } + # Keep the selected expert's reference operands; other slots must not alias its scalars. + bank["input_scale"][expert] = s + bank["weight_scale_2"][expert] = w2 bank["weight"][expert] = packed bank["linear_scale"][expert] = sf bank["g"] = torch.ones(n_experts, dtype=torch.float32) / bank["input_scale"] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_nope_query_scale.py b/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_nope_query_scale.py index ba1a50b72e6a..1c88d3a410ec 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_nope_query_scale.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/ops/attention/test_modeling_v2_beam_nope_query_scale.py @@ -50,11 +50,15 @@ def test_scale_against_float64() -> None: """The fp32 scale is within four fp32 units of the float64 expression at every position, so the bf16 result differs from a float64-scaled one only where the product sits within that distance of a bf16 rounding boundary.""" - positions = torch.arange(129, device="cuda") - q = torch.ones(129, 1, 1, dtype=torch.float32, device="cuda") + positions = torch.tensor([0, 128, 2047, 65535, 131071], device="cuda") + q = torch.ones(positions.numel(), 1, 1, dtype=torch.float32, device="cuda") scale32 = op(q, positions).view(-1) scale64 = torch.exp(0.1 * torch.log1p((positions.double() + 1) / 2048)) assert float(((scale32.double() - scale64).abs() / scale64).max()) <= 4 * 2.0**-23 + # Analytic BF16 goldens for unit queries at positions 2047, 65535 and 131071: + # round(2**0.1), round(33**0.1), round(65**0.1). Independent of op.reference. + expected = torch.tensor([1.0703125, 1.421875, 1.515625], dtype=torch.bfloat16, device="cuda") + torch.testing.assert_close(op(q.bfloat16(), positions)[2:, 0, 0], expected, rtol=0, atol=0) def test_scale_is_visible_at_nonzero_positions() -> None: From 22d1a8c1f07bb4d3834f6e94850121254d156cca Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 15:18:15 -0800 Subject: [PATCH 19/33] Report optional Beam source comparisons as separate skipped tests Signed-off-by: Maxim Gluhovskoi --- .../dense_block/test_dense_block.py | 29 +++++-- .../_components/moe_router/test_router.py | 80 ++++++++++++------- 2 files changed, 74 insertions(+), 35 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 5a7b45f8cf35..b312c7ee462b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -26,6 +26,8 @@ dict of tensors captured from the source block under the reference's record names (plus ``input`` and ``positions``). The final output and every replay are then also compared against the captures, and the generated input must equal theirs bit for bit. + Separate source-capture tests report SKIPPED when this variable is unset; an explicitly + configured directory must contain valid captures. Reference tests run independently. * ``BEAM_DENSE_BLOCK_EVIDENCE``: a directory that receives one JSON record per test case: every comparison with its rule, the op calls and CUDA kernels of the forward, and the device and software it ran on; and ``fp32-session.json``, the fp32 GEMM checks made @@ -312,10 +314,10 @@ def keep(name: str, value: torch.Tensor) -> None: return record -def source_captures(seed: int, length: int) -> dict | None: +def source_captures(seed: int, length: int) -> dict: directory = os.environ.get("BEAM_DENSE_BLOCK_SOURCE_CAPTURES") if not directory: - return None + pytest.skip("BEAM_DENSE_BLOCK_SOURCE_CAPTURES is not configured") captures = torch.load( Path(directory) / f"seed{seed}" / f"T{length}.pt", map_location="cuda", weights_only=True ) @@ -761,12 +763,6 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: final.update(reference="reference") results.append(final) results.extend(replay(blk, md, ref, "reference")) - captures = source_captures(seed, length) - if captures is not None: - from_source = compare("block_output", out, captures["block_output"], "block_output") - from_source.update(reference="source_captures") - results.append(from_source) - results.extend(replay(blk, md, captures, "source_captures")) _write_evidence( f"block-seed{seed}-T{length}", { @@ -781,6 +777,23 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: assert_all_pass(results) +@pytest.mark.skipif( + not os.environ.get("BEAM_DENSE_BLOCK_SOURCE_CAPTURES"), + reason="BEAM_DENSE_BLOCK_SOURCE_CAPTURES is not configured", +) +@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) +def test_block_matches_source_captures(seed, length, ieee_fp32) -> None: + """Optional source qualification, separate from the always-run reference comparisons.""" + captures = source_captures(seed, length) + blk = loaded_block(seed) + x, positions = case_inputs(seed, length) + with fresh_context(length) as md: + out = blk(x, positions, md) + results = [compare("block_output", out, captures["block_output"], "block_output")] + results.extend(replay(blk, md, captures, "source_captures")) + assert_all_pass(results) + + @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) def test_forward_runs_the_component_path(seed, length, ieee_fp32) -> None: """One case: the component's own composition ran, op by op, through thop attention on diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index e4c2494b5e8e..f376aa0cf399 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -26,7 +26,9 @@ ``fixtures/.pt`` (``hidden``, ``bias`` and the same names). The generated inputs must then equal the captured ones bit for bit, the op's ``reference`` must reproduce every captured tensor bit for bit, and the component is compared with the captures - under the same rules as with the reference. + under the same rules as with the reference. Separate source-capture tests report + SKIPPED when this variable is unset; a configured directory must contain valid files. + Ordinary reference and routing-invariant tests run independently. * ``BEAM_MOE_ROUTER_EVIDENCE``: a directory that receives one JSON record per test case: every comparison with its rule, hashes of the routed tensors, the torch calls and CUDA kernels of the forward, and the device and software it ran on. @@ -160,19 +162,19 @@ def route_hashes(route: dict) -> dict: # ----------------------------------------------------------------------------- source captures -def _captures_dir() -> Path | None: +def _captures_dir() -> Path: directory = os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES") - return Path(directory) if directory else None + if not directory: + pytest.skip("BEAM_MOE_ROUTER_SOURCE_CAPTURES is not configured") + return Path(directory) def _as_route(captured: dict) -> dict: return {name: captured[key] for name, key in CAPTURE_NAMES.items()} -def matrix_captures(seed: int, tokens: int) -> dict | None: +def matrix_captures(seed: int, tokens: int) -> dict: root = _captures_dir() - if root is None: - return None params = torch.load(root / f"seed{seed}" / "params.pt", weights_only=True) blob = torch.load(root / f"seed{seed}" / f"T{tokens}.pt", weights_only=True) return dict( @@ -183,21 +185,14 @@ def matrix_captures(seed: int, tokens: int) -> dict | None: ) -def fixture_captures(name: str) -> dict | None: +def fixture_captures(name: str) -> dict: root = _captures_dir() - if root is None: - return None blob = torch.load(root / "fixtures" / f"{name}.pt", weights_only=True) return dict(bias=blob["bias"], hidden=blob["hidden"], route=_as_route(blob)) -def check_against_captures( - captured: dict | None, inputs: dict, route: dict, reference: dict -) -> list: - """With captures: same input bytes, the reference reproduces them bitwise, and the - component meets the rules against them. Without: nothing.""" - if captured is None: - return [] +def check_against_captures(captured: dict, inputs: dict, route: dict, reference: dict) -> list: + """Same input bytes, bitwise reference reproduction, and component comparison rules.""" results = [] for name, tensor in inputs.items(): same = torch.equal(tensor.cpu(), captured[name].cpu()) @@ -583,7 +578,7 @@ def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: @pytest.mark.parametrize("seed,tokens", cases.CASES, ids=cases.CASE_IDS) def test_matrix(seed, tokens) -> None: """One case of the fixed matrix: the output contract, every step against the reference - (and the source captures when given), the routing invariants, and the evidence that + the routing invariants, and the evidence that the routing ran on the GPU in this component's op with no host synchronization.""" weight, bias, hidden_cpu = cases.matrix_case(seed, tokens) router = loaded_router(weight, bias) @@ -606,12 +601,6 @@ def test_matrix(seed, tokens) -> None: results = check_output_contract(weights, ids, hidden) results += cases.compare_routes(route, reference, "reference") results += check_invariants(route, sum_to_one=True) - results += check_against_captures( - matrix_captures(seed, tokens), - {"weight": weight, "bias": bias, "hidden": hidden_cpu}, - route, - reference, - ) host = [c["func"] for c in trace.calls if c["func"] in HOST_TRANSFERS] off_device = [ @@ -661,11 +650,51 @@ def test_matrix(seed, tokens) -> None: cases.assert_all_pass(results) +@pytest.mark.skipif( + not os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES"), + reason="BEAM_MOE_ROUTER_SOURCE_CAPTURES is not configured", +) +@pytest.mark.parametrize("seed,tokens", cases.CASES, ids=cases.CASE_IDS) +def test_matrix_matches_source_captures(seed, tokens) -> None: + captured = matrix_captures(seed, tokens) + weight, bias, hidden_cpu = cases.matrix_case(seed, tokens) + router = loaded_router(weight, bias) + hidden = hidden_cpu.cuda() + _, _, route = routed(router, hidden) + cases.assert_all_pass( + check_against_captures( + captured, + {"weight": weight, "bias": bias, "hidden": hidden_cpu}, + route, + reference_route(router, hidden), + ) + ) + + +@pytest.mark.skipif( + not os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES"), + reason="BEAM_MOE_ROUTER_SOURCE_CAPTURES is not configured", +) +@pytest.mark.parametrize("name", tuple(cases.fixtures())) +def test_fixture_matches_source_captures(name) -> None: + captured = fixture_captures(name) + route, reference, _ = route_fixture(name) + _, bias, hidden = cases.fixture_tensors(cases.fixtures()[name]) + cases.assert_all_pass( + check_against_captures( + captured, + {"bias": bias, "hidden": hidden}, + route, + reference, + ) + ) + + # ----------------------------------------------------------------------------- controlled fixtures def route_fixture(name: str) -> tuple[dict, dict, list]: - """(component route, reference route, comparisons) for one fixture, captures included.""" + """(component route, reference route, comparisons) independent of source captures.""" spec = cases.fixtures()[name] weight, bias, hidden_cpu = cases.fixture_tensors(spec) router = loaded_router(weight, bias) @@ -674,9 +703,6 @@ def route_fixture(name: str) -> tuple[dict, dict, list]: reference = reference_route(router, hidden) results = cases.compare_routes(route, reference, "reference") results += check_invariants(route, sum_to_one=False) - results += check_against_captures( - fixture_captures(name), {"bias": bias, "hidden": hidden_cpu}, route, reference - ) exact = torch.equal(route["logits"].cpu(), spec["logits"].float()) results.append( { From ba108e02d0548dacf0a486c9f370e2a0f780a370 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 15:27:51 -0800 Subject: [PATCH 20/33] Move Beam component evidence reporting to opt-in QA runner Signed-off-by: Maxim Gluhovskoi --- tests/scripts/beam_component_qa.py | 121 ++++++++++++++++++ .../models/beam/_beam_component_support.py | 64 +-------- .../models/beam/_components/README.md | 23 ++++ .../dense_block/test_dense_block.py | 40 ++---- .../models/beam/_components/moe/test_moe.py | 53 ++++---- .../_components/moe_router/test_router.py | 119 +++++++++-------- .../sparse_block/test_sparse_block.py | 51 ++++---- 7 files changed, 270 insertions(+), 201 deletions(-) create mode 100644 tests/scripts/beam_component_qa.py diff --git a/tests/scripts/beam_component_qa.py b/tests/scripts/beam_component_qa.py new file mode 100644 index 000000000000..2b17c7df40e5 --- /dev/null +++ b/tests/scripts/beam_component_qa.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +r"""Run explicitly selected component tests and save their pytest reports for manual QA. + +Example (run from the repository root with a compatible native build):: + + python tests/scripts/beam_component_qa.py --output /tmp/beam-router-qa -- \ + tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router -v + +The output directory must be new. Reports may contain local paths and scheduler IDs; +keep them outside the public repository. This script does not supply source captures, +change test selection or tolerances, or turn skipped qualification into a pass. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +from pathlib import Path + +import pytest + + +def environment(root: Path) -> dict: + """Identify the checkout and imported runtime separately, without loading a model.""" + try: + revision = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=5 + ) + source_revision = revision.stdout.strip() if revision.returncode == 0 else None + except (OSError, subprocess.SubprocessError): + source_revision = None + result = { + "source_revision": source_revision, + "python": sys.version, + "machine": platform.machine(), + "argv": sys.argv, + "slurm_job_id": os.environ.get("SLURM_JOB_ID"), + "slurm_step_id": os.environ.get("SLURM_STEP_ID"), + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + } + # Do not initialize CUDA just to report a collection failure or a skipped test. + for name in ("torch", "tensorrt_llm"): + module = sys.modules.get(name) + if module is not None: + result[name] = {"version": module.__version__, "file": module.__file__} + torch = sys.modules.get("torch") + if torch is not None and torch.cuda.is_initialized(): + result["cuda"] = { + "version": torch.version.cuda, + "device": torch.cuda.get_device_name(), + "capability": list(torch.cuda.get_device_capability()), + } + return result + + +class QaReport: + """Collect standard pytest outcomes and record_property metrics, including failures.""" + + def __init__(self, output: Path) -> None: + self.output = output + self.reports: list[dict] = [] + + def pytest_collectreport(self, report: pytest.CollectReport) -> None: + if report.failed or report.skipped: + self.reports.append( + { + "nodeid": report.nodeid, + "phase": "collection", + "outcome": report.outcome, + "details": str(report.longrepr), + } + ) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.reports.append( + { + "nodeid": report.nodeid, + "phase": report.when, + "outcome": report.outcome, + "duration": report.duration, + "properties": report.user_properties, + "details": str(report.longrepr) if report.longrepr else None, + } + ) + + def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: + record = { + "exit_code": int(exitstatus), + "collect_only": session.config.option.collectonly, + "tests_collected": session.testscollected, + "environment": environment(session.config.rootpath), + "reports": self.reports, + } + (self.output / "report.json").write_text( + json.dumps(record, indent=2, default=str) + "\n", encoding="utf-8" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path, help="New local QA report directory") + parser.add_argument( + "pytest_args", nargs=argparse.REMAINDER, help="-- followed by pytest selection/options" + ) + args = parser.parse_args() + pytest_args = args.pytest_args + if pytest_args[:1] == ["--"]: + pytest_args = pytest_args[1:] + if not pytest_args: + parser.error("provide an explicit test selection after --") + args.output.mkdir(parents=True, exist_ok=False) + return int(pytest.main(pytest_args, plugins=[QaReport(args.output)])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index 9bb1dc7db9c9..e6b1dceee6c6 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Test support shared by the Beam GPU tests under ``models/beam/``. -Only harness code lives here: the IEEE fp32 session guard, the optional evidence writer, -result bookkeeping that judges with the rules each test file passes in, op-path +Only harness code lives here: the IEEE fp32 session guard, result bookkeeping that +judges with the rules each test file passes in, op-path instrumentation, the one-request KV cache context of the block tests, and the generated configuration and state of the six-block small model. No reference math lives here; each reference stays in its own ``_*_reference.py`` or ``_*_cases.py`` module, and every rule @@ -19,18 +19,12 @@ import contextlib import hashlib -import json import math -import os -import platform import re -import subprocess -import sys import types import zlib from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence -from pathlib import Path from typing import TYPE_CHECKING, NoReturn import torch @@ -67,7 +61,7 @@ def math_state() -> dict: } -def ieee_fp32_session(write_evidence: Callable[[str, dict], None]) -> Iterator[dict]: +def ieee_fp32_session() -> Iterator[dict]: """Enforce IEEE fp32 and verify it before and after the module's tests.""" matmul = torch.backends.cuda.matmul saved = ( @@ -88,7 +82,6 @@ def ieee_fp32_session(write_evidence: Callable[[str, dict], None]) -> Iterator[d yield state after = fp32_gemm_probe() state.update(math_after=math_state(), fp32_gemm_probe_after=after) - write_evidence("fp32-session", {"fixture": "ieee_fp32", "math": state}) assert after <= 1e-5, f"fp32 GEMM stopped being IEEE fp32 (relative error {after:.3g})" finally: # Legacy setters can also change the newer precision controls. @@ -100,57 +93,6 @@ def ieee_fp32_session(write_evidence: Callable[[str, dict], None]) -> Iterator[d torch.backends.cudnn.fp32_precision = cudnn_precision -def git_revision() -> str | None: - import tensorrt_llm - - root = Path(tensorrt_llm.__file__).resolve().parents[1] - try: - out = subprocess.run( - ["git", "-C", str(root), "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=30, - ) - except (OSError, subprocess.SubprocessError): - return None - return out.stdout.strip() or None - - -def environment(ieee: dict, execution: Mapping[str, object], **extra: object) -> dict: - """The device and software a case ran on; ``extra`` entries follow ``execution``.""" - import tensorrt_llm - - device = torch.cuda.current_device() - return { - "device_name": torch.cuda.get_device_name(device), - "device_capability": list(torch.cuda.get_device_capability(device)), - "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), - "torch": torch.__version__, - "torch_cuda": torch.version.cuda, - "tensorrt_llm": tensorrt_llm.__version__, - "tensorrt_llm_file": tensorrt_llm.__file__, - "source_revision": git_revision(), - "python": sys.version.split()[0], - "machine": platform.machine(), - "argv": sys.argv, - "slurm_job_id": os.environ.get("SLURM_JOB_ID"), - "slurm_step_id": os.environ.get("SLURM_STEP_ID"), - "execution": dict(execution), - **extra, - "math": ieee, - } - - -def write_evidence(env_var: str, name: str, record: dict) -> None: - """Write ``record`` as ``.json`` into the directory named by ``env_var``, if set.""" - directory = os.environ.get(env_var) - if not directory: - return - path = Path(directory) - path.mkdir(parents=True, exist_ok=True) - (path / f"{name}.json").write_text(json.dumps(record, indent=1, default=str) + "\n") - - def tensor_sha256(t: torch.Tensor) -> str: flat = t.detach().contiguous().cpu().reshape(-1) h = hashlib.sha256() diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md index a61700b34aaf..347a24a9b49c 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md @@ -14,3 +14,26 @@ TP1 normalization and activation use their FlashInfer contracts; the TP4 input n and dense MLP use explicit FP32 Beam operators, so BF16 rounding can differ. TP1 also accepts a narrower RoPE schema. Compare each helper against its declared contract, not against the TP4 model as a bitwise reference. + +## Manual QA reports + +Tests expose comparison metrics with pytest's `record_property` fixture. Ordinary +pytest runs do not write custom evidence files or collect Git/Slurm metadata. +For a manual report, run the opt-in QA script from the repository root: + +```bash +python tests/scripts/beam_component_qa.py --output /tmp/beam-router-qa -- \ + tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router -v +``` + +Use a new output directory for each run. `report.json` records pytest's exit code, +collection/setup/call/teardown outcomes, comparison properties, and checkout/runtime +identity. A successful setup phase does not imply a successful test; inspect the +call and teardown outcomes too. Missing optional source captures remain explicit +skips; configured invalid captures fail. The script neither stages captures nor +changes numerical checks. Large suites still require their documented resources. + +This replaces the `BEAM_*_EVIDENCE` environment variables and per-case JSON files; +metadata is collected once per QA run. Dense weight-accounting checks still compare +all tensors directly, without the optional duplicate hashing pass. Keep reports +outside the public repository because they contain local paths and scheduler IDs. diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index b312c7ee462b..2869c100f942 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -28,17 +28,13 @@ compared against the captures, and the generated input must equal theirs bit for bit. Separate source-capture tests report SKIPPED when this variable is unset; an explicitly configured directory must contain valid captures. Reference tests run independently. -* ``BEAM_DENSE_BLOCK_EVIDENCE``: a directory that receives one JSON record per test case: - every comparison with its rule, the op calls and CUDA kernels of the forward, and the - device and software it ran on; and ``fp32-session.json``, the fp32 GEMM checks made - before and after the module's tests. +QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable. No model files. """ from __future__ import annotations -import functools import math import os import types @@ -58,7 +54,7 @@ import _beam_component_support as support # noqa: E402 import _dense_block_reference as reference # noqa: E402 -from _beam_component_support import assert_all_pass, tensor_sha256 # noqa: E402 +from _beam_component_support import assert_all_pass # noqa: E402 from _validating import validating # noqa: E402 from _components.dense_block import block as block_module # noqa: E501 @@ -212,14 +208,11 @@ def block_config(**overrides) -> types.SimpleNamespace: def ieee_fp32(): """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both before the module's tests and after them.""" - yield from support.ieee_fp32_session(_write_evidence) + yield from support.ieee_fp32_session() -def _environment(ieee: dict) -> dict: - return support.environment(ieee, EXECUTION) - - -_write_evidence = functools.partial(support.write_evidence, "BEAM_DENSE_BLOCK_EVIDENCE") +def _execution_state(ieee: dict) -> dict: + return {"math": ieee, "execution": dict(EXECUTION)} # ----------------------------------------------------------------------------- comparison @@ -523,11 +516,6 @@ def accounting(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> dict: assert torch.equal(loaded[name], state[name]), f"{name} differs from the reference's" assert not reference.state_problems(generated(seed)["state_cpu"]) record = {"tensors": len(loaded), "state_problems": []} - if os.environ.get("BEAM_DENSE_BLOCK_EVIDENCE"): - record["sha256"] = { - name: tensor_sha256(t) for name, t in generated(seed)["state_cpu"].items() - } - record["input_sha256"] = tensor_sha256(generated(seed)["input"]) return record @@ -748,7 +736,7 @@ def test_rejects_unsupported_inputs_before_dispatch(case) -> None: @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_block_matches_reference(seed, length, ieee_fp32) -> None: +def test_block_matches_reference(record_property, seed, length, ieee_fp32) -> None: """One case: the whole block, then every operation on its own input.""" blk = loaded_block(seed) accounting_record = accounting(seed, blk) @@ -763,7 +751,7 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: final.update(reference="reference") results.append(final) results.extend(replay(blk, md, ref, "reference")) - _write_evidence( + record_property( f"block-seed{seed}-T{length}", { "test": "test_block_matches_reference", @@ -771,7 +759,7 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: "accounting": accounting_record, "rules": {k: list(v) for k, v in RULES.items()}, "results": results, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(results) @@ -795,7 +783,7 @@ def test_block_matches_source_captures(seed, length, ieee_fp32) -> None: @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_forward_runs_the_component_path(seed, length, ieee_fp32) -> None: +def test_forward_runs_the_component_path(record_property, seed, length, ieee_fp32) -> None: """One case: the component's own composition ran, op by op, through thop attention on this GPU, with no SDPA and no built-in Beam module on the way.""" blk = loaded_block(seed) @@ -830,9 +818,9 @@ def test_forward_runs_the_component_path(seed, length, ieee_fp32) -> None: "attention_kernels": [k for k in kernels if support.ATTENTION_KERNEL.search(k)], "sdpa_cpu_ops": [n for n in cpu_ops if support.SDPA_OPS.search(n)], "kv_cache": dict(KV_CACHE, manager=type(md.kv_cache_manager).__name__, **cache_mapping), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), } - _write_evidence(f"path-seed{seed}-T{length}", record) + record_property(f"path-seed{seed}-T{length}", record) assert type(blk).__module__.endswith("dense_block.block") assert tuple(names) == EXPECTED_CALLS, names @@ -863,7 +851,7 @@ def test_forward_runs_the_component_path(seed, length, ieee_fp32) -> None: assert torch.equal(keys, k_in) and torch.equal(values, v_in) -def test_detects_a_wrong_residual_coefficient(ieee_fp32, monkeypatch) -> None: +def test_detects_a_wrong_residual_coefficient(record_property, ieee_fp32, monkeypatch) -> None: """The complete-block comparison rejects a zeroed final residual coefficient and accepts the unchanged block, on the same case and the same comparison.""" seed, length = 17, 17 @@ -881,14 +869,14 @@ def test_detects_a_wrong_residual_coefficient(ieee_fp32, monkeypatch) -> None: out = blk(x, positions, md) runs[label] = compare("block_output", out, ref["block_output"], "block_output") runs[label]["feed_forward_scales"] = list(blk.feed_forward_scales) - _write_evidence( + record_property( f"fault-seed{seed}-T{length}", { "test": "test_detects_a_wrong_residual_coefficient", "case": {"seed": seed, "length": length}, "fault": "final residual coefficient sqrt(2)/sqrt(3) -> 0.0, fixture-scoped", "runs": runs, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert runs["unchanged"]["passed"] is True, runs["unchanged"]["problems"] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py index 73d177714ecf..93a071479562 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -20,12 +20,7 @@ env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/moe -Optional environment: - -* ``BEAM_MOE_EVIDENCE``: a directory that receives one JSON record per compared case: - every comparison with its rule, the op and expert calls where they were recorded, and - the device and software it ran on; and ``fp32-session.json``, the fp32 GEMM checks made - before and after the module's tests. +QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). Dependencies: a TensorRT-LLM build for sm_103 and about 60 GiB of free device memory: one seed's generated state and the loaded MoE hold 18.75 GiB of expert weights each, and @@ -34,7 +29,6 @@ from __future__ import annotations -import functools import gc import types from collections import OrderedDict @@ -178,7 +172,7 @@ def moe_config(**overrides) -> types.SimpleNamespace: def ieee_fp32(): """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both before the module's tests and after them.""" - yield from support.ieee_fp32_session(_write_evidence) + yield from support.ieee_fp32_session() @pytest.fixture(scope="module", autouse=True) @@ -188,11 +182,8 @@ def device_memory(): release_cache() -def _environment(ieee: dict) -> dict: - return support.environment(ieee, EXECUTION) - - -_write_evidence = functools.partial(support.write_evidence, "BEAM_MOE_EVIDENCE") +def _execution_state(ieee: dict) -> dict: + return {"math": ieee, "execution": dict(EXECUTION)} # -------------------------------------------------- comparison @@ -589,7 +580,7 @@ def test_rejects_unsupported_inputs_before_dispatch(case, error, match) -> None: @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_moe_matches_reference(seed, length, ieee_fp32) -> None: +def test_moe_matches_reference(record_property, seed, length, ieee_fp32) -> None: """One case: the forward's own routing (the router op's mapping of that call) selects exactly the reference's groups and experts in order, its fp32 steps and the output are within their rules, and the call leaves its input and the parameters untouched.""" @@ -611,7 +602,7 @@ def test_moe_matches_reference(seed, length, ieee_fp32) -> None: results.extend(compare_routes(route, ref_route)) results.append(weight_invariants(route)) results.append(unchanged(x, x_before, versions, after)) - _write_evidence( + record_property( f"moe-seed{seed}-T{length}", { "test": "test_moe_matches_reference", @@ -620,7 +611,7 @@ def test_moe_matches_reference(seed, length, ieee_fp32) -> None: "results": results, "selected_experts": sorted(set(route["ids"].reshape(-1).tolist())), "input_sha256": tensor_sha256(x), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(results) @@ -629,7 +620,7 @@ def test_moe_matches_reference(seed, length, ieee_fp32) -> None: # -------------------------------------------------- tests: the component path -def test_forward_runs_the_component_path(ieee_fp32) -> None: +def test_forward_runs_the_component_path(record_property, ieee_fp32) -> None: """One case: the router op once, then BeamDispatchCombine, inside which every selected expert -- ascending, once each -- receives exactly its tokens' rows in token order and runs its own three projections and SwiGLU; every op's guard admits its arguments and no @@ -640,14 +631,14 @@ def test_forward_runs_the_component_path(ieee_fp32) -> None: with builtin_beam_forbidden(), validating(*OPS), recording(*OPS, experts=True) as calls: out = moe(x) torch.cuda.synchronize() - _write_evidence( + record_property( f"path-seed{seed}-T{length}", { "test": "test_forward_runs_the_component_path", "case": {"seed": seed, "length": length}, "moe": f"{type(moe).__module__}.{type(moe).__qualname__}", "op_calls": loggable(calls), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) @@ -695,7 +686,7 @@ def test_forward_runs_the_component_path(ieee_fp32) -> None: # -------------------------------------------------- tests: designed fixtures -def test_expert_identity_fixture(ieee_fp32, monkeypatch) -> None: +def test_expert_identity_fixture(record_property, ieee_fp32, monkeypatch) -> None: """Designed logits route four tokens to the same eight experts -- both ends of the id range and of four groups -- in four different orders. Every one of the eight gets all four rows, the other 248 are never asked for, the output matches the reference, and @@ -719,7 +710,7 @@ def test_expert_identity_fixture(ieee_fp32, monkeypatch) -> None: monkeypatch.undo() assert moe.experts[255] is not moe.experts[0] runs["restored"] = compare("moe_output", moe(x), ref_out, "moe_output") - _write_evidence( + record_property( "fixture-expert-identity", { "test": "test_expert_identity_fixture", @@ -728,7 +719,7 @@ def test_expert_identity_fixture(ieee_fp32, monkeypatch) -> None: "weights": route["weights"].tolist(), "expert_calls": loggable(asked), "runs": runs, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert route["ids"].tolist() == designed, route["ids"].tolist() @@ -744,7 +735,7 @@ def test_expert_identity_fixture(ieee_fp32, monkeypatch) -> None: assert runs["restored"]["passed"] is True, runs["restored"]["problems"] -def test_changed_routing_between_calls(ieee_fp32) -> None: +def test_changed_routing_between_calls(record_property, ieee_fp32) -> None: """Rows routed to expert set A, then rows routed to the disjoint set B, then A again: B's call routes as designed and asks for no expert of A, and the repeated A call reproduces the first one bit for bit -- nothing of one call's routing survives into @@ -767,7 +758,7 @@ def test_changed_routing_between_calls(ieee_fp32) -> None: } ) first, second, third = runs - _write_evidence( + record_property( "fixture-changed-routing", { "test": "test_changed_routing_between_calls", @@ -775,7 +766,7 @@ def test_changed_routing_between_calls(ieee_fp32) -> None: {"ids": r["ids"].tolist(), "experts": r["experts"], "result": r["vs_reference"]} for r in runs ], - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert first["ids"].tolist() == [designed_ids(SET_A, t) for t in range(len(RANKS))] @@ -806,7 +797,7 @@ def test_zero_input_gives_exact_zero() -> None: assert [c["expert_id"] for c in asked] == sorted(set(route["ids"].reshape(-1).tolist())) -def test_token_permutation_preserves_correspondence(ieee_fp32) -> None: +def test_token_permutation_preserves_correspondence(record_property, ieee_fp32) -> None: """Reversing the tokens reverses the routing and the output: each token keeps its own experts and its own output row.""" seed, length = 17, 17 @@ -828,18 +819,18 @@ def test_token_permutation_preserves_correspondence(ieee_fp32) -> None: compare("output_vs_reference", base, ref_out, "moe_output"), compare("permuted_output_vs_reference", permuted, ref_out[perm], "moe_output"), ] - _write_evidence( + record_property( f"permutation-seed{seed}-T{length}", { "test": "test_token_permutation_preserves_correspondence", "results": results, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(results) -def test_detects_routed_scaling_faults(ieee_fp32, monkeypatch) -> None: +def test_detects_routed_scaling_faults(record_property, ieee_fp32, monkeypatch) -> None: """The output comparison rejects the combination scaled by 1.0 or by 2.5 twice, and accepts the unchanged and the restored MoE, on the same case and the same rule.""" seed, length = 17, 17 @@ -852,12 +843,12 @@ def test_detects_routed_scaling_faults(ieee_fp32, monkeypatch) -> None: runs[f"scaled_by_{factor}"] = compare("moe_output", moe(x), ref_out, "moe_output") monkeypatch.undo() runs["restored"] = compare("moe_output", moe(x), ref_out, "moe_output") - _write_evidence( + record_property( f"fault-scaling-seed{seed}-T{length}", { "test": "test_detects_routed_scaling_faults", "runs": runs, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert dispatch_combine.ROUTED_SCALING_FACTOR == 2.5 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index f376aa0cf399..206a66304f37 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -29,9 +29,7 @@ under the same rules as with the reference. Separate source-capture tests report SKIPPED when this variable is unset; a configured directory must contain valid files. Ordinary reference and routing-invariant tests run independently. -* ``BEAM_MOE_ROUTER_EVIDENCE``: a directory that receives one JSON record per test case: - every comparison with its rule, hashes of the routed tensors, the torch calls and CUDA - kernels of the forward, and the device and software it ran on. +QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). Dependencies: a TensorRT-LLM build for sm_103. No model files. """ @@ -39,18 +37,13 @@ from __future__ import annotations import contextlib -import functools import os -import platform -import sys import types from pathlib import Path import pytest import torch -import tensorrt_llm - try: import tensorrt_llm._torch.models.modeling_beam as builtin_beam except ModuleNotFoundError as exc: @@ -122,39 +115,19 @@ def router_config(**overrides) -> types.SimpleNamespace: @pytest.fixture(scope="module", autouse=True) def ieee_fp32(): """Enforce IEEE fp32 and check the actual GEMM precision.""" - yield from support.ieee_fp32_session(_write_evidence) + yield from support.ieee_fp32_session() -# ----------------------------------------------------------------------------- evidence +# ----------------------------------------------------------------------------- comparison bookkeeping -def _environment() -> dict: - device = torch.cuda.current_device() +def _execution_state() -> dict: return { - "device_name": torch.cuda.get_device_name(device), - "device_capability": list(torch.cuda.get_device_capability(device)), - "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), - "torch": torch.__version__, - "torch_cuda": torch.version.cuda, - "tensorrt_llm": tensorrt_llm.__version__, - "tensorrt_llm_file": tensorrt_llm.__file__, - "python": sys.version.split()[0], - "machine": platform.machine(), - "argv": sys.argv, - "pid": os.getpid(), - "slurm_job_id": os.environ.get("SLURM_JOB_ID"), - "slurm_step_id": os.environ.get("SLURM_STEP_ID"), "execution": dict(EXECUTION), - "ambient_math": { - "cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision, - }, - "source_captures": os.environ.get("BEAM_MOE_ROUTER_SOURCE_CAPTURES"), + "math": {"cuda_matmul_fp32_precision": torch.backends.cuda.matmul.fp32_precision}, } -_write_evidence = functools.partial(support.write_evidence, "BEAM_MOE_ROUTER_EVIDENCE") - - def route_hashes(route: dict) -> dict: return {name: tensor_sha256(t) for name, t in sorted(route.items())} @@ -353,7 +326,9 @@ def check_invariants(route: dict, sum_to_one: bool) -> list: # ----------------------------------------------------------------------------- construction and loading -def test_construction_allocates_only_the_router_state() -> None: +def test_construction_allocates_only_the_router_state( + record_property, +) -> None: """Construction allocates exactly the two FP32 router tensors: no expert, attention, cache or model state, and no submodule.""" torch.cuda.synchronize() @@ -373,9 +348,9 @@ def test_construction_allocates_only_the_router_state() -> None: "submodules": [n for n, _ in router.named_modules() if n], "elements": elements, "cuda_bytes_allocated": allocated, - "environment": _environment(), + "execution_state": _execution_state(), } - _write_evidence("construction", record) + record_property("construction", record) assert params == {"weight": ((256, 5120), torch.float32, False)} assert buffers == {"e_score_correction_bias": ((256,), torch.float32)} assert record["submodules"] == [] @@ -409,11 +384,11 @@ def test_construction_rejects_a_cpu_device() -> None: @pytest.mark.parametrize("seed", cases.SEEDS) -def test_loading_copies_both_tensors_exactly(seed) -> None: +def test_loading_copies_both_tensors_exactly(record_property, seed) -> None: weight, bias, _ = cases.matrix_case(seed, 1) router = BeamMoeRouterBf16Sm103Tp1(router_config(), LAYER_IDX, device="cuda") router.load_weights(weight, bias) - _write_evidence( + record_property( f"loading-seed{seed}", { "test": "test_loading_copies_both_tensors_exactly", @@ -513,7 +488,9 @@ def test_forward_rejects_invalid_input(case, error, match) -> None: # ----------------------------------------------------------------------------- precision -def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: +def test_precision_path_is_fp32_without_tf32_or_autocast( + record_property, +) -> None: """Projection, sigmoid, selection scores, group scores and normalization are fp32, the projection is IEEE fp32 even when the caller has TF32 or bf16 autocast on, and the caller's global precision policy remains unchanged.""" @@ -540,7 +517,7 @@ def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: fp64_error = float((route["logits"].double() - fp64_logits).abs().max()) projection = [c for c in trace.calls if c["func"] == "trtllm.cublas_mm"] dtypes = {name: str(t.dtype) for name, t in route.items()} - _write_evidence( + record_property( "precision", { "test": "test_precision_path_is_fp32_without_tf32_or_autocast", @@ -551,7 +528,7 @@ def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: "logits_vs_fp64_max_abs": fp64_error, "bitwise_equal_to_plain_call": {k: torch.equal(route[k], plain[k]) for k in route}, "torch_calls": [c["func"] for c in trace.calls], - "environment": _environment(), + "execution_state": _execution_state(), }, ) assert ambient == after == "tf32", (ambient, after) @@ -576,7 +553,7 @@ def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: @pytest.mark.parametrize("seed,tokens", cases.CASES, ids=cases.CASE_IDS) -def test_matrix(seed, tokens) -> None: +def test_matrix(record_property, seed, tokens) -> None: """One case of the fixed matrix: the output contract, every step against the reference the routing invariants, and the evidence that the routing ran on the GPU in this component's op with no host synchronization.""" @@ -616,7 +593,7 @@ def test_matrix(seed, tokens) -> None: "torch.Tensor.gather", "torch.Tensor.sum", } - _write_evidence( + record_property( f"matrix-seed{seed}-T{tokens}", { "test": "test_matrix", @@ -638,7 +615,7 @@ def test_matrix(seed, tokens) -> None: "off_device_outputs": off_device, "sync_debug_mode": "error", "cuda_kernels": kernels, - "environment": _environment(), + "execution_state": _execution_state(), }, ) assert type(router).__module__.endswith("moe_router.router") @@ -716,8 +693,10 @@ def route_fixture(name: str) -> tuple[dict, dict, list]: return route, reference, results -def _record_fixture(name: str, test: str, route: dict, results: list, extra: dict) -> None: - _write_evidence( +def _record_fixture( + record_property, name: str, test: str, route: dict, results: list, extra: dict +) -> None: + record_property( f"fixture-{name}", { "test": test, @@ -731,12 +710,14 @@ def _record_fixture(name: str, test: str, route: dict, results: list, extra: dic "hashes": route_hashes(route), "results": results, **extra, - "environment": _environment(), + "execution_state": _execution_state(), }, ) -def test_bias_selects_but_never_weights() -> None: +def test_bias_selects_but_never_weights( + record_property, +) -> None: """The correction bias changes which experts are chosen, and the returned weights are still the normalized unbiased scores of the chosen experts.""" zero, _, zero_results = route_fixture("bias_zero") @@ -758,9 +739,16 @@ def test_bias_selects_but_never_weights() -> None: ), "weights_vs_normalized_biased_max_abs": leak_gap, } - _record_fixture("bias_zero", "test_bias_selects_but_never_weights", zero, zero_results, {}) _record_fixture( - "bias_shift", "test_bias_selects_but_never_weights", shift, shift_results, extra + record_property, "bias_zero", "test_bias_selects_but_never_weights", zero, zero_results, {} + ) + _record_fixture( + record_property, + "bias_shift", + "test_bias_selects_but_never_weights", + shift, + shift_results, + extra, ) cases.assert_all_pass(zero_results + shift_results) assert zero["groups"].tolist() == [[0, 1, 2, 3]] @@ -775,12 +763,15 @@ def test_bias_selects_but_never_weights() -> None: assert leak_gap > cases.RULES["weights"], f"bias in the weights would be invisible: {leak_gap}" -def test_group_boundary_moves_with_the_top_two_sum() -> None: +def test_group_boundary_moves_with_the_top_two_sum( + record_property, +) -> None: """Only the challenger group's second expert moves, and the group crosses the 4th/5th boundary; the group holding the single best expert stays out.""" route, _, results = route_fixture("group_boundary") group_scores = route["group_scores"] _record_fixture( + record_property, "group_boundary", "test_group_boundary_moves_with_the_top_two_sum", route, @@ -798,7 +789,9 @@ def test_group_boundary_moves_with_the_top_two_sum() -> None: assert 5 not in route["groups"].flatten().tolist() -def test_exclusion_masks_every_expert_of_dropped_groups() -> None: +def test_exclusion_masks_every_expert_of_dropped_groups( + record_property, +) -> None: """The best single expert sits in a group that loses; it and every other expert of the four dropped groups are -inf, although every retained selection score is negative.""" route, _, results = route_fixture("exclusion") @@ -806,6 +799,7 @@ def test_exclusion_masks_every_expert_of_dropped_groups() -> None: selected_scores = route["selection_scores"].gather(-1, ids) spike_score = float(route["selection_scores"][0, cases.EXCLUSION_SPIKE]) _record_fixture( + record_property, "exclusion", "test_exclusion_masks_every_expert_of_dropped_groups", route, @@ -820,11 +814,20 @@ def test_exclusion_masks_every_expert_of_dropped_groups() -> None: assert all(e // cases.PER_GROUP in route["groups"][0].tolist() for e in ids[0].tolist()) -def test_expert_boundary_moves_one_expert() -> None: +def test_expert_boundary_moves_one_expert( + record_property, +) -> None: """Expert 2 crosses the 8th/9th boundary within an unchanged group selection: it enters last and expert 97 leaves, every other position unchanged.""" route, _, results = route_fixture("expert_boundary") - _record_fixture("expert_boundary", "test_expert_boundary_moves_one_expert", route, results, {}) + _record_fixture( + record_property, + "expert_boundary", + "test_expert_boundary_moves_one_expert", + route, + results, + {}, + ) cases.assert_all_pass(results) assert route["groups"].tolist() == [[0, 1, 2, 3], [0, 1, 2, 3]] assert route["ids"].tolist() == [ @@ -833,7 +836,9 @@ def test_expert_boundary_moves_one_expert() -> None: ] -def test_ties_follow_native_topk_order() -> None: +def test_ties_follow_native_topk_order( + record_property, +) -> None: """Match the native reference in each batch shape, including valid boundary ties.""" route, _, results = route_fixture("ties") spec = cases.fixtures()["ties"] @@ -859,6 +864,7 @@ def test_ties_follow_native_topk_order() -> None: for i, name in enumerate(spec["rows"]) } _record_fixture( + record_property, "ties", "test_ties_follow_native_topk_order", route, @@ -868,7 +874,9 @@ def test_ties_follow_native_topk_order() -> None: cases.assert_all_pass(results) -def test_epsilon_is_the_source_denominator() -> None: +def test_epsilon_is_the_source_denominator( + record_property, +) -> None: """Weights divide by sum + 1e-20: a row whose scores underflow to 0 gets zero weights, not NaN, and a row whose selected scores sum to ~1e-20 sums to S / (S + 1e-20), well below 1 -- a denominator without the term, or with 1e-12 or 1e-6, gives 1 or ~0.""" @@ -878,6 +886,7 @@ def test_epsilon_is_the_source_denominator() -> None: expected_sum = (exact / (exact + 1e-20)).tolist() sums = route["weights"].sum(dim=-1).double().tolist() _record_fixture( + record_property, "epsilon", "test_epsilon_is_the_source_denominator", route, diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 5aca151c610e..740bda0891d9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -26,12 +26,7 @@ env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block -Optional environment: - -* ``BEAM_SPARSE_BLOCK_EVIDENCE``: a directory that receives one JSON record per test - case: every comparison with its rule, the op calls and CUDA kernels of the forward - where they were recorded, and the device and software it ran on; and - ``fp32-session.json``, the fp32 GEMM checks made before and after the module's tests. +QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 60 GiB of free device memory: one seed's generated state and the loaded block hold 18.75 GiB of @@ -40,7 +35,6 @@ from __future__ import annotations -import functools import gc import math import types @@ -264,7 +258,7 @@ def replaced(items, index: int, value) -> list: def ieee_fp32(): """The source's fp32 steps are IEEE fp32: refuse a process that substitutes TF32, both before the module's tests and after them.""" - yield from support.ieee_fp32_session(_write_evidence) + yield from support.ieee_fp32_session() @pytest.fixture(scope="module", autouse=True) @@ -274,12 +268,13 @@ def device_memory(): release_cache() -def _environment(ieee: dict) -> dict: +def _execution_state(ieee: dict) -> dict: layer_mask = [i for i, kept in enumerate(LAYER_MASK) if kept] - return support.environment(ieee, EXECUTION, kv_cache=dict(KV_CACHE, layer_mask=layer_mask)) - - -_write_evidence = functools.partial(support.write_evidence, "BEAM_SPARSE_BLOCK_EVIDENCE") + return { + "math": ieee, + "execution": dict(EXECUTION), + "kv_cache": dict(KV_CACHE, layer_mask=layer_mask), + } # -------------------------------------------------- comparison @@ -794,7 +789,7 @@ def test_rejects_unsupported_inputs_before_dispatch(case) -> None: blk(x, positions, md) -def test_cache_slot_is_the_managers_local_index(ieee_fp32) -> None: +def test_cache_slot_is_the_managers_local_index(record_property, ieee_fp32) -> None: """Logical layer 3 lives in slot 0 of a manager whose layer mask selects it alone: the attention op is called with local index 0, and the K and V it appended are read back through the manager under the logical index 3.""" @@ -815,14 +810,14 @@ def test_cache_slot_is_the_managers_local_index(ieee_fp32) -> None: out = blk(x, positions, md) torch.cuda.synchronize() keys, values = cache_rows(md, length) - _write_evidence( + record_property( f"cache-slot-seed{seed}-T{length}", { "test": "test_cache_slot_is_the_managers_local_index", "layout": layout, "step_local_layer_idx": step["local_layer_idx"], "op_calls": loggable(calls), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert isinstance(mgr, KVCacheManagerV2) @@ -889,7 +884,7 @@ def test_cache_slot_in_a_manager_holding_two_layers(ieee_fp32) -> None: @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_block_matches_reference(seed, length, ieee_fp32) -> None: +def test_block_matches_reference(record_property, seed, length, ieee_fp32) -> None: """One case, routing judged apart from numerical error: exact routing on the block's own MoE input; against the whole reference block, routing differences only on near ties and the output within its rule on the tokens routed alike; the MoE output within its rule @@ -903,7 +898,7 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: ref = reference_record(seed, x, positions) results = block_results(seed, out, captured, ref) results.append(unchanged(x, x_before, versions, after)) - _write_evidence( + record_property( f"block-seed{seed}-T{length}", { "test": "test_block_matches_reference", @@ -912,7 +907,7 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: "route_gap": ROUTE_GAP, "results": results, "input_sha256": tensor_sha256(x), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(results) @@ -921,7 +916,7 @@ def test_block_matches_reference(seed, length, ieee_fp32) -> None: # -------------------------------------------------- tests: the component path -def test_forward_runs_the_component_path(ieee_fp32) -> None: +def test_forward_runs_the_component_path(record_property, ieee_fp32) -> None: """One case: the component's own composition ran, op by op, with the experts' calls nested inside BeamDispatchCombine, through thop attention on this GPU, with no SDPA, no built-in Beam module and every op's guard admitting its arguments.""" @@ -948,9 +943,9 @@ def test_forward_runs_the_component_path(ieee_fp32) -> None: "attention_kernels": [k for k in kernels if support.ATTENTION_KERNEL.search(k)], "sdpa_cpu_ops": [n for n in cpu_ops if support.SDPA_OPS.search(n)], "kv_cache_manager": type(manager).__name__, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), } - _write_evidence(f"path-seed{seed}-T{length}", record) + record_property(f"path-seed{seed}-T{length}", record) assert type(blk).__module__.endswith("sparse_block.block") top = [c for c in calls if c["depth"] == 0] @@ -982,7 +977,7 @@ def test_forward_runs_the_component_path(ieee_fp32) -> None: # -------------------------------------------------- tests: fixtures -def test_causal_prefix_independence(ieee_fp32) -> None: +def test_causal_prefix_independence(record_property, ieee_fp32) -> None: """Two 129-token inputs sharing their first 17 rows: those rows reach the MoE bit for bit alike and are routed alike, and their outputs agree within the block rule (each expert's batch differs, so only the expert GEMMs' row counts change).""" @@ -1007,21 +1002,21 @@ def test_causal_prefix_independence(ieee_fp32) -> None: ] if not results[1]["passed"]: results[1]["problems"].append("the shared rows' MoE inputs differ") - _write_evidence( + record_property( f"prefix-seed{seed}-T{length}", { "test": "test_causal_prefix_independence", "prefix": prefix, "results": results, "suffix_ids_differ": not torch.equal(ids_b[prefix:], ids_a[prefix:]), - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(results) assert not torch.equal(out_b[0, prefix:], out_a[0, prefix:]), "the suffix changed nothing" -def test_residual_uses_block3_coefficients(ieee_fp32, monkeypatch) -> None: +def test_residual_uses_block3_coefficients(record_property, ieee_fp32, monkeypatch) -> None: """The residual entry with the block's coefficients is block 3's add bit for bit and differs from block 0's; and the block comparison rejects a block run with block 0's coefficients, and accepts the unchanged and the restored block.""" @@ -1055,13 +1050,13 @@ def test_residual_uses_block3_coefficients(ieee_fp32, monkeypatch) -> None: monkeypatch.undo() out, captured = run_block(blk, x, positions) runs[label] = block_results(seed, out, captured, ref) - _write_evidence( + record_property( f"fault-residual-seed{seed}-T{length}", { "test": "test_residual_uses_block3_coefficients", "fault": "block 0's residual coefficients in both adds, test-scoped", "runs": runs, - "environment": _environment(ieee_fp32), + "execution_state": _execution_state(ieee_fp32), }, ) assert_all_pass(runs["unchanged"]) From dbc872535dd33dd89c30a860dbd65a83beac6910 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 15:42:30 -0800 Subject: [PATCH 21/33] Correct new-token context length in cached-prefix test fixture Signed-off-by: Maxim Gluhovskoi --- .../_torch/modeling_v2/models/beam/_beam_component_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index e6b1dceee6c6..0918b79d9fd2 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -494,7 +494,7 @@ def fresh_context( ), seq_lens=torch.tensor([num_tokens] * num_requests, dtype=torch.int), request_ids=request_ids, - prompt_lens=[cached_tokens + num_tokens] * num_requests, + prompt_lens=[num_tokens] * num_requests, ) md.prepare() yield md From 3b4f2dcf1ab2ee5088a272682c2f83de6a23b88d Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 15:52:37 -0800 Subject: [PATCH 22/33] [refactor] Remove unused Beam component interfaces Signed-off-by: Maxim Gluhovskoi --- .../component.py | 11 ++----- .../full_model_nvfp4__sm_103__tp4/model.py | 33 ++++--------------- .../models/beam/test_component_contracts.py | 2 -- 3 files changed, 9 insertions(+), 37 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/component.py b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/component.py index 4c8660547915..523cacdf9eb3 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/component.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/component.py @@ -133,7 +133,9 @@ def tensor(self, name: str) -> torch.Tensor: """The rank-local storage of higher-precision tensor ``name``.""" spec = self.specs[name] if isinstance(spec, QuantSpec): - raise ValueError(f"{name} is an NVFP4 expert projection; see expert_storage()") + raise ValueError( + f"{name} is an NVFP4 expert projection; use expert_bank() for packed weights and scales" + ) return getattr(self, _attr(name)) def bank(self, bank: str, field: str = "weight") -> torch.Tensor: @@ -152,13 +154,6 @@ def expert_bank(self, layer: int) -> dict: for kind in _KINDS } - def expert_storage(self, layer: int, expert: int, proj: str) -> dict: - """Views of one expert projection's rank-local storage (``proj`` in ``PROJECTIONS``).""" - spec = self.specs.get(f"model.layers.{layer}.mlp.experts.{expert}.{proj}") - if not isinstance(spec, QuantSpec): - raise ValueError(f"layer {layer} expert {expert} {proj!r} is not held") - return {f: self.bank(spec.bank, f)[expert] for f in BANK_FIELDS} - # ------------------------------------------------------------------ loading def _load_expert(self, catalog, reader, spec: QuantSpec, digest) -> dict: diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py index a47b1c26d243..3bd4c989eaf9 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/model.py @@ -21,13 +21,12 @@ generation request appends one token. Each request's ``position_ids`` must be exactly ``past .. past + n - 1``. ``logit_rows`` selects the hidden rows that reach the final norm and the head before either runs: ``None`` (default) is the last row of each request, in request -order -- the next-token logits a sampler needs -- ``"all"`` every row, or explicit row -indices. +order -- the next-token logits a sampler needs -- ``"all"`` every row. The cache must be a BF16 ``KVCacheManagerV2`` holding all 52 logical layers in logical order with 2 kv heads of 128 per rank, a 512-token window on every sliding layer and the whole -sequence (``max_seq_len``) on the full-attention layers 5, 11, 17, 23, 29, 35, 41, 47 and 51 -(``attention_windows``). Input geometry, metadata and cache are validated before execution; absolute position +sequence (``max_seq_len``) on the full-attention layers 5, 11, 17, 23, 29, 35, 41, 47 and 51. +Input geometry, metadata and cache are validated before execution; absolute position values are compared with the request spans only in diagnostic mode. A call with T above ``max_tokens`` is rejected (the default ``MAX_NUM_TOKENS`` matches the supported deployment's native ``max_num_tokens``). @@ -60,7 +59,7 @@ #: Tokens per call of the supported deployment (its native ``max_num_tokens``). MAX_NUM_TOKENS = 8192 -LogitRows = Literal["all"] | Sequence[int] | None +LogitRows = Literal["all"] | None class BeamFullModelNvfp4Sm103Tp4(BeamCheckpointLayoutNvfp4Sm103Tp4): @@ -86,13 +85,6 @@ def __init__( self.vocab_size = config["vocab_size"] self._checked_manager = None - def attention_windows(self, max_seq_len: int) -> list[int]: - """Per logical layer: the attention window its cache must keep (512 or ``max_seq_len``).""" - return [ - self.window if self.schedule.is_sliding(layer) else max_seq_len - for layer in range(NUM_LAYERS) - ] - def check_cache_manager(self, manager: KVCacheManagerV2) -> None: """Raise ValueError unless ``manager`` holds every logical layer as this model needs. @@ -144,22 +136,9 @@ def _select_rows(self, logit_rows: LogitRows, spans: Sequence[tuple[int, int]]) end += n ends.append(end - 1) return ends - if isinstance(logit_rows, str): - if logit_rows != "all": - raise ValueError( - f"logit_rows must be None, 'all' or row indices, got {logit_rows!r}" - ) + if isinstance(logit_rows, str) and logit_rows == "all": return list(range(num_tokens)) - rows = list(logit_rows) - if ( - not rows - or any(type(r) is not int or not 0 <= r < num_tokens for r in rows) - or len(set(rows)) != len(rows) - ): - raise ValueError( - f"logit_rows must be distinct row indices in [0, {num_tokens}), got {logit_rows!r}" - ) - return rows + raise ValueError(f"logit_rows must be None or 'all', got {logit_rows!r}") def check_call( self, diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py index a2d5a85e4768..bd2d57f75149 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py @@ -48,8 +48,6 @@ def test_cache_allows_estimator_and_short_sequence_geometry(length, retention): max_seq_len=length, ) Component.check_cache_manager(component, manager) - # Cache retention does not change the mathematical attention window. - assert Component.attention_windows(component, length) == [512] * 51 + [length] @pytest.mark.parametrize( From 05c3b8390333767d8683f6f455843b1129491104 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 15:57:52 -0800 Subject: [PATCH 23/33] [docs] Clarify Beam family operator references Signed-off-by: Maxim Gluhovskoi --- .../_torch/modeling_v2/models/beam/dispatch_combine.py | 3 ++- .../modeling_v2/models/beam/_components/moe_router/router.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py index 303dc3db1b5c..6e80ce407d6e 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py @@ -18,7 +18,8 @@ is re-selected, no bias is added and no weight is re-normalized here. No residual, normalization or collective is applied. -The combination, as the source model computes it (``ops/moe/beam_dispatch_combine.py``):: +The combination follows the Beam family operator +``tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine``:: for each selected expert e, ascending: out = expert_provider(e, hidden[tokens routed to e]) [n, 5120] bf16 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py index ff5348a3fe62..7b9b4ad35240 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py @@ -11,7 +11,8 @@ dispatches no token and combines no expert output; the routed scaling factor belongs to that combine and is not applied here. It does not call the built-in Beam modeling. -The routing, every step fp32 (``ops/moe/beam_grouped_topk_router.py``):: +The routing uses FP32 at every step, through the Beam family operator +``tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router``:: logits = hidden.float() @ weight^T [T, 256] scores = sigmoid(logits) unbiased: the weights use these From 7295d5e7b795f8d70b2f90e68bf50def4be11b79 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 16:07:30 -0800 Subject: [PATCH 24/33] [fix] Validate Beam routing IDs and block cache retention Signed-off-by: Maxim Gluhovskoi --- .../modeling_v2/models/beam/_attention.py | 20 ++++- .../beam/ops/moe/beam_dispatch_combine.py | 18 ++++- .../beam/_components/dense_block/block.py | 1 + .../beam/_components/sparse_block/block.py | 1 + .../models/beam/test_component_contracts.py | 76 +++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_attention.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_attention.py index 254dbfcfa441..05b575546759 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_attention.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_attention.py @@ -175,6 +175,7 @@ def check_step( head_dim: int, device: torch.device, max_tokens: int, + attention_window: int | None, ) -> None: """Raise ValueError on any call input a block would otherwise compute wrongly. @@ -183,7 +184,8 @@ def check_step( ``arange(past, past + T)`` on that device; a prepared ``TrtllmAttentionMetadata`` holding one step of one request on a BF16 ``KVCacheManagerV2`` that holds logical layer ``layer_idx`` with ``num_kv_heads`` kv heads of ``head_dim`` (``check_metadata`` - states the steps and returns ``past``). + states the steps and returns ``past``). Cache retention must cover ``attention_window`` + (or the full sequence for None), capped by the manager's maximum sequence length. """ if not isinstance(hidden_states, torch.Tensor) or hidden_states.dtype != torch.bfloat16: raise ValueError("hidden_states must be a torch.bfloat16 tensor") @@ -204,6 +206,22 @@ def check_step( past = check_metadata( md, num_tokens, layer_idx=layer_idx, num_kv_heads=num_kv_heads, head_dim=head_dim ) + manager = md.kv_cache_manager + slot = manager.layer_offsets[layer_idx] + windows = manager.max_attention_window_vec + if slot >= len(windows): + raise ValueError(f"the KV cache manager has no retention window for layer {layer_idx}") + retained = windows[slot] + required = ( + manager.max_seq_len + if attention_window is None + else min(attention_window, manager.max_seq_len) + ) + if retained is not None and int(retained) < required: + raise ValueError( + f"layer {layer_idx} requires cache retention of at least {required} tokens; " + f"the cache keeps a window of {retained}" + ) expected = torch.arange( past, past + num_tokens, dtype=position_ids.dtype, device=position_ids.device ) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/ops/moe/beam_dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/beam/ops/moe/beam_dispatch_combine.py index 56c42284df3c..663d2ae3776c 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/ops/moe/beam_dispatch_combine.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/ops/moe/beam_dispatch_combine.py @@ -34,7 +34,8 @@ class BeamDispatchCombine(OpWrapper): The provider is called once per selected expert, with at least one row; an expert no token selected is never called. Returns `output`. `expert_ids`, `mixing_weights` and `hidden_states` are read, never written; the weights are used as given, without - re-normalization or bias. + re-normalization or bias. Expert ids must be nonnegative and distinct within each + token; ``is_valid`` checks these preconditions when catalog validation is enabled. """ ARCHS = frozenset({Arch.SM_103}) @@ -94,6 +95,21 @@ def __call__( acc.index_add_(0, expert_tokens, route.float()) return (acc * routed_scaling_factor).to(torch.bfloat16) + def is_valid( + self, + hidden_states: torch.Tensor, + expert_ids: torch.Tensor, + mixing_weights: torch.Tensor, + expert_provider: Callable[[int, torch.Tensor], torch.Tensor], + routed_scaling_factor: float, + ) -> None: + """Check routing values only when catalog validation is enabled.""" + assert torch.all(expert_ids >= 0), "expert_ids must be nonnegative" + ordered = expert_ids.sort(dim=-1).values + assert torch.all(ordered[:, 1:] != ordered[:, :-1]), ( + "expert_ids must be distinct within each token" + ) + def reference( self, hidden_states: torch.Tensor, diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py index 164a5acf80c7..45716a2b9aa6 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py @@ -220,6 +220,7 @@ def _check_step( head_dim=self.head_dim, device=self.input_layernorm.device, max_tokens=MAX_TOKENS, + attention_window=self.window, ) if not self._loaded: raise ValueError("load_weights must complete before running the block") diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py index 66a1439fa1b3..dd4d9d5c5247 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py @@ -248,6 +248,7 @@ def _check_step( head_dim=self.head_dim, device=self.input_layernorm.device, max_tokens=MAX_TOKENS, + attention_window=self.window, ) def step_args(self, md: TrtllmAttentionMetadata) -> dict: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py index bd2d57f75149..7f2b41d4dc60 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py @@ -515,3 +515,79 @@ def test_bringup_helpers_and_tensor_captures_are_not_runtime_apis(): assert not names.intersection({"capture", "layer_capture"}), source elif isinstance(node, ast.ImportFrom): assert not (node.module or "").startswith("_components"), source + + +@pytest.mark.parametrize( + "ids,error", + [ + ([[1, 0], [0, 2]], None), + ([[-1, 0]], "nonnegative"), + ([[1, 1]], "distinct within each token"), + ], +) +def test_dispatch_routing_preconditions(ids, error): + import torch + + from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( + beam_dispatch_combine as op, + ) + + ids = torch.tensor(ids, dtype=torch.int64) + hidden = torch.ones(ids.shape[0], 4, dtype=torch.bfloat16) + weights = torch.full(ids.shape, 0.5, dtype=torch.float32) + + def provider(expert, rows): + return rows * (expert + 1) + + args = (hidden, ids, weights, provider, 2.5) + if error is not None: + with pytest.raises(AssertionError, match=error): + op.is_valid(*args) + else: + op.is_valid(*args) + assert torch.equal(op(*args), op.reference(*args)) + + +@pytest.mark.parametrize( + "attention_window,max_seq_len,retained,accepted", + [ + (512, 8192, 511, False), + (512, 8192, 512, True), + (512, 8192, 1024, True), + (512, 128, 128, True), + (512, 128, 127, False), + (512, 8192, None, True), + (None, 8192, 512, False), + (None, 8192, 8192, True), + (None, 8192, None, True), + ], +) +def test_standalone_block_cache_retention( + monkeypatch, attention_window, max_seq_len, retained, accepted +): + import torch + + from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention + + # Metadata geometry is checked separately; use a non-identity logical-to-local mapping. + monkeypatch.setattr(_attention, "check_metadata", lambda *args, **kwargs: 0) + md = NS( + kv_cache_manager=NS( + layer_offsets={5: 1}, max_attention_window_vec=[0, retained], max_seq_len=max_seq_len + ) + ) + args = (torch.ones(1, 2, 4, dtype=torch.bfloat16), torch.arange(2).reshape(1, 2), md) + kwargs = dict( + layer_idx=5, + hidden_size=4, + num_kv_heads=2, + head_dim=128, + device=torch.device("cpu"), + max_tokens=17, + attention_window=attention_window, + ) + if accepted: + _attention.check_step(*args, **kwargs) + else: + with pytest.raises(ValueError, match="requires cache retention"): + _attention.check_step(*args, **kwargs) From 86937ede518eb2861343dcf217f48892644481d1 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 16:15:53 -0800 Subject: [PATCH 25/33] Align Beam TP1 block tests with served normalization and activation Signed-off-by: Maxim Gluhovskoi --- .../beam/_components/dense_block/block.py | 18 ++++++++------ .../dense_block/test_dense_block.py | 22 +++++++++-------- .../test_small_model_prefill.py | 24 ++++++++++++------- .../beam/_components/sparse_block/block.py | 10 +++++--- .../sparse_block/test_sparse_block.py | 6 ++++- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py index 45716a2b9aa6..921ed30118dd 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py @@ -39,14 +39,18 @@ import torch from torch import nn -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.modeling_v2.catalog.activation.flashinfer_silu_and_mul import ( # noqa: E501 - flashinfer_silu_and_mul, -) from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) from tensorrt_llm._torch.modeling_v2.models.beam import _attention from tensorrt_llm._torch.modeling_v2.models.beam._constants import SM as _SM +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( + swiglu_fp32, +) +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( + beam_rmsnorm_fp32, +) from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, depth_scales, @@ -259,7 +263,7 @@ def feed_forward(self, h: torch.Tensor) -> torch.Tensor: """Normalized input [T, hidden] -> dense SwiGLU output [T, hidden], before its post-norm.""" gate = cublas_mm(h, self.mlp_gate_proj.t()) up = cublas_mm(h, self.mlp_up_proj.t()) - act = flashinfer_silu_and_mul(torch.cat((gate, up), dim=-1)) + act = swiglu_fp32(gate, up)["act"] return cublas_mm(act, self.mlp_down_proj.t()) @torch.no_grad() @@ -275,7 +279,7 @@ def forward( x = hidden_states.view(num_tokens, self.hidden_size) positions = position_ids.view(num_tokens) - h = flashinfer_rmsnorm(x, self.input_layernorm, self.eps) + h = beam_rmsnorm_fp32(x, self.input_layernorm, self.eps) h = self.attention(h, positions, step) h = flashinfer_rmsnorm(h, self.post_attention_layernorm, self.eps) x = beam_depth_residual(h, x, *self.attention_scales) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 2869c100f942..9010d1b8b404 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -59,9 +59,6 @@ from _components.dense_block import block as block_module # noqa: E501 from _components.dense_block.block import BeamDenseBlockBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.activation.flashinfer_silu_and_mul import ( # noqa: E501 - flashinfer_silu_and_mul, -) from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( thop_attention, ) @@ -70,12 +67,18 @@ flashinfer_rmsnorm, ) from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( + swiglu_fp32, +) from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( beam_neox_rope, ) from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( beam_output_gate, ) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( + beam_rmsnorm_fp32, +) from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) @@ -156,7 +159,7 @@ #: The forward's op calls, in order: the composition this component claims to run. EXPECTED_CALLS = ( - "FlashinferRmsnorm", + "BeamRmsnormFp32", "CublasMm", "CublasMm", "CublasMm", @@ -172,19 +175,20 @@ "FlashinferRmsnorm", "CublasMm", "CublasMm", - "FlashinferSiluAndMul", + "SwigluFp32", "CublasMm", "FlashinferRmsnorm", "BeamDepthResidual", ) OPS = ( + beam_rmsnorm_fp32, flashinfer_rmsnorm, cublas_mm, beam_neox_rope, thop_attention, beam_output_gate, beam_depth_residual, - flashinfer_silu_and_mul, + swiglu_fp32, ) assert torch.cuda.is_available(), "the Beam dense block requires a CUDA device" @@ -355,7 +359,7 @@ def check(name: str, rule: str | None, actual: torch.Tensor, expected: str) -> d check( "input_layernorm", "rmsnorm", - flashinfer_rmsnorm(x, w["input_layernorm.weight"], eps), + beam_rmsnorm_fp32(x, w["input_layernorm.weight"], eps), "input_layernorm", ) ] @@ -467,9 +471,7 @@ def check(name: str, rule: str | None, actual: torch.Tensor, expected: str) -> d ) ) out.append(check("mlp_up", "projection", cublas_mm(h2, w["mlp.up_proj.weight"].t()), "mlp_up")) - act = flashinfer_silu_and_mul( - torch.cat((flat("mlp_gate", inter), flat("mlp_up", inter)), dim=-1) - ) + act = swiglu_fp32(flat("mlp_gate", inter), flat("mlp_up", inter))["act"] out.append(check("mlp_act", "swiglu_activation", act, "mlp_act")) out.append( check( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index 2de4fdd7d005..4751386c55f3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -49,9 +49,6 @@ BeamSixLayerBf16Sm103Tp1, prefill_context, ) -from tensorrt_llm._torch._experimental.modeling_v2.catalog.activation.flashinfer_silu_and_mul import ( # noqa: E501 - flashinfer_silu_and_mul, -) from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( thop_attention, ) @@ -80,6 +77,9 @@ from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 beam_grouped_topk_router, ) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( + beam_rmsnorm_fp32, +) from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) @@ -106,6 +106,7 @@ #: Every catalog entry the forward can call. OPS = ( + beam_rmsnorm_fp32, flashinfer_rmsnorm, cublas_mm, beam_neox_rope, @@ -113,7 +114,6 @@ thop_attention, beam_output_gate, beam_depth_residual, - flashinfer_silu_and_mul, beam_grouped_topk_router, beam_dispatch_combine, swiglu_fp32, @@ -189,7 +189,12 @@ def recording(model): events: list = [] current = [None] saved = [] - weight_args_from = {"CublasMm": 1, "FlashinferRmsnorm": 1, "BeamGroupedTopkRouter": 1} + weight_args_from = { + "CublasMm": 1, + "FlashinferRmsnorm": 1, + "BeamRmsnormFp32": 1, + "BeamGroupedTopkRouter": 1, + } for w in OPS: cls = type(w) original = cls.__call__ @@ -659,6 +664,7 @@ def test_forward_composes_the_six_blocks(seed, length) -> None: assert bitwise_equal(embedded[0], model.embed_tokens[ids[0]]) for i in range(6): block = [c for c in calls if c["block"] == i and c["op"] != "expert"] + assert block[0]["op"] == "BeamRmsnormFp32" attention = [c for c in block if c["op"] == "ThopAttention"] (att,) = attention assert att["kwargs"]["local_layer_idx"] == i @@ -669,8 +675,8 @@ def test_forward_composes_the_six_blocks(seed, length) -> None: depth_scales(i, False), ] qkv = att["qkv"] - k_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][2]["output"] - q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][1]["output"] + k_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][1]["output"] + q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][0]["output"] packed_q = qkv[:, : 48 * 128].reshape(length, 48, 128) packed_k = qkv[:, 48 * 128 : 56 * 128].reshape(length, 8, 128) if i < 5: @@ -709,7 +715,7 @@ def test_layer5_query_scale_is_visible() -> None: model(generate_ids(17, 129), positions_of(129), md) torch.cuda.synchronize() block = [c for c in calls if c["block"] == 5] - q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][1]["output"].view(129, 48, 128) + q_norm = [c for c in block if c["op"] == "FlashinferRmsnorm"][0]["output"].view(129, 48, 128) (att,) = [c for c in block if c["op"] == "ThopAttention"] packed_q = att["qkv"][:, : 48 * 128].reshape(129, 48, 128) changed = (packed_q != q_norm).flatten(1).any(dim=1) @@ -730,7 +736,7 @@ def observe(ids): out = model(ids, positions_of(ids.shape[1]), md) torch.cuda.synchronize() pre_ff = [c["output"] for c in calls if c["block"] == 3 and c["op"] == "FlashinferRmsnorm"][ - 4 + 3 ] return out, [x for kind, _, x in events if kind == "exit"], pre_ff diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py index dd4d9d5c5247..3f914a877d55 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py @@ -54,10 +54,14 @@ import torch from torch import nn -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( + flashinfer_rmsnorm, +) from tensorrt_llm._torch.modeling_v2.models.beam import _attention from tensorrt_llm._torch.modeling_v2.models.beam._constants import SM as _SM +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( + beam_rmsnorm_fp32, +) from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, depth_scales, @@ -290,7 +294,7 @@ def forward( x = hidden_states.view(num_tokens, self.hidden_size) positions = position_ids.view(num_tokens) - h = flashinfer_rmsnorm(x, self.input_layernorm, self.eps) + h = beam_rmsnorm_fp32(x, self.input_layernorm, self.eps) h = self.attention(h, positions, step) h = flashinfer_rmsnorm(h, self.post_attention_layernorm, self.eps) x = beam_depth_residual(h, x, *self.attention_scales) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 740bda0891d9..8abc467b49a9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -89,6 +89,9 @@ from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 beam_grouped_topk_router, ) +from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( + beam_rmsnorm_fp32, +) from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) @@ -176,7 +179,7 @@ #: The forward's top-level op calls, in order: the composition this component claims to #: run. The experts' calls nest inside BeamDispatchCombine. EXPECTED_CALLS = ( - "FlashinferRmsnorm", + "BeamRmsnormFp32", "CublasMm", "CublasMm", "CublasMm", @@ -199,6 +202,7 @@ EXPERT_CALLS = ("expert", "CublasMm", "CublasMm", "SwigluFp32", "CublasMm") #: Every catalog entry the block's forward can call. OPS = ( + beam_rmsnorm_fp32, flashinfer_rmsnorm, cublas_mm, beam_neox_rope, From ba79122724148bdba73a12e54795cf6e6457b770 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 16:28:23 -0800 Subject: [PATCH 26/33] Account for router projection in Beam sparse operator trace Signed-off-by: Maxim Gluhovskoi --- .../sparse_block/test_sparse_block.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 8abc467b49a9..8fe843b30586 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -954,11 +954,22 @@ def test_forward_runs_the_component_path(record_property, ieee_fp32) -> None: assert type(blk).__module__.endswith("sparse_block.block") top = [c for c in calls if c["depth"] == 0] assert tuple(c["op"] for c in top) == EXPECTED_CALLS, [c["op"] for c in top] - # Every other call ran inside BeamDispatchCombine: right after it, one level down. + # The router owns one nested projection; the dispatcher owns the expert calls. position = names.index("BeamDispatchCombine") - inner = len(calls) - len(top) + router_position = names.index("BeamGroupedTopkRouter") + assert position == router_position + 2 + assert (calls[router_position + 1]["op"], calls[router_position + 1]["depth"]) == ( + "CublasMm", + 1, + ) + inner = len(calls) - len(top) - 1 depths = [c["depth"] for c in calls] - assert depths == [0] * (position + 1) + [1] * inner + [0] * (len(calls) - position - 1 - inner) + assert depths == ( + [0] * (router_position + 1) + + [1, 0] + + [1] * inner + + [0] * (len(calls) - position - 1 - inner) + ) router = calls[names.index("BeamGroupedTopkRouter")] ids = router["output"]["ids"] selected = sorted(set(ids.reshape(-1).tolist())) From 001c66f3d2b85120ab834358402642a2c79c68cb Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Mon, 16 Nov 2026 16:33:26 -0800 Subject: [PATCH 27/33] Document shared serving operators in Beam test helpers Signed-off-by: Maxim Gluhovskoi --- .../_torch/modeling_v2/models/beam/_components/README.md | 9 +++++---- .../models/beam/_components/dense_block/block.py | 4 ++-- .../models/beam/_components/sparse_block/block.py | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md index 347a24a9b49c..8affd2039208 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md @@ -10,10 +10,11 @@ supports the local cached-generation tests. Use `LLM()` or `trtllm-serve` for se Run tests through pytest with the repository's test import plugin. Existing L0 selectors retain the small single-GPU tests; the large component suites remain manual-only. -TP1 normalization and activation use their FlashInfer contracts; the TP4 input norm -and dense MLP use explicit FP32 Beam operators, so BF16 rounding can differ. TP1 also -accepts a narrower RoPE schema. Compare each helper against its declared contract, -not against the TP4 model as a bitwise reference. +The dense and sparse helpers call the serving family's shared attention function, +input normalization (`beam_rmsnorm_fp32`), and activation (`swiglu_fp32`); the other +normalizations use the same FlashInfer entries as serving. Their TP1 geometry and +narrower RoPE schema remain distinct from TP4. Compare each helper against its +declared contract, not against the TP4 model as a bitwise reference. ## Manual QA reports diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py index 921ed30118dd..cdde5e05ddb9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Beam decoder blocks 0-2 / sm_103 / tp1: one dense, sliding-attention block, constructed directly. -This TP1 component uses FlashInfer normalization/activation kernels; see ``../README.md`` -for its numerical differences from the TP4 serving model. +This TP1 component shares attention, normalization and activation operators with serving; +see ``../README.md`` for the remaining geometry and configuration differences. A component, not a routed target. There is no ``routing.py``, no router-index entry and no ``ForCausalLM``: nothing in the engine builds this class, and it serves no request. diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py index 3f914a877d55..00d97bc41406 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Beam decoder blocks 3-5 / sm_103 / tp1: one sparse (MoE) block, constructed directly. -This TP1 component uses FlashInfer normalization/activation kernels; see ``../README.md`` -for its numerical differences from the TP4 serving model. +This TP1 component shares attention, normalization and activation operators with serving; +see ``../README.md`` for the remaining geometry and configuration differences. A component, not a routed target. There is no ``routing.py``, no router-index entry and no ``ForCausalLM``: nothing in the engine builds this class, and it serves no request. @@ -12,8 +12,8 @@ -- a context step after any cached prefix, or a one-token generation step -- with the attention batch state prepared on a ``KVCacheManagerV2`` that holds the block's logical layer. The forward is assembled from catalog entries, the attention sublayer -shared with the dense block component (``../_attention.py``) and the MoE component -(``../moe``); it does not call the built-in Beam modeling and owns no +shared with the dense block and serving in the production family ``_attention.py``. +Its MoE execution uses the checkout-only ``../moe`` helper; it does not call the built-in Beam modeling and owns no dense MLP. The block, with block 3's residual coefficients (block i's are in ``depth_scales``):: From 82ed6c94f7e7717139f4faab2f2881df093e237c Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Tue, 17 Nov 2026 11:30:48 -0800 Subject: [PATCH 28/33] [infra] Complete Beam ModelingV2 package migration Follow upstream #22286: migrate added Beam modules, imports, documentation and test references to tensorrt_llm._torch.modeling_v2. Preserve the package-location guard and current upstream test selection. Validation: 83 tracked package/CI-selection checks and 90 host-adapted component checks per branch; pre-commit passed. Native GPU/distributed qualification remains separate. Advisory Pyrefly unavailable. Signed-off-by: Maxim Gluhovskoi --- .../models/beam/_tp4/component_bf16.py | 11 ++----- .../models/beam/dispatch_combine.py | 2 +- .../beam/_components/dense_block/block.py | 13 +++----- .../dense_block/test_dense_block.py | 30 ++++++----------- .../models/beam/_components/moe/test_moe.py | 10 +++--- .../_components/moe_expert/test_expert.py | 4 +-- .../beam/_components/moe_router/router.py | 2 +- .../_components/moe_router/test_router.py | 2 +- .../six_layer_model/test_cached_generation.py | 6 ++-- .../test_small_model_prefill.py | 32 +++++++------------ .../beam/_components/sparse_block/block.py | 9 ++---- .../sparse_block/test_sparse_block.py | 30 +++++++---------- .../test_checkpoint_layout.py | 12 +++---- .../test_checkpoint_layout_nvfp4.py | 6 ++-- .../models/beam/test_component_contracts.py | 4 +-- .../models/beam/test_diagnostic_mode.py | 10 +++--- .../models/beam/test_dispatch_combine.py | 6 ++-- .../models/beam/test_distributed_loading.py | 16 ++++------ 18 files changed, 76 insertions(+), 129 deletions(-) diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py index bab6f5682562..7bc4733d126b 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/_tp4/component_bf16.py @@ -44,20 +44,14 @@ from torch import nn from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.distributed import AllReduce, allgather from tensorrt_llm._torch.modeling_v2._router_index import ( MODELING_V2_VALIDATE_ENV, step_contract_enabled, ) from tensorrt_llm._torch.modeling_v2.catalog.comm.allgather import allgather -from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import ( - allreduce, - allreduce_group_args, -) +from tensorrt_llm._torch.modeling_v2.catalog.comm.allreduce import allreduce, allreduce_group_args from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( beam_logit_softcap, ) @@ -70,7 +64,6 @@ beam_depth_residual, depth_scales, ) -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm.functional import AllReduceFusionOp from tensorrt_llm.mapping import Mapping as ParallelMapping diff --git a/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py b/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py index 6e80ce407d6e..1ad98d22f6ce 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py +++ b/tensorrt_llm/_torch/modeling_v2/models/beam/dispatch_combine.py @@ -19,7 +19,7 @@ normalization or collective is applied. The combination follows the Beam family operator -``tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine``:: +``tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine``:: for each selected expert e, ascending: out = expert_provider(e, hidden[tokens routed to e]) [n, 5120] bf16 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py index cdde5e05ddb9..7b03809ae9e9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/block.py @@ -39,18 +39,13 @@ import torch from torch import nn +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm from tensorrt_llm._torch.modeling_v2.models.beam import _attention from tensorrt_llm._torch.modeling_v2.models.beam._constants import SM as _SM -from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( - swiglu_fp32, -) -from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( - beam_rmsnorm_fp32, -) +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import swiglu_fp32 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import beam_rmsnorm_fp32 from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, depth_scales, diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 9010d1b8b404..052ddc543883 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -59,30 +59,20 @@ from _components.dense_block import block as block_module # noqa: E501 from _components.dense_block.block import BeamDenseBlockBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( - thop_attention, -) -from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( - swiglu_fp32, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( - beam_neox_rope, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.models.beam import _attention +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import swiglu_fp32 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_neox_rope import beam_neox_rope +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_output_gate import ( beam_output_gate, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( - beam_rmsnorm_fp32, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import beam_rmsnorm_fp32 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.resource_manager import DataType from tensorrt_llm.mapping import Mapping diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py index 93a071479562..c6fdafcdfef6 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -59,15 +59,15 @@ from _components.moe.moe import BeamMoeBf16Sm103Tp1 from _components.moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 # noqa: E501 from _components.moe_router.router import BeamMoeRouterBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch._experimental.modeling_v2.models.beam import dispatch_combine # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.models.beam import dispatch_combine # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 swiglu_fp32, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 beam_dispatch_combine, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 beam_grouped_topk_router, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py index 0b1c1e9c2824..4192de7c29c3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -36,8 +36,8 @@ from _validating import validating # noqa: E402 from _components.moe_expert.expert import WEIGHT_SHAPES, BeamMoeExpertBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import CublasMm, cublas_mm -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import CublasMm, cublas_mm +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 SwigluFp32, swiglu_fp32, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py index 7b9b4ad35240..935396f90c79 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/router.py @@ -12,7 +12,7 @@ that combine and is not applied here. It does not call the built-in Beam modeling. The routing uses FP32 at every step, through the Beam family operator -``tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router``:: +``tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router``:: logits = hidden.float() @ weight^T [T, 256] scores = sigmoid(logits) unbiased: the weights use these diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index 206a66304f37..c3c5744d0266 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -59,7 +59,7 @@ from _validating import validating # noqa: E402 from _components.moe_router.router import NORM_EPS, BeamMoeRouterBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 BeamGroupedTopkRouter, beam_grouped_topk_router, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py index 331c5c0c2163..3b6ff5f68d93 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py @@ -62,13 +62,11 @@ from _components.six_layer_model import generation # noqa: E501 from _components.six_layer_model import model as model_module # noqa: E501 from _components.six_layer_model.generation import BeamSmallModelGenerator, Lifecycle # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( - thop_attention, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention +from tensorrt_llm._torch.modeling_v2.models.beam import _attention from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState from tensorrt_llm._torch.pyexecutor.sampler import TorchSampler from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index 4751386c55f3..cde622fd010d 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -49,38 +49,30 @@ BeamSixLayerBf16Sm103Tp1, prefill_context, ) -from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( - thop_attention, -) -from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( # noqa: E501 beam_logit_softcap, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 swiglu_fp32, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( - beam_neox_rope, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_nope_query_scale import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_neox_rope import beam_neox_rope +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_nope_query_scale import ( # noqa: E501 beam_nope_query_scale, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_output_gate import ( beam_output_gate, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 beam_dispatch_combine, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 beam_grouped_topk_router, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( - beam_rmsnorm_fp32, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import beam_rmsnorm_fp32 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py index 00d97bc41406..a5d0b4f28349 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py @@ -54,14 +54,11 @@ import torch from torch import nn -from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm from tensorrt_llm._torch.modeling_v2.models.beam import _attention from tensorrt_llm._torch.modeling_v2.models.beam._constants import SM as _SM -from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( - beam_rmsnorm_fp32, -) +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import beam_rmsnorm_fp32 from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, depth_scales, diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 8fe843b30586..972b259bc824 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -67,35 +67,27 @@ from _components.moe.moe import BeamMoeBf16Sm103Tp1 from _components.sparse_block import block as block_module # noqa: E501 from _components.sparse_block.block import BeamSparseBlockBf16Sm103Tp1 # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.catalog.attention.thop_attention import ( - thop_attention, -) -from tensorrt_llm._torch._experimental.modeling_v2.catalog.gemm.cublas_mm import cublas_mm -from tensorrt_llm._torch._experimental.modeling_v2.catalog.norm.flashinfer_rmsnorm import ( - flashinfer_rmsnorm, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention +from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm +from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 swiglu_fp32, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_neox_rope import ( - beam_neox_rope, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.attention.beam_output_gate import ( +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_neox_rope import beam_neox_rope +from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_output_gate import ( beam_output_gate, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 beam_dispatch_combine, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_grouped_topk_router import ( # noqa: E501 beam_grouped_topk_router, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import ( - beam_rmsnorm_fp32, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( +from tensorrt_llm._torch.modeling_v2.models.beam.ops.norm.beam_rmsnorm_fp32 import beam_rmsnorm_fp32 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.residual.beam_depth_residual import ( beam_depth_residual, ) -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.resource_manager import DataType from tensorrt_llm.mapping import Mapping diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index 909cb788d4e7..dea1f52cded3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -30,13 +30,9 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 - layout_bf16 as layout, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 - loader_bf16 as loader, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4.layout_bf16 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import layout_bf16 as layout # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import loader_bf16 as loader # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam._tp4.layout_bf16 import ( # noqa: E501 BF16, F32, FP8, @@ -533,7 +529,7 @@ def _sm103() -> bool: @pytest.mark.skipif(not _sm103(), reason="the component is built for sm_103") def test_mapping_rejections_and_weight_state_guard() -> None: - from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4.component_bf16 import ( # noqa: E501 + from tensorrt_llm._torch.modeling_v2.models.beam._tp4.component_bf16 import ( # noqa: E501 BeamCheckpointLayoutBf16Sm103Tp4, ) from tensorrt_llm.mapping import Mapping diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py index 102529871c1b..29c4235902e3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout_nvfp4.py @@ -30,10 +30,10 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import ( # noqa: E501 layout_bf16 as bf16_layout, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 layout, loader, ) @@ -520,7 +520,7 @@ def _assert_gemm(out: torch.Tensor, a_codes, a_sf, w_packed, w_sf, alpha) -> Non @needs_sm103 @pytest.mark.parametrize("shared_fc1", [True, False]) def test_expert_provider_matches_independent_reference(shared_fc1, monkeypatch) -> None: - from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + from tensorrt_llm._torch.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 expert as provider, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py index 7f2b41d4dc60..914cac9b4c52 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_component_contracts.py @@ -528,7 +528,7 @@ def test_bringup_helpers_and_tensor_captures_are_not_runtime_apis(): def test_dispatch_routing_preconditions(ids, error): import torch - from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( + from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( beam_dispatch_combine as op, ) @@ -567,7 +567,7 @@ def test_standalone_block_cache_retention( ): import torch - from tensorrt_llm._torch._experimental.modeling_v2.models.beam import _attention + from tensorrt_llm._torch.modeling_v2.models.beam import _attention # Metadata geometry is checked separately; use a non-identity logical-to-local mapping. monkeypatch.setattr(_attention, "check_metadata", lambda *args, **kwargs: 0) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py index b75120e6d56b..99e5cedaa357 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_diagnostic_mode.py @@ -19,11 +19,9 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2._router_index import MODELING_V2_VALIDATE_ENV -from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( # noqa: E501 - component_bf16 as bf16, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2._router_index import MODELING_V2_VALIDATE_ENV +from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import component_bf16 as bf16 # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 component as nvfp4, ) @@ -207,7 +205,7 @@ def stop(md: object, layer: int) -> NoReturn: def test_full_model_validates_once_and_marks_every_block_call_checked( monkeypatch: pytest.MonkeyPatch, ) -> None: - from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 + from tensorrt_llm._torch.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( # noqa: E501 model as full, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py index b0b4f790cfeb..a2481477054c 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py @@ -28,11 +28,11 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2.models.beam import dispatch_combine # noqa: E501 -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.dispatch_combine import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam import dispatch_combine # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.dispatch_combine import ( # noqa: E501 dispatch_and_combine, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 +from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 beam_dispatch_combine, ) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py index 0a554c6fd1fa..7bbd7424459b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py @@ -30,15 +30,13 @@ def test_distributed_loading(tmp_path, monkeypatch, packed): from safetensors.torch import save_file - from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( - component_bf16 as bf16, - ) - from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import layout_bf16 as layout - from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import loader_bf16 as loader - from tensorrt_llm._torch._experimental.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( + from tensorrt_llm._torch.distributed.communicator import Distributed + from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import component_bf16 as bf16 + from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import layout_bf16 as layout + from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import loader_bf16 as loader + from tensorrt_llm._torch.modeling_v2.models.beam.full_model_nvfp4__sm_103__tp4 import ( component as nvfp4, ) - from tensorrt_llm._torch.distributed.communicator import Distributed from tensorrt_llm.mapping import Mapping rank = mpi_rank() @@ -114,9 +112,7 @@ def test_catalog_collectives() -> None: if mpi_disabled() or mpi_world_size() != 4 or torch.cuda.device_count() < 4: pytest.skip("requires MPI with four ranks and four visible GPUs on one node") - from tensorrt_llm._torch._experimental.modeling_v2.models.beam._tp4 import ( - component_bf16 as bf16, - ) + from tensorrt_llm._torch.modeling_v2.models.beam._tp4 import component_bf16 as bf16 from tensorrt_llm.mapping import Mapping rank = mpi_rank() From 44fb47691743c97b5f0468cb8e4b86e22f1ab5ac Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Tue, 17 Nov 2026 16:17:31 -0800 Subject: [PATCH 29/33] Skip Beam component suites on unsupported CUDA devices Signed-off-by: Maxim Gluhovskoi --- .../beam/_components/dense_block/test_dense_block.py | 5 +++-- .../modeling_v2/models/beam/_components/moe/test_moe.py | 5 +++-- .../models/beam/_components/moe_expert/test_expert.py | 5 +++-- .../models/beam/_components/moe_router/test_router.py | 5 +++-- .../_components/six_layer_model/test_cached_generation.py | 7 +++++-- .../six_layer_model/test_small_model_prefill.py | 5 +++-- .../beam/_components/sparse_block/test_sparse_block.py | 5 +++-- 7 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 052ddc543883..36df360a7b32 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -44,6 +44,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + try: import tensorrt_llm._torch.models.modeling_beam as builtin_beam except ModuleNotFoundError as exc: @@ -181,8 +184,6 @@ swiglu_fp32, ) -assert torch.cuda.is_available(), "the Beam dense block requires a CUDA device" - def block_config(**overrides) -> types.SimpleNamespace: fields = dict(BLOCK_CONFIG) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py index c6fdafcdfef6..bb3f923e8155 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py @@ -36,6 +36,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + try: import tensorrt_llm._torch.models.modeling_beam as builtin_beam except ModuleNotFoundError as exc: @@ -71,8 +74,6 @@ beam_grouped_topk_router, ) -assert torch.cuda.is_available(), "the Beam MoE tests require a CUDA device" - pytestmark = pytest.mark.high_cuda_memory # -------------------------------------------------- the fixed configuration diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py index 4192de7c29c3..c36103fc7838 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -29,6 +29,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + __extra_import_path__ = [".", "../..", "../../../.."] import _expert_cases as cases # noqa: E402 @@ -42,8 +45,6 @@ swiglu_fp32, ) -assert torch.cuda.is_available(), "the Beam MoE expert tests require a CUDA device" - BF16 = torch.bfloat16 _WEIGHT_BYTES = sum(2 * rows * cols for rows, cols in WEIGHT_SHAPES.values()) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index c3c5744d0266..cbd85171e95d 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -44,6 +44,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + try: import tensorrt_llm._torch.models.modeling_beam as builtin_beam except ModuleNotFoundError as exc: @@ -64,8 +67,6 @@ beam_grouped_topk_router, ) -assert torch.cuda.is_available(), "the Beam MoE router tests require a CUDA device" - #: The router as the checkpoint configures it. Only these fields are read. ROUTER_CONFIG = types.MappingProxyType( dict( diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py index 3b6ff5f68d93..530b9bf9686e 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py @@ -41,6 +41,8 @@ Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 130 GiB of free device memory (generated state and loaded model). No model files. +This is a manual QA suite, not scheduled CI; ensure that memory is available before running. +The high_cuda_memory marker is a label, not a free-memory check. """ from __future__ import annotations @@ -54,6 +56,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + __extra_import_path__ = ["../.."] import _beam_component_support as support # noqa: E402 @@ -73,8 +78,6 @@ from tensorrt_llm._torch.pyexecutor.seq_slot_manager import SeqSlotManager from tensorrt_llm.bindings.executor import FinishReason -assert torch.cuda.is_available(), "the Beam six-block model requires a CUDA device" - pytestmark = pytest.mark.high_cuda_memory VOCAB = 151936 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py index cde622fd010d..c76f506c7977 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py @@ -39,6 +39,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + __extra_import_path__ = ["../.."] import _beam_component_support as support # noqa: E402 @@ -80,8 +83,6 @@ from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.mapping import Mapping -assert torch.cuda.is_available(), "the Beam six-block model requires a CUDA device" - pytestmark = pytest.mark.high_cuda_memory # -------------------------------------------------- the fixed configuration diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py index 972b259bc824..c009ae7a4d90 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py @@ -43,6 +43,9 @@ import pytest import torch +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) + try: import tensorrt_llm._torch.models.modeling_beam as builtin_beam except ModuleNotFoundError as exc: @@ -92,8 +95,6 @@ from tensorrt_llm._torch.pyexecutor.resource_manager import DataType from tensorrt_llm.mapping import Mapping -assert torch.cuda.is_available(), "the Beam sparse block requires a CUDA device" - pytestmark = pytest.mark.high_cuda_memory # -------------------------------------------------- the fixed configuration From 479e66dc148c71ea3fb60ef3587fd6c52c0fe89f Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Tue, 17 Nov 2026 16:25:24 -0800 Subject: [PATCH 30/33] Isolate collectives in single-GPU Beam weight-state test Signed-off-by: Maxim Gluhovskoi --- .../full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index dea1f52cded3..bd71be4c2d6b 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -528,12 +528,15 @@ def _sm103() -> bool: @pytest.mark.skipif(not _sm103(), reason="the component is built for sm_103") -def test_mapping_rejections_and_weight_state_guard() -> None: +def test_mapping_rejections_and_weight_state_guard(monkeypatch: pytest.MonkeyPatch) -> None: from tensorrt_llm._torch.modeling_v2.models.beam._tp4.component_bf16 import ( # noqa: E501 BeamCheckpointLayoutBf16Sm103Tp4, ) from tensorrt_llm.mapping import Mapping + # This single-GPU test checks rank-local state, not inter-rank communication. + monkeypatch.setattr(BeamCheckpointLayoutBf16Sm103Tp4, "_init_collectives", lambda self: None) + cfg = model_config() for bad in ( dict(world_size=2, rank=0, tp_size=2, moe_tp_size=2), From a98df36ee5d281dcc6d81151312dab947311acce Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Wed, 18 Nov 2026 10:55:51 -0800 Subject: [PATCH 31/33] Move manual Beam qualification suites into integration tests Signed-off-by: Maxim Gluhovskoi --- tests/integration/defs/beam/README.md | 43 +++++++++++++++++++ tests/integration/defs/beam/__init__.py | 2 + .../defs/beam/_components/__init__.py | 2 + .../defs}/beam/_components/moe/__init__.py | 0 .../beam/_components/moe/_moe_reference.py | 0 .../defs}/beam/_components/moe/moe.py | 10 +++-- .../defs}/beam/_components/moe/test_moe.py | 15 ++++--- .../_components/six_layer_model/__init__.py | 0 .../_components/six_layer_model/generation.py | 0 .../beam/_components/six_layer_model/model.py | 5 ++- .../six_layer_model/test_cached_generation.py | 16 ++++--- .../test_small_model_prefill.py | 13 +++--- .../beam/_components/sparse_block/__init__.py | 0 .../sparse_block/_sparse_block_reference.py | 5 ++- .../beam/_components/sparse_block/block.py | 0 .../sparse_block/test_sparse_block.py | 16 ++++--- .../defs}/beam/test_distributed_loading.py | 4 +- .../test_lists/test-db/l0_b300.yml | 2 +- .../test-db/manual_b300_beam_components.yml | 21 --------- .../test-db/manual_b300_beam_distributed.yml | 19 -------- .../models/beam/_beam_component_support.py | 5 ++- .../models/beam/_components/README.md | 7 +-- 22 files changed, 111 insertions(+), 74 deletions(-) create mode 100644 tests/integration/defs/beam/README.md create mode 100644 tests/integration/defs/beam/__init__.py create mode 100644 tests/integration/defs/beam/_components/__init__.py rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/moe/__init__.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/moe/_moe_reference.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/moe/moe.py (96%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/moe/test_moe.py (99%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/six_layer_model/__init__.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/six_layer_model/generation.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/six_layer_model/model.py (99%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/six_layer_model/test_cached_generation.py (98%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/six_layer_model/test_small_model_prefill.py (98%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/sparse_block/__init__.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/sparse_block/_sparse_block_reference.py (99%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/sparse_block/block.py (100%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/_components/sparse_block/test_sparse_block.py (98%) rename tests/{unittest/_torch/modeling_v2/models => integration/defs}/beam/test_distributed_loading.py (97%) delete mode 100644 tests/integration/test_lists/test-db/manual_b300_beam_components.yml delete mode 100644 tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml diff --git a/tests/integration/defs/beam/README.md b/tests/integration/defs/beam/README.md new file mode 100644 index 000000000000..191484657d86 --- /dev/null +++ b/tests/integration/defs/beam/README.md @@ -0,0 +1,43 @@ + + +# Manual Beam qualification + +These checkout-only integration suites are run explicitly with pytest. They are +not scheduled in L0, nightly or LR CI. Lightweight component/operator tests stay +under `tests/unittest/_torch/modeling_v2/models/beam`; the registered model's +named accuracy gate stays under `tests/integration/defs/accuracy/modeling_v2/beam`. + +The TP1 helpers here compose the small components from the unit-test tree. They +are not serving APIs or a bitwise reference for TP4. Use `LLM()` or `trtllm-serve` +for serving. Tests retain their own numerical contracts and optional source-capture +requirements; unavailable optional captures are reported as skips. + +Run from the repository root with a matching installed/native TRT-LLM build. +The single-GPU suites require SM103 and generated weights, not a checkpoint: + +| Suite | Approximate free device memory | Command | +| --- | --- | --- | +| MoE | 60 GiB | `python -m pytest tests/integration/defs/beam/_components/moe -v` | +| Sparse block | 60 GiB | `python -m pytest tests/integration/defs/beam/_components/sparse_block -v` | +| Six-layer prefill | 200 GiB | `python -m pytest tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py -v` | +| Cached generation | 130 GiB | `python -m pytest tests/integration/defs/beam/_components/six_layer_model/test_cached_generation.py -v` | + +Run the distributed suite separately with four ranks and four visible GPUs on +one node. It uses generated tensors rather than a model checkpoint; a two-minute +outer timeout bounds collective failures and process startup: + +```bash +env -u TLLM_DISABLE_MPI timeout 120s mpirun -n 4 python -m pytest -q \ + tests/integration/defs/beam/test_distributed_loading.py +``` + +For optional QA reports, pass an explicit suite to the existing wrapper: + +```bash +python tests/scripts/beam_component_qa.py --output /tmp/beam-moe-qa -- \ + tests/integration/defs/beam/_components/moe -v +``` + +Use a new report directory outside the repository. Reports contain local runtime +and scheduler metadata. Inspect skipped cases and call/teardown results rather +than treating successful collection or setup as qualification. diff --git a/tests/integration/defs/beam/__init__.py b/tests/integration/defs/beam/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tests/integration/defs/beam/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/integration/defs/beam/_components/__init__.py b/tests/integration/defs/beam/_components/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tests/integration/defs/beam/_components/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/__init__.py b/tests/integration/defs/beam/_components/moe/__init__.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/moe/__init__.py rename to tests/integration/defs/beam/_components/moe/__init__.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py b/tests/integration/defs/beam/_components/moe/_moe_reference.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/moe/_moe_reference.py rename to tests/integration/defs/beam/_components/moe/_moe_reference.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/moe.py b/tests/integration/defs/beam/_components/moe/moe.py similarity index 96% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/moe/moe.py rename to tests/integration/defs/beam/_components/moe/moe.py index eb712a2a8c34..2b7ba9e0fa68 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/moe.py +++ b/tests/integration/defs/beam/_components/moe/moe.py @@ -39,10 +39,12 @@ from tensorrt_llm._torch.modeling_v2.models.beam import dispatch_combine from tensorrt_llm._torch.modeling_v2.models.beam._constants import SM as _SM -from ..moe_expert.expert import WEIGHT_SHAPES as EXPERT_WEIGHT_SHAPES -from ..moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 -from ..moe_router.router import BeamMoeRouterBf16Sm103Tp1 -from ..moe_router.router import check_config as check_router_config +__extra_import_path__ = ["~/tests/unittest/_torch/modeling_v2/models/beam"] + +from _components.moe_expert.expert import WEIGHT_SHAPES as EXPERT_WEIGHT_SHAPES +from _components.moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 +from _components.moe_router.router import BeamMoeRouterBf16Sm103Tp1 +from _components.moe_router.router import check_config as check_router_config #: Fields of the checkpoint's configuration this component reads beyond the router's. SUPPORTED_CONFIG = { diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py b/tests/integration/defs/beam/_components/moe/test_moe.py similarity index 99% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py rename to tests/integration/defs/beam/_components/moe/test_moe.py index bb3f923e8155..72e0f4cd21e7 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe/test_moe.py +++ b/tests/integration/defs/beam/_components/moe/test_moe.py @@ -18,7 +18,7 @@ Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE \\ - python -m pytest tests/unittest/_torch/modeling_v2/models/beam/_components/moe + python -m pytest tests/integration/defs/beam/_components/moe QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). @@ -45,7 +45,12 @@ if exc.name != "tensorrt_llm._torch.models.modeling_beam": raise builtin_beam = None # The standard-serving implementation is a separate PR. -__extra_import_path__ = [".", "../..", "../../../.."] +__extra_import_path__ = [ + ".", + "~/tests/integration/defs", + "~/tests/unittest/_torch/modeling_v2/models/beam", + "~/tests/unittest/_torch/modeling_v2", +] import _beam_component_support as support # noqa: E402 import _moe_reference as reference # noqa: E402 @@ -57,11 +62,11 @@ tensor_versions, unchanged, ) -from _validating import validating # noqa: E402 - -from _components.moe.moe import BeamMoeBf16Sm103Tp1 from _components.moe_expert.expert import BeamMoeExpertBf16Sm103Tp1 # noqa: E501 from _components.moe_router.router import BeamMoeRouterBf16Sm103Tp1 # noqa: E501 +from _validating import validating # noqa: E402 +from beam._components.moe.moe import BeamMoeBf16Sm103Tp1 + from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm from tensorrt_llm._torch.modeling_v2.models.beam import dispatch_combine # noqa: E501 from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/__init__.py b/tests/integration/defs/beam/_components/six_layer_model/__init__.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/__init__.py rename to tests/integration/defs/beam/_components/six_layer_model/__init__.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/generation.py b/tests/integration/defs/beam/_components/six_layer_model/generation.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/generation.py rename to tests/integration/defs/beam/_components/six_layer_model/generation.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/model.py b/tests/integration/defs/beam/_components/six_layer_model/model.py similarity index 99% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/model.py rename to tests/integration/defs/beam/_components/six_layer_model/model.py index 52dd5b98f6ab..2ec5343829be 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/model.py +++ b/tests/integration/defs/beam/_components/six_layer_model/model.py @@ -80,7 +80,10 @@ from tensorrt_llm.mapping import Mapping as ParallelMapping from tensorrt_llm.runtime.kv_cache_manager_v2 import BAD_PAGE_INDEX, LayerId, PageIndexMode -from ..dense_block import block as dense_block +__extra_import_path__ = ["~/tests/unittest/_torch/modeling_v2/models/beam"] + +from _components.dense_block import block as dense_block + from ..moe.moe import check_weights from ..moe.moe import weight_shapes as moe_weight_shapes from ..sparse_block import block as sparse_block diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py b/tests/integration/defs/beam/_components/six_layer_model/test_cached_generation.py similarity index 98% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py rename to tests/integration/defs/beam/_components/six_layer_model/test_cached_generation.py index 530b9bf9686e..e96eb426953e 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_cached_generation.py +++ b/tests/integration/defs/beam/_components/six_layer_model/test_cached_generation.py @@ -37,7 +37,7 @@ on ``PATH``):: env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ - tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model + tests/integration/defs/beam/_components/six_layer_model Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 130 GiB of free device memory (generated state and loaded model). No model files. @@ -59,14 +59,20 @@ if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) -__extra_import_path__ = ["../.."] +__extra_import_path__ = [ + "~/tests/integration/defs", + "~/tests/unittest/_torch/modeling_v2/models/beam", +] import _beam_component_support as support # noqa: E402 from _beam_component_support import bitwise_equal # noqa: E402 +from beam._components.six_layer_model import generation # noqa: E501 +from beam._components.six_layer_model import model as model_module # noqa: E501 +from beam._components.six_layer_model.generation import ( # noqa: E501 + BeamSmallModelGenerator, + Lifecycle, +) -from _components.six_layer_model import generation # noqa: E501 -from _components.six_layer_model import model as model_module # noqa: E501 -from _components.six_layer_model.generation import BeamSmallModelGenerator, Lifecycle # noqa: E501 from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py similarity index 98% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py rename to tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py index c76f506c7977..4ba248b40be7 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py @@ -22,7 +22,7 @@ Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ - tests/unittest/_torch/modeling_v2/models/beam/_components/six_layer_model + tests/integration/defs/beam/_components/six_layer_model Dependencies: a TensorRT-LLM build for sm_103 with flashinfer importable and about 200 GiB of free device memory: the generated state and the loaded model hold 62 GiB each, and one @@ -42,16 +42,19 @@ if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): pytest.skip("Beam component tests require a CUDA device with sm_103", allow_module_level=True) -__extra_import_path__ = ["../.."] +__extra_import_path__ = [ + "~/tests/integration/defs", + "~/tests/unittest/_torch/modeling_v2/models/beam", +] import _beam_component_support as support # noqa: E402 from _beam_component_support import bitwise_equal # noqa: E402 - -from _components.six_layer_model import model as model_module # noqa: E501 -from _components.six_layer_model.model import ( # noqa: E501 +from beam._components.six_layer_model import model as model_module # noqa: E501 +from beam._components.six_layer_model.model import ( # noqa: E501 BeamSixLayerBf16Sm103Tp1, prefill_context, ) + from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/__init__.py b/tests/integration/defs/beam/_components/sparse_block/__init__.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/__init__.py rename to tests/integration/defs/beam/_components/sparse_block/__init__.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py b/tests/integration/defs/beam/_components/sparse_block/_sparse_block_reference.py similarity index 99% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py rename to tests/integration/defs/beam/_components/sparse_block/_sparse_block_reference.py index 044e0b35bec7..75b242cd4711 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/_sparse_block_reference.py +++ b/tests/integration/defs/beam/_components/sparse_block/_sparse_block_reference.py @@ -25,7 +25,10 @@ import torch import torch.nn.functional as F -__extra_import_path__ = ["../moe", "../dense_block"] +__extra_import_path__ = [ + "../moe", + "~/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block", +] import _dense_block_reference as block_reference # noqa: E402 import _moe_reference as moe_reference # noqa: E402 diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py b/tests/integration/defs/beam/_components/sparse_block/block.py similarity index 100% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/block.py rename to tests/integration/defs/beam/_components/sparse_block/block.py diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py b/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py similarity index 98% rename from tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py rename to tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py index c009ae7a4d90..d10849e82f23 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py @@ -24,7 +24,7 @@ Invocation (fp32 steps need the TF32 overrides removed before torch is imported):: env -u TORCH_ALLOW_TF32_CUBLAS_OVERRIDE -u NVIDIA_TF32_OVERRIDE python -m pytest \\ - tests/unittest/_torch/modeling_v2/models/beam/_components/sparse_block + tests/integration/defs/beam/_components/sparse_block QA reports: use ``tests/scripts/beam_component_qa.py`` (see the component README). @@ -52,7 +52,13 @@ if exc.name != "tensorrt_llm._torch.models.modeling_beam": raise builtin_beam = None # The standard-serving implementation is a separate PR. -__extra_import_path__ = [".", "../..", "../moe", "../../../.."] +__extra_import_path__ = [ + ".", + "~/tests/integration/defs", + "../moe", + "~/tests/unittest/_torch/modeling_v2/models/beam", + "~/tests/unittest/_torch/modeling_v2", +] import _beam_component_support as support # noqa: E402 import _moe_reference as moe_reference # noqa: E402 @@ -66,10 +72,10 @@ unchanged, ) from _validating import validating # noqa: E402 +from beam._components.moe.moe import BeamMoeBf16Sm103Tp1 +from beam._components.sparse_block import block as block_module # noqa: E501 +from beam._components.sparse_block.block import BeamSparseBlockBf16Sm103Tp1 # noqa: E501 -from _components.moe.moe import BeamMoeBf16Sm103Tp1 -from _components.sparse_block import block as block_module # noqa: E501 -from _components.sparse_block.block import BeamSparseBlockBf16Sm103Tp1 # noqa: E501 from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py b/tests/integration/defs/beam/test_distributed_loading.py similarity index 97% rename from tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py rename to tests/integration/defs/beam/test_distributed_loading.py index 7bbd7424459b..d8d0a264f6a9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py +++ b/tests/integration/defs/beam/test_distributed_loading.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Manual four-GPU load-error qualification with one 32-byte generated tensor. +r"""Manual four-GPU load-error qualification with one 32-byte generated tensor. Run in a matching installed/native TRT-LLM environment on one node:: env -u TLLM_DISABLE_MPI timeout 120s mpirun -n 4 python -m pytest -q \\ - tests/unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py + tests/integration/defs/beam/test_distributed_loading.py This uses real MPI error exchange, catalog diagnostic agreement and CUDA copies. Only model geometry/quantization metadata are reduced; no checkpoint, inference, diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index d8092951d32b..c294a5323772 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -163,7 +163,7 @@ l0_b300: - unittest/_torch/modeling_v2/test_modeling_v2_no_stale_claims.py TIMEOUT (30) - unittest/_torch/modeling_v2/activation TIMEOUT (30) # Beam: lightweight single-GPU entries. High-memory MoE, sparse-block and six-layer - # suites belong to manual_b300_beam_components.yml, not L0 or nightly/LR. + # suites live in tests/integration/defs/beam and run manually, outside L0/nightly/LR. # Startup-inclusive measurements on one GB300 (ARM64, retained cache): operators # 58s, GEMM/FP4 entries <=23s, expert 35s and router 26s. The B300 x86 operator # entry measured 83s in CI. Each keeps an independent 30-minute ceiling. diff --git a/tests/integration/test_lists/test-db/manual_b300_beam_components.yml b/tests/integration/test_lists/test-db/manual_b300_beam_components.yml deleted file mode 100644 index beb10e18d52b..000000000000 --- a/tests/integration/test_lists/test-db/manual_b300_beam_components.yml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -version: 0.0.1 -# High-memory generated models; explicitly selected manual qualification, not L0. -manual_b300_beam_components: -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*b300*' - linux_distribution_name: ubuntu* - terms: - stage: manual - backend: pytorch - tests: - - unittest/_torch/modeling_v2/models/beam/_components/moe TIMEOUT (45) - - unittest/_torch/modeling_v2/models/beam/_components/sparse_block TIMEOUT (45) - - unittest/_torch/modeling_v2/models/beam/_components/six_layer_model TIMEOUT (45) diff --git a/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml b/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml deleted file mode 100644 index 26a5d9c83753..000000000000 --- a/tests/integration/test_lists/test-db/manual_b300_beam_distributed.yml +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -version: 0.0.1 -manual_b300_beam_distributed: -# Tiny generated-data test; launch with mpirun -n 4 as documented in the module. -- condition: - ranges: - system_gpu_count: - gte: 4 - lte: 4 - wildcards: - gpu: - - '*b300*' - linux_distribution_name: ubuntu* - terms: - stage: manual - backend: pytorch - tests: - - unittest/_torch/modeling_v2/models/beam/test_distributed_loading.py TIMEOUT (2) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py index 0918b79d9fd2..ea41b6bb8f94 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_beam_component_support.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Test support shared by the Beam GPU tests under ``models/beam/``. +"""Test support shared by the Beam GPU unit and integration tests. Only harness code lives here: the IEEE fp32 session guard, result bookkeeping that judges with the rules each test file passes in, op-path @@ -31,7 +31,8 @@ from torch.overrides import TorchFunctionMode, resolve_name if TYPE_CHECKING: - from _components.moe.moe import BeamMoeBf16Sm103Tp1 + __extra_import_path__ = ["~/tests/integration/defs"] + from beam._components.moe.moe import BeamMoeBf16Sm103Tp1 from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md index 8affd2039208..c71da3e787d9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md @@ -4,11 +4,12 @@ These TP1 components are checkout-only test helpers, not supported serving APIs or a reference for TP4 bitwise results. Their names describe the isolated component; device and geometry contracts are -documented in each helper. They do not register ModelingV2 targets. The six-layer prefill helper also -supports the local cached-generation tests. Use `LLM()` or `trtllm-serve` for serving. +documented in each helper. They do not register ModelingV2 targets. Use `LLM()` or `trtllm-serve` for serving. Run tests through pytest with the repository's test import plugin. Existing L0 selectors -retain the small single-GPU tests; the large component suites remain manual-only. +retain the small single-GPU tests. The large MoE, sparse-block, six-layer and +four-rank qualification suites live in +[the manual integration tree](../../../../../../integration/defs/beam/README.md). The dense and sparse helpers call the serving family's shared attention function, input normalization (`beam_rmsnorm_fp32`), and activation (`swiglu_fp32`); the other From 4a9b0602d5aa40c8523e642502aac31e08b22742 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Wed, 18 Nov 2026 14:42:32 -0800 Subject: [PATCH 32/33] test: simplify redundant Beam component coverage Remove test-only weight-state machinery, exact allocator accounting and signature introspection. Consolidate fixture/schema checks into functional construction checks and remove redundant dense/router report payloads. Keep direct dense weight comparisons and numerical tolerances unchanged. Keep the full operator matrices; transfer dispatch input-immutability and expert-row correspondence assertions there. The dispatch wrapper retains one 129-token smoke case; routing wrapper checks retain all three token lengths with one seed. Dense path profiling retains lengths 1 and 129 with one seed; dense numerical and optional source-capture matrices remain full. The repeated wrapper/profile seed combinations and exact internal allocation contracts are intentionally no longer tested. Historical generator/cache, NVFP4 reference and native distributed coverage remain unchanged. Validation: 83 tracked-snapshot CI-selection/package-layout checks per PR, pre-commit, six CPU-adapted dispatch arithmetic cases and a corrupt-row negative check passed. Native B300 execution not rerun. Pyrefly unavailable. Signed-off-by: Maxim Gluhovskoi --- .../test_small_model_prefill.py | 31 +++---- .../sparse_block/test_sparse_block.py | 24 +----- .../models/beam/_components/README.md | 2 +- .../dense_block/test_dense_block.py | 82 +++---------------- .../_components/moe_expert/test_expert.py | 16 +--- .../_components/moe_router/test_router.py | 57 ++----------- .../test_checkpoint_layout.py | 44 +--------- .../test_modeling_v2_beam_dispatch_combine.py | 11 +++ .../models/beam/test_dispatch_combine.py | 60 ++------------ 9 files changed, 49 insertions(+), 278 deletions(-) diff --git a/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py b/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py index 4ba248b40be7..5b449ab2c36e 100644 --- a/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py +++ b/tests/integration/defs/beam/_components/six_layer_model/test_small_model_prefill.py @@ -61,8 +61,8 @@ from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.beam_logit_softcap import ( # noqa: E501 beam_logit_softcap, ) -from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 - swiglu_fp32, +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( + swiglu_fp32, # noqa: E501 ) from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_neox_rope import beam_neox_rope from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_nope_query_scale import ( # noqa: E501 @@ -303,25 +303,6 @@ def manager_context(length: int, **overrides): # -------------------------------------------------- tests: configuration and construction -def test_configuration_is_the_fixed_scope() -> None: - assert model_module.NUM_LAYERS == 6 - assert model_module.LAYER_TYPES == tuple(CONFIG["layer_types"]) - assert model_module.MLP_LAYER_TYPES == tuple(CONFIG["mlp_layer_types"]) - assert model_module.ATTENTION_WINDOWS == (512,) * 5 + (1024,) - assert model_module.MAX_TOKENS == model_module.KV_CACHE["max_seq_len"] == 1024 - for field, value in model_module.SUPPORTED_CONFIG.items(): - assert CONFIG[field] == value, field - schema = model_module.weight_schema(model_config()) - assert len(schema) == 2394 - assert sum(1 for n in schema if ".mlp.experts." in n) == 3 * 256 * 3 - fp32 = sorted(n for n, (_, dtype) in schema.items() if dtype == torch.float32) - assert fp32 == sorted( - f"model.layers.{i}.mlp.gate.{t}" - for i in (3, 4, 5) - for t in ("weight", "e_score_correction_bias") - ) - - @pytest.mark.parametrize( "overrides,match", [ @@ -360,6 +341,14 @@ def test_constructs_six_distinct_blocks() -> None: assert model.layers[i].mlp.gate.weight.dtype == torch.float32 tensors = model.checkpoint_tensors() assert len(tensors) == 2394 + assert sum(".mlp.experts." in name for name in tensors) == 3 * 256 * 3 + assert sorted( + name for name, tensor in tensors.items() if tensor.dtype == torch.float32 + ) == sorted( + f"model.layers.{i}.mlp.gate.{name}" + for i in (3, 4, 5) + for name in ("weight", "e_score_correction_bias") + ) spans = sorted( (t.data_ptr(), t.data_ptr() + t.numel() * t.element_size()) for t in tensors.values() ) diff --git a/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py b/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py index d10849e82f23..6002c39e1142 100644 --- a/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py +++ b/tests/integration/defs/beam/_components/sparse_block/test_sparse_block.py @@ -80,8 +80,8 @@ from tensorrt_llm._torch.modeling_v2.catalog.attention.thop_attention import thop_attention from tensorrt_llm._torch.modeling_v2.catalog.gemm.cublas_mm import cublas_mm from tensorrt_llm._torch.modeling_v2.catalog.norm.flashinfer_rmsnorm import flashinfer_rmsnorm -from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( # noqa: E501 - swiglu_fp32, +from tensorrt_llm._torch.modeling_v2.models.beam.ops.activation.swiglu_fp32 import ( + swiglu_fp32, # noqa: E501 ) from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_neox_rope import beam_neox_rope from tensorrt_llm._torch.modeling_v2.models.beam.ops.attention.beam_output_gate import ( @@ -576,26 +576,6 @@ def cache_rows(md: TrtllmAttentionMetadata, length: int) -> tuple[torch.Tensor, # -------------------------------------------------- tests: configuration and construction -def test_configuration_is_the_fixed_scope() -> None: - """One configuration, no open choices: what the component accepts is what is tested.""" - assert CASES == ((17, 1), (17, 17), (17, 129), (41, 1), (41, 17), (41, 129)) - assert block_module.SUPPORTED_LAYER_IDX == LAYER_IDX - for field, value in block_module.SUPPORTED_CONFIG.items(): - assert BLOCK_CONFIG[field] == value, field - assert BLOCK_CONFIG["rope_parameters"]["sliding_attention"] == block_module.SUPPORTED_ROPE - assert BLOCK_CONFIG["layer_types"][LAYER_IDX] == "sliding_attention" - assert BLOCK_CONFIG["mlp_layer_types"][LAYER_IDX] == "sparse" - assert len(BLOCK_CONFIG["layer_types"]) == len(BLOCK_CONFIG["mlp_layer_types"]) == NUM_LAYERS - assert block_module.MAX_TOKENS == 1024 and max(LENGTHS) == reference.MAX_TOKENS - own = block_module.weight_shapes(block_config()) - assert list(own.items()) == list(reference.own_shapes().items()) - assert (reference.HEADS, reference.KV_HEADS, reference.HEAD_DIM) == (48, 8, 128) - assert reference.WINDOW == BLOCK_CONFIG["sliding_window"] == KV_CACHE["max_seq_len"] - assert reference.LAYER_IDX == LAYER_IDX and sum(LAYER_MASK) == 1 and LAYER_MASK[LAYER_IDX] - assert (reference.SEEDS, reference.LENGTHS) == (SEEDS, LENGTHS) - assert torch.cuda.get_device_capability() == SM, "this suite certifies sm_103 only" - - def test_block_constructs_alone() -> None: """The block is built directly from its configuration, without the model around it: its twelve attention and norm tensors and the MoE component, no dense MLP, and block 3's diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md index c71da3e787d9..ce3075055676 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/README.md @@ -19,7 +19,7 @@ declared contract, not against the TP4 model as a bitwise reference. ## Manual QA reports -Tests expose comparison metrics with pytest's `record_property` fixture. Ordinary +Some tests expose additional comparison metrics with pytest's `record_property` fixture. Ordinary pytest runs do not write custom evidence files or collect Git/Slurm metadata. For a manual report, run the opt-in QA script from the repository root: diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py index 36df360a7b32..a1485e5f83f6 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/dense_block/test_dense_block.py @@ -110,10 +110,6 @@ LENGTHS = (1, 17, 129) CASES = tuple((seed, length) for seed in SEEDS for length in LENGTHS) CASE_IDS = [f"seed{seed}-T{length}" for seed, length in CASES] -#: Eager, batch 1, one fresh context: no CUDA graph, no overlap scheduler, no cached prefix. -EXECUTION = types.MappingProxyType( - dict(eager=True, batch_size=1, fresh_context=True, cuda_graph=False, overlap_scheduler=False) -) #: The KV cache the attention op appends to: one bf16 layer, 32-token pages, 512 tokens per #: sequence (the window), one request. KV_CACHE = types.MappingProxyType(dict(tokens_per_block=32, max_seq_len=512, num_blocks=16)) @@ -206,10 +202,6 @@ def ieee_fp32(): yield from support.ieee_fp32_session() -def _execution_state(ieee: dict) -> dict: - return {"math": ieee, "execution": dict(EXECUTION)} - - # ----------------------------------------------------------------------------- comparison @@ -498,7 +490,7 @@ def check(name: str, rule: str | None, actual: torch.Tensor, expected: str) -> d return out -def accounting(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> dict: +def check_loaded_weights(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> None: """The fifteen tensors: the same names, order, shapes, dtypes and values on both sides.""" state = generated(seed)["state"] expected = reference.param_shapes() @@ -508,8 +500,6 @@ def accounting(seed: int, blk: BeamDenseBlockBf16Sm103Tp1) -> dict: assert tuple(loaded[name].shape) == shape and loaded[name].dtype == DTYPE, name assert torch.equal(loaded[name], state[name]), f"{name} differs from the reference's" assert not reference.state_problems(generated(seed)["state_cpu"]) - record = {"tensors": len(loaded), "state_problems": []} - return record # ----------------------------------------------------------------------------- op-path instrumentation @@ -565,26 +555,6 @@ def cache_rows(md: TrtllmAttentionMetadata, length: int) -> tuple[torch.Tensor, # ----------------------------------------------------------------------------- tests: configuration and construction -def test_configuration_is_the_fixed_scope() -> None: - """One configuration, no open choices: what the component accepts is what is tested.""" - assert CASES == ((17, 1), (17, 17), (17, 129), (41, 1), (41, 17), (41, 129)) - assert block_module.DEFAULT_LAYER_IDX == LAYER_IDX - for field, value in block_module.SUPPORTED_CONFIG.items(): - assert BLOCK_CONFIG[field] == value, field - assert BLOCK_CONFIG["rope_parameters"]["sliding_attention"] == block_module.SUPPORTED_ROPE - assert block_module.MAX_TOKENS == 1024 >= max(LENGTHS) - assert reference.BLOCK0.hidden == BLOCK_CONFIG["hidden_size"] - assert reference.BLOCK0.intermediate == BLOCK_CONFIG["intermediate_size"] - assert (reference.BLOCK0.heads, reference.BLOCK0.kv_heads, reference.BLOCK0.head_dim) == ( - 48, - 8, - 128, - ) - assert reference.BLOCK0.window == BLOCK_CONFIG["sliding_window"] == KV_CACHE["max_seq_len"] - assert (reference.SEEDS, reference.LENGTHS) == (SEEDS, LENGTHS) - assert torch.cuda.get_device_capability() == SM, "this suite certifies sm_103 only" - - def test_block_constructs_alone(ieee_fp32) -> None: """The block is built directly from its configuration, without the model around it.""" @@ -729,10 +699,10 @@ def test_rejects_unsupported_inputs_before_dispatch(case) -> None: @pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_block_matches_reference(record_property, seed, length, ieee_fp32) -> None: +def test_block_matches_reference(seed, length, ieee_fp32) -> None: """One case: the whole block, then every operation on its own input.""" blk = loaded_block(seed) - accounting_record = accounting(seed, blk) + check_loaded_weights(seed, blk) ref = reference_record(seed, length) x, positions = case_inputs(seed, length) results = [] @@ -744,17 +714,6 @@ def test_block_matches_reference(record_property, seed, length, ieee_fp32) -> No final.update(reference="reference") results.append(final) results.extend(replay(blk, md, ref, "reference")) - record_property( - f"block-seed{seed}-T{length}", - { - "test": "test_block_matches_reference", - "case": {"seed": seed, "length": length}, - "accounting": accounting_record, - "rules": {k: list(v) for k, v in RULES.items()}, - "results": results, - "execution_state": _execution_state(ieee_fp32), - }, - ) assert_all_pass(results) @@ -775,10 +734,11 @@ def test_block_matches_source_captures(seed, length, ieee_fp32) -> None: assert_all_pass(results) -@pytest.mark.parametrize("seed,length", CASES, ids=CASE_IDS) -def test_forward_runs_the_component_path(record_property, seed, length, ieee_fp32) -> None: +@pytest.mark.parametrize("length", [1, 129]) +def test_forward_runs_the_component_path(length, ieee_fp32) -> None: """One case: the component's own composition ran, op by op, through thop attention on this GPU, with no SDPA and no built-in Beam module on the way.""" + seed = 17 blk = loaded_block(seed) x, positions = case_inputs(seed, length) activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] @@ -802,19 +762,6 @@ def test_forward_runs_the_component_path(record_property, seed, length, ieee_fp3 qkv = attention[0].pop("qkv") if attention else None for c in calls: c.pop("qkv", None) - record = { - "test": "test_forward_runs_the_component_path", - "case": {"seed": seed, "length": length}, - "block": f"{type(blk).__module__}.{type(blk).__qualname__}", - "op_calls": calls, - "cuda_kernels": kernels, - "attention_kernels": [k for k in kernels if support.ATTENTION_KERNEL.search(k)], - "sdpa_cpu_ops": [n for n in cpu_ops if support.SDPA_OPS.search(n)], - "kv_cache": dict(KV_CACHE, manager=type(md.kv_cache_manager).__name__, **cache_mapping), - "execution_state": _execution_state(ieee_fp32), - } - record_property(f"path-seed{seed}-T{length}", record) - assert type(blk).__module__.endswith("dense_block.block") assert tuple(names) == EXPECTED_CALLS, names (call,) = attention @@ -834,8 +781,8 @@ def test_forward_runs_the_component_path(record_property, seed, length, ieee_fp3 # Logical block 0 in a one-layer manager: slot 0, which the core call indexes. assert cache_mapping == {"layer_offsets": {0: 0}, "num_local_layers": 1, "num_layers": 1} assert call["local_layer_idx"] == 0 - assert record["attention_kernels"], "no attention kernel ran on the GPU" - assert not record["sdpa_cpu_ops"], record["sdpa_cpu_ops"] + assert any(support.ATTENTION_KERNEL.search(k) for k in kernels), "no attention kernel ran" + assert not any(support.SDPA_OPS.search(n) for n in cpu_ops), cpu_ops assert torch.cuda.get_device_capability(x.device) == SM assert out.shape == (1, length, 5120) and bool(torch.isfinite(out.float()).all()) # The op appended exactly this call's post-RoPE K and V to the pool. @@ -844,11 +791,12 @@ def test_forward_runs_the_component_path(record_property, seed, length, ieee_fp3 assert torch.equal(keys, k_in) and torch.equal(values, v_in) -def test_detects_a_wrong_residual_coefficient(record_property, ieee_fp32, monkeypatch) -> None: +def test_detects_a_wrong_residual_coefficient(ieee_fp32, monkeypatch) -> None: """The complete-block comparison rejects a zeroed final residual coefficient and accepts the unchanged block, on the same case and the same comparison.""" seed, length = 17, 17 blk = loaded_block(seed) + check_loaded_weights(seed, blk) ref = reference_record(seed, length) x, positions = case_inputs(seed, length) runs = {} @@ -862,16 +810,6 @@ def test_detects_a_wrong_residual_coefficient(record_property, ieee_fp32, monkey out = blk(x, positions, md) runs[label] = compare("block_output", out, ref["block_output"], "block_output") runs[label]["feed_forward_scales"] = list(blk.feed_forward_scales) - record_property( - f"fault-seed{seed}-T{length}", - { - "test": "test_detects_a_wrong_residual_coefficient", - "case": {"seed": seed, "length": length}, - "fault": "final residual coefficient sqrt(2)/sqrt(3) -> 0.0, fixture-scoped", - "runs": runs, - "execution_state": _execution_state(ieee_fp32), - }, - ) assert runs["unchanged"]["passed"] is True, runs["unchanged"]["problems"] assert runs["faulted"]["passed"] is False and runs["faulted"]["violations"] > 0 assert runs["restored"][ diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py index c36103fc7838..631b065298c9 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_expert/test_expert.py @@ -23,7 +23,6 @@ from __future__ import annotations import contextlib -import gc from collections.abc import Iterator import pytest @@ -46,7 +45,6 @@ ) BF16 = torch.bfloat16 -_WEIGHT_BYTES = sum(2 * rows * cols for rows, cols in WEIGHT_SHAPES.values()) @pytest.fixture(scope="module", autouse=True) @@ -113,18 +111,8 @@ def loaded(weights: dict) -> BeamMoeExpertBf16Sm103Tp1: # ----------------------------------------------------------------------------- state -def _requested_cuda_bytes() -> int: - """Bytes the live tensors asked for; unlike memory_allocated, free of block rounding.""" - return torch.cuda.memory_stats()["requested_bytes.all.current"] - - -def test_construction_holds_one_expert_and_nothing_else() -> None: - gc.collect() - torch.cuda.synchronize() - before = _requested_cuda_bytes() +def test_construction_has_expected_parameters() -> None: expert = BeamMoeExpertBf16Sm103Tp1(device="cuda") - torch.cuda.synchronize() - assert _requested_cuda_bytes() - before == _WEIGHT_BYTES == 78_643_200 inventory = { name: (tuple(p.shape), p.dtype, p.device.type, p.requires_grad) for name, p in expert.named_parameters() @@ -134,8 +122,6 @@ def test_construction_holds_one_expert_and_nothing_else() -> None: "up_proj": ((2560, 5120), BF16, "cuda", False), "down_proj": ((5120, 2560), BF16, "cuda", False), } - assert sum(p.numel() for p in expert.parameters()) == 39_321_600 - assert list(expert.buffers()) == [] and list(expert.children()) == [] assert list(expert.checkpoint_tensors()) == list(WEIGHT_SHAPES) diff --git a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py index cbd85171e95d..8d7ac6da52c3 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/_components/moe_router/test_router.py @@ -5,7 +5,9 @@ Configuration under test, fixed in ``_router_cases.py``: 256 experts in 8 groups of 32, 4 groups kept, 8 experts per token, hidden 5120; FP32 router weight and correction bias, BF16 already-normalized input; one sm_103 GPU; eager; generated -parameters; seeds {17, 41} x tokens {1, 17, 129}, and the controlled fixtures there. +parameters; seed 17 x tokens {1, 17, 129}, and the controlled fixtures there. +The operator certification covers both seeds {17, 41}; optional source captures +also retain the full seed/token matrix. What is compared. Every call goes through the component's ``forward``; the op's mapping of that same call is captured, and each of its steps is compared with the op's own @@ -489,9 +491,7 @@ def test_forward_rejects_invalid_input(case, error, match) -> None: # ----------------------------------------------------------------------------- precision -def test_precision_path_is_fp32_without_tf32_or_autocast( - record_property, -) -> None: +def test_precision_path_is_fp32_without_tf32_or_autocast() -> None: """Projection, sigmoid, selection scores, group scores and normalization are fp32, the projection is IEEE fp32 even when the caller has TF32 or bf16 autocast on, and the caller's global precision policy remains unchanged.""" @@ -518,20 +518,6 @@ def test_precision_path_is_fp32_without_tf32_or_autocast( fp64_error = float((route["logits"].double() - fp64_logits).abs().max()) projection = [c for c in trace.calls if c["func"] == "trtllm.cublas_mm"] dtypes = {name: str(t.dtype) for name, t in route.items()} - record_property( - "precision", - { - "test": "test_precision_path_is_fp32_without_tf32_or_autocast", - "ambient_during_call": {"fp32_precision": ambient, "autocast_bf16": True}, - "ambient_after_call": after, - "projection_calls": projection, - "route_dtypes": dtypes, - "logits_vs_fp64_max_abs": fp64_error, - "bitwise_equal_to_plain_call": {k: torch.equal(route[k], plain[k]) for k in route}, - "torch_calls": [c["func"] for c in trace.calls], - "execution_state": _execution_state(), - }, - ) assert ambient == after == "tf32", (ambient, after) assert len(projection) == 1, projection assert projection[0]["math"] == { @@ -553,12 +539,10 @@ def test_precision_path_is_fp32_without_tf32_or_autocast( # ----------------------------------------------------------------------------- the fixed matrix -@pytest.mark.parametrize("seed,tokens", cases.CASES, ids=cases.CASE_IDS) -def test_matrix(record_property, seed, tokens) -> None: - """One case of the fixed matrix: the output contract, every step against the reference - the routing invariants, and the evidence that - the routing ran on the GPU in this component's op with no host synchronization.""" - weight, bias, hidden_cpu = cases.matrix_case(seed, tokens) +@pytest.mark.parametrize("tokens", cases.TOKENS) +def test_wrapper_routes_on_gpu(tokens) -> None: + """The wrapper preserves routing and keeps token calculations on the GPU.""" + weight, bias, hidden_cpu = cases.matrix_case(17, tokens) router = loaded_router(weight, bias) hidden = hidden_cpu.cuda() torch.cuda.synchronize() @@ -594,31 +578,6 @@ def test_matrix(record_property, seed, tokens) -> None: "torch.Tensor.gather", "torch.Tensor.sum", } - record_property( - f"matrix-seed{seed}-T{tokens}", - { - "test": "test_matrix", - "case": {"seed": seed, "tokens": tokens}, - "component": f"{type(router).__module__}.{type(router).__qualname__}", - "op": f"{BeamGroupedTopkRouter.__module__}.{BeamGroupedTopkRouter.__qualname__}", - "rules": dict(cases.RULES, norm_atol=cases.NORM_ATOL), - "results": results, - "hashes": route_hashes(route), - "inputs": { - "weight": tensor_sha256(weight), - "bias": tensor_sha256(bias), - "hidden": tensor_sha256(hidden_cpu), - }, - "ids": route["ids"].tolist(), - "groups": route["groups"].tolist(), - "torch_calls": trace.calls, - "host_transfers": host, - "off_device_outputs": off_device, - "sync_debug_mode": "error", - "cuda_kernels": kernels, - "execution_state": _execution_state(), - }, - ) assert type(router).__module__.endswith("moe_router.router") assert torch.cuda.get_device_capability(hidden.device) == SM assert not host, f"routing moved token data to the host: {host}" diff --git a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py index bd71be4c2d6b..b168cde56429 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/full_model_nvfp4__sm_103__tp4/test_checkpoint_layout.py @@ -498,37 +498,12 @@ def test_wrong_destination_and_short_read_raise(checkpoint) -> None: # ------------------------------------------------------------------ component guards -def _tensor_state(tensor: torch.Tensor) -> tuple: - return ( - id(tensor), - tensor._version, - tensor.data_ptr(), - tuple(tensor.shape), - tuple(tensor.stride()), - tensor.dtype, - tensor.device, - ) - - -def _weight_state(model) -> tuple: - """Snapshot parameter identity, version and storage metadata for the test.""" - return tuple((name, _tensor_state(p)) for name, p in model.named_parameters()) - - -def _check_weight_state(model, state: tuple) -> None: - """Check that a loaded model still matches the test snapshot.""" - if not model.usable: - raise RuntimeError("the component is not usable: no complete load") - if _weight_state(model) != state: - raise RuntimeError("the weights changed after the test snapshot") - - def _sm103() -> bool: return torch.cuda.is_available() and torch.cuda.get_device_capability() == (10, 3) @pytest.mark.skipif(not _sm103(), reason="the component is built for sm_103") -def test_mapping_rejections_and_weight_state_guard(monkeypatch: pytest.MonkeyPatch) -> None: +def test_mapping_rejections_and_unloaded_model(monkeypatch: pytest.MonkeyPatch) -> None: from tensorrt_llm._torch.modeling_v2.models.beam._tp4.component_bf16 import ( # noqa: E501 BeamCheckpointLayoutBf16Sm103Tp4, ) @@ -552,25 +527,8 @@ def test_mapping_rejections_and_weight_state_guard(monkeypatch: pytest.MonkeyPat model = BeamCheckpointLayoutBf16Sm103Tp4(cfg, mapping, layers=(0,)) assert model.vocab_interval == (37984, 75968) assert tuple(model.tensor("model.layers.0.self_attn.k_proj.weight").shape) == (256, 5120) - state = _weight_state(model) - with pytest.raises(RuntimeError): # no complete load: unusable - _check_weight_state(model, state) with pytest.raises(RuntimeError): model.embed(torch.zeros(1, dtype=torch.int64, device="cuda")) - # White-box: mark usable to exercise the guard without a four-rank load. - model.usable = True - _check_weight_state(model, state) - model.tensor("model.layers.0.input_layernorm.weight").add_(0) # in place: version changes - with pytest.raises(RuntimeError): - _check_weight_state(model, state) - state = _weight_state(model) - name = "model__layers__0__self_attn__q_norm__weight" - old = getattr(model, name) - setattr( - model, name, torch.nn.Parameter(old.detach().clone(), requires_grad=False) - ) # same-shape replacement - with pytest.raises(RuntimeError): - _check_weight_state(model, state) @pytest.mark.cpu_only diff --git a/tests/unittest/_torch/modeling_v2/models/beam/ops/moe/test_modeling_v2_beam_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/ops/moe/test_modeling_v2_beam_dispatch_combine.py index d2d5efcedfa9..af6ce700952f 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/ops/moe/test_modeling_v2_beam_dispatch_combine.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/ops/moe/test_modeling_v2_beam_dispatch_combine.py @@ -32,10 +32,21 @@ def test_certified_cells(cell, seed, certifying) -> None: # noqa: F811 — pyte cases.TOP_K, ) called, referenced = experts.fresh(), experts.fresh() + before = (x.clone(), ids.clone(), w.clone()) out = op(x, ids, w, called, spec["routed_scaling_factor"]) ref = op.reference(x, ids, w, referenced, spec["routed_scaling_factor"]) assert out.dtype == torch.bfloat16 and out.shape == (spec["tokens"], cases.HIDDEN) assert cases.bitwise_equal(out, ref) + assert out.device == x.device and bool(torch.isfinite(out).all()) + for actual, saved in zip((x, ids, w), before): + assert cases.bitwise_equal(actual, saved) + assignments = 0 + for expert, rows, _ in called.calls: + tokens = (ids == expert).any(dim=1).nonzero().squeeze(1) + assert rows.shape[0] == tokens.numel() >= 1 + assert cases.bitwise_equal(rows, x[tokens]) + assignments += rows.shape[0] + assert assignments == ids.numel() # One call per selected expert, ascending, on both sides. assert ( [e for e, _, _ in called.calls] diff --git a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py index a2481477054c..62c5108e3646 100644 --- a/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py +++ b/tests/unittest/_torch/modeling_v2/models/beam/test_dispatch_combine.py @@ -23,17 +23,11 @@ from __future__ import annotations -import inspect - import pytest import torch -from tensorrt_llm._torch.modeling_v2.models.beam import dispatch_combine # noqa: E501 -from tensorrt_llm._torch.modeling_v2.models.beam.dispatch_combine import ( # noqa: E501 - dispatch_and_combine, -) -from tensorrt_llm._torch.modeling_v2.models.beam.ops.moe.beam_dispatch_combine import ( # noqa: E501 - beam_dispatch_combine, +from tensorrt_llm._torch.modeling_v2.models.beam.dispatch_combine import ( + dispatch_and_combine, # noqa: E501 ) __extra_import_path__ = ["."] @@ -102,38 +96,6 @@ def run_checked( # ----------------------------------------------------------------------------- the interface -def test_public_callable_and_provider_interface() -> None: - params = list(inspect.signature(dispatch_and_combine).parameters) - assert params == ["hidden_states", "expert_ids", "mixing_weights", "expert_provider"] - assert ( - dispatch_combine.HIDDEN_SIZE, - dispatch_combine.SUPPORTED_CONFIG["num_experts"], - dispatch_combine.EXPERTS_PER_TOKEN, - dispatch_combine.ROUTED_SCALING_FACTOR, - ) == (5120, 256, 8, 2.5) - - -def test_component_holds_no_parameters_and_allocates_nothing_persistent( - experts: cases.AffineExperts, -) -> None: - """No router parameter, expert matrix, module or configuration object exists to build: - the component is a function, its entry holds no state, and after a call only the - returned tensor remains allocated.""" - assert inspect.isfunction(dispatch_and_combine) - assert vars(beam_dispatch_combine) == {} - assert not any( - isinstance(v, (torch.nn.Module, torch.Tensor)) for v in vars(dispatch_combine).values() - ) - x, ids, w = (t.cuda() for t in cases.matrix_case(17, 17)) - a, b = experts.a, experts.b - torch.cuda.synchronize() - before = torch.cuda.memory_stats()["requested_bytes.all.current"] - out = dispatch_and_combine(x, ids, w, lambda e, rows: (rows.float() * a[e] + b[e]).to(BF16)) - torch.cuda.synchronize() - after = torch.cuda.memory_stats()["requested_bytes.all.current"] - assert after - before == out.numel() * out.element_size() - - _T = 17 @@ -235,21 +197,9 @@ def test_rejects_provider_outputs_outside_the_contract( # ----------------------------------------------------------------------------- the matrix -@pytest.mark.parametrize("tokens", cases.TOKENS) -@pytest.mark.parametrize("seed", cases.SEEDS) -def test_matrix_matches_the_source_combination( - seed: int, tokens: int, experts: cases.AffineExperts -) -> None: - out, calls, gold = run_checked(*cases.matrix_case(seed, tokens), experts) - assert cases.bitwise_equal(out, gold["output"]), f"seed {seed}, {tokens} tokens" - # Each expert's output went back to its own tokens: the scattered golden is different. - if tokens > 1: - wrong = cases.golden( - *(t.cuda() for t in cases.matrix_case(seed, tokens)), - experts.fresh(), - fault="wrong_scatter_mapping", - ) - assert not torch.equal(wrong["output"], out) +def test_wrapper_matches_source_combination(experts: cases.AffineExperts) -> None: + out, _, gold = run_checked(*cases.matrix_case(17, 129), experts) + assert cases.bitwise_equal(out, gold["output"]) # ----------------------------------------------------------------------------- fixtures From 43cf5fc5440e6637e17f93d80774bfc9139d8c3f Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Wed, 18 Nov 2026 18:50:09 -0800 Subject: [PATCH 33/33] docs: keep B300 test timing comments platform-specific Signed-off-by: Maxim Gluhovskoi --- tests/integration/test_lists/test-db/l0_b300.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index c294a5323772..aa7f19e7a658 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -164,9 +164,8 @@ l0_b300: - unittest/_torch/modeling_v2/activation TIMEOUT (30) # Beam: lightweight single-GPU entries. High-memory MoE, sparse-block and six-layer # suites live in tests/integration/defs/beam and run manually, outside L0/nightly/LR. - # Startup-inclusive measurements on one GB300 (ARM64, retained cache): operators - # 58s, GEMM/FP4 entries <=23s, expert 35s and router 26s. The B300 x86 operator - # entry measured 83s in CI. Each keeps an independent 30-minute ceiling. + # The B300 x86 operator entry measured 83s in CI. Each entry keeps an independent + # 30-minute ceiling. - unittest/_torch/modeling_v2/models/beam/ops TIMEOUT (30) - unittest/_torch/modeling_v2/models/beam/test_modeling_v2_beam_cublas_mm.py TIMEOUT (30) - unittest/_torch/modeling_v2/models/beam/test_modeling_v2_beam_nvfp4_gemm.py TIMEOUT (30)