Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions 3rdparty/vendor_patches/flashinfer-prims-ts.patch
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
diff --git a/_compile_cache.py b/_compile_cache.py
index b374a65311402982d818d80e9145ab40ed9727a1..cdfa76294f2d854117db62ad36724254006f8cb8 100644
index b374a65..cdfa762 100644
--- a/_compile_cache.py
+++ b/_compile_cache.py
@@ -1,3 +1,4 @@
Expand Down Expand Up @@ -194,7 +194,7 @@ index b374a65311402982d818d80e9145ab40ed9727a1..cdfa76294f2d854117db62ad36724254
f"prims_ts_{family}",
f"{kind}_sm{major}{minor}_{key}",
diff --git a/block_sparse.py b/block_sparse.py
index fa055618705216cddf6b7f3e9ed4c1d680c2af9a..38de4926a045f17c5b6f206f4a20ecbe2532a1c7 100644
index fa05561..38de492 100644
--- a/block_sparse.py
+++ b/block_sparse.py
@@ -1,3 +1,4 @@
Expand Down Expand Up @@ -256,19 +256,19 @@ index fa055618705216cddf6b7f3e9ed4c1d680c2af9a..38de4926a045f17c5b6f206f4a20ecbe
q: torch.Tensor,
paged_kv_cache: PagedKVCache,
diff --git a/context.py b/context.py
index d3f104e4e34b1bcb7ada697f9e76f7013d6ca100..1ec1080b942f60fad22e3e4a1627cad9592697ad 100644
index 898859e..564760b 100644
--- a/context.py
+++ b/context.py
@@ -48,7 +48,7 @@ from ._hardware import get_max_active_clusters

import torch
@@ -56,7 +56,7 @@ from .vc_attention import (
vc_scale_shapes,
)

-from flashinfer.api_logging import flashinfer_experimental_api
+from flashinfer.api_logging import flashinfer_api as flashinfer_experimental_api


if TYPE_CHECKING:
@@ -493,7 +493,7 @@ def _validate_device(device: torch.device) -> int:
@@ -505,7 +505,7 @@ def _validate_device(device: torch.device) -> int:
# Rubin runs through the sm_100f family target; a CuTe DSL older than 4.8
# cannot emit for it unless CUTE_DSL_ARCH=sm_100f is set before import.
if capability == (10, 7):
Expand All @@ -278,7 +278,7 @@ index d3f104e4e34b1bcb7ada697f9e76f7013d6ca100..1ec1080b942f60fad22e3e4a1627cad9
require_cute_dsl_arch(device_index)
return device_index
diff --git a/decode.py b/decode.py
index 4fb096f39849a3801349758360e71b092ee200e9..f91814650c48dcf5b116273742880ac722fed527 100644
index 4fb096f..f918146 100644
--- a/decode.py
+++ b/decode.py
@@ -1,3 +1,4 @@
Expand Down Expand Up @@ -327,15 +327,15 @@ index 4fb096f39849a3801349758360e71b092ee200e9..f91814650c48dcf5b116273742880ac7
q: torch.Tensor,
paged_kv_cache: PagedKVCache,
diff --git a/kernels/fmha_context/fmha_resources.py b/kernels/fmha_context/fmha_resources.py
index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2fe30a922 100644
index fbd1f12..2efae0a 100644
--- a/kernels/fmha_context/fmha_resources.py
+++ b/kernels/fmha_context/fmha_resources.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2026 by FlashInfer team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -1199,8 +1200,6 @@ class GmemQKVResource(MemoryResource):
@@ -1310,8 +1311,6 @@ class GmemQKVResource(MemoryResource):
if cutlass.const_expr(self.cfg.has_uniform_varlen):
seqlen_k = Int32(self.cfg.uniform_seq_len_k)
else:
Expand All @@ -344,7 +344,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
seqlen_k = _load_runtime_seq_len_kv(
self.seqlens_kv, self.max_seq_len_kv, batch_coord
)
@@ -1223,8 +1222,6 @@ class GmemQKVResource(MemoryResource):
@@ -1334,8 +1333,6 @@ class GmemQKVResource(MemoryResource):
kv_request_begin = batch_coord * Int32(self.block_table_row_stride)
kv_page_idx_ub = Int32(self.cfg.max_num_pages_per_seq_kv - 1)
else:
Expand All @@ -353,7 +353,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
kv_request_begin, kv_page_idx_ub = _load_block_table_row_bounds(
Int32(self.block_table_row_stride),
self.cfg,
@@ -1300,13 +1297,9 @@ class GmemQKVResource(MemoryResource):
@@ -1411,13 +1408,9 @@ class GmemQKVResource(MemoryResource):
kv_request_begin = batch_coord * Int32(self.block_table_row_stride)
kv_page_idx_ub = Int32(self.cfg.max_num_pages_per_seq_kv - 1)
else:
Expand All @@ -367,7 +367,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
kv_request_begin, kv_page_idx_ub = _load_block_table_row_bounds(
Int32(self.block_table_row_stride),
self.cfg,
@@ -1743,8 +1736,6 @@ class SmemPageOffsetsKvResource(MemoryResource):
@@ -1854,8 +1847,6 @@ class SmemPageOffsetsKvResource(MemoryResource):
kv_request_begin: Int32,
kv_page_idx_ub: Int32,
) -> None:
Expand All @@ -376,7 +376,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
cfg = self.cfg
# Context's K/V tile index is kv_tile_start + loop_offset; for V the
# producer runs one tile ahead in TAIL, so reuse the same expression
@@ -1835,8 +1826,6 @@ class SmemPageOffsetsKvResource(MemoryResource):
@@ -1946,8 +1937,6 @@ class SmemPageOffsetsKvResource(MemoryResource):
tile_offset: cutlass.Constexpr[int] = 0,
) -> cutlass.Array:
"""Retain one tile's page IDs after its SMEM window is released."""
Expand All @@ -385,7 +385,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
tile_idx = _resolve_kv_tile_idx_context(
stage_info, kv_tile_start, tile_offset=tile_offset
)
@@ -5770,3 +5759,14 @@ class GmemOResource(MemoryResource):
@@ -6541,3 +6530,14 @@ class GmemOResource(MemoryResource):
prims.cp_async_bulk_commit_group()
if cutlass.const_expr(self.cfg.gmem_o_store_wait_after_write):
prims.cp_async_bulk_wait_group(0, read=True)
Expand All @@ -401,7 +401,7 @@ index d17584b725e24235ce0ec90caac174a29d4ab965..af69ee7d51bc1179352d0bf2ba71fad2
+ _resolve_kv_tile_idx_context,
+)
diff --git a/kernels/fmha_decode/fmha_decode_resources/tmem_transformed_kv.py b/kernels/fmha_decode/fmha_decode_resources/tmem_transformed_kv.py
index b22f0f413b7312bab9a5f8e296255fe9aeabe901..a4a87200ca5d3ace5cf899caa9672a1a32c5289e 100644
index b22f0f4..a4a8720 100644
--- a/kernels/fmha_decode/fmha_decode_resources/tmem_transformed_kv.py
+++ b/kernels/fmha_decode/fmha_decode_resources/tmem_transformed_kv.py
@@ -1,3 +1,4 @@
Expand All @@ -419,7 +419,7 @@ index b22f0f413b7312bab9a5f8e296255fe9aeabe901..a4a87200ca5d3ace5cf899caa9672a1a
from .helpers_common import (
_TASK_CACHE_TMEM_BASE_OFFSET,
diff --git a/mla_decode.py b/mla_decode.py
index b62bd1081fbae0e18e0950008bae87c8986677b4..7a305b1ec6f2e860b3627e6d71c562e231c98a52 100644
index b62bd10..7a305b1 100644
--- a/mla_decode.py
+++ b/mla_decode.py
@@ -1,3 +1,4 @@
Expand Down Expand Up @@ -742,7 +742,7 @@ index b62bd1081fbae0e18e0950008bae87c8986677b4..7a305b1ec6f2e860b3627e6d71c562e2
def get_prims_ts_sparse_mla_decode_workspace_size(*plan_args, **plan_kwargs):
"""Return workspace bytes using the same arguments as wrapper.plan().
diff --git a/q_token_kv_block_sparse_metadata.py b/q_token_kv_block_sparse_metadata.py
index e2bbef13c0f552030875e6b433e0fcdb20686cf6..17025dbc49ef997993009ddab7ff2250308f035c 100644
index e2bbef1..17025db 100644
--- a/q_token_kv_block_sparse_metadata.py
+++ b/q_token_kv_block_sparse_metadata.py
@@ -1,3 +1,4 @@
Expand Down
10 changes: 5 additions & 5 deletions 3rdparty/vendor_sources.lock.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ vendors:
patch_digest: sha256:bc9b96a69873e0ddfcba66ab899c666cf52766976f24d850048d2a0bf8d967ff
digest: sha256-tree-v1:d2d382335ccb68b55078523e63835c28b699607227afb81308cb7bb1d85845ac
flashinfer-prims-ts:
url: https://github.com/yuxianq/flashinfer.git
branch: trtllm-prims-ts-dev-20261012
commit: aa709061230a58858375c513056e7733fa74294a
url: https://github.com/harrisonzhy/flashinfer.git
branch: harrisonzhy/trtllm-prims-ts-dev_vc-attn-20261029
Comment thread
harrisonzhy marked this conversation as resolved.
Resolved
commit: 7087b06c22ceb116b2236a938038a00b39f752d1
source: flashinfer/attention/prims_ts
destination: tensorrt_llm/_torch/attention/backends/prims_ts
include:
- '**/*.py'
patch: 3rdparty/vendor_patches/flashinfer-prims-ts.patch
patch_digest: sha256:5cbcaf2b60029742c55f622eadc84b32f49e574038c417972955706157715141
digest: sha256-tree-v1:fa20dc33e52628519dba7f071f5b28a4eb7963768adb6b46eb8000e4fdb805c4
patch_digest: sha256:b4e502ee5bf3d4b4477d42d96d6fa953f1819e82398b40d17812f6fda7165b22
digest: sha256-tree-v1:de7e56b151091c462d22b3c578cc6703167d0a7de995807f83cd25f3f5736354
openengine-proto:
url: https://github.com/ai-dynamo/openengine.git
tag: v0.1.0
Expand Down
55 changes: 53 additions & 2 deletions docs/source/features/visualgen-quantized-attention.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ A recipe is the tuple `(qk_dtype, v_dtype, (q_block_size, k_block_size, v_block_
| `TRTLLM` | `fp8` | `fp8` | `(1, 1, 1)`, `(1, 4, 1)`, `(1, 16, 1)` | SageAttention (FP8 QK) for Blackwell |
| `TRTLLM` | `bf16` | `fp8` | `(0, 0, 0)` | PrimTS QK16PV8 for Blackwell (`TLLM_FMHA_LIBS=+prims_ts`) |
| `TRTLLM` | `fp8` | `fp8` | `(0, 0, 0)` | PrimTS FP8 for Blackwell (`TLLM_FMHA_LIBS=+prims_ts`) |
| `TRTLLM` | `bf16` | `fp8` | `(0, 0, 1)` | VC-Attention-QK16 for Blackwell, V as E4M3 tile residuals with per-channel scales (`vc_attention: true`, `TLLM_FMHA_LIBS=+prims_ts`) |
| `TRTLLM` | `fp8` | `fp8` | `(128, 128, 1)` | VC-Attention-QK8 for Blackwell, Hadamard-rotated E4M3 Q/K scaled per 128 tokens and the same V treatment (`vc_attention: true`, `TLLM_FMHA_LIBS=+prims_ts`) |
| `CUDNN` | `fp8` | `fp8` | `(0, 0, 0)` | cuDNN FP8 |
| `CUDNN` | `mxfp8` | `mxfp8` | `(0, 0, 0)` | cuDNN MXFP8 |
| `CUTEDSL` | `bf16` | `fp8` | `(0, 0, 0)` | QK16PV8 |
Expand All @@ -40,7 +42,7 @@ A recipe is the tuple `(qk_dtype, v_dtype, (q_block_size, k_block_size, v_block_
| `FLASHINFER` | `nvfp4` | `fp8` | `(0, 0, 0)` | NVFP4 Q/K |
| `FLASHINFER` | `nvfp4` | `nvfp4` | `(0, 0, 0)` | NVFP4 Attention |

The `TRTLLM` backend picks the kernel from the block sizes: SageAttention quantizes per block, and the `(0, 0, 0)` per-tensor recipes run on the PrimTS FMHA library, which `TLLM_FMHA_LIBS=+prims_ts` enables.
The `TRTLLM` backend picks the kernel from the block sizes: SageAttention quantizes per block, and the `(0, 0, 0)` per-tensor recipes run on the PrimTS FMHA library, which `TLLM_FMHA_LIBS=+prims_ts` enables. VC-Attention fixes its block sizes to the recipe's scaling granularity; leave them at `0` and the validator sets them, see [VC-Attention in the `TRTLLM` backend](#vc-attention-in-the-trtllm-backend).

### Choosing and Tuning a Recipe

Expand All @@ -53,6 +55,7 @@ Video quality is generally more sensitive to BMM1 accuracy than BMM2 accuracy, s
- On B300/GB300, start with MXFP8 when optimizing the quality-throughput balance. SageAttention with FP8 Q/K remains an alternative when the `TRTLLM` backend is preferred for the surrounding workload.
- For SageAttention with INT8 Q/K, the default `(2, 16, 1)` or `(1, 16, 1)` block-size recipes works well for most cases. Use `(1, 4, 1)` when video quality is not satisfactory.
- The `CUDNN` backend requires Q/K and V to use the same format. Use `fp8` for per-tensor scaling or `mxfp8` for block scaling.
- For VC-Attention (`vc_attention: true`), `vc_repair_budget: 0.005` (0.5% of the tokens as repair rows) runs faster than the default tile means at unchanged quality; `0.005` to `0.02` is the useful range.
- For `CUTEDSL` MXFP8 or NVFP4 recipes, `v_block_size: 1` uses a separate V scale per head and channel, while `v_block_size: 0` uses one tensor-wide V scale. Try the per-channel variant when the tensor-wide scale loses quality.

After a recipe meets the quality target, benchmark its end-to-end throughput with the production workload.
Expand All @@ -68,6 +71,8 @@ After a recipe meets the quality target, benchmark its end-to-end throughput wit
| `v_block_size` | int ≥ 0 | `0` | V block size on the hidden dimension. `0` = one tensor-wide V scale; `1` = one scale per channel. Keep `0` if `v_dtype` defines its own scaling format (e.g., `mxfp8` or `nvfp4`) |
| `smooth_k` | bool | `False` | SageAttention only. Subtract the per-channel K mean before quantizing K. |
| `smooth_v` | bool | `False` | SageAttention only. Subtract the per-channel V mean before quantizing V and add it back to the output. Applied where attention runs on the PrimTS kernels; see [SageAttention in the `TRTLLM` backend](#sageattention-in-the-trtllm-backend). |
| `vc_attention` | bool | `false` | Run the recipe as VC-Attention on the `TRTLLM` backend. See [VC-Attention in the `TRTLLM` backend](#vc-attention-in-the-trtllm-backend). |
| `vc_repair_budget` | float in `[0, 1)` | `0.0` | Fraction of tokens VC-Attention appends as V repair rows instead of restoring tile means. `0` restores the means. `vc_attention` only. |

Routing (`tensorrt_llm/_torch/visual_gen/attention/backends/utils.py`) forwards the validated `quant_attention_config` into the backend constructor: `TrtllmAttention` for `TRTLLM`, `CuDNNAttention` for `CUDNN`, `FlashInferAttention` for `FLASHINFER`, and the dense `CuTeDSLAttention` FMHA backend for `CUTEDSL`.

Expand Down Expand Up @@ -156,6 +161,52 @@ attention_config:

For Hopper SageAttention, set `q_block_size` to `2`.

## VC-Attention in the `TRTLLM` backend

**What it does.** VC-Attention ([arXiv 2609.15810](https://arxiv.org/abs/2609.15810)) quantizes V around its tile means on the PrimTS FMHA library:

- **V** is stored per 128-token K/V tile as a bf16 mean plus E4M3 residuals with one scale per head and channel; the kernel adds the mean back inside the softmax. A k-means permutation groups similar value rows into the same tile.
- **Q and K** stay bf16 under VC-Attention-QK16 (`qk_dtype: bf16`). Under VC-Attention-QK8 (`qk_dtype: fp8`) they are Hadamard-rotated E4M3 with one scale per 128-token block.
- **V-Smooth schedule.** Grouping and demeaning run over the first quarter of the denoising steps, regrouping every fourth step; later steps run plain per-channel E4M3 V. The pipeline publishes the step, so the window follows the sampler's step count.
- **Repair rows.** `vc_repair_budget` replaces the tile means by appending that fraction of the tokens with the largest residual as extra K/V rows. `0.005` is faster than the means at unchanged quality.

**Requirements and behavior.**

- B200/GB200 and B300/GB300 with `TLLM_FMHA_LIBS=+prims_ts`; other GPUs and backends reject `vc_attention`.
- Separate Q, K and V, equal head counts, head dimension 128, dense mask. Not composable with `sparse_attention_config`, `smooth_k` or `smooth_v`.
- The recipe fixes the block sizes, `(0, 0, 1)` for QK16 and `(128, 128, 1)` for QK8; the validator sets them from the defaults and rejects other values.
- With CUDA graphs the V-Smooth phase is a graph key, so regrouping, smoothing and plain steps replay separate graphs.

**Configuration.**

```python
from tensorrt_llm import VisualGenArgs
from tensorrt_llm.visual_gen import AttentionConfig, QuantAttentionConfig

args = VisualGenArgs(
model="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
attention_config=AttentionConfig(
backend="TRTLLM",
quant_attention_config=QuantAttentionConfig(
qk_dtype="fp8",
v_dtype="fp8",
vc_attention=True,
vc_repair_budget=0.005,
),
),
)
```

```yaml
attention_config:
backend: TRTLLM
quant_attention_config:
qk_dtype: fp8
v_dtype: fp8
vc_attention: true
vc_repair_budget: 0.005
```

## FP8 and MXFP8 in the cuDNN Backend

**What it does.** The `CUDNN` backend uses cuDNN fused SDPA. Its quantized recipes use the same format for BMM1 and BMM2:
Expand Down Expand Up @@ -285,5 +336,5 @@ attention_config:
## Interaction With Other Features

- **Linear-layer quantization** (`VisualGenArgs.quant_config`, e.g. FP8 block scales or NVFP4) is independent and can be combined with any attention recipe.
- **Sparse attention.** On `CUTEDSL`, quantized attention and Video Sparse Attention (VSA) are mutually exclusive and rejected by the validator. On `TRTLLM`, Skip Softmax uses the same backend and the SageAttention unit tests exercise the two together.
- **Sparse attention.** On `CUTEDSL`, quantized attention and Video Sparse Attention (VSA) are mutually exclusive and rejected by the validator. On `TRTLLM`, Skip Softmax uses the same backend and the SageAttention unit tests exercise the two together. VC-Attention (`vc_attention: true`) is rejected together with any `sparse_attention_config`.
- **Parallelism.** SageAttention is covered by a multi-GPU Ulysses test (`tests/unittest/_torch/visual_gen/multi_gpu/test_ulysses_sage_attention.py`). The CuTe DSL dense backend produces LSE, so it also composes with Attention2D / Ring context parallelism; the TRTLLM Sage path does not expose LSE through this wrapper.
4 changes: 4 additions & 0 deletions tensorrt_llm/_torch/attention/backends/fmha/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class Fmha(ABC):
supports_skip_correction: ClassVar[bool] = False
supports_block_sparse_inputs: ClassVar[bool] = False
supports_per_tensor_fp8: ClassVar[bool] = False
supports_vc_attention: ClassVar[bool] = False
supports_workspace_reclamation: bool = False
supports_fp4_mla: ClassVar[bool] = False
# Serves a generation call whose QK-norm / RoPE / KV-append prologue the attention module deferred to the
Expand Down Expand Up @@ -136,6 +137,9 @@ def is_supported(
):
logger.debug(f"{type(self).__name__} does not run a deferred attention prologue.")
return False
if getattr(forward_args, "vc_attention", False) and not self.supports_vc_attention:
logger.debug(f"{type(self).__name__} does not support VC-Attention.")
return False
return self._is_supported(q, k, v, metadata, forward_args, phase=phase)

def _is_supported(
Expand Down
Loading